Ticket #394: phrase-settling-pond.patch

matcher/Makefile.mk

@@ -3 +3 @@
         matcher/andnotpostlist.h\
         matcher/branchpostlist.h\
         matcher/collapser.h\
+        matcher/exactphrasecheck.h\
         matcher/exactphrasepostlist.h\
         matcher/externalpostlist.h\
         matcher/extraweightpostlist.h\
@@ -38 +39 @@
         matcher/andnotpostlist.cc\
         matcher/branchpostlist.cc\
         matcher/collapser.cc\
+        matcher/exactphrasecheck.cc\
         matcher/exactphrasepostlist.cc\
         matcher/externalpostlist.cc\
         matcher/localmatch.cc\

matcher/exactphrasecheck.cc

@@ +1 @@
+/** @file exactphrasecheck.cc
+ * @brief Check if terms form a particular exact phrase.
+ */
+/* Copyright (C) 2006,2007,2009 Olly Betts
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ */
+
+// FIXME: this could probably share code with ExactPhrasePostList.
+
+#include <config.h>
+
+#include "exactphrasecheck.h"
+#include "positionlist.h"
+#include "postlist.h"
+#include "omassert.h"
+#include "omdebug.h"
+
+#include <algorithm>
+#include <vector>
+
+using namespace std;
+
+namespace {
+
+class TermCompare {
+    const Xapian::Database & db;
+    vector<string> & terms;
+
+  public:
+    TermCompare(const Xapian::Database & db_,
+                vector<string> & terms_)
+        : db(db_), terms(terms_) { }
+
+    bool operator()(unsigned a, unsigned b) const {
+        return db.get_collection_freq(terms[a]) < db.get_collection_freq(terms[b]);
+    }
+};
+
+}
+
+ExactPhraseCheck::ExactPhraseCheck(const Xapian::Database & db_,
+                                   const vector<string> &terms_)
+    : db(db_), terms(terms_)
+{
+    if (terms.empty()) {
+        poslists = NULL;
+        order = NULL;
+        return;
+    }
+
+    AssertRel(terms.size(),>,1);
+    size_t n = terms_.size();
+    poslists = new PositionList*[n];
+    try {
+        order = new unsigned[n];
+    } catch (...) {
+        delete [] poslists;
+        throw;
+    }
+    for (size_t i = 0; i < n; ++i) {
+        poslists[i] = NULL;
+        order[i] = unsigned(i);
+    }
+
+    // We often don't need to read all the position lists, so rather than using
+    // the shortest position lists first, we approximate by using the terms
+    // with the lowest collection freq first.  Overall this should give a
+    // similar order.
+    sort(order, order + terms.size(), TermCompare(db, terms));
+}
+
+ExactPhraseCheck::~ExactPhraseCheck()
+{
+    delete [] poslists;
+    delete [] order;
+}
+
+bool
+ExactPhraseCheck::start_position_list(unsigned i, Xapian::docid did)
+{
+    AssertRel(i,<,terms.size());
+    unsigned index = order[i];
+    // FIXME: nasty hacking around with internals and refcount - we should
+    // just add a new Databse::Internal method to do what we want.
+    Xapian::PositionIterator p = db.positionlist_begin(did, terms[index]);
+    PositionList * tmp = p.internal.get();
+    if (!tmp)
+        return false;
+    ++tmp->ref_count;
+    p.internal = poslists[i];
+    poslists[i] = tmp;
+    poslists[i]->index = index;
+    return true;
+}
+
+bool
+ExactPhraseCheck::operator()(Xapian::docid did)
+{
+    DEBUGCALL(MATCH, bool, "ExactPhraseCheck::operator()", did);
+
+    AssertRel(terms.size(),>,1);
+
+    bool result = false;
+    // If the first term we check only occurs too close to the start of the
+    // document, we only need to read one term's positions.  E.g. search for
+    // "ripe mango" when the only occurrence of 'mango' in the current document
+    // is at position 0.
+    if (!start_position_list(0, did))
+        goto done;
+    poslists[0]->skip_to(poslists[0]->index);
+    if (poslists[0]->at_end()) goto done;
+
+    // If we get here, we'll need to read the positionlists for at least two
+    // terms, so check the true positionlist length for the two terms with the
+    // lowest wdf and if necessary swap them so the true shorter one is first.
+    if (!start_position_list(1, did))
+        goto done;
+    if (poslists[0]->get_size() < poslists[1]->get_size()) {
+        poslists[1]->skip_to(poslists[1]->index);
+        if (poslists[1]->at_end()) goto done;
+        swap(poslists[0], poslists[1]);
+    }
+
+    {
+        unsigned read_hwm = 1;
+        Xapian::termpos idx0 = poslists[0]->index;
+        do {
+            Xapian::termpos base = poslists[0]->get_position() - idx0;
+            unsigned i = 1;
+            while (true) {
+                if (i > read_hwm) {
+                    read_hwm = i;
+                    if (!start_position_list(i, did))
+                        goto done;
+                    // FIXME: consider comparing with poslist[0] and swapping
+                    // if less common.  Should we allow for the number of positions
+                    // we've read from poslist[0] already?
+                }
+                Xapian::termpos required = base + poslists[i]->index;
+                poslists[i]->skip_to(required);
+                if (poslists[i]->at_end()) goto done;
+                if (poslists[i]->get_position() != required) break;
+                if (++i == terms.size()) {
+                    result = true;
+                    goto done;
+                }
+            }
+            poslists[0]->next();
+        } while (!poslists[0]->at_end());
+    }
+done:
+    for (size_t i = 0; i < terms.size(); ++i) {
+        delete poslists[i];
+        poslists[i] = NULL;
+    }
+    RETURN(result);
+}

matcher/multimatch.cc

@@ -46 +46 @@
 
 #include "weightinternal.h"
 
+#include "exactphrasecheck.h"
+
 #include <xapian/errorhandler.h>
 #include <xapian/matchspy.h>
 #include <xapian/version.h> // For XAPIAN_HAS_REMOTE_BACKEND
@@ -356 +358 @@
     map<string, Xapian::MSet::Internal::TermFreqAndWeight> * termfreqandwts_ptr;
     termfreqandwts_ptr = &termfreqandwts;
 
+    vector<string> pool_terms;
     Xapian::termcount total_subqs = 0;
     // Keep a count of matches which we know exist, but we won't see.  This
     // occurs when a submatch is remote, and returns a lower bound on the
@@ -365 +368 @@
     for (size_t i = 0; i != leaves.size(); ++i) {
         PostList *pl;
         try {
+            if (!is_remote[i]) pool_terms.clear();
             pl = leaves[i]->get_postlist_and_term_info(this,
                                                        termfreqandwts_ptr,
-                                                       &total_subqs);
+                                                       &total_subqs,
+                                                       pool_terms);
             if (termfreqandwts_ptr && !termfreqandwts.empty())
                 termfreqandwts_ptr = NULL;
             if (is_remote[i]) {
@@ -522 +527 @@
     // Is the mset a valid heap?
     bool is_heap = false;
 
+    size_t SETTLING_POND_SIZE = 0;
+    if (!pool_terms.empty()) {
+        const char * sps = getenv("POND_SIZE");
+        SETTLING_POND_SIZE = sps ? atoi(sps) : 100000;
+    }
+    ExactPhraseCheck phrase_check(db, pool_terms);
+    // FIXME: a min/max heap is probably a better choice here (notably more
+    // compact) but the STL doesn't provide one so we'd have to find an
+    // implementation or write one.
+    multimap<Xapian::weight, Xapian::Internal::MSetItem> settling_pond;
     while (true) {
         bool pushback;
 
@@ -649 +664 @@
             new_item.wt = wt;
         }
 
+        if (SETTLING_POND_SIZE) {
+            if (items.size() >= max_msize) {
+                // Settling pond handling...
+                multimap<Xapian::weight, Xapian::Internal::MSetItem>::iterator it;
+                it = settling_pond.upper_bound(-min_weight);
+                settling_pond.erase(it, settling_pond.end());
+
+                settling_pond.insert(make_pair(-new_item.wt, new_item));
+                if (settling_pond.size() < SETTLING_POND_SIZE) {
+                    continue;
+                }
+
+                // Take the last item off the heap, which will have a reasonably
+                // high weight in general.
+                it = settling_pond.begin();
+                swap(new_item, it->second);
+                settling_pond.erase(it);
+            }
+            if (!phrase_check(new_item.did)) continue;
+        }
+
         pushback = true;
 
         // Perform collapsing on key if requested.
@@ -811 +847 @@
         }
     }
 
+    multimap<Xapian::weight, Xapian::Internal::MSetItem>::iterator it;
+    for (it = settling_pond.begin(); it != settling_pond.end(); ++it) {
+        const Xapian::Internal::MSetItem & new_item = it->second;
+        if (new_item.wt < min_weight) break;
+        if (!phrase_check(new_item.did)) continue;
+
+        {
+            ++docs_matched;
+            if (items.size() >= max_msize) {
+                items.push_back(new_item);
+                if (!is_heap) {
+                    is_heap = true;
+                    make_heap(items.begin(), items.end(), mcmp);
+                } else {
+                    push_heap<vector<Xapian::Internal::MSetItem>::iterator,
+                              MSetCmp>(items.begin(), items.end(), mcmp);
+                }
+                pop_heap<vector<Xapian::Internal::MSetItem>::iterator,
+                         MSetCmp>(items.begin(), items.end(), mcmp);
+                items.pop_back();
+
+                min_item = items.front();
+                if (sort_by == REL || sort_by == REL_VAL) {
+                    if (docs_matched >= check_at_least) {
+                        if (sort_by == REL) {
+                            // We're done if this is a forward boolean match
+                            // with only one database (bodgetastic, FIXME
+                            // better if we can!)
+                            if (rare(max_possible == 0 && sort_forward)) {
+                                // In the multi database case, MergePostList
+                                // currently processes each database
+                                // sequentially (which actually may well be
+                                // more efficient) so the docids in general
+                                // won't arrive in order.
+                                // FIXME: is this still good here:
+                                // if (leaves.size() == 1) break;
+                            }
+                        }
+                        if (min_item.wt > min_weight) {
+                            LOGLINE(MATCH, "Setting min_weight to " <<
+                                    min_item.wt << " from " << min_weight);
+                            min_weight = min_item.wt;
+                        }
+                    }
+                }
+            } else {
+                items.push_back(new_item);
+                is_heap = false;
+                if (sort_by == REL && items.size() == max_msize) {
+                    if (docs_matched >= check_at_least) {
+                        // We're done if this is a forward boolean match
+                        // with only one database (bodgetastic, FIXME
+                        // better if we can!)
+                        if (rare(max_possible == 0 && sort_forward)) {
+                            // In the multi database case, MergePostList
+                            // currently processes each database
+                            // sequentially (which actually may well be
+                            // more efficient) so the docids in general
+                            // won't arrive in order.
+                            // FIXME: if (leaves.size() == 1) break;
+                        }
+                    }
+                }
+            }
+        }
+
+        // Keep a track of the greatest weight we've seen.
+        if (new_item.wt > greatest_wt) {
+            greatest_wt = new_item.wt;
+#ifdef XAPIAN_HAS_REMOTE_BACKEND
+            const unsigned int multiplier = db.internal.size();
+            unsigned int db_num = (new_item.did - 1) % multiplier;
+            if (is_remote[db_num]) {
+                // Note that the greatest weighted document came from a remote
+                // database, and which one.
+                greatest_wt_subqs_db_num = db_num;
+            } else
+#endif
+            {
+                greatest_wt_subqs_matched = pl->count_matching_subqs();
+#ifdef XAPIAN_HAS_REMOTE_BACKEND
+                greatest_wt_subqs_db_num = UINT_MAX;
+#endif
+            }
+            if (percent_cutoff) {
+                Xapian::weight w = new_item.wt * percent_cutoff_factor;
+                if (w > min_weight) {
+                    min_weight = w;
+                    if (!is_heap) {
+                        is_heap = true;
+                        make_heap<vector<Xapian::Internal::MSetItem>::iterator,
+                                  MSetCmp>(items.begin(), items.end(), mcmp);
+                    }
+                    while (!items.empty() && items.front().wt < min_weight) {
+                        pop_heap<vector<Xapian::Internal::MSetItem>::iterator,
+                                 MSetCmp>(items.begin(), items.end(), mcmp);
+                        Assert(items.back().wt < min_weight);
+                        items.pop_back();
+                    }
+#ifdef XAPIAN_ASSERTIONS_PARANOID
+                    vector<Xapian::Internal::MSetItem>::const_iterator i;
+                    for (i = items.begin(); i != items.end(); ++i) {
+                        Assert(i->wt >= min_weight);
+                    }
+#endif
+                }
+            }
+        }
+    }
+
+
     // done with posting list tree
     delete pl;
 

matcher/localmatch.cc

@@ -89 +89 @@
 PostList *
 LocalSubMatch::get_postlist_and_term_info(MultiMatch * matcher,
         map<string, Xapian::MSet::Internal::TermFreqAndWeight> * termfreqandwts,
-        Xapian::termcount * total_subqs_ptr)
+        Xapian::termcount * total_subqs_ptr,
+        std::vector<std::string> & pool_terms)
 {
     DEBUGCALL(MATCH, PostList *, "LocalSubMatch::get_postlist_and_term_info",
               matcher << ", [termfreqandwts], [total_subqs_ptr]");
@@ -98 +99 @@
     // Build the postlist tree for the query.  This calls
     // LocalSubMatch::postlist_from_op_leaf_query() for each term in the query,
     // which builds term_info as a side effect.
-    QueryOptimiser opt(*db, *this, matcher);
+    QueryOptimiser opt(*db, *this, matcher, pool_terms);
     PostList * pl = opt.optimise_query(&orig_query);
     *total_subqs_ptr = opt.get_total_subqueries();
 

matcher/exactphrasecheck.h

@@ +1 @@
+/** @file exactphrasecheck.cc
+ * @brief Check if terms form a particular exact phrase.
+ */
+/* Copyright (C) 2006 Olly Betts
+ * Copyright (C) 2009 Lemur Consulting Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ */
+
+#ifndef XAPIAN_INCLUDED_EXACTPHRASEPOSTLIST_H
+#define XAPIAN_INCLUDED_EXACTPHRASEPOSTLIST_H
+
+#include "xapian/database.h"
+
+#include <string>
+#include <vector>
+
+typedef Xapian::PositionIterator::Internal PositionList;
+
+/** Check for an exact phrase using positional information.
+ *
+ *  Tests if the terms occur somewhere in the document in the order given
+ *  and at adjacent term positions.
+ */
+class ExactPhraseCheck {
+    Xapian::Database db;
+
+    std::vector<std::string> terms;
+
+    PositionList ** poslists;
+
+    unsigned * order;
+
+    /// Start reading from the i-th position list.
+    bool start_position_list(unsigned i, Xapian::docid did);
+
+  public:
+    ExactPhraseCheck(const Xapian::Database & db_,
+                     const std::vector<std::string> &terms_);
+
+    ~ExactPhraseCheck();
+
+    /// Test if the specified document contains the terms as an exact phrase.
+    bool operator()(Xapian::docid did);
+};
+
+#endif

matcher/localmatch.h

@@ -81 +81 @@
     /// Get PostList and term info.
     PostList * get_postlist_and_term_info(MultiMatch *matcher,
         std::map<string, Xapian::MSet::Internal::TermFreqAndWeight> *termfreqandwts,
-        Xapian::termcount * total_subqs_ptr);
+        Xapian::termcount * total_subqs_ptr,
+        std::vector<std::string> & pool_terms);
 
     /** Convert a postlist into a synonym postlist.
      */

matcher/remotesubmatch.cc

@@ -64 +64 @@
 PostList *
 RemoteSubMatch::get_postlist_and_term_info(MultiMatch *,
         map<string, Xapian::MSet::Internal::TermFreqAndWeight> * termfreqandwts,
-        Xapian::termcount * total_subqs_ptr)
+        Xapian::termcount * total_subqs_ptr,
+        std::vector<std::string> &)
 {
     DEBUGCALL(MATCH, PostList *, "RemoteSubMatch::get_postlist_and_term_info",
               "[matcher], " << (void*)termfreqandwts << ", " << (void*)total_subqs_ptr);

matcher/queryoptimiser.cc

@@ -29 +29 @@
 #include "emptypostlist.h"
 #include "exactphrasepostlist.h"
 #include "externalpostlist.h"
+#include "leafpostlist.h"
 #include "multiandpostlist.h"
 #include "multimatch.h"
 #include "omassert.h"
@@ -49 +50 @@
 using namespace std;
 
 PostList *
-QueryOptimiser::do_subquery(const Xapian::Query::Internal * query, double factor)
+QueryOptimiser::do_subquery(const Xapian::Query::Internal * query, double factor,
+                            bool top_and)
 {
     DEBUGCALL(MATCH, PostList *, "QueryOptimiser::do_subquery",
               query << ", " << factor);
@@ -79 +81 @@
         case Xapian::Query::OP_FILTER:
         case Xapian::Query::OP_NEAR:
         case Xapian::Query::OP_PHRASE:
-            RETURN(do_and_like(query, factor));
+            RETURN(do_and_like(query, factor, top_and));
 
         case Xapian::Query::OP_OR:
         case Xapian::Query::OP_XOR:
@@ -99 +101 @@
 
         case Xapian::Query::OP_AND_NOT: {
             AssertEq(query->subqs.size(), 2);
-            PostList * l = do_subquery(query->subqs[0], factor);
-            PostList * r = do_subquery(query->subqs[1], 0.0);
+            PostList * l = do_subquery(query->subqs[0], factor, top_and);
+            PostList * r = do_subquery(query->subqs[1], 0.0, false);
             RETURN(new AndNotPostList(l, r, matcher, db_size));
         }
 
         case Xapian::Query::OP_AND_MAYBE: {
             AssertEq(query->subqs.size(), 2);
-            PostList * l = do_subquery(query->subqs[0], factor);
-            PostList * r = do_subquery(query->subqs[1], factor);
+            PostList * l = do_subquery(query->subqs[0], factor, top_and);
+            PostList * r = do_subquery(query->subqs[1], factor, false);
             RETURN(new AndMaybePostList(l, r, matcher, db_size));
         }
 
@@ -140 +142 @@
             AssertEq(query->subqs.size(), 1);
             double sub_factor = factor;
             if (sub_factor != 0.0) sub_factor *= query->get_dbl_parameter();
-            RETURN(do_subquery(query->subqs[0], sub_factor));
+            RETURN(do_subquery(query->subqs[0], sub_factor, top_and));
         }
 
         default:
@@ -163 +165 @@
 };
 
 PostList *
-QueryOptimiser::do_and_like(const Xapian::Query::Internal *query, double factor)
+QueryOptimiser::do_and_like(const Xapian::Query::Internal *query, double factor,
+                            bool top_and)
 {
     DEBUGCALL(MATCH, PostList *, "QueryOptimiser::do_and_like",
               query << ", " << factor);
@@ -195 +198 @@
             pl = new NearPostList(pl, window, terms);
         } else if (window == filter.end - filter.begin) {
             AssertEq(filter.op, Xapian::Query::OP_PHRASE);
-            pl = new ExactPhrasePostList(pl, terms);
+            if (top_and) {
+                vector<PostList *>::const_iterator j;
+                for (j = terms.begin(); j != terms.end(); ++j) {
+                    // FIXME: avoid dynamic_cast<> here.
+                    LeafPostList * lpl = dynamic_cast<LeafPostList*>(*j);
+                    if (!lpl || lpl->term.empty()) goto cannot_pool;
+                    pool_terms.push_back(lpl->term);
+                }
+                top_and = false;
+            } else {
+cannot_pool:
+                pl = new ExactPhrasePostList(pl, terms);
+            }
         } else {
             AssertEq(filter.op, Xapian::Query::OP_PHRASE);
             pl = new PhrasePostList(pl, window, terms);
@@ -244 +259 @@
         if (is_and_like(subq->op)) {
             do_and_like(subq, factor, and_plists, pos_filters);
         } else {
-            PostList * pl = do_subquery(subq, factor);
+            PostList * pl = do_subquery(subq, factor, false);
             and_plists.push_back(pl);
         }
     }
@@ -255 +270 @@
         size_t begin = end - queries.size();
         Xapian::termcount window = query->parameter;
 
+        if (window == queries.size()) {
+        }
         pos_filters.push_back(PosFilter(op, begin, end, window));
     }
 }
@@ -335 +352 @@
 
     Xapian::Query::Internal::subquery_list::const_iterator q;
     for (q = queries.begin(); q != queries.end(); ++q) {
-        postlists.push_back(do_subquery(*q, factor));
+        postlists.push_back(do_subquery(*q, factor, false));
     }
 
     if (op == Xapian::Query::OP_ELITE_SET) {

matcher/queryoptimiser.h

@@ -44 +44 @@
 
     MultiMatch * matcher;
 
+    std::vector<std::string> & pool_terms;
+
     /** How many leaf subqueries there are.
      *
      *  Used for scaling percentages when the highest weighted document doesn't
@@ -59 +61 @@
      *  @return         A PostList subtree.
      */
     PostList * do_subquery(const Xapian::Query::Internal * query,
-                           double factor);
+                           double factor, bool top_and);
 
     /** Optimise an AND-like Xapian::Query::Internal subtree into a PostList
      *  subtree.
@@ -69 +71 @@
      *
      *  @return         A PostList subtree.
      */
-    PostList * do_and_like(const Xapian::Query::Internal *query, double factor);
+    PostList * do_and_like(const Xapian::Query::Internal *query, double factor,
+                           bool top_and);
 
     /** Optimise an AND-like Xapian::Query::Internal subtree into a PostList
      *  subtree.
@@ -107 +110 @@
   public:
     QueryOptimiser(const Xapian::Database::Internal & db_,
                    LocalSubMatch & localsubmatch_,
-                   MultiMatch * matcher_)
+                   MultiMatch * matcher_,
+                   std::vector<std::string> & pool_terms_)
         : db(db_), db_size(db.get_doccount()), localsubmatch(localsubmatch_),
-          matcher(matcher_), total_subqs(0) { }
+          matcher(matcher_), pool_terms(pool_terms_), total_subqs(0) { }
 
     PostList * optimise_query(Xapian::Query::Internal * query) {
-        return do_subquery(query, 1.0);
+        return do_subquery(query, 1.0, true);
     }
 
     Xapian::termcount get_total_subqueries() const { return total_subqs; }

matcher/remotesubmatch.h

@@ -72 +72 @@
     PostList * get_postlist_and_term_info(MultiMatch *matcher,
         std::map<std::string,
                  Xapian::MSet::Internal::TermFreqAndWeight> *termfreqandwts,
-        Xapian::termcount * total_subqs_ptr);
+        Xapian::termcount * total_subqs_ptr,
+        std::vector<std::string> & pool_terms);
 
     /// Get percentage factor - only valid after get_postlist_and_term_info().
     double get_percent_factor() const { return percent_factor; }

common/leafpostlist.h

@@ -47 +47 @@
 
     bool need_doclength;
 
+  public: // FIXME: avoid having to make term public.
     /// The term name for this postlist ("" for an alldocs postlist).
     std::string term;
 
+  protected:
     /// Only constructable as a base class for derived classes.
     LeafPostList(const std::string & term_)
         : weight(0), need_doclength(false), term(term_) { }

common/submatch.h

@@ -76 +76 @@
     virtual PostList * get_postlist_and_term_info(MultiMatch *matcher,
         std::map<std::string,
                  Xapian::MSet::Internal::TermFreqAndWeight> *termfreqandwts,
-        Xapian::termcount * total_subqs_ptr)
+        Xapian::termcount * total_subqs_ptr,
+        std::vector<std::string> & pool_terms)
         = 0;
 };