Ticket #394: phrase-settling-pond-update-20120911.patch

xapian-core/api/leafpostlist.cc

@@ -102 +102 @@
 {
     return 1;
 }
+
+std::string
+LeafPostList::get_termname() const
+{
+    return term;
+}

xapian-core/api/leafpostlist.h

@@ -86 +86 @@
     TermFreqs get_termfreq_est_using_stats(
         const Xapian::Weight::Internal & stats) const;
 
+    virtual std::string get_termname() const;
+
     Xapian::termcount count_matching_subqs() const;
 };
 

xapian-core/api/postlist.cc

@@ -78 +78 @@
     return 0;
 }
 
+std::string
+PostList::get_termname() const
+{
+    return std::string();
+}
+
 }

xapian-core/api/postlist.h

@@ -194 +194 @@
     /// Count the number of leaf subqueries which match at the current position.
     virtual Xapian::termcount count_matching_subqs() const;
 
+    /// If this is a term, return the name, otherwise return empty string.
+    virtual std::string get_termname() const;
+
     /// Return a string description of this object.
     virtual std::string get_description() const = 0;
 };

xapian-core/api/queryinternal.cc

@@ -248 +248 @@
                   Xapian::termcount window_)
             : op_(op__), begin(begin_), end(end_), window(window_) { }
 
-        PostList * postlist(PostList * pl, const vector<PostList*>& pls) const;
+        PostList * postlist(PostList * pl, const vector<PostList*>& pls,
+                            QueryOptimiser * qopt) const;
     };
 
     list<PosFilter> pos_filters;
@@ -264 +265 @@
 };
 
 PostList *
-AndContext::PosFilter::postlist(PostList * pl, const vector<PostList*>& pls) const
+AndContext::PosFilter::postlist(PostList * pl, const vector<PostList*>& pls,
+                                QueryOptimiser * qopt) const
 try {
     vector<PostList *>::const_iterator terms_begin = pls.begin() + begin;
     vector<PostList *>::const_iterator terms_end = pls.begin() + end;
@@ -273 +275 @@
         pl = new NearPostList(pl, window, terms_begin, terms_end);
     } else if (window == end - begin) {
         AssertEq(op_, Xapian::Query::OP_PHRASE);
-        pl = new ExactPhrasePostList(pl, terms_begin, terms_end);
+        if (qopt->top_and) {
+            vector<PostList *>::const_iterator j;
+            for (j = terms_begin; j != terms_end; ++j) {
+                const string & term = (*j)->get_termname();
+                if (term.empty()) {
+                    // FIXME: Currently all the subqueries must be terms.
+                    qopt->pool_terms.clear();
+                    goto cannot_pool;
+                }
+                qopt->pool_terms.push_back(term);
+            }
+            // We can currently only handle hoisting out one phrase check.
+            // FIXME: Gather a list of checks, not a list of the terms in one
+            // check.
+            qopt->top_and = false;
+        } else {
+cannot_pool:
+            pl = new ExactPhrasePostList(pl, terms_begin, terms_end);
+        }
     } else {
         AssertEq(op_, Xapian::Query::OP_PHRASE);
         pl = new PhrasePostList(pl, window, terms_begin, terms_end);
@@ -308 +328 @@
     list<PosFilter>::const_iterator i;
     for (i = pos_filters.begin(); i != pos_filters.end(); ++i) {
         const PosFilter & filter = *i;
-        pl.reset(filter.postlist(pl.release(), pls));
+        pl.reset(filter.postlist(pl.release(), pls, qopt));
     }
 
     // Empty pls so our destructor doesn't delete them all!
@@ -491 +511 @@
                                       QueryOptimiser * qopt,
                                       double factor) const
 {
+    bool top_and = qopt->top_and;
+    qopt->top_and = false;
     ctx.add_postlist(postlist(qopt, factor));
+    qopt->top_and = top_and;
 }
 
 void
@@ -499 +522 @@
                                   QueryOptimiser * qopt,
                                   double factor) const
 {
+    bool top_and = qopt->top_and;
+    qopt->top_and = false;
     ctx.add_postlist(postlist(qopt, factor));
+    qopt->top_and = top_and;
 }
 
 namespace Internal {
@@ -1147 +1173 @@
     LOGCALL(QUERY, PostingIterator::Internal *, "QueryAndNot::postlist", qopt | factor);
     // FIXME: Combine and-like side with and-like stuff above.
     AutoPtr<PostList> l(subqueries[0].internal->postlist(qopt, factor));
+    bool top_and = qopt->top_and;
+    qopt->top_and = false;
     OrContext ctx(subqueries.size() - 1);
     do_or_like(ctx, qopt, 0.0, 0, 1);
     AutoPtr<PostList> r(ctx.postlist(qopt));
+    qopt->top_and = top_and;
     RETURN(new AndNotPostList(l.release(), r.release(),
                               qopt->matcher, qopt->db_size));
 }
@@ -1180 +1209 @@
     LOGCALL(QUERY, PostingIterator::Internal *, "QueryAndMaybe::postlist", qopt | factor);
     // FIXME: Combine and-like side with and-like stuff above.
     AutoPtr<PostList> l(subqueries[0].internal->postlist(qopt, factor));
+    bool top_and = qopt->top_and;
+    qopt->top_and = false;
     OrContext ctx(subqueries.size() - 1);
     do_or_like(ctx, qopt, factor, 0, 1);
     AutoPtr<PostList> r(ctx.postlist(qopt));
+    qopt->top_and = top_and;
     RETURN(new AndMaybePostList(l.release(), r.release(),
                                 qopt->matcher, qopt->db_size));
 }

xapian-core/common/submatch.h

@@ -76 +76 @@
     virtual PostList * get_postlist_and_term_info(MultiMatch *matcher,
         std::map<std::string,
                  Xapian::MSet::Internal::TermFreqAndWeight> *termfreqandwts,
-        Xapian::termcount * total_subqs_ptr)
+        Xapian::termcount * total_subqs_ptr,
+        std::vector<std::string> & pool_terms)
         = 0;
 };
 

xapian-core/matcher/Makefile.mk

@@ -4 +4 @@
         matcher/branchpostlist.h\
         matcher/collapser.h\
         matcher/const_database_wrapper.h\
+        matcher/exactphrasecheck.h\
         matcher/exactphrasepostlist.h\
         matcher/externalpostlist.h\
         matcher/extraweightpostlist.h\
@@ -42 +43 @@
         matcher/branchpostlist.cc\
         matcher/collapser.cc\
         matcher/const_database_wrapper.cc\
+        matcher/exactphrasecheck.cc\
         matcher/exactphrasepostlist.cc\
         matcher/externalpostlist.cc\
         matcher/localsubmatch.cc\

new file xapian-core/matcher/exactphrasecheck.cc

@@ +1 @@
+/** @file exactphrasecheck.cc
+ * @brief Check if terms form a particular exact phrase.
+ */
+/* Copyright (C) 2006,2007,2009,2012 Olly Betts
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ */
+
+// FIXME: this could probably share code with ExactPhrasePostList.
+
+#include <config.h>
+
+#include "exactphrasecheck.h"
+
+#include "debuglog.h"
+#include "omassert.h"
+#include "backends/positionlist.h"
+
+#include <algorithm>
+#include <vector>
+
+using namespace std;
+
+class TermCompare {
+    const Xapian::Database & db;
+    vector<string> & terms;
+
+  public:
+    TermCompare(const Xapian::Database & db_,
+                vector<string> & terms_)
+        : db(db_), terms(terms_) { }
+
+    bool operator()(unsigned a, unsigned b) const {
+        return db.get_collection_freq(terms[a]) < db.get_collection_freq(terms[b]);
+    }
+};
+
+ExactPhraseCheck::ExactPhraseCheck(const Xapian::Database & db_,
+                                   const vector<string> &terms_)
+    : db(db_), terms(terms_)
+{
+    if (terms.empty()) {
+        poslists = NULL;
+        order = NULL;
+        return;
+    }
+
+    AssertRel(terms.size(),>,1);
+    size_t n = terms_.size();
+    poslists = new PositionList*[n];
+    try {
+        order = new unsigned[n];
+    } catch (...) {
+        delete [] poslists;
+        throw;
+    }
+    for (size_t i = 0; i < n; ++i) {
+        poslists[i] = NULL;
+        order[i] = unsigned(i);
+    }
+
+    // We often don't need to read all the position lists, so rather than using
+    // the shortest position lists first, we approximate by using the terms
+    // with the lowest collection freq first.  Overall this should give a
+    // similar order.
+    sort(order, order + terms.size(), TermCompare(db, terms));
+}
+
+ExactPhraseCheck::~ExactPhraseCheck()
+{
+    delete [] poslists;
+    delete [] order;
+}
+
+bool
+ExactPhraseCheck::start_position_list(unsigned i, Xapian::docid did)
+{
+    AssertRel(i,<,terms.size());
+    unsigned index = order[i];
+    // FIXME: nasty hacking around with internals and ref counts - we should
+    // just add a new Database::Internal method to do what we want.
+    Xapian::PositionIterator p = db.positionlist_begin(did, terms[index]);
+    PositionList * tmp = p.internal;
+    if (!tmp)
+        return false;
+    ++tmp->_refs;
+    p.internal = poslists[i];
+    poslists[i] = tmp;
+    poslists[i]->index = index;
+    return true;
+}
+
+bool
+ExactPhraseCheck::operator()(Xapian::docid did)
+{
+    LOGCALL(MATCH, bool, "ExactPhraseCheck::operator()", did);
+
+    if (terms.size() <= 1) RETURN(true);
+
+    // We often don't need to read all the position lists, so rather than using
+
+    AssertRel(terms.size(),>,1);
+
+    bool result = false;
+    // If the first term we check only occurs too close to the start of the
+    // document, we only need to read one term's positions.  E.g. search for
+    // "ripe mango" when the only occurrence of 'mango' in the current document
+    // is at position 0.
+    if (!start_position_list(0, did))
+        goto done;
+    poslists[0]->skip_to(poslists[0]->index);
+    if (poslists[0]->at_end()) goto done;
+
+    // If we get here, we'll need to read the positionlists for at least two
+    // terms, so check the true positionlist length for the two terms with the
+    // lowest wdf and if necessary swap them so the true shorter one is first.
+    if (!start_position_list(1, did))
+        goto done;
+    if (poslists[0]->get_size() < poslists[1]->get_size()) {
+        poslists[1]->skip_to(poslists[1]->index);
+        if (poslists[1]->at_end()) goto done;
+        swap(poslists[0], poslists[1]);
+    }
+
+    {
+        unsigned read_hwm = 1;
+        Xapian::termpos idx0 = poslists[0]->index;
+        do {
+            Xapian::termpos base = poslists[0]->get_position() - idx0;
+            unsigned i = 1;
+            while (true) {
+                if (i > read_hwm) {
+                    read_hwm = i;
+                    if (!start_position_list(i, did))
+                        goto done;
+                    // FIXME: consider comparing with poslist[0] and swapping
+                    // if less common.  Should we allow for the number of positions
+                    // we've read from poslist[0] already?
+                }
+                Xapian::termpos required = base + poslists[i]->index;
+                poslists[i]->skip_to(required);
+                if (poslists[i]->at_end()) goto done;
+                if (poslists[i]->get_position() != required) break;
+                if (++i == terms.size()) {
+                    result = true;
+                    goto done;
+                }
+            }
+            poslists[0]->next();
+        } while (!poslists[0]->at_end());
+    }
+done:
+    for (size_t i = 0; i < terms.size(); ++i) {
+        delete poslists[i];
+        poslists[i] = NULL;
+    }
+    RETURN(result);
+}

new file xapian-core/matcher/exactphrasecheck.h

@@ +1 @@
+/** @file exactphrasecheck.cc
+ * @brief Check if terms form a particular exact phrase.
+ */
+/* Copyright (C) 2006,2012 Olly Betts
+ * Copyright (C) 2009 Lemur Consulting Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ */
+
+#ifndef XAPIAN_INCLUDED_EXACTPHRASEPOSTLIST_H
+#define XAPIAN_INCLUDED_EXACTPHRASEPOSTLIST_H
+
+#include "xapian/database.h"
+
+#include <string>
+#include <vector>
+
+typedef Xapian::PositionIterator::Internal PositionList;
+
+/** Check for an exact phrase using positional information.
+ *
+ *  Tests if the terms occur somewhere in the document in the order given
+ *  and at adjacent term positions.
+ */
+class ExactPhraseCheck {
+    Xapian::Database db;
+
+    std::vector<std::string> terms;
+
+    PositionList ** poslists;
+
+    unsigned * order;
+
+    /// Start reading from the i-th position list.
+    bool start_position_list(unsigned i, Xapian::docid did);
+
+  public:
+    ExactPhraseCheck(const Xapian::Database & db_,
+                     const std::vector<std::string> &terms_);
+
+    ~ExactPhraseCheck();
+
+    /// Test if the specified document contains the terms as an exact phrase.
+    bool operator()(Xapian::docid did);
+};
+
+#endif

xapian-core/matcher/localsubmatch.cc

@@ -68 +68 @@
 PostList *
 LocalSubMatch::get_postlist_and_term_info(MultiMatch * matcher,
         map<string, Xapian::MSet::Internal::TermFreqAndWeight> * termfreqandwts,
-        Xapian::termcount * total_subqs_ptr)
+        Xapian::termcount * total_subqs_ptr,
+        std::vector<std::string> & pool_terms)
 {
     LOGCALL(MATCH, PostList *, "LocalSubMatch::get_postlist_and_term_info", matcher | termfreqandwts | total_subqs_ptr);
     (void)matcher;
@@ -82 +83 @@
 
     PostList * pl;
     {
-        QueryOptimiser opt(*db, *this, matcher);
+        QueryOptimiser opt(*db, *this, matcher, pool_terms);
         pl = query.internal->postlist(&opt, 1.0);
         *total_subqs_ptr = opt.get_total_subqs();
     }

xapian-core/matcher/localsubmatch.h

@@ -89 +89 @@
     PostList * get_postlist_and_term_info(MultiMatch *matcher,
         std::map<std::string,
                  Xapian::MSet::Internal::TermFreqAndWeight> *termfreqandwts,
-        Xapian::termcount * total_subqs_ptr);
+        Xapian::termcount * total_subqs_ptr,
+        std::vector<std::string> & pool_terms);
 
     /** Convert a postlist into a synonym postlist.
      */

xapian-core/matcher/multimatch.cc

@@ -46 +46 @@
 #include "valuestreamdocument.h"
 #include "weight/weightinternal.h"
 
+#include "exactphrasecheck.h"
+
 #include <xapian/errorhandler.h>
 #include <xapian/matchspy.h>
 #include <xapian/version.h> // For XAPIAN_HAS_REMOTE_BACKEND
@@ -355 +357 @@
     map<string, Xapian::MSet::Internal::TermFreqAndWeight> * termfreqandwts_ptr;
     termfreqandwts_ptr = &termfreqandwts;
 
+    vector<string> pool_terms;
     Xapian::termcount total_subqs = 0;
     // Keep a count of matches which we know exist, but we won't see.  This
     // occurs when a submatch is remote, and returns a lower bound on the
@@ -364 +367 @@
     for (size_t i = 0; i != leaves.size(); ++i) {
         PostList *pl;
         try {
+            if (!is_remote[i]) pool_terms.clear();
             pl = leaves[i]->get_postlist_and_term_info(this,
                                                        termfreqandwts_ptr,
-                                                       &total_subqs);
+                                                       &total_subqs,
+                                                       pool_terms);
             if (termfreqandwts_ptr && !termfreqandwts.empty())
                 termfreqandwts_ptr = NULL;
             if (is_remote[i]) {
@@ -525 +530 @@
     // Is the mset a valid heap?
     bool is_heap = false;
 
+    size_t SETTLING_POND_SIZE = 0;
+    if (!pool_terms.empty()) {
+        const char * sps = getenv("POND_SIZE");
+        SETTLING_POND_SIZE = sps ? atoi(sps) : 100000;
+    }
+    ExactPhraseCheck phrase_check(db, pool_terms);
+    // FIXME: a min/max heap is probably a better choice here (notably more
+    // compact) but the STL doesn't provide one so we'd have to find an
+    // implementation or write one.
+    multimap<double, Xapian::Internal::MSetItem> settling_pond;
     while (true) {
         bool pushback;
 
@@ -646 +661 @@
             new_item.wt = wt;
         }
 
+        if (SETTLING_POND_SIZE) {
+            if (items.size() >= max_msize) {
+                // Settling pond handling...
+                multimap<double, Xapian::Internal::MSetItem>::iterator it;
+                it = settling_pond.upper_bound(-min_weight);
+                settling_pond.erase(it, settling_pond.end());
+
+                settling_pond.insert(make_pair(-new_item.wt, new_item));
+                if (settling_pond.size() < SETTLING_POND_SIZE) {
+                    continue;
+                }
+
+                // Take the last item off the heap, which will have a reasonably
+                // high weight in general.
+                it = settling_pond.begin();
+                swap(new_item, it->second);
+                settling_pond.erase(it);
+            }
+            if (!phrase_check(new_item.did)) continue;
+        }
+
         pushback = true;
 
         // Perform collapsing on key if requested.
@@ -808 +844 @@
         }
     }
 
+    multimap<double, Xapian::Internal::MSetItem>::iterator it;
+    for (it = settling_pond.begin(); it != settling_pond.end(); ++it) {
+        const Xapian::Internal::MSetItem & new_item = it->second;
+        if (new_item.wt < min_weight) break;
+        if (!phrase_check(new_item.did)) continue;
+
+        {
+            ++docs_matched;
+            if (items.size() >= max_msize) {
+                items.push_back(new_item);
+                if (!is_heap) {
+                    is_heap = true;
+                    make_heap(items.begin(), items.end(), mcmp);
+                } else {
+                    push_heap<vector<Xapian::Internal::MSetItem>::iterator,
+                              MSetCmp>(items.begin(), items.end(), mcmp);
+                }
+                pop_heap<vector<Xapian::Internal::MSetItem>::iterator,
+                         MSetCmp>(items.begin(), items.end(), mcmp);
+                items.pop_back();
+
+                min_item = items.front();
+                if (sort_by == REL || sort_by == REL_VAL) {
+                    if (docs_matched >= check_at_least) {
+                        if (sort_by == REL) {
+                            // We're done if this is a forward boolean match
+                            // with only one database (bodgetastic, FIXME
+                            // better if we can!)
+                            if (rare(max_possible == 0 && sort_forward)) {
+                                // In the multi database case, MergePostList
+                                // currently processes each database
+                                // sequentially (which actually may well be
+                                // more efficient) so the docids in general
+                                // won't arrive in order.
+                                // FIXME: is this still good here:
+                                // if (leaves.size() == 1) break;
+                            }
+                        }
+                        if (min_item.wt > min_weight) {
+                            LOGLINE(MATCH, "Setting min_weight to " <<
+                                    min_item.wt << " from " << min_weight);
+                            min_weight = min_item.wt;
+                        }
+                    }
+                }
+            } else {
+                items.push_back(new_item);
+                is_heap = false;
+                if (sort_by == REL && items.size() == max_msize) {
+                    if (docs_matched >= check_at_least) {
+                        // We're done if this is a forward boolean match
+                        // with only one database (bodgetastic, FIXME
+                        // better if we can!)
+                        if (rare(max_possible == 0 && sort_forward)) {
+                            // In the multi database case, MergePostList
+                            // currently processes each database
+                            // sequentially (which actually may well be
+                            // more efficient) so the docids in general
+                            // won't arrive in order.
+                            // FIXME: if (leaves.size() == 1) break;
+                        }
+                    }
+                }
+            }
+        }
+
+        // Keep a track of the greatest weight we've seen.
+        if (new_item.wt > greatest_wt) {
+            greatest_wt = new_item.wt;
+#ifdef XAPIAN_HAS_REMOTE_BACKEND
+            const unsigned int multiplier = db.internal.size();
+            unsigned int db_num = (new_item.did - 1) % multiplier;
+            if (is_remote[db_num]) {
+                // Note that the greatest weighted document came from a remote
+                // database, and which one.
+                greatest_wt_subqs_db_num = db_num;
+            } else
+#endif
+            {
+                greatest_wt_subqs_matched = pl->count_matching_subqs();
+#ifdef XAPIAN_HAS_REMOTE_BACKEND
+                greatest_wt_subqs_db_num = UINT_MAX;
+#endif
+            }
+            if (percent_cutoff) {
+                double w = new_item.wt * percent_cutoff_factor;
+                if (w > min_weight) {
+                    min_weight = w;
+                    if (!is_heap) {
+                        is_heap = true;
+                        make_heap<vector<Xapian::Internal::MSetItem>::iterator,
+                                  MSetCmp>(items.begin(), items.end(), mcmp);
+                    }
+                    while (!items.empty() && items.front().wt < min_weight) {
+                        pop_heap<vector<Xapian::Internal::MSetItem>::iterator,
+                                 MSetCmp>(items.begin(), items.end(), mcmp);
+                        Assert(items.back().wt < min_weight);
+                        items.pop_back();
+                    }
+#ifdef XAPIAN_ASSERTIONS_PARANOID
+                    vector<Xapian::Internal::MSetItem>::const_iterator i;
+                    for (i = items.begin(); i != items.end(); ++i) {
+                        Assert(i->wt >= min_weight);
+                    }
+#endif
+                }
+            }
+        }
+    }
+
+
     // done with posting list tree
     pl.reset(NULL);
 

xapian-core/matcher/queryoptimiser.h

@@ -49 +49 @@
     Xapian::termcount total_subqs;
 
   public:
+    std::vector<std::string> & pool_terms;
+
+    bool top_and;
+
     const Xapian::Database::Internal & db;
 
     Xapian::doccount db_size;
@@ -57 +61 @@
 
     QueryOptimiser(const Xapian::Database::Internal & db_,
                    LocalSubMatch & localsubmatch_,
-                   MultiMatch * matcher_)
+                   MultiMatch * matcher_,
+                   std::vector<std::string> & pool_terms_)
         : localsubmatch(localsubmatch_), total_subqs(0),
-          db(db_), db_size(db.get_doccount()), matcher(matcher_) { }
+          pool_terms(pool_terms_), top_and(true), db(db_),
+          db_size(db.get_doccount()), matcher(matcher_) { }
 
     void inc_total_subqs() { ++total_subqs; }
 

xapian-core/matcher/remotesubmatch.cc

@@ -62 +62 @@
 PostList *
 RemoteSubMatch::get_postlist_and_term_info(MultiMatch *,
         map<string, Xapian::MSet::Internal::TermFreqAndWeight> * termfreqandwts,
-        Xapian::termcount * total_subqs_ptr)
+        Xapian::termcount * total_subqs_ptr,
+        std::vector<std::string> &)
 {
     LOGCALL(MATCH, PostList *, "RemoteSubMatch::get_postlist_and_term_info", Literal("[matcher]") | termfreqandwts | total_subqs_ptr);
     Xapian::MSet mset;

xapian-core/matcher/remotesubmatch.h

@@ -72 +72 @@
     PostList * get_postlist_and_term_info(MultiMatch *matcher,
         std::map<std::string,
                  Xapian::MSet::Internal::TermFreqAndWeight> *termfreqandwts,
-        Xapian::termcount * total_subqs_ptr);
+        Xapian::termcount * total_subqs_ptr,
+        std::vector<std::string> & pool_terms);
 
     /// Get percentage factor - only valid after get_postlist_and_term_info().
     double get_percent_factor() const { return percent_factor; }