Ticket #394: phrase-settling-pond-update-20120913.patch

xapian-core/api/postlist.cc

@@ -1 +1 @@
 /** @file postlist.cc
  * @brief Abstract base class for postlists.
  */
-/* Copyright (C) 2007,2009,2011 Olly Betts
+/* Copyright (C) 2007,2009,2011,2012 Olly Betts
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -53 +53 @@
 }
 
 PositionList *
-PostList::read_position_list()
+PostList::read_position_list(Xapian::docid)
 {
     throw Xapian::UnimplementedError("OP_NEAR and OP_PHRASE only currently support terms as subqueries");
 }

xapian-core/api/postlist.h

@@ -1 +1 @@
 /** @file postlist.h
  * @brief Abstract base class for postlists.
  */
-/* Copyright (C) 2007,2008,2009,2011 Olly Betts
+/* Copyright (C) 2007,2008,2009,2011,2012 Olly Betts
  * Copyright (C) 2009 Lemur Consulting Ltd
  *
  * This program is free software; you can redistribute it and/or
@@ -113 +113 @@
      */
     virtual double recalc_maxweight() = 0;
 
-    /** Read the position list for the term in the current document and
-     *  return a pointer to it (owned by the PostList).
+    /** Read the position list for the term and return a pointer to it (owned
+     *  by the PostList).
+     *
+     *  The position list is for the specified document id, or the current
+     *  document if no document id is specified.
      *
      *  The default implementation throws Xapian::UnimplementedError.
      */
-    virtual PositionList * read_position_list();
+    virtual PositionList * read_position_list(Xapian::docid other_did = 0);
 
     /** Read the position list for the term in the current document and
      *  return a pointer to it (not owned by the PostList).

xapian-core/api/queryinternal.cc

@@ -248 +248 @@
                   Xapian::termcount window_)
             : op_(op__), begin(begin_), end(end_), window(window_) { }
 
-        PostList * postlist(PostList * pl, const vector<PostList*>& pls) const;
+        PostList * postlist(PostList * pl, const vector<PostList*>& pls,
+                            QueryOptimiser * qopt) const;
     };
 
     list<PosFilter> pos_filters;
@@ -264 +265 @@
 };
 
 PostList *
-AndContext::PosFilter::postlist(PostList * pl, const vector<PostList*>& pls) const
+AndContext::PosFilter::postlist(PostList * pl, const vector<PostList*>& pls,
+                                QueryOptimiser * qopt) const
 try {
     vector<PostList *>::const_iterator terms_begin = pls.begin() + begin;
     vector<PostList *>::const_iterator terms_end = pls.begin() + end;
@@ -273 +275 @@
         pl = new NearPostList(pl, window, terms_begin, terms_end);
     } else if (window == end - begin) {
         AssertEq(op_, Xapian::Query::OP_PHRASE);
-        pl = new ExactPhrasePostList(pl, terms_begin, terms_end);
+        if (qopt->top_and) {
+            vector<PostList *>::const_iterator j;
+            for (j = terms_begin; j != terms_end; ++j) {
+                qopt->pool_terms.push_back(*j);
+            }
+            // We can currently only handle hoisting out one phrase check.
+            // FIXME: Gather a list of checks, not a list of the subqueries in
+            // one check.
+            qopt->top_and = false;
+        } else {
+            pl = new ExactPhrasePostList(pl, terms_begin, terms_end);
+        }
     } else {
         AssertEq(op_, Xapian::Query::OP_PHRASE);
         pl = new PhrasePostList(pl, window, terms_begin, terms_end);
@@ -308 +321 @@
     list<PosFilter>::const_iterator i;
     for (i = pos_filters.begin(); i != pos_filters.end(); ++i) {
         const PosFilter & filter = *i;
-        pl.reset(filter.postlist(pl.release(), pls));
+        pl.reset(filter.postlist(pl.release(), pls, qopt));
     }
 
     // Empty pls so our destructor doesn't delete them all!
@@ -491 +504 @@
                                       QueryOptimiser * qopt,
                                       double factor) const
 {
+    bool top_and = qopt->top_and;
+    qopt->top_and = false;
     ctx.add_postlist(postlist(qopt, factor));
+    qopt->top_and = top_and;
 }
 
 void
@@ -499 +515 @@
                                   QueryOptimiser * qopt,
                                   double factor) const
 {
+    bool top_and = qopt->top_and;
+    qopt->top_and = false;
     ctx.add_postlist(postlist(qopt, factor));
+    qopt->top_and = top_and;
 }
 
 namespace Internal {
@@ -1147 +1166 @@
     LOGCALL(QUERY, PostingIterator::Internal *, "QueryAndNot::postlist", qopt | factor);
     // FIXME: Combine and-like side with and-like stuff above.
     AutoPtr<PostList> l(subqueries[0].internal->postlist(qopt, factor));
+    bool top_and = qopt->top_and;
+    qopt->top_and = false;
     OrContext ctx(subqueries.size() - 1);
     do_or_like(ctx, qopt, 0.0, 0, 1);
     AutoPtr<PostList> r(ctx.postlist(qopt));
+    qopt->top_and = top_and;
     RETURN(new AndNotPostList(l.release(), r.release(),
                               qopt->matcher, qopt->db_size));
 }
@@ -1180 +1202 @@
     LOGCALL(QUERY, PostingIterator::Internal *, "QueryAndMaybe::postlist", qopt | factor);
     // FIXME: Combine and-like side with and-like stuff above.
     AutoPtr<PostList> l(subqueries[0].internal->postlist(qopt, factor));
+    bool top_and = qopt->top_and;
+    qopt->top_and = false;
     OrContext ctx(subqueries.size() - 1);
     do_or_like(ctx, qopt, factor, 0, 1);
     AutoPtr<PostList> r(ctx.postlist(qopt));
+    qopt->top_and = top_and;
     RETURN(new AndMaybePostList(l.release(), r.release(),
                                 qopt->matcher, qopt->db_size));
 }

xapian-core/backends/brass/brass_alldocspostlist.cc

@@ -64 +64 @@
 }
 
 PositionList *
-BrassAllDocsPostList::read_position_list()
+BrassAllDocsPostList::read_position_list(Xapian::docid)
 {
-    LOGCALL(DB, PositionList *, "BrassAllDocsPostList::read_position_list", NO_ARGS);
+    LOGCALL(DB, PositionList *, "BrassAllDocsPostList::read_position_list", "[docid]");
     throw Xapian::InvalidOperationError("BrassAllDocsPostList::read_position_list() not meaningful");
 }
 

xapian-core/backends/brass/brass_alldocspostlist.h

@@ -46 +46 @@
 
     Xapian::termcount get_wdf() const;
 
-    PositionList *read_position_list();
+    PositionList *read_position_list(Xapian::docid other_did = 0);
 
     PositionList *open_position_list() const;
 

xapian-core/backends/brass/brass_postlist.cc

@@ -785 +785 @@
 }
 
 PositionList *
-BrassPostList::read_position_list()
+BrassPostList::read_position_list(Xapian::docid other_did)
 {
-    LOGCALL(DB, PositionList *, "BrassPostList::read_position_list", NO_ARGS);
+    LOGCALL(DB, PositionList *, "BrassPostList::read_position_list", other_did);
+    if (other_did == 0)
+        other_did = did;
     Assert(this_db.get());
-    positionlist.read_data(&this_db->position_table, did, term);
+    positionlist.read_data(&this_db->position_table, other_did, term);
     RETURN(&positionlist);
 }
 

xapian-core/backends/brass/brass_postlist.h

@@ -252 +252 @@
          */
         Xapian::termcount get_wdf() const { Assert(have_started); return wdf; }
 
-        /** Get the list of positions of the term in the current document.
+        /** Get the list of positions of the term.
          */
-        PositionList *read_position_list();
+        PositionList * read_position_list(Xapian::docid other_did = 0);
 
         /** Get the list of positions of the term in the current document.
          */

xapian-core/backends/chert/chert_alldocspostlist.cc

@@ -64 +64 @@
 }
 
 PositionList *
-ChertAllDocsPostList::read_position_list()
+ChertAllDocsPostList::read_position_list(Xapian::docid)
 {
-    LOGCALL(DB, PositionList *, "ChertAllDocsPostList::read_position_list", NO_ARGS);
+    LOGCALL(DB, PositionList *, "ChertAllDocsPostList::read_position_list", "[docid]");
     throw Xapian::InvalidOperationError("ChertAllDocsPostList::read_position_list() not meaningful");
 }
 

xapian-core/backends/chert/chert_alldocspostlist.h

@@ -46 +46 @@
 
     Xapian::termcount get_wdf() const;
 
-    PositionList *read_position_list();
+    PositionList *read_position_list(Xapian::docid other_did = 0);
 
     PositionList *open_position_list() const;
 

xapian-core/backends/chert/chert_modifiedpostlist.cc

@@ -24 +24 @@
 #include "chert_database.h"
 #include "debuglog.h"
 
-ChertModifiedPostList::~ChertModifiedPostList()
-{
-    delete poslist;
-}
-
 void
 ChertModifiedPostList::skip_deletes(double w_min)
 {
@@ -82 +77 @@
 }
 
 PositionList *
-ChertModifiedPostList::read_position_list()
+ChertModifiedPostList::read_position_list(Xapian::docid other_did)
 {
+    if (other_did != 0) {
+        return ChertPostList::read_position_list(other_did);
+    }
     if (it != mods.end() && (ChertPostList::at_end() || it->first <= ChertPostList::get_docid())) {
-        if (poslist) {
-            delete poslist;
-            poslist = NULL;
-        }
-        poslist = this_db->open_position_list(it->first, term);
-        return poslist;
+        return ChertPostList::read_position_list(it->first);
     }
     return ChertPostList::read_position_list();
 }

xapian-core/backends/chert/chert_modifiedpostlist.h

@@ -36 +36 @@
     map<Xapian::docid, pair<char, Xapian::termcount> >::const_iterator it;
     //@}
 
-    /// Pointer to PositionList returned from read_position_list to be deleted.
-    PositionList * poslist;
-
     /// Skip over deleted documents after a next() or skip_to().
     void skip_deletes(double w_min);
 
@@ -48 +45 @@
                           const string & term_,
                           const map<Xapian::docid, pair<char, Xapian::termcount> > & mods_)
         : ChertPostList(this_db_, term_, true),
-          mods(mods_), it(mods.begin()), poslist(0)
+          mods(mods_), it(mods.begin())
     { }
 
-    ~ChertModifiedPostList();
-
     Xapian::doccount get_termfreq() const;
 
     Xapian::docid get_docid() const;
@@ -61 +56 @@
 
     Xapian::termcount get_wdf() const;
 
-    PositionList *read_position_list();
+    PositionList *read_position_list(Xapian::docid other_did = 0);
 
     PositionList *open_position_list() const;
 

xapian-core/backends/chert/chert_postlist.cc

@@ -785 +785 @@
 }
 
 PositionList *
-ChertPostList::read_position_list()
+ChertPostList::read_position_list(Xapian::docid other_did)
 {
-    LOGCALL(DB, PositionList *, "ChertPostList::read_position_list", NO_ARGS);
+    LOGCALL(DB, PositionList *, "ChertPostList::read_position_list", other_did);
+    if (other_did == 0)
+        other_did = did;
     Assert(this_db.get());
-    positionlist.read_data(&this_db->position_table, did, term);
+    positionlist.read_data(&this_db->position_table, other_did, term);
     RETURN(&positionlist);
 }
 

xapian-core/backends/chert/chert_postlist.h

@@ -251 +251 @@
          */
         Xapian::termcount get_wdf() const { Assert(have_started); return wdf; }
 
-        /** Get the list of positions of the term in the current document.
+        /** Get the list of positions of the term.
          */
-        PositionList *read_position_list();
+        PositionList * read_position_list(Xapian::docid other_did = 0);
 
         /** Get the list of positions of the term in the current document.
          */

xapian-core/backends/contiguousalldocspostlist.cc

@@ -60 +60 @@
 }
 
 PositionList *
-ContiguousAllDocsPostList::read_position_list()
+ContiguousAllDocsPostList::read_position_list(Xapian::docid)
 {
     // Throws the same exception.
     return ContiguousAllDocsPostList::open_position_list();

xapian-core/backends/contiguousalldocspostlist.h

@@ -68 +68 @@
     Xapian::termcount get_wdf() const;
 
     /// Throws InvalidOperationError.
-    PositionList *read_position_list();
+    PositionList *read_position_list(Xapian::docid other_did = 0);
 
     /// Throws InvalidOperationError.
     PositionList * open_position_list() const;

xapian-core/backends/inmemory/inmemory_database.cc

@@ -78 +78 @@
 //////////////
 
 InMemoryPostList::InMemoryPostList(intrusive_ptr<const InMemoryDatabase> db_,
-                                   const InMemoryTerm & imterm,
+                                   const InMemoryTerm & imterm_,
                                    const std::string & term_)
         : LeafPostList(term_),
+          imterm(imterm_),
           pos(imterm.docs.begin()),
-          end(imterm.docs.end()),
-          termfreq(imterm.term_freq),
           started(false),
           db(db_)
 {
-    while (pos != end && !pos->valid) ++pos;
+    while (pos != imterm.docs.end() && !pos->valid) ++pos;
 }
 
 Xapian::doccount
 InMemoryPostList::get_termfreq() const
 {
-    return termfreq;
+    return imterm.term_freq;
 }
 
 Xapian::docid
@@ -112 +111 @@
     if (started) {
         Assert(!at_end());
         ++pos;
-        while (pos != end && !pos->valid) ++pos;
+        while (pos != imterm.docs.end() && !pos->valid) ++pos;
     } else {
         started = true;
     }
@@ -141 +140 @@
 InMemoryPostList::at_end() const
 {
     if (db->is_closed()) InMemoryDatabase::throw_database_closed();
-    return (pos == end);
+    return (pos == imterm.docs.end());
 }
 
 string
 InMemoryPostList::get_description() const
 {
-    return "InMemoryPostList " + str(termfreq);
+    return "InMemoryPostList " + str(imterm.term_freq);
 }
 
 Xapian::termcount
@@ -158 +157 @@
 }
 
 PositionList *
-InMemoryPostList::read_position_list()
+InMemoryPostList::read_position_list(Xapian::docid other_did)
 {
     if (db->is_closed()) InMemoryDatabase::throw_database_closed();
-    mypositions.set_data(pos->positions);
+    if (other_did == 0) {
+        mypositions.set_data(pos->positions);
+    } else {
+        vector<InMemoryPosting>::const_iterator p = imterm.docs.begin();
+        while (p != imterm.docs.end() && (!p->valid || p->did < other_did))
+            ++p;
+        if (p != imterm.docs.end() && p->did == other_did)
+            mypositions.set_data(p->positions);
+        else
+            mypositions.set_data(OmDocumentTerm::term_positions());
+    }
     return &mypositions;
 }
 
@@ -329 +338 @@
 }
 
 PositionList *
-InMemoryAllDocsPostList::read_position_list()
+InMemoryAllDocsPostList::read_position_list(Xapian::docid)
 {
     throw Xapian::UnimplementedError("Can't open position list for all docs iterator");
 }

xapian-core/backends/inmemory/inmemory_database.h

@@ -136 +136 @@
 class InMemoryPostList : public LeafPostList {
     friend class InMemoryDatabase;
     private:
+        const InMemoryTerm & imterm;
         vector<InMemoryPosting>::const_iterator pos;
-        vector<InMemoryPosting>::const_iterator end;
-        Xapian::doccount termfreq;
         bool started;
 
         /** List of positions of the current term.
@@ -156 +155 @@
         Xapian::docid       get_docid() const;     // Gets current docid
         Xapian::termcount   get_doclength() const; // Length of current document
         Xapian::termcount   get_wdf() const;       // Within Document Frequency
-        PositionList * read_position_list();
+        PositionList * read_position_list(Xapian::docid other_did = 0);
         PositionList * open_position_list() const;
 
         PostList *next(double w_min); // Moves to next docid
@@ -185 +184 @@
         Xapian::docid       get_docid() const;     // Gets current docid
         Xapian::termcount   get_doclength() const; // Length of current document
         Xapian::termcount   get_wdf() const;       // Within Document Frequency
-        PositionList * read_position_list();
+        PositionList * read_position_list(Xapian::docid other_did = 0);
         PositionList * open_position_list() const;
 
         PostList *next(double w_min);      // Moves to next docid

xapian-core/backends/remote/net_postlist.cc

@@ -52 +52 @@
 }
 
 PositionList *
-NetworkPostList::read_position_list()
-{
-    lastposlist = db->open_position_list(lastdocid, term);
-    return lastposlist.get();
-}
-
-PositionList *
 NetworkPostList::open_position_list() const
 {
     return db->open_position_list(lastdocid, term);

xapian-core/backends/remote/net_postlist.h

@@ -45 +45 @@
 
     Xapian::docid lastdocid;
     Xapian::termcount lastwdf;
-    Xapian::Internal::intrusive_ptr<PositionList> lastposlist;
 
     Xapian::doccount termfreq;
 
@@ -79 +78 @@
     /// Get the Within Document Frequency of the term in the current document.
     Xapian::termcount get_wdf() const;
 
-    /// Read the position list for the term in the current document and
-    /// return a pointer to it (owned by the PostList).
-    PositionList * read_position_list();
+    // We don't need NetworkPostList::read_position_list() since
+    // read_position_list() is only used by the matcher and remote matches are
+    // run as local matches on the remote end, and the results serialised.
 
     /// Read the position list for the term in the current document and
     /// return a pointer to it (not owned by the PostList).

xapian-core/common/submatch.h

@@ -1 +1 @@
 /** @file submatch.h
  *  @brief base class for sub-matchers
  */
-/* Copyright (C) 2006,2007,2009,2011 Olly Betts
+/* Copyright (C) 2006,2007,2009,2011,2012 Olly Betts
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -76 +76 @@
     virtual PostList * get_postlist_and_term_info(MultiMatch *matcher,
         std::map<std::string,
                  Xapian::MSet::Internal::TermFreqAndWeight> *termfreqandwts,
-        Xapian::termcount * total_subqs_ptr)
+        Xapian::termcount * total_subqs_ptr,
+        std::vector<PostList*> & pool_terms)
         = 0;
 };
 

xapian-core/matcher/Makefile.mk

@@ -4 +4 @@
         matcher/branchpostlist.h\
         matcher/collapser.h\
         matcher/const_database_wrapper.h\
+        matcher/exactphrasecheck.h\
         matcher/exactphrasepostlist.h\
         matcher/externalpostlist.h\
         matcher/extraweightpostlist.h\
@@ -42 +43 @@
         matcher/branchpostlist.cc\
         matcher/collapser.cc\
         matcher/const_database_wrapper.cc\
+        matcher/exactphrasecheck.cc\
         matcher/exactphrasepostlist.cc\
         matcher/externalpostlist.cc\
         matcher/localsubmatch.cc\

new file xapian-core/matcher/exactphrasecheck.cc

@@ +1 @@
+/** @file exactphrasecheck.cc
+ * @brief Check if terms form a particular exact phrase.
+ */
+/* Copyright (C) 2006,2007,2009,2012 Olly Betts
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ */
+
+// FIXME: this could probably share code with ExactPhrasePostList.
+
+#include <config.h>
+
+#include "exactphrasecheck.h"
+
+#include "debuglog.h"
+#include "omassert.h"
+#include "api/postlist.h"
+#include "backends/positionlist.h"
+
+#include <algorithm>
+#include <vector>
+
+using namespace std;
+
+class TermCompare {
+    vector<PostList*> & terms;
+
+  public:
+    TermCompare(vector<PostList*> & terms_) : terms(terms_) { }
+
+    bool operator()(unsigned a, unsigned b) const {
+        return terms[a]->get_termfreq_est() < terms[b]->get_termfreq_est();
+    }
+};
+
+ExactPhraseCheck::ExactPhraseCheck(const Xapian::Database & db_,
+                                   const vector<PostList*> &terms_)
+    : db(db_), terms(terms_)
+{
+    if (terms.empty()) {
+        poslists = NULL;
+        order = NULL;
+        return;
+    }
+
+    AssertRel(terms.size(),>,1);
+    size_t n = terms.size();
+    poslists = new PositionList*[n];
+    try {
+        order = new unsigned[n];
+    } catch (...) {
+        delete [] poslists;
+        throw;
+    }
+    for (size_t i = 0; i < n; ++i) order[i] = unsigned(i);
+}
+
+ExactPhraseCheck::~ExactPhraseCheck()
+{
+    delete [] poslists;
+    delete [] order;
+}
+
+void
+ExactPhraseCheck::start_position_list(unsigned i, Xapian::docid did)
+{
+    AssertRel(i,<,terms.size());
+    unsigned index = order[i];
+    poslists[i] = terms[index]->read_position_list(did);
+    poslists[i]->index = index;
+}
+
+bool
+ExactPhraseCheck::operator()(Xapian::docid did)
+{
+    LOGCALL(MATCH, bool, "ExactPhraseCheck::operator()", did);
+
+    if (terms.size() <= 1) RETURN(true);
+
+    // We often don't need to read all the position lists, so rather than using
+    // the shortest position lists first, we approximate by using the terms
+    // with the lowest wdf first.  This will typically give the same or a very
+    // similar order.
+    sort(order, order + terms.size(), TermCompare(terms));
+
+    AssertRel(terms.size(),>,1);
+
+    // If the first term we check only occurs too close to the start of the
+    // document, we only need to read one term's positions.  E.g. search for
+    // "ripe mango" when the only occurrence of 'mango' in the current document
+    // is at position 0.
+    start_position_list(0, did);
+    poslists[0]->skip_to(poslists[0]->index);
+    if (poslists[0]->at_end()) RETURN(false);
+
+    // If we get here, we'll need to read the positionlists for at least two
+    // terms, so check the true positionlist length for the two terms with the
+    // lowest wdf and if necessary swap them so the true shorter one is first.
+    start_position_list(1, did);
+    if (poslists[0]->get_size() < poslists[1]->get_size()) {
+        poslists[1]->skip_to(poslists[1]->index);
+        if (poslists[1]->at_end()) RETURN(false);
+        swap(poslists[0], poslists[1]);
+    }
+
+    {
+        unsigned read_hwm = 1;
+        Xapian::termpos idx0 = poslists[0]->index;
+        do {
+            Xapian::termpos base = poslists[0]->get_position() - idx0;
+            unsigned i = 1;
+            while (true) {
+                if (i > read_hwm) {
+                    read_hwm = i;
+                    start_position_list(i, did);
+                    // FIXME: consider comparing with poslist[0] and swapping
+                    // if less common.  Should we allow for the number of positions
+                    // we've read from poslist[0] already?
+                }
+                Xapian::termpos required = base + poslists[i]->index;
+                poslists[i]->skip_to(required);
+                if (poslists[i]->at_end()) RETURN(false);
+                if (poslists[i]->get_position() != required) break;
+                if (++i == terms.size()) RETURN(true);
+            }
+            poslists[0]->next();
+        } while (!poslists[0]->at_end());
+    }
+    RETURN(false);
+}

new file xapian-core/matcher/exactphrasecheck.h

@@ +1 @@
+/** @file exactphrasecheck.h
+ * @brief Check if terms form a particular exact phrase.
+ */
+/* Copyright (C) 2006,2012 Olly Betts
+ * Copyright (C) 2009 Lemur Consulting Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ */
+
+#ifndef XAPIAN_INCLUDED_EXACTPHRASEPOSTLIST_H
+#define XAPIAN_INCLUDED_EXACTPHRASEPOSTLIST_H
+
+#include "xapian/database.h"
+
+#include <string>
+#include <vector>
+
+typedef Xapian::PositionIterator::Internal PositionList;
+typedef Xapian::PostingIterator::Internal PostList;
+
+/** Check for an exact phrase using positional information.
+ *
+ *  Tests if the terms occur somewhere in the document in the order given
+ *  and at adjacent term positions.
+ */
+class ExactPhraseCheck {
+    Xapian::Database db;
+
+    std::vector<PostList*> terms;
+
+    PositionList ** poslists;
+
+    unsigned * order;
+
+    /// Start reading from the i-th position list.
+    void start_position_list(unsigned i, Xapian::docid did);
+
+  public:
+    ExactPhraseCheck(const Xapian::Database & db_,
+                     const std::vector<PostList*> &terms_);
+
+    ~ExactPhraseCheck();
+
+    /// Test if the specified document contains the terms as an exact phrase.
+    bool operator()(Xapian::docid did);
+};
+
+#endif

xapian-core/matcher/exactphrasepostlist.cc

@@ -102 +102 @@
         swap(poslists[0], poslists[1]);
     }
 
-    unsigned read_hwm = 1;
-    Xapian::termpos idx0 = poslists[0]->index;
-    do {
-        Xapian::termpos base = poslists[0]->get_position() - idx0;
-        unsigned i = 1;
-        while (true) {
-            if (i > read_hwm) {
-                read_hwm = i;
-                start_position_list(i);
-                // FIXME: consider comparing with poslist[0] and swapping
-                // if less common.  Should we allow for the number of positions
-                // we've read from poslist[0] already?
+    {
+        unsigned read_hwm = 1;
+        Xapian::termpos idx0 = poslists[0]->index;
+        do {
+            Xapian::termpos base = poslists[0]->get_position() - idx0;
+            unsigned i = 1;
+            while (true) {
+                if (i > read_hwm) {
+                    read_hwm = i;
+                    start_position_list(i);
+                    // FIXME: consider comparing with poslist[0] and swapping
+                    // if less common.  Should we allow for the number of positions
+                    // we've read from poslist[0] already?
+                }
+                Xapian::termpos required = base + poslists[i]->index;
+                poslists[i]->skip_to(required);
+                if (poslists[i]->at_end()) RETURN(false);
+                if (poslists[i]->get_position() != required) break;
+                if (++i == terms.size()) RETURN(true);
             }
-            Xapian::termpos required = base + poslists[i]->index;
-            poslists[i]->skip_to(required);
-            if (poslists[i]->at_end()) RETURN(false);
-            if (poslists[i]->get_position() != required) break;
-            if (++i == terms.size()) RETURN(true);
-        }
-        poslists[0]->next();
-    } while (!poslists[0]->at_end());
+            poslists[0]->next();
+        } while (!poslists[0]->at_end());
+    }
     RETURN(false);
 }
 

xapian-core/matcher/externalpostlist.cc

@@ -115 +115 @@
 }
 
 PositionList *
-ExternalPostList::read_position_list()
+ExternalPostList::read_position_list(Xapian::docid)
 {
     return NULL;
 }

xapian-core/matcher/externalpostlist.h

@@ -74 +74 @@
 
     double recalc_maxweight();
 
-    PositionList * read_position_list();
+    PositionList * read_position_list(Xapian::docid other_did = 0);
 
     PositionList * open_position_list() const;
 

xapian-core/matcher/localsubmatch.cc

@@ -1 +1 @@
 /** @file localsubmatch.cc
  *  @brief SubMatch class for a local database.
  */
-/* Copyright (C) 2006,2007,2009,2010,2011 Olly Betts
+/* Copyright (C) 2006,2007,2009,2010,2011,2012 Olly Betts
  * Copyright (C) 2007,2008,2009 Lemur Consulting Ltd
  *
  * This program is free software; you can redistribute it and/or modify
@@ -68 +68 @@
 PostList *
 LocalSubMatch::get_postlist_and_term_info(MultiMatch * matcher,
         map<string, Xapian::MSet::Internal::TermFreqAndWeight> * termfreqandwts,
-        Xapian::termcount * total_subqs_ptr)
+        Xapian::termcount * total_subqs_ptr,
+        std::vector<PostList*> & pool_terms)
 {
     LOGCALL(MATCH, PostList *, "LocalSubMatch::get_postlist_and_term_info", matcher | termfreqandwts | total_subqs_ptr);
     (void)matcher;
@@ -82 +83 @@
 
     PostList * pl;
     {
-        QueryOptimiser opt(*db, *this, matcher);
+        QueryOptimiser opt(*db, *this, matcher, pool_terms);
         pl = query.internal->postlist(&opt, 1.0);
         *total_subqs_ptr = opt.get_total_subqs();
     }

xapian-core/matcher/localsubmatch.h

@@ -1 +1 @@
 /** @file localsubmatch.h
  *  @brief SubMatch class for a local database.
  */
-/* Copyright (C) 2006,2007,2009,2010,2011 Olly Betts
+/* Copyright (C) 2006,2007,2009,2010,2011,2012 Olly Betts
  * Copyright (C) 2007 Lemur Consulting Ltd
  *
  * This program is free software; you can redistribute it and/or modify
@@ -89 +89 @@
     PostList * get_postlist_and_term_info(MultiMatch *matcher,
         std::map<std::string,
                  Xapian::MSet::Internal::TermFreqAndWeight> *termfreqandwts,
-        Xapian::termcount * total_subqs_ptr);
+        Xapian::termcount * total_subqs_ptr,
+        std::vector<PostList*> & pool_terms);
 
     /** Convert a postlist into a synonym postlist.
      */

xapian-core/matcher/multimatch.cc

@@ -2 +2 @@
  *
  * Copyright 1999,2000,2001 BrightStation PLC
  * Copyright 2001,2002 Ananova Ltd
- * Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2010,2011 Olly Betts
+ * Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012 Olly Betts
  * Copyright 2003 Orange PCS Ltd
  * Copyright 2003 Sam Liddicott
  * Copyright 2007,2008,2009 Lemur Consulting Ltd
@@ -46 +46 @@
 #include "valuestreamdocument.h"
 #include "weight/weightinternal.h"
 
+#include "exactphrasecheck.h"
+
 #include <xapian/errorhandler.h>
 #include <xapian/matchspy.h>
 #include <xapian/version.h> // For XAPIAN_HAS_REMOTE_BACKEND
@@ -355 +357 @@
     map<string, Xapian::MSet::Internal::TermFreqAndWeight> * termfreqandwts_ptr;
     termfreqandwts_ptr = &termfreqandwts;
 
+    vector<PostList*> pool_terms;
     Xapian::termcount total_subqs = 0;
     // Keep a count of matches which we know exist, but we won't see.  This
     // occurs when a submatch is remote, and returns a lower bound on the
@@ -364 +367 @@
     for (size_t i = 0; i != leaves.size(); ++i) {
         PostList *pl;
         try {
+            if (!is_remote[i]) pool_terms.clear();
             pl = leaves[i]->get_postlist_and_term_info(this,
                                                        termfreqandwts_ptr,
-                                                       &total_subqs);
+                                                       &total_subqs,
+                                                       pool_terms);
             if (termfreqandwts_ptr && !termfreqandwts.empty())
                 termfreqandwts_ptr = NULL;
             if (is_remote[i]) {
@@ -525 +530 @@
     // Is the mset a valid heap?
     bool is_heap = false;
 
+    size_t SETTLING_POND_SIZE = 0;
+    if (!pool_terms.empty()) {
+        const char * sps = getenv("POND_SIZE");
+        SETTLING_POND_SIZE = sps ? atoi(sps) : 100000;
+    }
+    ExactPhraseCheck phrase_check(db, pool_terms);
+    // FIXME: a min/max heap is probably a better choice here (notably more
+    // compact) but the STL doesn't provide one so we'd have to find an
+    // implementation or write one.
+    multimap<double, Xapian::Internal::MSetItem> settling_pond;
     while (true) {
         bool pushback;
 
@@ -646 +661 @@
             new_item.wt = wt;
         }
 
+        if (SETTLING_POND_SIZE) {
+            if (items.size() >= max_msize) {
+                // Settling pond handling...
+                multimap<double, Xapian::Internal::MSetItem>::iterator it;
+                it = settling_pond.upper_bound(-min_weight);
+                settling_pond.erase(it, settling_pond.end());
+
+                settling_pond.insert(make_pair(-new_item.wt, new_item));
+                if (settling_pond.size() < SETTLING_POND_SIZE) {
+                    continue;
+                }
+
+                // Take the last item off the heap, which will have a reasonably
+                // high weight in general.
+                it = settling_pond.begin();
+                swap(new_item, it->second);
+                settling_pond.erase(it);
+            }
+            if (!phrase_check(new_item.did)) continue;
+        }
+
         pushback = true;
 
         // Perform collapsing on key if requested.
@@ -808 +844 @@
         }
     }
 
+    multimap<double, Xapian::Internal::MSetItem>::iterator it;
+    for (it = settling_pond.begin(); it != settling_pond.end(); ++it) {
+        const Xapian::Internal::MSetItem & new_item = it->second;
+        if (new_item.wt < min_weight) break;
+        if (!phrase_check(new_item.did)) continue;
+
+        {
+            ++docs_matched;
+            if (items.size() >= max_msize) {
+                items.push_back(new_item);
+                if (!is_heap) {
+                    is_heap = true;
+                    make_heap(items.begin(), items.end(), mcmp);
+                } else {
+                    push_heap<vector<Xapian::Internal::MSetItem>::iterator,
+                              MSetCmp>(items.begin(), items.end(), mcmp);
+                }
+                pop_heap<vector<Xapian::Internal::MSetItem>::iterator,
+                         MSetCmp>(items.begin(), items.end(), mcmp);
+                items.pop_back();
+
+                min_item = items.front();
+                if (sort_by == REL || sort_by == REL_VAL) {
+                    if (docs_matched >= check_at_least) {
+                        if (sort_by == REL) {
+                            // We're done if this is a forward boolean match
+                            // with only one database (bodgetastic, FIXME
+                            // better if we can!)
+                            if (rare(max_possible == 0 && sort_forward)) {
+                                // In the multi database case, MergePostList
+                                // currently processes each database
+                                // sequentially (which actually may well be
+                                // more efficient) so the docids in general
+                                // won't arrive in order.
+                                // FIXME: is this still good here:
+                                // if (leaves.size() == 1) break;
+                            }
+                        }
+                        if (min_item.wt > min_weight) {
+                            LOGLINE(MATCH, "Setting min_weight to " <<
+                                    min_item.wt << " from " << min_weight);
+                            min_weight = min_item.wt;
+                        }
+                    }
+                }
+            } else {
+                items.push_back(new_item);
+                is_heap = false;
+                if (sort_by == REL && items.size() == max_msize) {
+                    if (docs_matched >= check_at_least) {
+                        // We're done if this is a forward boolean match
+                        // with only one database (bodgetastic, FIXME
+                        // better if we can!)
+                        if (rare(max_possible == 0 && sort_forward)) {
+                            // In the multi database case, MergePostList
+                            // currently processes each database
+                            // sequentially (which actually may well be
+                            // more efficient) so the docids in general
+                            // won't arrive in order.
+                            // FIXME: if (leaves.size() == 1) break;
+                        }
+                    }
+                }
+            }
+        }
+
+        // Keep a track of the greatest weight we've seen.
+        if (new_item.wt > greatest_wt) {
+            greatest_wt = new_item.wt;
+#ifdef XAPIAN_HAS_REMOTE_BACKEND
+            const unsigned int multiplier = db.internal.size();
+            unsigned int db_num = (new_item.did - 1) % multiplier;
+            if (is_remote[db_num]) {
+                // Note that the greatest weighted document came from a remote
+                // database, and which one.
+                greatest_wt_subqs_db_num = db_num;
+            } else
+#endif
+            {
+                greatest_wt_subqs_matched = pl->count_matching_subqs();
+#ifdef XAPIAN_HAS_REMOTE_BACKEND
+                greatest_wt_subqs_db_num = UINT_MAX;
+#endif
+            }
+            if (percent_cutoff) {
+                double w = new_item.wt * percent_cutoff_factor;
+                if (w > min_weight) {
+                    min_weight = w;
+                    if (!is_heap) {
+                        is_heap = true;
+                        make_heap<vector<Xapian::Internal::MSetItem>::iterator,
+                                  MSetCmp>(items.begin(), items.end(), mcmp);
+                    }
+                    while (!items.empty() && items.front().wt < min_weight) {
+                        pop_heap<vector<Xapian::Internal::MSetItem>::iterator,
+                                 MSetCmp>(items.begin(), items.end(), mcmp);
+                        Assert(items.back().wt < min_weight);
+                        items.pop_back();
+                    }
+#ifdef XAPIAN_ASSERTIONS_PARANOID
+                    vector<Xapian::Internal::MSetItem>::const_iterator i;
+                    for (i = items.begin(); i != items.end(); ++i) {
+                        Assert(i->wt >= min_weight);
+                    }
+#endif
+                }
+            }
+        }
+    }
+
+
     // done with posting list tree
     pl.reset(NULL);
 

xapian-core/matcher/multixorpostlist.h

@@ -101 +101 @@
 
     double recalc_maxweight();
 
-    PositionList * read_position_list() {
+    PositionList * read_position_list(Xapian::docid = 0) {
         return NULL;
     }
 

xapian-core/matcher/queryoptimiser.h

@@ -1 +1 @@
 /** @file queryoptimiser.h
  * @brief Details passed around while building PostList tree from Query tree
  */
-/* Copyright (C) 2007,2008,2009,2010,2011 Olly Betts
+/* Copyright (C) 2007,2008,2009,2010,2011,2012 Olly Betts
  * Copyright (C) 2008 Lemur Consulting Ltd
  *
  * This program is free software; you can redistribute it and/or
@@ -49 +49 @@
     Xapian::termcount total_subqs;
 
   public:
+    std::vector<PostList*> & pool_terms;
+
+    bool top_and;
+
     const Xapian::Database::Internal & db;
 
     Xapian::doccount db_size;
@@ -57 +61 @@
 
     QueryOptimiser(const Xapian::Database::Internal & db_,
                    LocalSubMatch & localsubmatch_,
-                   MultiMatch * matcher_)
+                   MultiMatch * matcher_,
+                   std::vector<PostList*> & pool_terms_)
         : localsubmatch(localsubmatch_), total_subqs(0),
-          db(db_), db_size(db.get_doccount()), matcher(matcher_) { }
+          pool_terms(pool_terms_), top_and(true), db(db_),
+          db_size(db.get_doccount()), matcher(matcher_) { }
 
     void inc_total_subqs() { ++total_subqs; }
 

xapian-core/matcher/remotesubmatch.cc

@@ -1 +1 @@
 /** @file remotesubmatch.cc
  *  @brief SubMatch class for a remote database.
  */
-/* Copyright (C) 2006,2007,2009,2010,2011 Olly Betts
+/* Copyright (C) 2006,2007,2009,2010,2011,2012 Olly Betts
  * Copyright (C) 2007,2008 Lemur Consulting Ltd
  *
  * This program is free software; you can redistribute it and/or modify
@@ -62 +62 @@
 PostList *
 RemoteSubMatch::get_postlist_and_term_info(MultiMatch *,
         map<string, Xapian::MSet::Internal::TermFreqAndWeight> * termfreqandwts,
-        Xapian::termcount * total_subqs_ptr)
+        Xapian::termcount * total_subqs_ptr,
+        std::vector<PostList*> &)
 {
-    LOGCALL(MATCH, PostList *, "RemoteSubMatch::get_postlist_and_term_info", Literal("[matcher]") | termfreqandwts | total_subqs_ptr);
+    LOGCALL(MATCH, PostList *, "RemoteSubMatch::get_postlist_and_term_info", Literal("[matcher]") | termfreqandwts | total_subqs_ptr | Literal("[pool_terms]"));
     Xapian::MSet mset;
     db->get_mset(mset, matchspies);
     percent_factor = mset.internal->percent_factor;

xapian-core/matcher/remotesubmatch.h

@@ -1 +1 @@
 /** @file remotesubmatch.h
  *  @brief SubMatch class for a remote database.
  */
-/* Copyright (C) 2006,2007,2009,2011 Olly Betts
+/* Copyright (C) 2006,2007,2009,2011,2012 Olly Betts
  * Copyright (C) 2007,2008 Lemur Consulting Ltd
  *
  * This program is free software; you can redistribute it and/or modify
@@ -72 +72 @@
     PostList * get_postlist_and_term_info(MultiMatch *matcher,
         std::map<std::string,
                  Xapian::MSet::Internal::TermFreqAndWeight> *termfreqandwts,
-        Xapian::termcount * total_subqs_ptr);
+        Xapian::termcount * total_subqs_ptr,
+        std::vector<PostList*> & pool_terms);
 
     /// Get percentage factor - only valid after get_postlist_and_term_info().
     double get_percent_factor() const { return percent_factor; }

xapian-core/matcher/selectpostlist.h

@@ -54 +54 @@
         double get_weight() const { return source->get_weight(); }
         Xapian::termcount get_doclength() const { return source->get_doclength(); }
         double recalc_maxweight() { return source->recalc_maxweight(); }
-        PositionList * read_position_list() { return source->read_position_list(); }
+        PositionList * read_position_list(Xapian::docid other_did = 0) {
+            return source->read_position_list(other_did);
+        }
         PositionList * open_position_list() const { return source->open_position_list(); }
         bool at_end() const { return source->at_end(); }
 

xapian-core/matcher/valuerangepostlist.cc

@@ -103 +103 @@
 }
 
 PositionList *
-ValueRangePostList::read_position_list()
+ValueRangePostList::read_position_list(Xapian::docid)
 {
     Assert(db);
     return NULL;

xapian-core/matcher/valuerangepostlist.h

@@ -73 +73 @@
 
     double recalc_maxweight();
 
-    PositionList * read_position_list();
+    PositionList * read_position_list(Xapian::docid other_did = 0);
 
     PositionList * open_position_list() const;