Ticket #164: weak_skip_to.patch

matcher/branchpostlist.h

@@ -56 +56 @@
         /** Utility method, to call recalc_maxweight() and do the pruning
          *  if a next() or skip_to() returns non-NULL result.
          */
-        void handle_prune(PostList *&kid, PostList *ret) {
+        bool handle_prune(PostList *&kid, PostList *ret) {
             if (ret) {
                 delete kid;
                 kid = ret;
 
                 // now tell matcher that maximum weights need recalculation.
                 matcher->recalc_maxweight();
+                return true;
             }
+            return false;
         }
 
     public:
@@ -123 +125 @@
     return true;
 }
 
+inline bool
+weak_skip_to_handling_prune(PostList * & pl, Xapian::docid did, Xapian::weight w_min,
+                            MultiMatch *matcher, bool & found)
+{
+    PostList *p = pl->weak_skip_to(did, w_min, found);
+    if (!p) return false;
+    delete pl;
+    pl = p;
+    // now tell matcher that maximum weights need recalculation.
+    if (matcher) matcher->recalc_maxweight();
+    return true;
+}
+
 #endif /* OM_HGUARD_BRANCHPOSTLIST_H */

matcher/andpostlist.h

@@ -37 +37 @@
     protected:
         Xapian::docid head;
         Xapian::weight lmax, rmax;
+
+        bool l_weak;
+        bool r_weak;
+
     private:
         Xapian::doccount dbsize;
 
-        void process_next_or_skip_to(Xapian::weight w_min, PostList *ret);
+        void process_next_or_skip_to(Xapian::weight w_min, Xapian::docid did);
+        void process_half_weak_next_or_skip_to(Xapian::weight w_min, Xapian::docid did);
+
+        /// If r is weak, but l isn't, swap r and l.
+        /// Return true if a swap happened.
+        bool swap_to_make_l_weak();
     public:
         Xapian::doccount get_termfreq_max() const;
         Xapian::doccount get_termfreq_min() const;

matcher/valuerangepostlist.cc

@@ -100 +100 @@
 ValueRangePostList::next(Xapian::weight)
 {
     Assert(db);
-    Xapian::docid lastdocid = db->get_lastdocid();
+    AssertParanoid(lastdocid == db->get_lastdocid());
     while (current < lastdocid) {
         try {
             if (++current == 0) break;
@@ -125 +125 @@
     return next(w_min);
 }
 
+PostList *
+ValueRangePostList::weak_skip_to(Xapian::docid did, Xapian::weight,
+                                 bool & found)
+{
+    Assert(db);
+    AssertParanoid(lastdocid == db->get_lastdocid());
+    if (did <= current) {
+        found = true;
+        return NULL;
+    }
+    current = did;
+    if (current > lastdocid) {
+        db = NULL;
+        found = true;
+        return NULL;
+    }
+
+    try {
+        AutoPtr<Xapian::Document::Internal> doc(db->open_document(current, true));
+        string v = doc->get_value(valno);
+        if (v >= begin && v <= end) {
+            found = true;
+            return NULL;
+        }
+    } catch (const Xapian::DocNotFoundError &) {
+        // Document doesn't exist.
+    }
+    // Haven't determined the position of the next match, other than knowing that
+    // it's not at document "did".
+    found = false;
+    return NULL;
+}
+
 bool
 ValueRangePostList::at_end() const
 {

matcher/valuerangepostlist.h

@@ -33 +33 @@
 
     Xapian::docid current;
 
+    /// Flag, true iff \a current points to a document which satisfies the restriction.
+    bool current_valid;
+
+    /// Highest docid used in the database.
+    Xapian::docid lastdocid;
+
     /// Disallow copying.
     ValueRangePostList(const ValueRangePostList &);
 
@@ -43 +49 @@
     ValueRangePostList(const Xapian::Database::Internal *db_,
                        Xapian::valueno valno_,
                        const std::string &begin_, const std::string &end_)
-        : db(db_), valno(valno_), begin(begin_), end(end_), current(0) { }
+        : db(db_), valno(valno_), begin(begin_), end(end_), current(0), 
+          current_valid(false), lastdocid(db->get_lastdocid())
+    { }
 
     Xapian::doccount get_termfreq_min() const;
 
@@ -69 +77 @@
 
     PostList * skip_to(Xapian::docid, Xapian::weight w_min);
 
+    PostList * weak_skip_to(Xapian::docid did, Xapian::weight w_min, bool & found);
+
+    bool prefer_weak_skip_to() { return true; }
+
     bool at_end() const;
 
     string get_description() const;

matcher/andpostlist.cc

@@ -27 +27 @@
 #include "omdebug.h"
 
 inline void
-AndPostList::process_next_or_skip_to(Xapian::weight w_min, PostList *ret)
+AndPostList::process_next_or_skip_to(Xapian::weight w_min, Xapian::docid did)
 {
     DEBUGCALL(MATCH, void, "AndPostList::process_next_or_skip_to",
-              w_min << ", " << ret);
+              w_min << ", " << did);
+
     head = 0;
-    handle_prune(r, ret);
+    if (did == 0) {
+        if (handle_prune(r, r->next(w_min - lmax)))
+            swap_to_make_l_weak();
+    } else {
+        if  (handle_prune(r, r->skip_to(did, w_min - lmax)))
+            swap_to_make_l_weak();
+    }
+
     DEBUGLINE(MATCH, "r at_end = " << r->at_end());
     if (r->at_end()) return;
 
@@ -41 +49 @@
     Xapian::docid rhead = r->get_docid();
     DEBUGLINE(MATCH, "rhead " << rhead);
     DEBUGLINE(MATCH, "w_min " << w_min << " rmax " << rmax);
-    skip_to_handling_prune(l, rhead, w_min - rmax, matcher);
+    if (skip_to_handling_prune(l, rhead, w_min - rmax, matcher))
+        swap_to_make_l_weak();
     DEBUGLINE(MATCH, "l at_end = " << l->at_end());
     if (l->at_end()) return;
 
@@ -52 +61 @@
         if (lhead < rhead) {
             // FIXME: CSE these w_min values?
             // But note that lmax and rmax may change on recalc_maxweight...
-            skip_to_handling_prune(l, rhead, w_min - rmax, matcher);
+            if (skip_to_handling_prune(l, rhead, w_min - rmax, matcher))
+                swap_to_make_l_weak();
             DEBUGLINE(MATCH, "l at_end = " << l->at_end());
             if (l->at_end()) {
                 head = 0;
@@ -61 +71 @@
             lhead = l->get_docid();
             DEBUGLINE(MATCH, "lhead " << lhead);
         } else {
-            skip_to_handling_prune(r, lhead, w_min - lmax, matcher);
+            if (skip_to_handling_prune(r, lhead, w_min - lmax, matcher))
+                swap_to_make_l_weak();
             DEBUGLINE(MATCH, "r at_end = " << r->at_end());
             if (r->at_end()) {
                 head = 0;
@@ -76 +87 @@
     return;
 }
 
+inline void
+AndPostList::process_half_weak_next_or_skip_to(Xapian::weight w_min, Xapian::docid did)
+{
+    DEBUGCALL(MATCH, void, "AndPostList::process_half_weak_next_or_skip_to",
+              w_min << ", " << did);
+
+    bool pruned = false;
+
+    head = 0;
+    if (did == 0) {
+        pruned = handle_prune(r, r->next(w_min - lmax));
+    } else {
+        pruned = handle_prune(r, r->skip_to(did, w_min - lmax));
+    }
+
+    DEBUGLINE(MATCH, "r at_end = " << r->at_end());
+    if (r->at_end()) return;
+
+    // r has just been advanced by next or skip_to so must be > head
+    // (and head is the current position of l)
+    Xapian::docid rhead = r->get_docid();
+    DEBUGLINE(MATCH, "rhead " << rhead);
+    DEBUGLINE(MATCH, "w_min " << w_min << " rmax " << rmax);
+    bool found = false;
+    Xapian::docid lhead = rhead;
+
+    while (true) {
+        if (pruned && swap_to_make_l_weak()) {
+            DEBUGLINE(MATCH, "swapped l and r - swapping lhead and rhead");
+            Xapian::docid tmp = lhead;
+            lhead = rhead;
+            rhead = tmp;
+        }
+
+        if (lhead == rhead) {
+            if (found) break;
+            // Not found, therefore l needs to be advanced, therefore so does
+            // r, but it's probably more efficient to move r first.
+            pruned = next_handling_prune(r, w_min - lmax, matcher);
+            DEBUGLINE(MATCH, "r at_end = " << r->at_end());
+            if (r->at_end()) return;
+            rhead = r->get_docid();
+            DEBUGLINE(MATCH, "rhead " << rhead);
+        } else if (lhead < rhead) {
+            pruned = weak_skip_to_handling_prune(l, rhead, w_min - rmax, matcher, found);
+            DEBUGLINE(MATCH, "l at_end = " << l->at_end());
+            if (l->at_end()) return;
+            lhead = l->get_docid();
+            DEBUGLINE(MATCH, "l found = " << found << ", lhead = " << lhead);
+        } else if (lhead > rhead) {
+            pruned = skip_to_handling_prune(r, lhead, w_min - lmax, matcher);
+            DEBUGLINE(MATCH, "r at_end = " << r->at_end());
+            if (r->at_end()) return;
+            rhead = r->get_docid();
+            DEBUGLINE(MATCH, "rhead " << rhead);
+        }
+    }
+
+    head = lhead;
+    return;
+}
+
 AndPostList::AndPostList(PostList *left_, PostList *right_,
                          MultiMatch *matcher_,
                          Xapian::doccount dbsize_,
@@ -90 +163 @@
         lmax = l->get_maxweight();
         rmax = r->get_maxweight();
     }
+
+    swap_to_make_l_weak();
 }
 
+bool
+AndPostList::swap_to_make_l_weak()
+{
+    DEBUGCALL(MATCH, bool, "AndPostList::swap_to_make_l_weak", "");
+    l_weak = l->prefer_weak_skip_to();
+    r_weak = r->prefer_weak_skip_to();
+    DEBUGLINE(MATCH, "l_weak=" << l_weak << ", r_weak=" << r_weak);
+    if (l_weak != r_weak) {
+        if (r_weak) {
+            // Swap, so that l is weak if only one subtree is weak. 
+            PostList * tmp_pl = r; r = l; l = tmp_pl;
+            Xapian::weight tmp_wt = rmax; rmax = lmax; lmax = tmp_wt;
+            l_weak = true;
+            r_weak = false;
+            AssertEq(l_weak, l->prefer_weak_skip_to());
+            AssertEq(r_weak, r->prefer_weak_skip_to());
+            RETURN(true);
+        }
+    }
+    RETURN(false);
+}
+
 PostList *
 AndPostList::next(Xapian::weight w_min)
 {
     DEBUGCALL(MATCH, PostList *, "AndPostList::next", w_min);
-    process_next_or_skip_to(w_min, r->next(w_min - lmax));
+    if (l_weak && !r_weak) {
+        process_half_weak_next_or_skip_to(w_min, 0);
+    } else {
+        process_next_or_skip_to(w_min, 0);
+    }
     RETURN(NULL);
 }
 
@@ -104 +205 @@
 AndPostList::skip_to(Xapian::docid did, Xapian::weight w_min)
 {
     DEBUGCALL(MATCH, PostList *, "AndPostList::skip_to", did << ", " << w_min);
-    if (did > head)
-        process_next_or_skip_to(w_min, r->skip_to(did, w_min - lmax));
+    if (did > head) {
+        if (l_weak && !r_weak) {
+            process_half_weak_next_or_skip_to(w_min, did);
+        } else {
+            process_next_or_skip_to(w_min, did);
+        }
+    }
     RETURN(NULL);
 }
 

common/postlist.h

@@ -180 +180 @@
          */
         virtual Internal *skip_to(Xapian::docid, Xapian::weight w_min) = 0;
 
+        /** Attempt to find the first entry in the postlist after a given
+         *  location.
+         *
+         *  This attempts to find the first entry in the postlist at or after a
+         *  given location, but if it can't quickly determine which document
+         *  this is will give up and report failure, rather than spend a great
+         *  deal of time looking for the document.
+         *
+         *  If the first entry in the postlist is at the given location, this
+         *  call must find it.
+         *
+         *  If the first entry is found, the \a found parameter is set to true,
+         *  and the postlist is left in exactly the same state as it would be
+         *  if \a skip_to() had been called.
+         *
+         *  If the first entry is not found, the \a found parameter is set to
+         *  false, and the postlist is left pointing to no document, but the
+         *  next time that next() is called, the postlist will be moved to the
+         *  first entry after that specified in this call.  (Or, if the next
+         *  call is to skip_to(), the skip_to() will behave as if the postlist
+         *  currently pointed to a document in the range defined by that
+         *  specified in this call and the location of the actual next entry in
+         *  the list.)  If \a get_docid() is called immediately after this
+         *  method has returned having set \a found to false, a value for the
+         *  lowest possible document ID which might be the next matching entry
+         *  should be returned.  (This can always be calculated cheaply,
+         *  because \a did is always a valid return value here.)
+         *
+         *  @param found This is a flag, which will be set to true if the next
+         *  entry is found, or if there is determined to be no next entry, and
+         *  to false otherwise.
+         */
+        virtual Internal * weak_skip_to(Xapian::docid did, Xapian::weight w_min,
+                                        bool & found) {
+            found = true;
+            return skip_to(did, w_min);
+        }
+
+        /** Flag, true if we should try weak_skip_to operations rather than skip_to
+         *  when possible when calling this postlist.
+         */
+        virtual bool prefer_weak_skip_to() { return false; }
+
         /// Returns true if we're off the end of the list
         virtual bool at_end() const = 0;