Ticket #340: patch

matcher/externalpostlist.cc

@@ -25 +25 @@
 
 #include <xapian/postingsource.h>
 
+#include "multimatch.h"
 #include "omassert.h"
 #include "omdebug.h"
 
@@ -32 +33 @@
 
 ExternalPostList::ExternalPostList(const Xapian::Database & db,
                                    Xapian::PostingSource *source_,
-                                   double factor_)
-    : source(source_), source_is_owned(false), current(0), factor(factor_)
+                                   double factor_,
+                                   MultiMatch * matcher_)
+    : source(source_), source_is_owned(false), current(0), factor(factor_),
+        matcher(matcher_)
 {
     Assert(source);
     Xapian::PostingSource * newsource = source->clone();
@@ -51 +54 @@
     }
 }
 
+void
+ExternalPostList::notify_new_maxweight()
+{
+    matcher->recalc_maxweight();
+}
+
 Xapian::doccount
 ExternalPostList::get_termfreq_min() const
 {

matcher/queryoptimiser.cc

@@ -68 +68 @@
             Assert(query->external_source);
             Xapian::Database wrappeddb(new ConstDatabaseWrapper(&db));
             RETURN(new ExternalPostList(wrappeddb,
-                                        query->external_source, factor));
+                                        query->external_source, factor,
+                                        matcher));
         }
 
         case Xapian::Query::OP_AND:

matcher/externalpostlist.h

@@ -28 +28 @@
     class PostingSource;
 }
 
+class MultiMatch;
+
 class ExternalPostList : public PostList {
     Xapian::PostingSource * source;
     bool source_is_owned;
@@ -36 +38 @@
 
     double factor;
 
+    /// The matcher to notify when maximum weights change.
+    MultiMatch * matcher;
+
     /// Disallow copying.
     ExternalPostList(const ExternalPostList &);
 
@@ -47 +52 @@
   public:
     ExternalPostList(const Xapian::Database & db,
                      Xapian::PostingSource *source_,
-                     double factor_);
+                     double factor_,
+                     MultiMatch * matcher_);
     ~ExternalPostList();
 
+    void notify_new_maxweight();
+
     Xapian::doccount get_termfreq_min() const;
 
     Xapian::doccount get_termfreq_est() const;

docs/postingsource.rst

@@ -1 +1 @@
 
 .. Copyright (C) 2008,2009 Olly Betts
+.. Copyright (C) 2009 Lemur Consulting Ltd
 
 =====================
 Xapian::PostingSource
@@ -72 +73 @@
 These methods have default implementations which always return 0, for
 convenience when deriving "weight-less" subclasses.
 
+If the maximum weight that can be returned by the PostingSource has decreased
+significantly, it may be worthwhile to notify the Xapian matcher about this, so
+that it can use this information for optmisations, allowing the match process
+to finish faster.  You can notify the matcher about this by calling the
+``PostingSource::notify_new_maxweight()`` method.  However, you should take
+care not to call this method too frequently, as the resulting computation can
+be moderately expensive.
+
+It is therefore advisable only to call this after a significant decrease in
+maxweight.  Profiling your postingsource will often be the best way to
+determine how often to call it.  If it's cheap to calculate the maxweight in
+your posting source, a reasonable strategy is to decide a maximum number of
+times to call this method (eg, 10) and then to call it whenever
+``floor(new_maxweight * 10 / original_maxweight)`` changes; this ensures that
+only reasonably significant drops result in a recalculation of the weights.
+Alternatively, you could simply impose a lower limit on the number of documents
+processed before re-calling this.
+
 The ``at_end()`` method checks if the current iteration position is past the
 last entry::
 

include/xapian/postingsource.h

@@ -29 +29 @@
 #include <string>
 #include <map>
 
+// Declaration of an opaque internal type.
+class ExternalPostList;
+
 namespace Xapian {
 
 /** Base class which provides an "external" source of postings.
@@ -43 +46 @@
     /// Don't allow copying.
     PostingSource(const PostingSource &);
 
+    /// The ExternalPostList which notifications should be sent to.
+    ExternalPostList * externalpl;
+
+    /// Set a pointer to the external postlist.
+    void register_externalpl(ExternalPostList * externalpl_) {
+        externalpl = externalpl_;
+    }
+
+    // ExternalPostList needs to be able to call register_externalpl.
+    friend class ::ExternalPostList;
+
   protected:
     /// Allow subclasses to be instantiated.
     PostingSource() { }
 
+    /** Notify the matcher that the maximum weight has decreased.
+     *
+     *  This will potentially cause the maxweights throughout the query tree
+     *  to be re-evaluated, and can therefore be quite expensive.  On the
+     *  other hand, it will allow the new reduced maxweight to be used to
+     *  trigger query optimisations, which can greatly reduce the query
+     *  processing time.
+     *
+     *  It is therefore advisable only to call this after a significant
+     *  decrease in maxweight.
+     */
+    void notify_new_maxweight();
+
   public:
     // Destructor.
     virtual ~PostingSource();

api/postingsource.cc

@@ -27 +27 @@
 
 #include "database.h"
 #include "document.h"
+#include "matcher/externalpostlist.h"
+
 #include "xapian/document.h"
 #include "xapian/error.h"
 #include "xapian/queryparser.h" // For sortable_unserialise().
@@ -40 +42 @@
 
 namespace Xapian {
 
+void
+PostingSource::notify_new_maxweight()
+{
+    externalpl->notify_new_maxweight();
+}
+
 PostingSource::~PostingSource() { }
 
 Xapian::weight