Ticket #199: matchspy3.patch

xapian-maintainer-tools/win32msvc/win32_api.mak

@@ -23 +23 @@
     $(INTDIR)/errorhandler.obj \
     $(INTDIR)/expanddecider.obj \
     $(INTDIR)/leafpostlist.obj \
+    $(INTDIR)/matchspy.obj \
     $(INTDIR)/omdatabase.obj \
     $(INTDIR)/omdocument.obj \
     $(INTDIR)/omenquire.obj \
@@ -51 +52 @@
     $(INTDIR)/errorhandler.cc\
     $(INTDIR)/expanddecider.cc\
     $(INTDIR)/leafpostlist.cc\
+    $(INTDIR)/matchspy.cc \
     $(INTDIR)/omdatabase.cc\
     $(INTDIR)/omdocument.cc\
     $(INTDIR)/omenquire.cc\

xapian-core/matcher/multimatch.cc

@@ -5 +5 @@
  * Copyright 2002,2003,2004,2005,2006,2007,2008,2009 Olly Betts
  * Copyright 2003 Orange PCS Ltd
  * Copyright 2003 Sam Liddicott
- * Copyright 2007,2008 Lemur Consulting Ltd
+ * Copyright 2007,2008,2009 Lemur Consulting Ltd
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -47 +47 @@
 #include "weightinternal.h"
 
 #include <xapian/errorhandler.h>
+#include <xapian/matchspy.h>
 #include <xapian/version.h> // For XAPIAN_HAS_REMOTE_BACKEND
 
 #ifdef XAPIAN_HAS_REMOTE_BACKEND
@@ -174 +175 @@
     }
 }
 
+/// Class which applies several match spies in turn.
+class MultipleMatchSpy : public Xapian::MatchSpy {
+  private:
+    /// List of match spies to call, in order.
+    const std::vector<Xapian::MatchSpy *> & spies;
+
+  public:
+    MultipleMatchSpy(const std::vector<Xapian::MatchSpy *> & spies_)
+            : spies(spies_) {}
+
+    /** Implementation of virtual operator().
+     *
+     *  This implementation calls all the spies in turn.
+     */
+    void operator()(const Xapian::Document &doc, Xapian::weight wt);
+};
+
+void 
+MultipleMatchSpy::operator()(const Xapian::Document &doc, Xapian::weight wt) {
+    LOGCALL_VOID(MATCH, "MultipleMatchSpy::operator()", doc << ", " << wt);
+    vector<Xapian::MatchSpy *>::const_iterator i;
+    for (i = spies.begin(); i != spies.end(); ++i) {
+        (**i)(doc, wt);
+    }
+}
+
 ////////////////////////////////////
 // Initialisation and cleaning up //
 ////////////////////////////////////
@@ -191 +218 @@
                        const Xapian::Sorter * sorter_,
                        Xapian::ErrorHandler * errorhandler_,
                        Xapian::Weight::Internal & stats,
-                       const Xapian::Weight * weight_)
+                       const Xapian::Weight * weight_,
+                       const vector<Xapian::MatchSpy *> & matchspies_)
         : db(db_), query(query_),
           collapse_max(collapse_max_), collapse_key(collapse_key_),
           percent_cutoff(percent_cutoff_), weight_cutoff(weight_cutoff_),
@@ -199 +227 @@
           sort_key(sort_key_), sort_by(sort_by_),
           sort_value_forward(sort_value_forward_), sorter(sorter_),
           errorhandler(errorhandler_), weight(weight_),
-          is_remote(db.internal.size())
+          is_remote(db.internal.size()),
+          matchspies(matchspies_)
 {
     DEBUGCALL(MATCH, void, "MultiMatch", db_ << ", " << query_ << ", " <<
               qlen << ", " << (omrset ? *omrset : Xapian::RSet()) << ", " <<
@@ -229 +258 @@
                 rem_db->set_query(query, qlen, collapse_max, collapse_key,
                                   order, sort_key, sort_by, sort_value_forward,
                                   percent_cutoff, weight_cutoff, weight,
-                                  subrsets[i]);
+                                  subrsets[i], matchspies);
                 bool decreasing_relevance =
                     (sort_by == REL || sort_by == REL_VAL);
-                smatch = new RemoteSubMatch(rem_db, decreasing_relevance);
+                smatch = new RemoteSubMatch(rem_db, decreasing_relevance, matchspies);
                 is_remote[i] = true;
             } else {
 #endif /* XAPIAN_HAS_REMOTE_BACKEND */
@@ -277 +306 @@
                      Xapian::MSet & mset,
                      const Xapian::Weight::Internal & stats,
                      const Xapian::MatchDecider *mdecider,
-                     const Xapian::MatchDecider *matchspy)
+                     const Xapian::MatchDecider *matchspy_legacy)
 {
     DEBUGCALL(MATCH, void, "MultiMatch::get_mset", first << ", " << maxitems
               << ", " << check_at_least << ", ...");
@@ -403 +432 @@
     Xapian::doccount matches_lower_bound = 0;
     Xapian::doccount matches_estimated   = pl->get_termfreq_est();
 
-    if (mdecider == NULL && matchspy == NULL) {
+    if (mdecider == NULL && matchspy_legacy == NULL) {
         // If we have a matcher decider or match spy, the lower bound must be
         // set to 0 as we could discard all hits.  Otherwise set it to the
         // minimum number of entries which the postlist could return.
         matches_lower_bound = pl->get_termfreq_min();
     }
 
+    // Prepare the matchspy
+    Xapian::MatchSpy *matchspy = NULL;
+    MultipleMatchSpy multispy(matchspies);
+    if (!matchspies.empty()) {
+        if (matchspies.size() == 1) {
+            matchspy = matchspies[0];
+        } else {
+            matchspy = &multispy;
+        }
+    }
+
     // Check if any results have been asked for (might just be wanting
     // maxweight).
     if (check_at_least == 0) {
@@ -437 +477 @@
         return;
     }
 
-    // Number of documents considered by a decider or matchspy.
+    // Number of documents considered by a decider or matchspy_legacy.
     Xapian::doccount decider_considered = 0;
-    // Number of documents denied by the decider or matchspy.
+    // Number of documents denied by the decider or matchspy_legacy.
     Xapian::doccount decider_denied = 0;
 
     // Set max number of results that we want - this is used to decide
@@ -545 +585 @@
             // VAL, then new_item.wt won't yet be set, but that doesn't
             // matter since it's not used by the sort function.
             if (!mcmp(new_item, min_item)) {
-                if (matchspy == NULL && mdecider == NULL && !collapser) {
+                if (mdecider == NULL && !collapser && matchspy_legacy == NULL) {
                     // Document was definitely suitable for mset - no more
                     // processing needed.
                     LOGLINE(MATCH, "Making note of match item which sorts lower than min_item");
@@ -563 +603 @@
                     continue;
                 }
                 // We can't drop the item, because we need to show it
-                // to the matchspy, test whether the mdecider would
+                // to the matchspy_legacy, test whether the mdecider would
                 // accept it, and/or test whether it would be collapsed.
                 LOGLINE(MATCH, "Keeping candidate which sorts lower than min_item for further investigation");
             }
         }
 
         // Use the match spy and/or decision functors (if specified).
-        if (matchspy != NULL || mdecider != NULL) {
+        if (matchspy != NULL || mdecider != NULL || matchspy_legacy != NULL) {
             const unsigned int multiplier = db.internal.size();
             Assert(multiplier != 0);
             Xapian::doccount n = (did - 1) % multiplier; // which actual database
@@ -584 +624 @@
                 Xapian::Document mydoc(doc.get());
 
                 ++decider_considered;
-                if (matchspy && !matchspy->operator()(mydoc)) {
+                if (matchspy_legacy && !matchspy_legacy->operator()(mydoc)) {
                     ++decider_denied;
                     continue;
                 }
@@ -592 +632 @@
                     ++decider_denied;
                     continue;
                 }
+                if (matchspy) {
+                    if (!calculated_weight) {
+                        wt = pl->get_weight();
+                        new_item.wt = wt;
+                        calculated_weight = true;
+                    }
+                    matchspy->operator()(mydoc, wt);
+                }
             }
         }
 
@@ -882 +930 @@
                     ", matches_upper_bound=" << matches_upper_bound);
         }
 
-        if (matchspy || mdecider) {
+        if (mdecider || matchspy_legacy) {
             if (!percent_cutoff) {
                 if (!collapser) {
                     // We're not collapsing or doing a percentage cutoff, so
@@ -946 +994 @@
                 matches_estimated = matches_lower_bound;
         }
 
-        if (collapser || matchspy || mdecider) {
+        if (collapser || mdecider || matchspy_legacy) {
             LOGLINE(MATCH, "Clamping estimate between bounds: "
                     "matches_lower_bound = " << matches_lower_bound <<
                     ", matches_estimated = " << matches_estimated <<
@@ -962 +1010 @@
                 matches_estimated = docs_matched;
         }
 
-        if (collapser && !matchspy && !mdecider && !percent_cutoff) {
+        if (collapser && !mdecider && !percent_cutoff && !matchspy_legacy) {
             AssertRel(docs_matched,<=,uncollapsed_upper_bound);
             if (docs_matched > uncollapsed_lower_bound)
                 uncollapsed_lower_bound = docs_matched;

xapian-core/matcher/remotesubmatch.cc

@@ -27 +27 @@
 #include "remote-database.h"
 #include "weightinternal.h"
 
-RemoteSubMatch::RemoteSubMatch(RemoteDatabase *db_, bool decreasing_relevance_)
-        : db(db_), decreasing_relevance(decreasing_relevance_)
+RemoteSubMatch::RemoteSubMatch(RemoteDatabase *db_,
+                               bool decreasing_relevance_,
+                               const vector<Xapian::MatchSpy *> & matchspies_)
+        : db(db_),
+          decreasing_relevance(decreasing_relevance_),
+          matchspies(matchspies_)
 {
     DEBUGCALL(MATCH, void, "RemoteSubMatch",
-              db_ << ", " << decreasing_relevance_);
+              db_ << ", " << decreasing_relevance_ << ", " <<
+              "matchspies");
 }
 
 bool
@@ -64 +69 @@
     DEBUGCALL(MATCH, PostList *, "RemoteSubMatch::get_postlist_and_term_info",
               "[matcher], " << (void*)termfreqandwts << ", " << (void*)total_subqs_ptr);
     Xapian::MSet mset;
-    db->get_mset(mset);
+    db->get_mset(mset, matchspies);
     percent_factor = mset.internal->percent_factor;
     if (termfreqandwts) *termfreqandwts = mset.internal->termfreqandwts;
     // For remote databases we report percent_factor rather than counting the

xapian-core/matcher/remotesubmatch.h

@@ -26 +26 @@
 #include "remote-database.h"
 #include "xapian/weight.h"
 
+class Xapian::MatchSpy;
+
 /// Class for performing matching on a remote database.
 class RemoteSubMatch : public SubMatch {
     /// Don't allow assignment.
@@ -46 +48 @@
     /// The factor to use to convert weights to percentages.
     double percent_factor;
 
+    /// The matchspies to use.
+    const vector<Xapian::MatchSpy *> & matchspies;
+
   public:
     /// Constructor.
-    RemoteSubMatch(RemoteDatabase *db_, bool decreasing_relevance_);
+    RemoteSubMatch(RemoteDatabase *db_,
+                   bool decreasing_relevance_,
+                   const vector<Xapian::MatchSpy *> & matchspies);
 
     /// Fetch and collate statistics.
     bool prepare_match(bool nowait, Xapian::Weight::Internal & total_stats);
@@ -69 +76 @@
     double get_percent_factor() const { return percent_factor; }
 
     /// Short-cut for single remote match.
-    void get_mset(Xapian::MSet & mset) { db->get_mset(mset); }
+    void get_mset(Xapian::MSet & mset) { db->get_mset(mset, matchspies); }
 };
 
 #endif /* XAPIAN_INCLUDED_REMOTESUBMATCH_H */

xapian-core/docs/categorisation.rst

@@ -1 +1 @@
 
 .. Copyright (C) 2007 Olly Betts
+.. Copyright (C) 2009 Lemur Consulting Ltd
 
 =============================
 Xapian Categorisation Support
@@ -14 +15 @@
 lists of category values which feature in matching documents.  There are
 numerous potential uses this can be put to, but a common one is to offer the
 user the ability to narrow down their search by filtering it to only include
-documents with a particular value of a particular category.
+documents with a particular value of a particular category.  This is often
+referred to as ``faceted search``.
 
 Some categories are numeric and can take many different values (examples
 include price, width, and height).  The number of different values will often
@@ -43 +45 @@
 Searching
 ---------
 
-At search time, you need to pass a ``Xapian::MatchSpy`` object to
-``Xapian::Enquire::get_mset()``, like so::
+At search time, you need to pass a ``Xapian::ValueCountMatchSpy`` object for
+each category you want to look at to ``Xapian::Enquire::add_matchspy()``, like
+so::
 
-    Xapian::MatchSpy spy;
+    Xapian::ValueCountMatchSpy spy0(0);
+    Xapian::ValueCountMatchSpy spy1(1);
+    Xapian::ValueCountMatchSpy spy3(3);
 
-    spy.add_category(0);
-    spy.add_category(1);
-    spy.add_category(3);
-
     Xapian::Enquire enq(db);
+    enq.add_matchspy(spy0);
+    enq.add_matchspy(spy1);
+    enq.add_matchspy(spy3);
 
     enq.set_query(query);
 
     Xapian::MSet mset = enq.get_mset(0, 10, 10000, NULL, NULL, &spy);
 
-The ``10000`` in the call to ``get_mset`` tells Xapian to check at least
+The ``10000`` in the call to ``get_mset()`` tells Xapian to check at least
 10000 documents, so the ``spy`` object will be passed at least 10000 documents
-to tally category information from (unless less than 10000 documents match
-the query, in which case it will see all of them).  Setting this higher will
-make the counts exact, but Xapian will have to do more work for most queries
-so searches will be slower.
+to tally category information from (unless fewer than 10000 documents match the
+query, in which case it will see all of them).  Setting this higher will make
+the counts exact, but Xapian will have to do more work for most queries so
+searches will be slower.
 
-The ``spy`` object now contains the category information.  You can find out
-how many documents it looked at by calling ``spy.get_total()``.  You can
-read the values for category ``cat_no`` like this::
+The ``spy`` objects now contain the category information.  You can find out how
+many documents they looked at by calling ``spy0.get_total()``.  (All the spies
+will have looked at the same number of documents.)  You can read the values
+from, say, ``spy0`` like this::
 
-    const map<string, size_t> & cat = spy.get_categories(cat_no);
+    const map<string, size_t> & cat = spy0.get_values();
     map<string, size_t>::const_iterator i;
     for (i = cat.begin(); i != cat.end(); ++i) {
         cout << i->first << ": " << i->second << endl;
     }
 
-You calculate the score for category ``cat_no`` like so::
+You can calculate a score to indicate how evenly spread the values are using
+the ``score_evenness`` function like so::
 
-    double score = spy.score_categorisation(cat_num);
+    double score = Xapian::score_evenness(spy0);
 
 Or if you prefer categories with 4 or 5 values::
 
-    double score = spy.score_categorisation(cat_num, 4.5);
+    double score = Xapian::score_evenness(spy0, 4.5);
 
 The smaller the score, the better - a perfectly even split with exactly the
 number of entries asked (or with no preference given for the number of entries)
@@ -89 +95 @@
 application, but to give you a rough idea, a suitable threshold is likely to be
 less than one.
 
-The scoring uses a sum of squared differences (currently that is - this should
+The scoring uses a sum of squared differences (currently, that is - this should
 probably be regarded as an implementation detail which could change in the
 future if we find a better algorithm).
 
-You would build ranges from numeric values for value ``cat_no``, asking for at
-most ``num_ranges`` ranges like so::
+You can build ranges from numeric values for the values returned from spy
+``spy0``, asking for at most ``num_ranges`` ranges like so::
 
-    bool result = spy.build_numeric_ranges(cat_no, num_ranges);
+    std::map<Xapian::NumericRange, Xapian::doccount> result;
+    Xapian::doccount values_seen;
+    values_seen = build_numeric_ranges(result, spy0.get_values(), num_ranges);
 
-If ranges could not be built (for example, because all documents have the
-same value for ``cat_no``), ``false`` is returned.  Otherwise ``true`` is
-returned, and the spy object's category map for value ``cat_no`` is modified
-to consist of ranges.  Keys are now built of strings returned by
-``Xapian::sortable_serialise()`` - either a single string if there is only
-one number in a particular range, or for a range a string padded to 9 bytes
-with zero bytes, with a second string appended.
+Here, ``result`` will be filled with a set of numeric ranges (holding at most
+``num_ranges`` ranges), and ``values_seen`` will be the count of the number of
+values seen (note - this may be different from the number of documents seen by
+the matchspy, since some may have no value stored in the slot).
 
+If there are no values seen by the spy, ``result`` will be empty.  If all the
+values seen by the spy are the same, ``result`` will contain a single entry,
+with a single range with the same start and end points.
+
 Restricting by category values
 ------------------------------
 
-If you're using the categorisation to offer the user choices for narrowing
-down their search results, you then need to be able to apply a suitable
-filter.
+If you're using the categorisation to offer the user choices for narrowing down
+their search results, you then need to be able to apply a suitable filter.
 
-For a range, the best way is to use ``Xapian::Query::OP_VALUE_RANGE`` to
+For a range, the easiest way is to use ``Xapian::Query::OP_VALUE_RANGE`` to
 build a filter query, and then combine this with the user's query using
 ``Xapian::Query::OP_FILTER``.
 
-For a single value, you could use ``Xapian::Query::OP_VALUE_RANGE`` with
-the same start and end, or ``Xapian::MatchDecider``, but it's probably
-most efficient to also index the categories as suitably prefixed boolean
-terms and use those for filtering.
+For a single value, you could use ``Xapian::Query::OP_VALUE_RANGE`` with the
+same start and end, or ``Xapian::MatchDecider``, but it's probably most
+efficient to also index the categories as suitably prefixed boolean terms and
+use those for filtering.
 
 Current Limitations
 ===================
 
-It's not currently possible to build logarithmic ranges without writing
-your own subclass.
-
-It's not possible to try building different ranges because the original
-data is overwritten.  If it's actually useful to do this, the API needs
-adjusting.
+It's not currently possible to build logarithmic ranges with
+``build_numeric_ranges``.

xapian-core/docs/Makefile.am

@@ -17 +17 @@
  bm25.html code_structure.html queryparser.html \
  quickstartexpand.cc.html quickstartindex.cc.html quickstartsearch.cc.html
 
-RSTDOCS = admin_notes.rst deprecation.rst glossary.rst \
+RSTDOCS = admin_notes.rst categorisation.rst deprecation.rst glossary.rst \
  postingsource.rst replication.rst replication_protocol.rst \
  sorting.rst serialisation.rst spelling.rst synonyms.rst \
  termgenerator.rst valueranges.rst

xapian-core/tests/api_matchspy.cc

@@ +1 @@
+/** @file api_matchspy.cc
+ * @brief tests of MatchSpy usage
+ */
+/* Copyright 2007,2009 Lemur Consulting Ltd
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
+ * USA
+ */
+
+#include <config.h>
+
+#include "api_matchspy.h"
+
+#include <xapian.h>
+
+#include "str.h"
+#include <cmath>
+#include <map>
+#include <vector>
+
+#include "backendmanager.h"
+#include "testsuite.h"
+#include "testutils.h"
+#include "apitest.h"
+
+using namespace std;
+
+// #######################################################################
+// # Tests start here
+
+class SimpleMatchSpy : public Xapian::MatchSpy {
+  public:
+    // Vector which will be filled with all the document contents seen.
+    std::vector<std::string> seen;
+
+    void operator()(const Xapian::Document &doc,
+                    Xapian::weight) {
+        // Note that this is not recommended usage of get_data() - you
+        // generally shouldn't call get_data() from inside a MatchSpy, because
+        // it is (likely to be) a slow operation resulting in considerable IO.
+        seen.push_back(doc.get_data());
+    }
+};
+
+// Basic test of a matchspy.
+DEFINE_TESTCASE(matchspy1, backend && !remote) {
+    Xapian::Database db(get_database("apitest_simpledata"));
+    Xapian::Enquire enquire(db);
+    enquire.set_query(Xapian::Query("this"));
+
+    SimpleMatchSpy myspy;
+
+    Xapian::MSet nospymset = enquire.get_mset(0, 100);
+    enquire.add_matchspy(&myspy);
+    Xapian::MSet spymset = enquire.get_mset(0, 100);
+
+    // Check that the match estimates aren't affected by the matchspy.
+    TEST_EQUAL(nospymset, spymset);
+
+    vector<bool> docid_checked(db.get_lastdocid());
+
+    // Check that we get the expected number of matches, and that the stored
+    // document contents are right.
+    Xapian::MSetIterator i = spymset.begin();
+    TEST(i != spymset.end());
+    TEST_EQUAL(spymset.size(), 6);
+    TEST_EQUAL(myspy.seen.size(), spymset.size());
+
+    std::sort(myspy.seen.begin(), myspy.seen.end());
+
+    std::vector<std::string> seen2;
+    for ( ; i != spymset.end(); ++i) {
+        const Xapian::Document doc(i.get_document());
+        seen2.push_back(doc.get_data());
+    }
+    std::sort(seen2.begin(), seen2.end());
+
+    TEST_EQUAL(myspy.seen.size(), seen2.size());
+    std::vector<std::string>::const_iterator j = myspy.seen.begin();
+    std::vector<std::string>::const_iterator j2 = seen2.begin();
+    for (; j != myspy.seen.end(); ++j, ++j2) {
+        TEST_EQUAL(*j, *j2);
+    }
+
+    return true;
+}
+
+static string values_to_repr(const map<string, Xapian::doccount> & cat) {
+    string resultrepr("|");
+    map<string, Xapian::doccount>::const_iterator i;
+    for (i = cat.begin(); i != cat.end(); ++i) {
+        resultrepr += i->first;
+        resultrepr += ':';
+        resultrepr += str(i->second);
+        resultrepr += '|';
+    }
+    return resultrepr;
+}
+
+DEFINE_TESTCASE(matchspy2, writable)
+{
+    if (get_dbtype() == "remotetcp" || get_dbtype() == "remoteprog") {
+        SKIP_TEST("Test not supported for remote backend");
+    }
+
+    Xapian::WritableDatabase db = get_writable_database("");
+    for (int c = 1; c <= 25; ++c) {
+        Xapian::Document doc;
+        doc.set_data("Document " + str(c));
+        int factors = 0;
+        for (int factor = 1; factor <= c; ++factor) {
+            doc.add_term("all");
+            if (c % factor == 0) {
+                doc.add_term("XFACT" + str(factor));
+                ++factors;
+            }
+        }
+
+        // Number of factors.
+        doc.add_value(0, str(factors));
+        // Units digits.
+        doc.add_value(1, str(c % 10));
+        // Constant.
+        doc.add_value(2, "fish");
+        // Number of digits.
+        doc.add_value(3, str(str(c).size()));
+
+        db.add_document(doc);
+    }
+
+    Xapian::ValueCountMatchSpy spy0(0);
+    Xapian::ValueCountMatchSpy spy1(1);
+    Xapian::ValueCountMatchSpy spy3(3);
+
+    Xapian::Enquire enq(db);
+
+    enq.set_query(Xapian::Query("all"));
+
+    enq.add_matchspy(&spy0);
+    enq.add_matchspy(&spy1);
+    enq.add_matchspy(&spy3);
+    Xapian::MSet mset = enq.get_mset(0, 10);
+
+    TEST_EQUAL(spy0.get_total(), 25);
+    TEST_EQUAL(spy1.get_total(), 25);
+    TEST_EQUAL(spy3.get_total(), 25);
+
+    static const char * results[] = {
+        "|1:1|2:9|3:3|4:7|5:1|6:3|8:1|",
+        "|0:2|1:3|2:3|3:3|4:3|5:3|6:2|7:2|8:2|9:2|",
+        "|1:9|2:16|",
+    };
+    TEST_STRINGS_EQUAL(values_to_repr(spy0.get_values()), results[0]);
+    TEST_STRINGS_EQUAL(values_to_repr(spy1.get_values()), results[1]);
+    TEST_STRINGS_EQUAL(values_to_repr(spy3.get_values()), results[2]);
+                       
+    {
+        // Test scoring evenness returns scores with the natural ordering.
+        double score0 = Xapian::score_evenness(spy0);
+        tout << "score0 = " << score0 << endl;
+        double score1 = Xapian::score_evenness(spy1);
+        tout << "score1 = " << score1 << endl;
+        double score3 = Xapian::score_evenness(spy3);
+        tout << "score3 = " << score3 << endl;
+        // 1 is obviously best, and 0 obviously worst.
+        TEST(score1 < score3);
+        TEST(score3 < score0);
+
+        // Check that the using the expanded form gives the same results.
+        double score0_check = Xapian::score_evenness(spy0.get_values(), spy0.get_total());
+        tout << "score0_check = " << score0_check << endl;
+        TEST_EQUAL(score0, score0_check);
+    }
+
+    {
+        // Test scoring evenness and about 7 categories returns scores with the
+        // natural ordering.
+        double score0 = Xapian::score_evenness(spy0, 7);
+        tout << "score0 = " << score0 << endl;
+        double score1 = Xapian::score_evenness(spy1, 7);
+        tout << "score1 = " << score1 << endl;
+        double score3 = Xapian::score_evenness(spy3, 7);
+        tout << "score3 = " << score3 << endl;
+        // 3 is clearly worst - 0 is arguably a little better than 1 (0 is the
+        // requested size, but 1 has a much more even split).
+        TEST(score0 < score1);
+        TEST(score1 < score3);
+
+        // Check that the using the expanded form gives the same results.
+        double score0_check = Xapian::score_evenness(spy0.get_values(), spy0.get_total());
+        tout << "score0_check = " << score0_check << endl;
+        TEST_EQUAL(score0, score0_check);
+    }
+
+    return true;
+}
+
+DEFINE_TESTCASE(matchspy3, writable)
+{
+    if (get_dbtype() == "remotetcp" || get_dbtype() == "remoteprog") {
+        SKIP_TEST("Test not supported for remote backend");
+    }
+
+    Xapian::WritableDatabase db = get_writable_database("");
+    for (int c = 1; c <= 25; ++c) {
+        Xapian::Document doc;
+        doc.set_data("Document " + str(c));
+        int factors = 0;
+        for (int factor = 1; factor <= c; ++factor) {
+            doc.add_term("all");
+            if (c % factor == 0) {
+                doc.add_term("XFACT" + str(factor));
+                ++factors;
+            }
+        }
+
+        // Number of factors.
+        doc.add_value(0, Xapian::sortable_serialise(factors));
+        // Units digits.
+        doc.add_value(1, Xapian::sortable_serialise(c % 10));
+        // (x + 1/3)*(x + 1/3).
+        doc.add_value(2, Xapian::sortable_serialise((c + 1.0/3.0) * (c + 1.0/3.0)));
+        // Reciprocal.
+        doc.add_value(3, Xapian::sortable_serialise(floor(100.0 / c)));
+
+        db.add_document(doc);
+    }
+
+    Xapian::ValueCountMatchSpy spy0(0);
+    Xapian::ValueCountMatchSpy spy1(1);
+    Xapian::ValueCountMatchSpy spy2(2);
+    Xapian::ValueCountMatchSpy spy3(3);
+
+    Xapian::Enquire enq(db);
+
+    enq.set_query(Xapian::Query("all"));
+
+    enq.add_matchspy(&spy0);
+    enq.add_matchspy(&spy1);
+    enq.add_matchspy(&spy2);
+    enq.add_matchspy(&spy3);
+    Xapian::MSet mset = enq.get_mset(0, 10);
+
+    TEST_EQUAL(spy0.get_total(), 25);
+    TEST_EQUAL(spy1.get_total(), 25);
+    TEST_EQUAL(spy2.get_total(), 25);
+    TEST_EQUAL(spy3.get_total(), 25);
+
+    static const string results[] = {
+        "|100:1|200:9|300:3|400:7|500:1|600:3|800:1|",
+        "|0..200:8|300..400:6|500..700:7|800..900:4|",
+        "|177..8711:9|10677..17777:4|20544..26677:3|30044..37377:3|41344..49877:3|54444..59211:2|64177:1|",
+        "|400..900:15|1000..1600:5|2000..2500:2|3300:1|5000:1|10000:1|",
+        ""
+    };
+    std::vector<Xapian::ValueCountMatchSpy *> spies;
+    spies.push_back(&spy0);
+    spies.push_back(&spy1);
+    spies.push_back(&spy2);
+    spies.push_back(&spy3);
+    for (Xapian::valueno v = 0; !results[v].empty(); ++v) {
+        Xapian::doccount total_seen;
+        std::map<Xapian::NumericRange, Xapian::doccount> ranges;
+        total_seen = Xapian::build_numeric_ranges(ranges, spies[v]->get_values(), 7);
+        if (results[v] == "|") {
+            TEST_EQUAL(total_seen, 0);
+            continue;
+        }
+        TEST_NOT_EQUAL(total_seen, 0);
+        TEST(ranges.size() <= 7);
+        string resultrepr("|");
+        map<Xapian::NumericRange, Xapian::doccount>::const_iterator i;
+        for (i = ranges.begin(); i != ranges.end(); ++i) {
+            if (i->first.get_lower() != i->first.get_upper()) {
+                resultrepr += str(floor(i->first.get_lower() * 100));
+                resultrepr += "..";
+                resultrepr += str(floor(i->first.get_upper() * 100));
+            } else {
+                double start = floor(i->first.get_lower() * 100);
+                resultrepr += str(start);
+            }
+            resultrepr += ':';
+            resultrepr += str(i->second);
+            resultrepr += '|';
+        }
+        tout << "value " << v << endl;
+        TEST_STRINGS_EQUAL(resultrepr, results[v]);
+    }
+
+    return true;
+}
+
+DEFINE_TESTCASE(matchspy4, writable)
+{
+    if (get_dbtype() == "remotetcp" || get_dbtype() == "remoteprog") {
+        SKIP_TEST("Test not supported for remote backend");
+    }
+
+    Xapian::WritableDatabase db = get_writable_database("");
+    for (int c = 1; c <= 25; ++c) {
+        Xapian::Document doc;
+        doc.set_data("Document " + str(c));
+        int factors = 0;
+        for (int factor = 1; factor <= c; ++factor) {
+            doc.add_term("all");
+            if (c % factor == 0) {
+                doc.add_term("XFACT" + str(factor));
+                ++factors;
+            }
+        }
+
+        // Number of factors.
+        doc.add_value(0, str(factors));
+        // Units digits.
+        doc.add_value(1, str(c % 10));
+        // Constant.
+        doc.add_value(2, "fish");
+        // Number of digits.
+        doc.add_value(3, str(str(c).size()));
+
+        db.add_document(doc);
+    }
+
+    Xapian::ValueCountMatchSpy spy0(0);
+    Xapian::ValueCountMatchSpy spy1(1);
+    Xapian::ValueCountMatchSpy spy3(3);
+
+    Xapian::Enquire enq(db);
+
+    enq.set_query(Xapian::Query("all"));
+
+    enq.add_matchspy(&spy0);
+    enq.add_matchspy(&spy1);
+    enq.add_matchspy(&spy3);
+    Xapian::MSet mset = enq.get_mset(0, 10);
+
+    TEST_EQUAL(spy0.get_total(), 25);
+    TEST_EQUAL(spy1.get_total(), 25);
+    TEST_EQUAL(spy3.get_total(), 25);
+
+    static const char * results[] = {
+        "|2:9|4:7|3:3|6:3|1:1|5:1|8:1|",
+        "|1:3|2:3|3:3|4:3|5:3|0:2|6:2|7:2|8:2|9:2|",
+        "|",
+        "|2:16|1:9|",
+        NULL
+    };
+    std::vector<Xapian::ValueCountMatchSpy *> spies;
+    spies.push_back(&spy0);
+    spies.push_back(&spy1);
+    spies.push_back(NULL);
+    spies.push_back(&spy3);
+    for (Xapian::valueno v = 0; results[v]; ++v) {
+        tout << "value " << v << endl;
+        std::vector<Xapian::StringAndFrequency> allvals;
+
+        Xapian::ValueCountMatchSpy * spy = spies[v];
+        if (spy != NULL)
+            spy->get_top_values(allvals, 100);
+        string allvals_str("|");
+        for (size_t i = 0; i < allvals.size(); ++i) {
+            allvals_str += allvals[i].get_string();
+            allvals_str += ':';
+            allvals_str += str(allvals[i].get_frequency());
+            allvals_str += '|';
+        }
+        tout << allvals_str << endl;
+        TEST_STRINGS_EQUAL(allvals_str, results[v]);
+
+        std::vector<Xapian::StringAndFrequency> vals;
+        for (size_t i = 0; i < allvals.size(); ++i) {
+            tout << "i " << i << endl;
+            if (spy != NULL)
+                spy->get_top_values(vals, i);
+            for (size_t j = 0; j < vals.size(); j++) {
+                tout << "j " << j << endl;
+                TEST_EQUAL(vals[j].get_string(), allvals[j].get_string());
+                TEST_EQUAL(vals[j].get_frequency(), allvals[j].get_frequency());
+            }
+        }
+    }
+
+    return true;
+}
+
+// Test builtin match spies
+DEFINE_TESTCASE(matchspy5, backend)
+{
+    Xapian::Database db(get_database("apitest_simpledata"));
+    Xapian::Enquire enquire(db);
+    enquire.set_query(Xapian::Query("this"));
+
+    Xapian::ValueCountMatchSpy myspy1(1);
+    Xapian::ValueCountMatchSpy myspy2(1);
+
+    enquire.add_matchspy(&myspy1);
+    enquire.add_matchspy(&myspy2);
+    Xapian::MSet mymset = enquire.get_mset(0, 100);
+    TEST_EQUAL(mymset.size(), 6);
+
+    const std::map<std::string, Xapian::doccount> & vals1 = myspy1.get_values();
+    const std::map<std::string, Xapian::doccount> & vals2 = myspy2.get_values();
+
+    TEST_EQUAL(vals1.size(), 2);
+    TEST(vals1.find("h") != vals1.end());
+    TEST(vals1.find("n") != vals1.end());
+    TEST_EQUAL(vals1.find("h")->second, 5);
+    TEST_EQUAL(vals1.find("n")->second, 1);
+
+    TEST_EQUAL(vals2.size(), 2);
+    TEST(vals2.find("h") != vals2.end());
+    TEST(vals2.find("n") != vals2.end());
+    TEST_EQUAL(vals2.find("h")->second, 5);
+    TEST_EQUAL(vals2.find("n")->second, 1);
+
+    return true;
+}
+
+class MySpy : public Xapian::MatchSpy {
+    void operator()(const Xapian::Document &, Xapian::weight) {
+    }
+};
+
+// Test exceptions from matchspy base class, and get_description method.
+DEFINE_TESTCASE(matchspy6, !backend)
+{
+    MySpy spy;
+
+    TEST_EXCEPTION(Xapian::UnimplementedError, spy.clone());
+    TEST_EXCEPTION(Xapian::UnimplementedError, spy.name());
+    TEST_EXCEPTION(Xapian::UnimplementedError, spy.serialise());
+    TEST_EXCEPTION(Xapian::UnimplementedError,
+                   spy.unserialise(std::string(),
+                                   Xapian::SerialisationContext()));
+    TEST_EXCEPTION(Xapian::UnimplementedError, spy.serialise_results());
+    TEST_EXCEPTION(Xapian::UnimplementedError,
+                   spy.merge_results(std::string()));
+    TEST_EQUAL(spy.get_description(), "Xapian::MatchSpy()");
+
+    return true;
+}
+
+/// Test that NumericRange comparisons work correctly.
+DEFINE_TESTCASE(numericrange1, !backend)
+{
+    Xapian::NumericRange n1(0, 0);
+    Xapian::NumericRange n2(0, 1);
+    Xapian::NumericRange n3(1, 1);
+    Xapian::NumericRange n4(2, 1);
+
+    TEST(!(n1 < n1));
+    TEST(n1 < n2);
+    TEST(!(n2 < n1));
+    TEST(n2 < n3);
+    TEST(!(n3 < n2));
+    TEST(n3 < n4);
+    TEST(!(n4 < n3));
+    return true;
+}

xapian-core/tests/Makefile.am

@@ -115 +115 @@
  api_collapse.cc \
  api_db.cc \
  api_generated.cc \
+ api_matchspy.cc \
  api_metadata.cc \
  api_nodb.cc \
  api_opsynonym.cc \

xapian-core/include/xapian/enquire.h

@@ -4 +4 @@
 /* Copyright 1999,2000,2001 BrightStation PLC
  * Copyright 2001,2002 Ananova Ltd
  * Copyright 2002,2003,2004,2005,2006,2007,2008,2009 Olly Betts
+ * Copyright 2009 Lemur Consulting Ltd
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -39 +40 @@
 class Document;
 class ErrorHandler;
 class ExpandDecider;
+class MatchSpy;
 class MSetIterator;
 class Query;
 class Weight;
@@ -689 +691 @@
          */
         const Xapian::Query & get_query() const;
 
+        /** Add a matchspy.
+         *
+         *  This matchspy will be called with some of the documents which match
+         *  the query, during the match process.  Exactly which of the matching
+         *  documents are passed to it depends on exactly when certain
+         *  optimisations occur during the match process, but it can be
+         *  controlled to some extent by setting the @a checkatleast parameter
+         *  to @a get_mset().
+         *
+         *  In particular, if there are enough matching documents, at least the
+         *  number specified by @a checkatleast will be passed to the matchspy.
+         *  This means that you can force the matchspy to be shown all matching
+         *  documents by setting @a checkatleast to the number of documents in
+         *  the database.
+         *
+         *  @param spy       The MatchSpy subclass to add.  The caller must
+         *                   ensure that this remains valid while the Enquire
+         *                   object remains active, or until @a
+         *                   clear_matchspies() is called.
+         */
+        void add_matchspy(MatchSpy * spy);
+
+        /** Remove all the matchspies.
+         */
+        void clear_matchspies();
+
         /** Set the weighting scheme to use for queries.
          *
          *  @param weight_  the new weighting scheme.  If no weighting scheme

xapian-core/include/xapian/serialisationcontext.h

@@ -31 +31 @@
 // Forward declarations.
 class Weight;
 class PostingSource;
+class MatchSpy;
 
 /** A context for serialisation.
  *
@@ -92 +93 @@
      */
     const Xapian::PostingSource *
             get_posting_source(const std::string & name) const;
+
+    /// Register a user-defined match spy class.
+    void register_match_spy(const Xapian::MatchSpy &spy);
+
+    /** Get a match spy given a name.
+     *
+     *  The returned match spy is owned by the context object.
+     *
+     *  Returns NULL if the match spy could not be found.
+     */
+    const Xapian::MatchSpy *
+            get_match_spy(const std::string & name) const;
 };
 
 }

xapian-core/include/xapian/matchspy.h

@@ +1 @@
+/** @file matchspy.h
+ * @brief MatchSpy implementation.
+ */
+/* Copyright (C) 2007,2008 Olly Betts
+ * Copyright (C) 2007,2009 Lemur Consulting Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ */
+
+#ifndef XAPIAN_INCLUDED_MATCHSPY_H
+#define XAPIAN_INCLUDED_MATCHSPY_H
+
+#include <xapian/enquire.h>
+#include <xapian/visibility.h>
+
+#include <string>
+#include <map>
+#include <set>
+#include <string>
+#include <vector>
+
+namespace Xapian {
+
+class Document;
+class SerialisationContext;
+
+/** Abstract base class for match spies.
+ *
+ *  The subclasses will generally accumulate information seen during the match,
+ *  to calculate aggregate functions, or other profiles of the matching
+ *  documents.
+ */
+class XAPIAN_VISIBILITY_DEFAULT MatchSpy {
+  private:
+    /// Don't allow assignment.
+    void operator=(const MatchSpy &);
+
+    /// Don't allow copying.
+    MatchSpy(const MatchSpy &);
+
+  protected:
+    /// Default constructor, needed by subclass constructors.
+    MatchSpy() {}
+
+  public:
+    /** Virtual destructor, because we have virtual methods. */
+    virtual ~MatchSpy();
+
+    /** Register a document with the match spy.
+     *
+     *  This is called by the matcher once with each document seen by the
+     *  matcher during the match process.  Note that the matcher will often not
+     *  see all the documents which match the query, due to optimisations which
+     *  allow low-weighted documents to be skipped, and allow the match process
+     *  to be terminated early.
+     *
+     *  @param doc The document seen by the match spy.
+     *  @param wt The weight of the document.
+     */
+    virtual void operator()(const Xapian::Document &doc,
+                            Xapian::weight wt) = 0;
+
+    /** Clone the match spy.
+     *
+     *  The clone should inherit the configuration of the parent, but need not
+     *  inherit the state.  ie, the clone does not need to be passed
+     *  information about the results seen by the parent.
+     *
+     *  If you don't want to support the remote backend in your match spy, you
+     *  can use the default implementation which simply throws
+     *  Xapian::UnimplementedError.
+     *
+     *  Note that the returned object will be deallocated by Xapian after use
+     *  with "delete".  It must therefore have been allocated with "new".
+     */
+    virtual MatchSpy * clone() const;
+
+    /** Return the name of this match spy.
+     *
+     *  This name is used by the remote backend.  It is passed with the
+     *  serialised parameters to the remote server so that it knows which class
+     *  to create.
+     *
+     *  Return the full namespace-qualified name of your class here - if your
+     *  class is called MyApp::FooMatchSpy, return "MyApp::FooMatchSpy" from
+     *  this method.
+     *
+     *  If you don't want to support the remote backend in your match spy, you
+     *  can use the default implementation which simply throws
+     *  Xapian::UnimplementedError.
+     */
+    virtual std::string name() const;
+
+    /** Return this object's parameters serialised as a single string.
+     *
+     *  If you don't want to support the remote backend in your match spy, you
+     *  can use the default implementation which simply throws
+     *  Xapian::UnimplementedError.
+     */
+    virtual std::string serialise() const;
+
+    /** Unserialise parameters.
+     *
+     *  This method unserialises parameters serialised by the @a serialise()
+     *  method and allocates and returns a new object initialised with them.
+     *
+     *  If you don't want to support the remote backend in your match spy, you
+     *  can use the default implementation which simply throws
+     *  Xapian::UnimplementedError.
+     *
+     *  Note that the returned object will be deallocated by Xapian after use
+     *  with "delete".  It must therefore have been allocated with "new".
+     */
+    virtual MatchSpy * unserialise(const std::string & s,
+                                   const SerialisationContext & context) const;
+
+    /** Serialise the results of this match spy.
+     *
+     *  If you don't want to support the remote backend in your match spy, you
+     *  can use the default implementation which simply throws
+     *  Xapian::UnimplementedError.
+     */
+    virtual std::string serialise_results() const;
+
+    /** Unserialise some results, and merge them into this matchspy.
+     *
+     *  The order in which results are merged should not be significant, since
+     *  this order is not specified (and will vary depending on the speed of
+     *  the search in each sub-database).
+     *
+     *  If you don't want to support the remote backend in your match spy, you
+     *  can use the default implementation which simply throws
+     *  Xapian::UnimplementedError.
+     */
+    virtual void merge_results(const std::string & s);
+
+    /** Return a string describing this object.
+     *
+     *  This default implementation returns a generic answer, to avoid forcing
+     *  those deriving their own MatchSpy subclasses from having to implement
+     *  this (they may not care what get_description() gives for their
+     *  subclass).
+     */
+    virtual std::string get_description() const;
+};
+
+
+/** A string with a corresponding frequency.
+ */
+class XAPIAN_VISIBILITY_DEFAULT StringAndFrequency {
+    std::string str;
+    Xapian::doccount frequency;
+  public:
+    StringAndFrequency(std::string str_, Xapian::doccount frequency_)
+            : str(str_), frequency(frequency_) {}
+
+    std::string get_string() const { return str; }
+    Xapian::doccount get_frequency() const { return frequency; }
+};
+
+
+/// Class for counting the frequencies of values in the matching documents.
+class XAPIAN_VISIBILITY_DEFAULT ValueCountMatchSpy : public MatchSpy {
+  protected:
+    /// The slot to count.
+    Xapian::valueno slot;
+
+    /// Total number of documents seen by the match spy.
+    Xapian::doccount total;
+
+    /// The values seen so far, together with their frequency.
+    std::map<std::string, Xapian::doccount> values;
+
+  public:
+    /// Construct an empty ValueCountMatchSpy.
+    ValueCountMatchSpy() : slot(Xapian::BAD_VALUENO), total(0) {}
+
+    /** Construct a MatchSpy which counts the values in a particular slot.
+     *
+     *  Further slots can be added by calling @a add_slot().
+     */
+    ValueCountMatchSpy(Xapian::valueno slot_)
+            : slot(slot_), total(0) {
+    }
+
+    /// Return the values seen in the slot.
+    const std::map<std::string, Xapian::doccount> & get_values() const {
+        return values;
+    }
+
+    /** Return the total number of documents tallied. */
+    size_t get_total() const {
+        return total;
+    }
+
+    /** Get the most frequent values in the slot.
+     *
+     *  @param result A vector which will be filled with the most frequent
+     *                values, in descending order of frequency.  Values with
+     *                the same frequency will be sorted in ascending
+     *                alphabetical order.
+     *
+     *  @param maxvalues The maximum number of values to return.
+     */
+    void get_top_values(std::vector<StringAndFrequency> & result,
+                        size_t maxvalues) const;
+
+    /** Implementation of virtual operator().
+     *
+     *  This implementation tallies values for a matching document.
+     */
+    void operator()(const Xapian::Document &doc, Xapian::weight wt);
+
+    virtual MatchSpy * clone() const;
+    virtual std::string name() const;
+    virtual std::string serialise() const;
+    virtual MatchSpy * unserialise(const std::string & s,
+                                   const SerialisationContext & context) const;
+    virtual std::string serialise_results() const;
+    virtual void merge_results(const std::string & s);
+    virtual std::string get_description() const;
+};
+
+
+/** A numeric range.
+ *
+ *  This is used to represent ranges of values returned by the match spies.
+ */
+class XAPIAN_VISIBILITY_DEFAULT NumericRange {
+    /// The lower value in the range.
+    double lower;
+
+    /// The upper value in the range.
+    double upper;
+
+  public:
+    NumericRange(double lower_, double upper_)
+            : lower(lower_), upper(upper_) {}
+
+    double get_lower() const { return lower; }
+    double get_upper() const { return upper; }
+
+    bool operator<(const NumericRange & other) const {
+        if (lower < other.lower) return true;
+        if (lower > other.lower) return false;
+        return (upper < other.upper);
+    }
+};
+
+
+/** Return a score reflecting how evenly divided a set of values is.
+ *
+ *  If you don't want to show a poor categorisation, or have multiple
+ *  categories and only space in your user interface to show a few, you want to
+ *  be able to decide how "good" a categorisation is.  One definition of "good"
+ *  is that it offers a fairly even split of the available values, and
+ *  (optionally) about a specified number of options.
+ *
+ *  @param values The values making up the categorisation, together with their
+ *  frequencies.
+ *
+ *  @param total The total number of documents seen.
+ *
+ *  @param desired_no_of_categories The desired number of categories - this is
+ *  a floating point value, so you can ask for 5.5 if you'd like "about 5 or 6
+ *  categories".  The default is to desire the number of categories that there
+ *  actually are, so the score then only reflects how even the split is.
+
+ *  @return A score for the categorisation for the value - lower is better,
+ *  with a perfectly even split across the right number of categories scoring
+ *  0.
+ */
+//@{
+double XAPIAN_VISIBILITY_DEFAULT score_evenness(
+        const std::map<std::string, Xapian::doccount> & values,
+        Xapian::doccount total,
+        double desired_no_of_categories = 0.0);
+double XAPIAN_VISIBILITY_DEFAULT score_evenness(
+        const std::map<Xapian::NumericRange, Xapian::doccount> & values,
+        Xapian::doccount total,
+        double desired_no_of_categories = 0.0);
+double XAPIAN_VISIBILITY_DEFAULT score_evenness(
+        const ValueCountMatchSpy & spy,
+        double desired_no_of_categories = 0.0);
+//@}
+
+
+/** Turn a category containing sort-encoded numeric values into a set of
+ *  ranges.
+ *
+ *  For "continuous" values (such as price, height, weight, etc), there will
+ *  usually be too many different values to offer the user, and the user won't
+ *  want to restrict to an exact value anyway.
+ *
+ *  This method produces a set of NumericRange objects for a particular value
+ *  number.
+ *
+ *  @param result     Used to return the resulting ranges.
+ *  @param values     The values representing the initial numbers.
+ *  @param max_ranges Group into at most this many ranges.
+ *
+ *  @return The number of values seen.
+ */
+doccount XAPIAN_VISIBILITY_DEFAULT build_numeric_ranges(
+        std::map<Xapian::NumericRange, Xapian::doccount> & result,
+        const std::map<std::string, Xapian::doccount> & values,
+        size_t max_ranges);
+
+}
+
+#endif // XAPIAN_INCLUDED_MATCHSPY_H

xapian-core/include/Makefile.mk

@@ -19 +19 @@
         include/xapian/enquire.h\
         include/xapian/errorhandler.h\
         include/xapian/expanddecider.h\
+        include/xapian/matchspy.h\
         include/xapian/positioniterator.h\
         include/xapian/postingiterator.h\
         include/xapian/postingsource.h\

xapian-core/include/xapian.h

@@ -45 +45 @@
 // Searching
 #include <xapian/enquire.h>
 #include <xapian/expanddecider.h>
+#include <xapian/matchspy.h>
 #include <xapian/postingsource.h>
 #include <xapian/query.h>
 #include <xapian/queryparser.h>

xapian-core/net/serialise.cc

@@ -203 +203 @@
     const map<string, Xapian::MSet::Internal::TermFreqAndWeight> &termfreqandwts
         = mset.internal->termfreqandwts;
 
+    result += encode_length(termfreqandwts.size());
     map<string, Xapian::MSet::Internal::TermFreqAndWeight>::const_iterator j;
     for (j = termfreqandwts.begin(); j != termfreqandwts.end(); ++j) {
         result += encode_length(j->first.size());
@@ -215 +216 @@
 }
 
 Xapian::MSet
-unserialise_mset(const string &s)
+unserialise_mset(const char ** p, const char * p_end)
 {
-    const char * p = s.data();
-    const char * p_end = p + s.size();
+    Xapian::doccount firstitem = decode_length(p, p_end, false);
+    Xapian::doccount matches_lower_bound = decode_length(p, p_end, false);
+    Xapian::doccount matches_estimated = decode_length(p, p_end, false);
+    Xapian::doccount matches_upper_bound = decode_length(p, p_end, false);
+    Xapian::doccount uncollapsed_lower_bound = decode_length(p, p_end, false);
+    Xapian::doccount uncollapsed_estimated = decode_length(p, p_end, false);
+    Xapian::doccount uncollapsed_upper_bound = decode_length(p, p_end, false);
+    Xapian::weight max_possible = unserialise_double(p, p_end);
+    Xapian::weight max_attained = unserialise_double(p, p_end);
 
-    Xapian::doccount firstitem = decode_length(&p, p_end, false);
-    Xapian::doccount matches_lower_bound = decode_length(&p, p_end, false);
-    Xapian::doccount matches_estimated = decode_length(&p, p_end, false);
-    Xapian::doccount matches_upper_bound = decode_length(&p, p_end, false);
-    Xapian::doccount uncollapsed_lower_bound = decode_length(&p, p_end, false);
-    Xapian::doccount uncollapsed_estimated = decode_length(&p, p_end, false);
-    Xapian::doccount uncollapsed_upper_bound = decode_length(&p, p_end, false);
-    Xapian::weight max_possible = unserialise_double(&p, p_end);
-    Xapian::weight max_attained = unserialise_double(&p, p_end);
+    double percent_factor = unserialise_double(p, p_end);
 
-    double percent_factor = unserialise_double(&p, p_end);
-
     vector<Xapian::Internal::MSetItem> items;
-    size_t msize = decode_length(&p, p_end, false);
+    size_t msize = decode_length(p, p_end, false);
     while (msize-- > 0) {
-        Xapian::weight wt = unserialise_double(&p, p_end);
-        Xapian::docid did = decode_length(&p, p_end, false);
-        size_t len = decode_length(&p, p_end, true);
-        string key(p, len);
-        p += len;
+        Xapian::weight wt = unserialise_double(p, p_end);
+        Xapian::docid did = decode_length(p, p_end, false);
+        size_t len = decode_length(p, p_end, true);
+        string key(*p, len);
+        *p += len;
         items.push_back(Xapian::Internal::MSetItem(wt, did, key,
-                                                   decode_length(&p, p_end, false)));
+                                                   decode_length(p, p_end, false)));
     }
 
+    size_t terminfosize = decode_length(p, p_end, false);
     map<string, Xapian::MSet::Internal::TermFreqAndWeight> terminfo;
-    while (p != p_end) {
+    while (terminfosize-- > 0) {
         Xapian::MSet::Internal::TermFreqAndWeight tfaw;
-        size_t len = decode_length(&p, p_end, true);
-        string term(p, len);
-        p += len;
-        tfaw.termfreq = decode_length(&p, p_end, false);
-        tfaw.termweight = unserialise_double(&p, p_end);
+        size_t len = decode_length(p, p_end, true);
+        string term(*p, len);
+        *p += len;
+        tfaw.termfreq = decode_length(p, p_end, false);
+        tfaw.termweight = unserialise_double(p, p_end);
         terminfo.insert(make_pair(term, tfaw));
     }
 

xapian-core/net/remoteserver.cc

@@ -25 +25 @@
 #include "xapian/database.h"
 #include "xapian/enquire.h"
 #include "xapian/error.h"
+#include "xapian/matchspy.h"
 #include "xapian/valueiterator.h"
 
 #include "safeerrno.h"
@@ -354 +355 @@
     send_message(REPLY_UPDATE, message);
 }
 
+/** Structure holding a list of match spies.
+ *
+ *  The main reason for the existence of this structure is to make it easy to
+ *  ensure that the match spies are all deleted after use.
+ */
+struct MatchSpyList {
+    vector<Xapian::MatchSpy *> spies;
+
+    ~MatchSpyList() {
+        vector<Xapian::MatchSpy *>::const_iterator i;
+        for (i = spies.begin(); i != spies.end(); ++i) {
+            delete *i;
+        }
+    }
+};
+
 void
 RemoteServer::msg_query(const string &message_in)
 {
@@ -405 +422 @@
 
     // Unserialise the Weight object.
     len = decode_length(&p, p_end, true);
-    const Xapian::Weight * wttype = ctx.get_weighting_scheme(string(p, len));
+    string wtname(p, len);
+    p += len;
+
+    const Xapian::Weight * wttype = ctx.get_weighting_scheme(wtname);
     if (wttype == NULL) {
         // Note: user weighting schemes should be registered by adding them to
         // a SerialisationContext, and setting the context using
         // RemoteServer::set_context().
         throw Xapian::InvalidArgumentError("Weighting scheme " +
-                                           string(p, len) + " not registered");
+                                           wtname + " not registered");
     }
-    p += len;
 
     len = decode_length(&p, p_end, true);
     AutoPtr<Xapian::Weight> wt(wttype->unserialise(string(p, len)));
     p += len;
 
     // Unserialise the RSet object.
-    Xapian::RSet rset = unserialise_rset(string(p, p_end - p));
+    len = decode_length(&p, p_end, true);
+    Xapian::RSet rset = unserialise_rset(string(p, len));
+    p += len;
 
+    // Unserialise the MatchSpy objects.
+    vector<Xapian::MatchSpy *>::size_type spycount = decode_length(&p, p_end, false);
+    MatchSpyList matchspies;
+    while (spycount != 0) {
+        len = decode_length(&p, p_end, true);
+        string spytype(p, len);
+        const Xapian::MatchSpy * spyclass = ctx.get_match_spy(spytype);
+        if (spyclass == NULL) {
+            throw Xapian::InvalidArgumentError("Match spy " + spytype +
+                                               " not registered");
+        }
+        p += len;
+
+        len = decode_length(&p, p_end, true);
+        matchspies.spies.push_back(spyclass->unserialise(string(p, len), ctx));
+        p += len;
+
+        --spycount;
+    }
+
     Xapian::Weight::Internal local_stats;
     MultiMatch match(*db, query.get(), qlen, &rset, collapse_max, collapse_key,
                      percent_cutoff, weight_cutoff, order,
                      sort_key, sort_by, sort_value_forward, NULL,
-                     NULL, local_stats, wt.get());
+                     NULL, local_stats, wt.get(), matchspies.spies);
 
     send_message(REPLY_STATS, serialise_stats(local_stats));
 
@@ -448 +489 @@
     Xapian::MSet mset;
     match.get_mset(first, maxitems, check_at_least, mset, total_stats, 0, 0);
 
-    send_message(REPLY_RESULTS, serialise_mset(mset));
+    message = serialise_mset(mset);
+
+    for (vector<Xapian::MatchSpy *>::const_iterator i = matchspies.spies.begin();
+         i != matchspies.spies.end(); ++i) {
+        string spy_results = (*i)->serialise_results();
+        message += encode_length(spy_results.size());
+        message += spy_results;
+    }
+    send_message(REPLY_RESULTS, message);
 }
 
 void

xapian-core/common/omenquireinternal.h

@@ -3 +3 @@
  * Copyright 1999,2000,2001 BrightStation PLC
  * Copyright 2001,2002 Ananova Ltd
  * Copyright 2002,2003,2004,2005,2006,2007,2008,2009 Olly Betts
+ * Copyright 2009 Lemur Consulting Ltd
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -159 +160 @@
          */
         mutable Weight * weight;
 
+        vector<MatchSpy *> spies;
+
         Internal(const Xapian::Database &databases, ErrorHandler * errorhandler_);
         ~Internal();
 
@@ -174 +177 @@
         const Query & get_query();
         MSet get_mset(Xapian::doccount first, Xapian::doccount maxitems,
                       Xapian::doccount check_at_least,
-                      const RSet *omrset, const MatchDecider *mdecider,
-                      const MatchDecider *matchspy) const;
+                      const RSet *omrset,
+                      const MatchDecider *mdecider,
+                      const MatchDecider *matchspy_legacy) const;
         ESet get_eset(Xapian::termcount maxitems, const RSet & omrset, int flags,
                       double k, const ExpandDecider *edecider) const;
 

xapian-core/common/multimatch.h

@@ -2 +2 @@
  *
  * Copyright 1999,2000,2001 BrightStation PLC
  * Copyright 2002,2003,2004,2005,2006,2007,2009 Olly Betts
+ * Copyright 2009 Lemur Consulting Ltd
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -71 +72 @@
         /** Is each sub-database remote? */
         vector<bool> is_remote;
 
+        /// The matchspies to use.
+        const vector<Xapian::MatchSpy *> & matchspies;
+
         /** get the maxweight that the postlist pl may return, calling
          *  recalc_maxweight if recalculate_w_max is set, and unsetting it.
          *  Must only be called on the top of the postlist tree.
@@ -110 +114 @@
                    const Xapian::Sorter * sorter_,
                    Xapian::ErrorHandler * errorhandler,
                    Xapian::Weight::Internal & stats,
-                   const Xapian::Weight *wtscheme);
+                   const Xapian::Weight *wtscheme,
+                   const vector<Xapian::MatchSpy *> & matchspies_);
 
         void get_mset(Xapian::doccount first,
                       Xapian::doccount maxitems,
@@ -118 +123 @@
                       Xapian::MSet & mset,
                       const Xapian::Weight::Internal & stats,
                       const Xapian::MatchDecider * mdecider,
-                      const Xapian::MatchDecider * matchspy);
+                      const Xapian::MatchDecider * matchspy_legacy);
 
         /** Called by postlists to indicate that they've rearranged themselves
          *  and the maxweight now possible is smaller.

xapian-core/common/remote-database.h

@@ -143 +143 @@
      * @param weight_cutoff             Weight cutoff.
      * @param wtscheme                  Weighting scheme.
      * @param omrset                    The rset.
+     * @param matchspies                The matchspies to use.  NULL if none.
      */
     void set_query(const Xapian::Query::Internal *query,
                    Xapian::termcount qlen,
@@ -154 +155 @@
                    bool sort_value_forward,
                    int percent_cutoff, Xapian::weight weight_cutoff,
                    const Xapian::Weight *wtscheme,
-                   const Xapian::RSet &omrset);
+                   const Xapian::RSet &omrset,
+                   const vector<Xapian::MatchSpy *> & matchspies);
 
     /** Get the stats from the remote server.
      *
@@ -169 +171 @@
                            const Xapian::Weight::Internal &stats);
 
     /// Get the MSet from the remote server.
-    void get_mset(Xapian::MSet &mset);
+    void get_mset(Xapian::MSet &mset,
+                  const vector<Xapian::MatchSpy *> & matchspies);
 
     /// Get remote termlist.
     TermList * open_term_list(Xapian::docid did) const;

xapian-core/common/remoteprotocol.h

@@ -41 +41 @@
 // 30.6: Support for OP_VALUE_GE and OP_VALUE_LE in query serialisation
 // 31: Clean up for Xapian 1.1.0
 // 32: Serialise termfreq and reltermfreqs together in serialise_stats.
-#define XAPIAN_REMOTE_PROTOCOL_MAJOR_VERSION 32
+// 33: Support for passing matchspies over the remote connection.
+#define XAPIAN_REMOTE_PROTOCOL_MAJOR_VERSION 33
 #define XAPIAN_REMOTE_PROTOCOL_MINOR_VERSION 0
 
 /** Message types (client -> server).

xapian-core/common/serialisationcontextinternal.h

@@ -31 +31 @@
 namespace Xapian {
     class Weight;
     class PostingSource;
+    class MatchSpy;
 }
 
 class Xapian::SerialisationContext::Internal
@@ -41 +42 @@
     /// Registered external posting sources.
     std::map<std::string, Xapian::PostingSource *> postingsources;
 
+    /// Registered match spies.
+    std::map<std::string, Xapian::MatchSpy *> matchspies;
+
     /// Add the standard default weighting schemes and posting sources.
     void add_defaults();
 
@@ -50 +54 @@
     /// Clear all registered posting sources from the context.
     void clear_posting_sources();
 
+    /// Clear all registered match spies from the context.
+    void clear_match_spies();
+
   public:
     Internal();
     ~Internal();
@@ -77 +84 @@
      */
     const Xapian::PostingSource *
             get_posting_source(const std::string & name) const;
+
+    /// Register a user-defined match spy class.
+    void register_match_spy(const Xapian::MatchSpy &spy);
+
+    /** Get a match spy given a name.
+     *
+     *  The returned match spy is owned by the context object.
+     *
+     *  Returns NULL if the match spy could not be found.
+     */
+    const Xapian::MatchSpy *
+            get_match_spy(const std::string & name) const;
+
 };
 
 #endif // XAPIAN_INCLUDED_SERIALISATIONCONTEXTINTERNAL_H

xapian-core/common/serialise.h

@@ -132 +132 @@
 
 /** Unserialise a serialised Xapian::MSet object.
  *
- *  @param s    The string to unserialise.
+ *  @param p     Pointer to pointer to start of the string to unserialise.
+ *  @param p_end Pointer to end of the string to unserialise.
  *
  *  @return     The unserialised Xapian::MSet object.
  */
-Xapian::MSet unserialise_mset(const std::string &s);
+Xapian::MSet unserialise_mset(const char ** p, const char * p_end);
 
 /** Serialise a Xapian::RSet object.
  *

xapian-core/api/Makefile.mk

@@ -16 +16 @@
         api/errorhandler.cc\
         api/expanddecider.cc\
         api/leafpostlist.cc\
+        api/matchspy.cc\
         api/omdatabase.cc\
         api/omdocument.cc\
         api/omenquire.cc\

xapian-core/api/omenquire.cc

@@ -3 +3 @@
  * Copyright 1999,2000,2001 BrightStation PLC
  * Copyright 2001,2002 Ananova Ltd
  * Copyright 2002,2003,2004,2005,2006,2007,2008,2009 Olly Betts
- * Copyright 2007 Lemur Consulting Ltd
+ * Copyright 2007,2009 Lemur Consulting Ltd
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -642 +642 @@
 Enquire::Internal::get_mset(Xapian::doccount first, Xapian::doccount maxitems,
                             Xapian::doccount check_at_least, const RSet *rset,
                             const MatchDecider *mdecider,
-                            const MatchDecider *matchspy) const
+                            const MatchDecider *matchspy_legacy) const
 {
     DEBUGCALL(API, MSet, "Enquire::Internal::get_mset", first << ", " <<
               maxitems << ", " << check_at_least << ", " << rset << ", " <<
-              mdecider << ", " << matchspy);
+              mdecider << ", " << matchspy_legacy);
 
     if (percent_cutoff && (sort_by == VAL || sort_by == VAL_REL)) {
         throw Xapian::UnimplementedError("Use of a percentage cutoff while sorting primary by value isn't currently supported");
@@ -661 +661 @@
                        collapse_max, collapse_key,
                        percent_cutoff, weight_cutoff,
                        order, sort_key, sort_by, sort_value_forward, sorter,
-                       errorhandler, stats, weight);
+                       errorhandler, stats, weight, spies);
     // Run query and put results into supplied Xapian::MSet object.
     MSet retval;
     match.get_mset(first, maxitems, check_at_least, retval,
-                   stats, mdecider, matchspy);
+                   stats, mdecider, matchspy_legacy);
 
     Assert(weight->name() != "bool" || retval.get_max_possible() == 0);
 
@@ -893 +893 @@
 }
 
 void
+Enquire::add_matchspy(MatchSpy * spy) {
+    DEBUGAPICALL(void, "Xapian::Enquire::add_matchspy", spy);
+    internal->spies.push_back(spy);
+}
+
+void
+Enquire::clear_matchspies() {
+    DEBUGAPICALL(const Xapian::Query &, "Xapian::Enquire::clear_matchspies", "");
+    internal->spies.clear();
+}
+
+void
 Enquire::set_weighting_scheme(const Weight &weight_)
 {
     DEBUGAPICALL(void, "Xapian::Enquire::set_weighting_scheme", "[Weight]");

xapian-core/api/serialisationcontext.cc

@@ -24 +24 @@
 #include "xapian/serialisationcontext.h"
 
 #include "xapian/error.h"
+#include "xapian/matchspy.h"
 #include "xapian/postingsource.h"
 #include "xapian/weight.h"
 
@@ -78 +79 @@
     RETURN(internal->get_weighting_scheme(name));
 }
 
-
 void
 SerialisationContext::register_posting_source(const Xapian::PostingSource &source)
 {
@@ -93 +93 @@
     RETURN(internal->get_posting_source(name));
 }
 
+void
+SerialisationContext::register_match_spy(const Xapian::MatchSpy &spy)
+{
+    LOGCALL_VOID(API, "Xapian::SerialisationContext::register_match_spy", spy.name());
+    internal->register_match_spy(spy);
+}
 
+const Xapian::MatchSpy *
+SerialisationContext::get_match_spy(const string & name) const
+{
+    LOGCALL(API, const Xapian::MatchSpy *, "Xapian::SerialisationContext::get_match_spy", name);
+    RETURN(internal->get_match_spy(name));
+}
+
+
 SerialisationContext::Internal::Internal()
         : Xapian::Internal::RefCntBase(),
           wtschemes(),
@@ -106 +120 @@
 {
     clear_weighting_schemes();
     clear_posting_sources();
+    clear_match_spies();
 }
 
 void
@@ -128 +143 @@
     postingsources[source->name()] = source;
     source = new Xapian::FixedWeightPostingSource(0.0);
     postingsources[source->name()] = source;
+
+    Xapian::MatchSpy * spy;
+    spy = new Xapian::ValueCountMatchSpy();
+    matchspies[spy->name()] = spy;
 }
 
 void
@@ -149 +168 @@
 }
 
 void
+SerialisationContext::Internal::clear_match_spies()
+{
+    map<string, Xapian::MatchSpy *>::const_iterator i;
+    for (i = matchspies.begin(); i != matchspies.end(); ++i) {
+        delete i->second;
+    }
+}
+
+void
 SerialisationContext::Internal::register_weighting_scheme(const Xapian::Weight &wt)
 {
     string wtname = wt.name();
@@ -220 +248 @@
     return i->second;
 }
 
+void
+SerialisationContext::Internal::register_match_spy(const Xapian::MatchSpy &spy)
+{
+    string spyname = spy.name();
+    if (spyname.empty()) {
+        throw Xapian::InvalidOperationError("Unable to register match spy - name() method returns empty string.");
+    }
+
+    map<string, Xapian::MatchSpy *>::const_iterator i;
+    i = matchspies.find(spyname);
+    if (i != matchspies.end()) {
+        delete i->second;
+    }
+
+    Xapian::MatchSpy * spyclone = spy.clone();
+    if (!spyclone) {
+        matchspies.erase(spyname);
+        throw Xapian::InvalidOperationError("Unable to register match spy - clone() method returns NULL.");
+    }
+    try {
+        matchspies[spyname] = spyclone;
+    } catch(...) {
+        delete spyclone;
+        matchspies.erase(spyname);
+        throw;
+    }
 }
+
+const Xapian::MatchSpy *
+SerialisationContext::Internal::get_match_spy(const string & name) const
+{
+    map<string, Xapian::MatchSpy *>::const_iterator i;
+    i = matchspies.find(name);
+    if (i == matchspies.end()) {
+        return NULL;
+    }
+    return i->second;
+}
+
+}

xapian-core/api/matchspy.cc

@@ +1 @@
+/** @file matchspy.cc
+ * @brief MatchSpy implementation.
+ */
+/* Copyright (C) 2007,2008,2009 Olly Betts
+ * Copyright (C) 2007,2009 Lemur Consulting Ltd
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ */
+
+#include <config.h>
+#include <xapian/matchspy.h>
+
+#include <xapian/document.h>
+#include <xapian/error.h>
+#include <xapian/queryparser.h>
+#include <xapian/serialisationcontext.h>
+
+#include <map>
+#include <string>
+#include <vector>
+
+#include "autoptr.h"
+#include "debuglog.h"
+#include "omassert.h"
+#include "serialise.h"
+#include "stringutils.h"
+#include "str.h"
+
+#include <float.h>
+#include <math.h>
+
+
+using namespace std;
+
+namespace Xapian {
+
+MatchSpy::~MatchSpy() {}
+
+MatchSpy *
+MatchSpy::clone() const {
+    throw UnimplementedError("MatchSpy not suitable for use with remote searches - clone() method unimplemented");
+}
+
+string
+MatchSpy::name() const {
+    throw UnimplementedError("MatchSpy not suitable for use with remote searches - name() method unimplemented");
+}
+
+string
+MatchSpy::serialise() const {
+    throw UnimplementedError("MatchSpy not suitable for use with remote searches - serialise() method unimplemented");
+}
+
+MatchSpy *
+MatchSpy::unserialise(const string &, const SerialisationContext &) const {
+    throw UnimplementedError("MatchSpy not suitable for use with remote searches - unserialise() method unimplemented");
+}
+
+string
+MatchSpy::serialise_results() const {
+    throw UnimplementedError("MatchSpy not suitable for use with remote searches - serialise_results() method unimplemented");
+}
+
+void
+MatchSpy::merge_results(const string &) {
+    throw UnimplementedError("MatchSpy not suitable for use with remote searches - merge_results() method unimplemented");
+}
+
+string
+MatchSpy::get_description() const {
+    return "Xapian::MatchSpy()";
+}
+
+
+/** Compare two StringAndFrequency objects.
+ *
+ *  The comparison is firstly by frequency (higher is better), then by string
+ *  (earlier lexicographic sort is better).
+ */
+class StringAndFreqCmpByFreq {
+  public:
+    /// Default constructor
+    StringAndFreqCmpByFreq() {}
+
+    /// Return true if a has a higher frequency than b.
+    /// If equal, compare by the str, to provide a stable sort order.
+    bool operator()(const StringAndFrequency &a,
+                    const StringAndFrequency &b) const {
+        if (a.get_frequency() > b.get_frequency()) return true;
+        if (a.get_frequency() < b.get_frequency()) return false;
+        if (a.get_string() > b.get_string()) return false;
+        return true;
+    }
+};
+
+/** Get the most frequent items from a map from string to frequency.
+ *
+ *  This takes input such as that returned by @a
+ *  ValueCountMatchSpy::get_values(), and returns a vector of the most
+ *  frequent items in the input.
+ *
+ *  @param result A vector which will be filled with the most frequent
+ *                items, in descending order of frequency.  Items with
+ *                the same frequency will be sorted in ascending
+ *                alphabetical order.
+ *
+ *  @param items The map from string to frequency, from which the most
+ *               frequent items will be selected.
+ *
+ *  @param maxitems The maximum number of items to return.
+ */
+static void
+get_most_frequent_items(vector<StringAndFrequency> & result,
+                        const map<string, doccount> & items,
+                        size_t maxitems)
+{
+    result.clear();
+    result.reserve(maxitems);
+    StringAndFreqCmpByFreq cmpfn;
+    bool is_heap(false);
+
+    for (map<string, doccount>::const_iterator i = items.begin();
+         i != items.end(); i++) {
+        Assert(result.size() <= maxitems);
+        result.push_back(StringAndFrequency(i->first, i->second));
+        if (result.size() > maxitems) {
+            // Make the list back into a heap.
+            if (is_heap) {
+                // Only the new element isn't in the right place.
+                push_heap(result.begin(), result.end(), cmpfn);
+            } else {
+                // Need to build heap from scratch.
+                make_heap(result.begin(), result.end(), cmpfn);
+                is_heap = true;
+            }
+            pop_heap(result.begin(), result.end(), cmpfn);
+            result.pop_back();
+        }
+    }
+
+    if (is_heap) {
+        sort_heap(result.begin(), result.end(), cmpfn);
+    } else {
+        sort(result.begin(), result.end(), cmpfn);
+    }
+}
+
+void
+ValueCountMatchSpy::operator()(const Document &doc, weight) {
+    ++total;
+    string val(doc.get_value(slot));
+    if (!val.empty()) ++values[val];
+}
+
+void
+ValueCountMatchSpy::get_top_values(vector<StringAndFrequency> & result,
+                                   size_t maxvalues) const
+{
+    get_most_frequent_items(result, values, maxvalues);
+}
+
+MatchSpy *
+ValueCountMatchSpy::clone() const {
+    return new ValueCountMatchSpy(slot);
+}
+
+string
+ValueCountMatchSpy::name() const {
+    return "Xapian::ValueCountMatchSpy";
+}
+
+string
+ValueCountMatchSpy::serialise() const {
+    string result;
+    result += encode_length(slot);
+    return result;
+}
+
+MatchSpy *
+ValueCountMatchSpy::unserialise(const string & s,
+                                const SerialisationContext &) const{
+    const char * p = s.data();
+    const char * end = p + s.size();
+
+    valueno new_slot = decode_length(&p, end, false);
+    if (p != end) {
+        throw NetworkError("Junk at end of serialised ValueCountMatchSpy");
+    }
+
+    return new ValueCountMatchSpy(new_slot);
+}
+
+string
+ValueCountMatchSpy::serialise_results() const {
+    LOGCALL(REMOTE, string, "ValueCountMatchSpy::serialise_results", "");
+    string result;
+    result += encode_length(total);
+    result += encode_length(values.size());
+    for (map<string, doccount>::const_iterator i = values.begin();
+         i != values.end(); ++i) {
+        result += encode_length(i->first.size());
+        result += i->first;
+        result += encode_length(i->second);
+    }
+    RETURN(result);
+}
+
+void
+ValueCountMatchSpy::merge_results(const string & s) {
+    LOGCALL_VOID(REMOTE, "ValueCountMatchSpy::merge_results", s);
+    const char * p = s.data();
+    const char * end = p + s.size();
+
+    total += decode_length(&p, end, false);
+
+    map<string, doccount>::size_type items = decode_length(&p, end, false);
+    while (p != end) {
+        while(items != 0) {
+            size_t vallen = decode_length(&p, end, true);
+            string val(p, vallen);
+            p += vallen;
+            doccount freq = decode_length(&p, end, false);
+            values[val] += freq;
+            --items;
+        }
+    }
+}
+
+string
+ValueCountMatchSpy::get_description() const {
+    return "Xapian::ValueCountMatchSpy(" + str(total) +
+            " docs seen, looking in " + str(values.size()) + " slots)";
+}
+
+
+inline double sqrd(double x) { return x * x; }
+
+/** Calculate a score based on how evenly distributed the frequencies of a set
+ *  of values are.
+ */
+template<class T> double
+do_score_evenness(const map<T, doccount> & values,
+                  doccount total,
+                  double desired_no_of_categories)
+{
+    if (total == 0) return 0.0;
+
+    size_t total_unset = total;
+    double score = 0.0;
+
+    if (desired_no_of_categories <= 0.0)
+        desired_no_of_categories = values.size();
+
+    double avg = double(total) / desired_no_of_categories;
+
+    typename map<T, doccount>::const_iterator i;
+    for (i = values.begin(); i != values.end(); ++i) {
+        size_t count = i->second;
+        total_unset -= count;
+        score += sqrd(count - avg);
+    }
+    if (total_unset) score += sqrd(total_unset - avg);
+
+    // Scale down so the total number of items doesn't make a difference.
+    score /= sqrd(total);
+
+    // Bias towards returning the number of categories requested.
+    score += 0.01 * sqrd(desired_no_of_categories - values.size());
+
+    return score;
+}
+
+double score_evenness(const map<string, doccount> & values,
+                      doccount total,
+                      double desired_no_of_categories) {
+    return do_score_evenness(values, total, desired_no_of_categories);
+}
+
+double score_evenness(const map<NumericRange, doccount> & values,
+                      doccount total,
+                      double desired_no_of_categories) {
+    return do_score_evenness(values, total, desired_no_of_categories);
+}
+
+double score_evenness(const ValueCountMatchSpy & spy,
+                      double desired_no_of_categories) {
+    return do_score_evenness(spy.get_values(), spy.get_total(),
+                             desired_no_of_categories);
+}
+
+
+/** A bucket, used when building numeric ranges.
+ */
+struct bucketval {
+    size_t count;
+    double min, max;
+
+    bucketval() : count(0), min(DBL_MAX), max(-DBL_MAX) { }
+
+    void update(size_t n, double value) {
+        count += n;
+        if (value < min) min = value;
+        if (value > max) max = value;
+    }
+};
+
+doccount build_numeric_ranges(map<NumericRange, doccount> & result,
+                              const map<string, doccount> & values,
+                              size_t max_ranges)
+{
+    double lo = DBL_MAX, hi = -DBL_MAX;
+    result.clear();
+
+    map<double, doccount> histo;
+    doccount total_set = 0;
+    map<string, doccount>::const_iterator i;
+    for (i = values.begin(); i != values.end(); ++i) {
+        if (i->first.size() == 0) continue;
+        double v = sortable_unserialise(i->first.c_str());
+        if (v < lo) lo = v;
+        if (v > hi) hi = v;
+        doccount count = i->second;
+        histo[v] = count;
+        total_set += count;
+    }
+
+    if (total_set == 0) {
+        // No set values.
+        return total_set;
+    }
+    if (lo == hi) {
+        // All set values are the same.
+        NumericRange range(lo, hi);
+        result[range] = total_set;
+        return total_set;
+    }
+
+    double sizeby = max(fabs(hi), fabs(lo));
+    // E.g. if sizeby = 27.4 and max_ranges = 7, we want to split into units of
+    // width 1.0 which we may then coalesce if there are too many used buckets.
+    double unit = pow(10.0, floor(log10(sizeby / max_ranges) - 0.2));
+    double start = floor(lo / unit) * unit;
+    // Can happen due to FP rounding (e.g. lo = 11.95, unit = 0.01).
+    if (start > lo) start = lo;
+    size_t n_buckets = size_t(ceil(hi / unit) - floor(lo / unit));
+
+    bool scaleby2 = true;
+    vector<bucketval> bucket(n_buckets + 1);
+    while (true) {
+        size_t n_used = 0;
+        map<double, doccount>::const_iterator j;
+        for (j = histo.begin(); j != histo.end(); ++j) {
+            double v = j->first;
+            size_t b = size_t(floor((v - start) / unit));
+            if (b > n_buckets) b = n_buckets; // FIXME - Hacky workaround to ensure that b is in range.
+            if (bucket[b].count == 0) ++n_used;
+            bucket[b].update(j->second, v);
+        }
+
+        if (n_used <= max_ranges) break;
+
+        unit *= scaleby2 ? 2.0 : 2.5;
+        scaleby2 = !scaleby2;
+        start = floor(lo / unit) * unit;
+        // Can happen due to FP rounding (e.g. lo = 11.95, unit = 0.01).
+        if (start > lo) start = lo;
+        n_buckets = size_t(ceil(hi / unit) - floor(lo / unit));
+        bucket.resize(0);
+        bucket.resize(n_buckets + 1);
+    }
+
+    map<string, doccount> discrete_categories;
+    for (size_t b = 0; b < bucket.size(); ++b) {
+        if (bucket[b].count == 0) continue;
+        NumericRange range(bucket[b].min, bucket[b].max);
+        result[range] = bucket[b].count;
+    }
+
+    return total_set;
+}
+
+}

xapian-core/backends/remote/remote-database.cc

@@ -43 +43 @@
 #include <vector>
 
 #include "xapian/error.h"
+#include "xapian/matchspy.h"
 
 using namespace std;
 
@@ -536 +537 @@
                          bool sort_value_forward,
                          int percent_cutoff, Xapian::weight weight_cutoff,
                          const Xapian::Weight *wtscheme,
-                         const Xapian::RSet &omrset)
+                         const Xapian::RSet &omrset,
+                         const vector<Xapian::MatchSpy *> & matchspies)
 {
     string tmp = query->serialise();
     string message = encode_length(tmp.size());
@@ -561 +563 @@
     message += encode_length(tmp.size());
     message += tmp;
 
-    message += serialise_rset(omrset);
+    tmp = serialise_rset(omrset);
+    message += encode_length(tmp.size());
+    message += tmp;
 
+    message += encode_length(matchspies.size());
+    for (vector<Xapian::MatchSpy *>::const_iterator i = matchspies.begin();
+         i != matchspies.end(); ++i) {
+
+        tmp = (*i)->name();
+        if (tmp.size() == 0) {
+            throw Xapian::UnimplementedError("MatchSpy not suitable for use with remote searches - name() method returned empty string");
+        }
+        message += encode_length(tmp.size());
+        message += tmp;
+
+        tmp = (*i)->serialise();
+        message += encode_length(tmp.size());
+        message += tmp;
+    }
+
     send_message(MSG_QUERY, message);
 }
 
@@ -592 +612 @@
 }
 
 void
-RemoteDatabase::get_mset(Xapian::MSet &mset)
+RemoteDatabase::get_mset(Xapian::MSet &mset,
+                         const vector<Xapian::MatchSpy *> & matchspies)
 {
     string message;
     get_message(message, REPLY_RESULTS);
-    mset = unserialise_mset(message);
+    const char * p = message.data();
+    const char * p_end = p + message.size();
+    mset = unserialise_mset(&p, p_end);
+
+    for (vector<Xapian::MatchSpy *>::const_iterator i = matchspies.begin();
+         i != matchspies.end(); ++i) {
+        if (p == p_end)
+            throw Xapian::NetworkError("Expected serialised matchspy");
+        size_t len = decode_length(&p, p_end, true);
+        string spyresults = string(p, len);
+        p += len;
+        (*i)->merge_results(spyresults);
+    }
+    if (p != p_end)
+        throw Xapian::NetworkError("Junk at end of mset");
 }
 
 void

xapian-bindings/csharp/Makefile.am

@@ -26 +26 @@
         Flint.cs \
         InMemory.cs \
         MatchDecider.cs \
+        MatchSpy.cs \
         MSet.cs \
         MSetIterator.cs \
         MultiValueSorter.cs \
@@ -37 +38 @@
         QueryParser.cs \
         Remote.cs \
         RSet.cs \
+        SWIGTYPE_p_std__mapT_Xapian__NumericRange_unsigned_int_t.cs \
+        SWIGTYPE_p_std__mapT_std__string_unsigned_int_t.cs \
         SWIGTYPE_p_std__string.cs \
         SWIGTYPE_p_std__vectorT_std__string_t.cs \
         SWIGTYPE_p_std__vectorT_Xapian__Query_t.cs \
+        SWIGTYPE_p_std__vectorT_Xapian__StringAndFrequency_t.cs \
         SerialisationContext.cs \
         SimpleStopper.cs \
         Sorter.cs \
@@ -49 +53 @@
         TermGenerator.cs \
         TermIterator.cs \
         TradWeight.cs \
+        ValueCountMatchSpy.cs \
         ValueIterator.cs \
         ValueRangeProcessor.cs \
         Version.cs \

xapian-bindings/python/pythontest2.py

@@ -1289 +1289 @@
     enq.set_query(xapian.Query('foo'))
     enq.get_mset(0, 10)
 
+def test_matchspy():
+    """Test use of matchspies.
+
+    """
+    db = setup_database()
+    query = xapian.Query(xapian.Query.OP_OR, "was", "it")
+    enq = xapian.Enquire(db)
+    enq.set_query(query)
+
+    def set_matchspy_deref(enq):
+        """Set a matchspy, and then drop the reference, to check that it
+        doesn't get deleted too soon.
+        """
+        spy = xapian.ValueCountMatchSpy(0)
+        enq.add_matchspy(spy)
+        del spy
+    set_matchspy_deref(enq)
+    mset = enq.get_mset(0, 10)
+    expect(len(mset), 5)
+
+    spy = xapian.ValueCountMatchSpy(0)
+    enq.add_matchspy(spy)
+    mset = enq.get_mset(0, 10)
+    expect(spy.get_values_as_dict(), {'zero': 1})
+    expect(spy.get_total(), 5)
+    expect(spy.get_top_values(10), [('zero', 1)])
+
 # Run all tests (ie, callables with names starting "test_").
 if not runtests(globals(), sys.argv[1:]):
     sys.exit(1)

xapian-bindings/python/util.i

@@ -172 +172 @@
     }
 }
 
+%{
+/* Typemap for returning a map of ints keyed by strings: converts to a dict.
+ * This is used for @a ValueCountMatchSpy::get_values().
+ * The GIL must be held when this is called.
+ */
+PyObject *
+value_map_to_dict(const std::map<std::string, Xapian::doccount> & vals)
+{
+    PyObject * result = PyDict_New();
+    if (result == 0) {
+        return NULL;
+    }
+
+    std::map<std::string, Xapian::doccount>::const_iterator i;
+    for (i = vals.begin(); i != vals.end(); ++i) {
+        PyObject * str = PyString_FromStringAndSize((*i).first.data(),
+                                                    (*i).first.size());
+        if (str == 0) {
+            Py_DECREF(result);
+            result = NULL;
+            return NULL;
+        }
+
+        PyObject * l = PyInt_FromLong((*i).second);
+        if (l == 0) {
+            Py_DECREF(str);
+            Py_DECREF(result);
+            result = NULL;
+            return NULL;
+        }
+
+        if (PyDict_SetItem(result, str, l) == -1) {
+            Py_DECREF(result);
+            result = NULL;
+            return NULL;
+        }
+        Py_DECREF(str);
+        Py_DECREF(l);
+    }
+    return result;
+}
+%}
+
+/** Typemap pair for getting the return value from @a ValueCountMatchSpy::get_top_values().
+ */
+%typemap(in, numinputs=0) std::vector<Xapian::StringAndFrequency> & result (std::vector<Xapian::StringAndFrequency> temp) {
+    $1 = &temp;
+}
+%typemap(argout) std::vector<Xapian::StringAndFrequency> & result {
+    Py_DECREF($result);
+    $result = PyList_New($1->size());
+    size_t pos = 0;
+    for (std::vector<Xapian::StringAndFrequency>::const_iterator i = $1->begin();
+         i != $1->end(); ++i) {
+        PyObject * str = PyString_FromStringAndSize((*i).get_string().data(),
+                                                    (*i).get_string().size());
+        if (str == 0) {
+            Py_DECREF($result);
+            $result = NULL;
+            SWIG_fail;
+        }
+
+        PyObject * l = PyInt_FromLong((*i).get_frequency());
+        if (l == 0) {
+            Py_DECREF($result);
+            Py_DECREF(str);
+            $result = NULL;
+            SWIG_fail;
+        }
+
+        PyObject *t = PyTuple_New(2);
+        if (t == 0) {
+            Py_DECREF($result);
+            Py_DECREF(str);
+            Py_DECREF(l);
+            $result = NULL;
+            SWIG_fail;
+        }
+        PyTuple_SetItem(t, 0, str);
+        PyTuple_SetItem(t, 1, l);
+
+        PyList_SetItem($result, pos++, t);
+    }
+}
+
 %typedef PyObject *LangSpecificListType;
 
 %inline %{

xapian-bindings/python/extra.i

@@ -22 +22 @@
  */
 %}
 
+%extend ValueCountMatchSpy {
+    %feature("nothread") get_values_as_dict;
+    %exception get_values_as_dict {
+        try {
+            $action
+        } catch (...) {
+            Xapian::SetPythonException();
+            SWIG_fail;
+        }
+    }
+    PyObject * get_values_as_dict() {
+        return value_map_to_dict($self->get_values());
+    }
+}
+
 %pythoncode %{
 
 # Set the documentation format - this is used by tools like "epydoc" to decide
@@ -1076 +1091 @@
 __all__ = tuple(__all__)
 
 
+# Fix up Enquire so that it keeps a python reference to the deciders supplied
+# to it so that they won't be deleted before the Enquire object.  This hack can
+# probably be removed once xapian bug #186 is fixed.
+_enquire_add_matchspy_orig = Enquire.add_matchspy
+def _enquire_match_spy_add(self, decider):
+    if not hasattr(self, '_deciders'):
+        self._deciders = []
+    self._deciders.append(decider)
+    _enquire_add_matchspy_orig(self, decider)
+_enquire_match_spy_add.__doc__ = Enquire.add_matchspy.__doc__
+Enquire.add_matchspy = _enquire_match_spy_add
+
+_enquire_clear_matchspies_orig = Enquire.clear_matchspies
+def _enquire_match_spies_clear(self):
+    _enquire_clear_matchspies_orig(self, decider)
+    if hasattr(self, '_deciders'):
+        del self._deciders
+_enquire_match_spies_clear.__doc__ = Enquire.clear_matchspies.__doc__
+Enquire.clear_matchspies = _enquire_match_spies_clear
+
+
+
 # Remove static methods which shouldn't be in the API.
 del Document_unserialise
 del Query_unserialise

xapian-bindings/xapian.i

@@ -356 +356 @@
 #endif
 
 class Database;
+class MatchSpy;
 class Query;
 class Sorter;
 
@@ -367 +368 @@
     void set_query(const Query & query, termcount qlen = 0);
     const Query& get_query();
 
+    void add_matchspy(MatchSpy * spy);
+    void clear_matchspies();
+
     void set_weighting_scheme(const Weight& weight);
     void set_collapse_key(Xapian::valueno collapse_key,
                           Xapian::doccount collapse_max = 1);
@@ -440 +444 @@
 
 }
 
+%ignore Xapian::SerialisationContext::operator=;
+%include <xapian/serialisationcontext.h>
+
 /* Generated code won't compile if directors are enabled.  Disable for now
  * while we investigate.
  *
@@ -466 +473 @@
 %warnfilter(842) Xapian::TradWeight::unserialise;
 %include <xapian/weight.h>
 
+%ignore Xapian::NumericRange::operator<;
+%include <xapian/matchspy.h>
+
 namespace Xapian {
 
 // xapian/database.h
@@ -747 +757 @@
 %include <xapian/replication.h>
 %include <xapian/valuesetmatchdecider.h>
 
-%ignore Xapian::SerialisationContext::operator=;
-%include <xapian/serialisationcontext.h>
-
 namespace Xapian {
 
 #if defined SWIGPYTHON