Ticket #100: patch

xapian-maintainer-tools/win32msvc/win32_matcher.mak

@@ -32 +32 @@
                  $(INTDIR)\multimatch.obj \
                  $(INTDIR)\expand.obj \
                  $(INTDIR)\stats.obj \
+                 $(INTDIR)\matchcmp.obj \
                  $(INTDIR)\mergepostlist.obj \
                  $(INTDIR)\msetpostlist.obj \
                  $(INTDIR)\msetcmp.obj \
@@ -166 +167 @@
    $(CPP_PROJ) $**
 <<
 
+"$(INTDIR)\matchcmp.obj" : ".\matchcmp.cc"
+       $(CPP) @<<
+   $(CPP_PROJ) $**
+<<
 
 "$(INTDIR)\mergepostlist.obj" : ".\mergepostlist.cc"
        $(CPP) @<<

xapian-core/matcher/Makefile.mk

@@ -43 +43 @@
         matcher/expandweight.cc\
         matcher/filterpostlist.cc\
         matcher/localmatch.cc\
+        matcher/matchcmp.cc\
         matcher/mergepostlist.cc\
         matcher/msetcmp.cc\
         matcher/msetpostlist.cc\

xapian-core/matcher/multimatch.cc

@@ -86 +86 @@
                        bool sort_value_forward_,
                        Xapian::ErrorHandler * errorhandler_,
                        StatsGatherer * gatherer_,
-                       const Xapian::Weight * weight_)
+                       const Xapian::Weight * weight_,
+                       const Xapian::MatchCmp * match_cmp_)
         : gatherer(gatherer_), db(db_), query(query_),
           collapse_key(collapse_key_), percent_cutoff(percent_cutoff_),
           weight_cutoff(weight_cutoff_), order(order_),
           sort_key(sort_key_), sort_by(sort_by_),
           sort_value_forward(sort_value_forward_),
           errorhandler(errorhandler_), weight(weight_),
+          match_cmp(match_cmp_),
           is_remote(db.internal.size())
 {
     DEBUGCALL(MATCH, void, "MultiMatch", db_ << ", " << query_ << ", " <<
@@ -139 +141 @@
                 is_remote[i] = true;
                 rem_db->set_query(query, qlen, collapse_key, order, sort_key,
                                   sort_by, sort_value_forward, percent_cutoff,
-                                  weight_cutoff, weight, subrsets[i]);
+                                  weight_cutoff, weight, match_cmp,
+                                  subrsets[i]);
                 bool decreasing_relevance =
                     (sort_by == REL || sort_by == REL_VAL);
                 smatch = new RemoteSubMatch(rem_db, gatherer.get(),
@@ -193 +196 @@
 }
 
 string
-MultiMatch::get_collapse_key(PostList *pl, Xapian::docid did,
-                             Xapian::valueno keyno, Xapian::Internal::RefCntPtr<Xapian::Document::Internal> &doc)
+MultiMatch::get_collapse_key(PostList *pl,
+                             Xapian::valueno keyno,
+                             const Xapian::Internal::MSetItem & item)
 {
-    DEBUGCALL(MATCH, string, "MultiMatch::get_collapse_key", pl << ", " << did << ", " << keyno << ", [doc]");
+    DEBUGCALL(MATCH, string, "MultiMatch::get_collapse_key", pl << ", " << item.did << ", " << keyno << ", [doc]");
+
+    // FIXME - this gets the collapse key from the postlist.  This only
+    // actually happens for Mset postlists (or mergepostlists containing an
+    // mset postlist), since this is the only case where the collapse key has
+    // been read already.  Instead, we should use the document object stored in
+    // the mset item.
+
     const string *key = pl->get_collapse_key();
     if (key) RETURN(*key);
-    if (doc.get() == 0) {
-        unsigned int multiplier = db.internal.size();
-        Assert(multiplier != 0);
-        Xapian::doccount n = (did - 1) % multiplier; // which actual database
-        Xapian::docid m = (did - 1) / multiplier + 1; // real docid in that database
-
-        Xapian::Internal::RefCntPtr<Xapian::Document::Internal> temp(db.internal[n]->open_document(m, true));
-        doc = temp;
-    }
-    RETURN(doc->get_value(keyno));
+    RETURN(item.get_document()->get_value(keyno));
 }
 
 Xapian::weight
@@ -231 +233 @@
 void
 MultiMatch::get_mset(Xapian::doccount first, Xapian::doccount maxitems,
                      Xapian::doccount check_at_least,
-                     Xapian::MSet & mset, const Xapian::MatchDecider *mdecider)
+                     Xapian::MSet & mset,
+                     const Xapian::MatchDecider *mdecider)
 {
     DEBUGCALL(MATCH, void, "MultiMatch::get_mset", first << ", " << maxitems
               << ", " << check_at_least << ", ...");
@@ -362 +365 @@
 
     /// Comparison functor for sorting MSet
     bool sort_forward = (order != Xapian::Enquire::DESCENDING);
-    MSetCmp mcmp(get_msetcmp_function(sort_by, sort_forward, sort_value_forward));
+    mset_cmp cmpfn = get_msetcmp_function(sort_by, sort_forward, sort_value_forward);
+    MSetCmp mcmp(cmpfn, sort_key, match_cmp);
 
+    /// Number of databases making up the database we're using.
+    const unsigned int db_internal_size = db.internal.size();
+    Assert(db_internal_size != 0);
+
     // Perform query
 
     // We form the mset in two stages.  In the first we fill up our working
@@ -410 +418 @@
         if (min_item.wt > 0.0) wt = pl->get_weight();
 
         DEBUGLINE(MATCH, "Candidate document id " << did << " wt " << wt);
-        Xapian::Internal::MSetItem new_item(wt, did);
-        if (sort_by != REL) {
-            const unsigned int multiplier = db.internal.size();
-            Assert(multiplier != 0);
-            Xapian::doccount n = (new_item.did - 1) % multiplier; // which actual database
-            Xapian::docid m = (new_item.did - 1) / multiplier + 1; // real docid in that database
-            Xapian::Internal::RefCntPtr<Xapian::Document::Internal> doc(db.internal[n]->open_document(m, true));
-            new_item.sort_key = doc->get_value(sort_key);
-        }
+        Xapian::Internal::RefCntPtr<Xapian::Database::Internal> internal_db = db.internal[((did - 1) % db_internal_size)]; // database holding document
+        Xapian::docid internal_docid = (did - 1) / db_internal_size + 1; // docid in internal_db
+        Xapian::Internal::MSetItem new_item(wt, did, internal_db, internal_docid);
 
         // Test if item has high enough weight (or sort key) to get into
         // proto-mset.
         if (sort_by != REL || min_item.wt > 0.0)
             if (!mcmp(new_item, min_item)) continue;
 
-        Xapian::Internal::RefCntPtr<Xapian::Document::Internal> doc;
-
         // Use the decision functor if any.
         if (mdecider != NULL) {
             const unsigned int multiplier = db.internal.size();
@@ -435 +435 @@
             // If the results are from a remote database, then the functor will
             // already have been applied there so we can skip this step.
             if (!is_remote[n]) {
-                if (doc.get() == 0) {
-                    Xapian::docid m = (did - 1) / multiplier + 1; // real docid in that database
-
-                    Xapian::Internal::RefCntPtr<Xapian::Document::Internal> temp(db.internal[n]->open_document(m, true));
-                    doc = temp;
-                }
-                Xapian::Document mydoc(doc.get());
+                Xapian::Document mydoc(new_item.get_document().get());
                 if (!mdecider->operator()(mydoc)) continue;
             }
         }
@@ -457 +451 @@
 
         // Perform collapsing on key if requested.
         if (collapse_key != Xapian::BAD_VALUENO) {
-            new_item.collapse_key = get_collapse_key(pl, did, collapse_key,
-                                                     doc);
+            new_item.collapse_key = get_collapse_key(pl, collapse_key, new_item);
 
             // Don't collapse on null key
             if (!new_item.collapse_key.empty()) {

xapian-core/matcher/msetcmp.h

@@ -20 +20 @@
 
 #include "omenquireinternal.h"
 
+class MSetCmp;
+
 // typedef for MSetItem comparison function.
 typedef bool (* mset_cmp)(const Xapian::Internal::MSetItem &,
-                          const Xapian::Internal::MSetItem &);
+                          const Xapian::Internal::MSetItem &,
+                          const MSetCmp *);
 
 /// Select the appropriate msetcmp function.
 mset_cmp get_msetcmp_function(Xapian::Enquire::Internal::sort_setting sort_by, bool sort_forward, bool sort_value_forward);
@@ -31 +34 @@
 class MSetCmp {
     mset_cmp fn;
   public:
-    MSetCmp(mset_cmp fn_) : fn(fn_) { }
+    Xapian::valueno sort_key;
+    const Xapian::MatchCmp * cmp;
+
+    /// Construct an instance of MSetCmp using a built-in comparison function.
+    MSetCmp(mset_cmp fn_,
+            Xapian::valueno sort_key_,
+            const Xapian::MatchCmp * cmp_)
+        : fn(fn_), sort_key(sort_key_), cmp(cmp_) { }
+
     /// Return true if MSetItem a should be ranked above MSetItem b.
     bool operator()(const Xapian::Internal::MSetItem &a,
-                    const Xapian::Internal::MSetItem &b) const {
-        return fn(a, b);
+                    const Xapian::Internal::MSetItem &b) const {
+        return fn(a, b, this);
     }
 };
+

xapian-core/matcher/msetcmp.cc

@@ -20 +20 @@
 
 #include <config.h>
 #include "msetcmp.h"
+#include <string>
 
 /* We use templates to generate the 14 different comparison functions
  * which we need.  This avoids having to write them all out by hand.
@@ -46 +47 @@
 // Order by relevance, then docid.
 template<bool FORWARD_DID> bool
 msetcmp_by_relevance(const Xapian::Internal::MSetItem &a,
-                     const Xapian::Internal::MSetItem &b)
+                     const Xapian::Internal::MSetItem &b,
+                     const MSetCmp *)
 {
     if (a.wt > b.wt) return true;
     if (a.wt < b.wt) return false;
@@ -56 +58 @@
 // Order by value, then docid.
 template<bool FORWARD_VALUE, bool FORWARD_DID> bool
 msetcmp_by_value(const Xapian::Internal::MSetItem &a,
-                 const Xapian::Internal::MSetItem &b)
+                 const Xapian::Internal::MSetItem &b,
+                 const MSetCmp * cmp)
 {
     if (!FORWARD_VALUE) {
         // We want dummy did 0 to compare worse than any other.
         if (a.did == 0) return false;
         if (b.did == 0) return true;
     }
-    if (a.sort_key > b.sort_key) return FORWARD_VALUE;
-    if (a.sort_key < b.sort_key) return !FORWARD_VALUE;
+    std::string sort_key_a = a.get_document()->get_value(cmp->sort_key);
+    std::string sort_key_b = b.get_document()->get_value(cmp->sort_key);
+    if (sort_key_a > sort_key_b) return FORWARD_VALUE;
+    if (sort_key_a < sort_key_b) return !FORWARD_VALUE;
     return msetcmp_by_did<FORWARD_DID, FORWARD_VALUE>(a, b);
 }
 
 // Order by value, then relevance, then docid.
 template<bool FORWARD_VALUE, bool FORWARD_DID> bool
 msetcmp_by_value_then_relevance(const Xapian::Internal::MSetItem &a,
-                                const Xapian::Internal::MSetItem &b)
+                                const Xapian::Internal::MSetItem &b,
+                                const MSetCmp * cmp)
 {
     if (!FORWARD_VALUE) {
         // two special cases to make min_item compares work when did == 0
         if (a.did == 0) return false;
         if (b.did == 0) return true;
     }
-    if (a.sort_key > b.sort_key) return FORWARD_VALUE;
-    if (a.sort_key < b.sort_key) return !FORWARD_VALUE;
+    std::string sort_key_a = a.get_document()->get_value(cmp->sort_key);
+    std::string sort_key_b = b.get_document()->get_value(cmp->sort_key);
+    if (sort_key_a > sort_key_b) return FORWARD_VALUE;
+    if (sort_key_a < sort_key_b) return !FORWARD_VALUE;
     if (a.wt > b.wt) return true;
     if (a.wt < b.wt) return false;
     return msetcmp_by_did<FORWARD_DID, FORWARD_VALUE>(a, b);
@@ -88 +96 @@
 // Order by relevance, then value, then docid.
 template<bool FORWARD_VALUE, bool FORWARD_DID> bool
 msetcmp_by_relevance_then_value(const Xapian::Internal::MSetItem &a,
-                                const Xapian::Internal::MSetItem &b)
+                                const Xapian::Internal::MSetItem &b,
+                                const MSetCmp * cmp)
 {
     if (!FORWARD_VALUE) {
         // two special cases to make min_item compares work when did == 0
@@ -97 +106 @@
     }
     if (a.wt > b.wt) return true;
     if (a.wt < b.wt) return false;
-    if (a.sort_key > b.sort_key) return FORWARD_VALUE;
-    if (a.sort_key < b.sort_key) return !FORWARD_VALUE;
+    std::string sort_key_a = a.get_document()->get_value(cmp->sort_key);
+    std::string sort_key_b = b.get_document()->get_value(cmp->sort_key);
+    if (sort_key_a > sort_key_b) return FORWARD_VALUE;
+    if (sort_key_a < sort_key_b) return !FORWARD_VALUE;
     return msetcmp_by_did<FORWARD_DID, FORWARD_VALUE>(a, b);
 }
 
+template<bool FORWARD_DID> bool
+msetcmp_by_user_cmpfn(const Xapian::Internal::MSetItem &a,
+                      const Xapian::Internal::MSetItem &b,
+                      const MSetCmp * cmp)
+{
+    // We want dummy did 0 to compare worse than any other.
+    if (a.did == 0) return false;
+    if (b.did == 0) return true;
+
+    if (a.did == b.did) return false; // FIXME - can this ever happen?  If not, convert this to Assert(a.did != b.did).
+    Xapian::MatchItem item_a(a);
+    Xapian::MatchItem item_b(b);
+    int c = (cmp->cmp->cmp)(item_a, item_b);
+    if (c < 0)
+        return true;
+    else if (c > 0)
+        return false;
+    if (FORWARD_DID) {
+        return (a.did < b.did);
+    } else {
+        return (a.did > b.did);
+    }
+}
+
 static mset_cmp mset_cmp_table[] = {
     // Xapian::Enquire::Internal::REL
     msetcmp_by_relevance<false>,
@@ -122 +157 @@
     msetcmp_by_relevance_then_value<true, true>,
     msetcmp_by_relevance_then_value<false, true>,
     msetcmp_by_relevance_then_value<true, false>,
-    msetcmp_by_relevance_then_value<false, false>
+    msetcmp_by_relevance_then_value<false, false>,
+    // Xapian::Enquire::Internal::USER
+    msetcmp_by_user_cmpfn<false>,
+    msetcmp_by_user_cmpfn<false>,
+    msetcmp_by_user_cmpfn<true>,
+    msetcmp_by_user_cmpfn<true>
 };
 
 mset_cmp get_msetcmp_function(Xapian::Enquire::Internal::sort_setting sort_by, bool sort_forward, bool sort_value_forward) {

xapian-core/tests/internaltest.cc

@@ -385 +385 @@
         255.5,
         256.125,
         257.03125,
+        0.12268031290495594321l,
+        0.12268031290495592933l,
     };
 
     check_double_serialisation(0.0);

xapian-core/tests/api_db.cc

@@ -1264 +1264 @@
 // set_sort_by_value
 // set_sort_by_value_then_relevance
 // set_sort_by_relevance_then_value
+// User comparison functions
 static bool test_sortrel1()
 {
     Xapian::Enquire enquire(get_database("apitest_sortrel"));
@@ -1279 +1280 @@
     const Xapian::docid order7[] = { 7,9,8,6,5,4,2,1,3 };
     const Xapian::docid order8[] = { 7,6,2,9,5,1,8,4,3 };
     const Xapian::docid order9[] = { 2,6,7,1,5,9,3,4,8 };
+    const Xapian::docid order10[] = { 7,9,4,5,6,1,2,3,8 };
+    const Xapian::docid order11[] = { 8,1,2,3,4,5,6,7,9 };
+    const Xapian::docid order12[] = { 9,7,6,5,4,3,2,1,8 };
+    const Xapian::docid order13[] = { 8,3,2,1,6,5,4,9,7 };
 
     Xapian::MSet mset;
     size_t i;
@@ -1289 +1294 @@
         TEST_EQUAL(*mset[i], order1[i]);
     }
 
+    enquire.set_sort_by_cmpfn(Xapian::IntegerMatchCmp(100));
+
+    mset = enquire.get_mset(0, 10);
+    TEST_EQUAL(mset.size(), sizeof(order10) / sizeof(Xapian::docid));
+    for (i = 0; i < sizeof(order10) / sizeof(Xapian::docid); ++i) {
+        TEST_EQUAL(*mset[i], order10[i]);
+    }
+
+    enquire.set_sort_by_cmpfn(Xapian::IntegerMatchCmp(100, true));
+
+    mset = enquire.get_mset(0, 10);
+    TEST_EQUAL(mset.size(), sizeof(order10) / sizeof(Xapian::docid));
+    for (i = 0; i < sizeof(order10) / sizeof(Xapian::docid); ++i) {
+        TEST_EQUAL(*mset[i], order10[i]);
+    }
+
+    enquire.set_sort_by_cmpfn(Xapian::IntegerMatchCmp(100, false));
+
+    mset = enquire.get_mset(0, 10);
+    TEST_EQUAL(mset.size(), sizeof(order11) / sizeof(Xapian::docid));
+    for (i = 0; i < sizeof(order11) / sizeof(Xapian::docid); ++i) {
+        TEST_EQUAL(*mset[i], order11[i]);
+    }
+
     enquire.set_sort_by_value_then_relevance(1);
 
     mset = enquire.get_mset(0, 10);
@@ -1305 +1334 @@
         TEST_EQUAL(*mset[i], order1[i]);
     }
 
+    enquire.set_sort_by_cmpfn(Xapian::IntegerMatchCmp(100));
+    enquire.set_docid_order(Xapian::Enquire::DESCENDING);
+
+    mset = enquire.get_mset(0, 10);
+    TEST_EQUAL(mset.size(), sizeof(order12) / sizeof(Xapian::docid));
+    for (i = 0; i < sizeof(order12) / sizeof(Xapian::docid); ++i) {
+        TEST_EQUAL(*mset[i], order12[i]);
+    }
+
+    enquire.set_sort_by_cmpfn(Xapian::IntegerMatchCmp(100, true));
+    enquire.set_docid_order(Xapian::Enquire::DESCENDING);
+
+    mset = enquire.get_mset(0, 10);
+    TEST_EQUAL(mset.size(), sizeof(order12) / sizeof(Xapian::docid));
+    for (i = 0; i < sizeof(order12) / sizeof(Xapian::docid); ++i) {
+        TEST_EQUAL(*mset[i], order12[i]);
+    }
+
+    enquire.set_sort_by_cmpfn(Xapian::IntegerMatchCmp(100, false));
+    enquire.set_docid_order(Xapian::Enquire::DESCENDING);
+
+    mset = enquire.get_mset(0, 10);
+    TEST_EQUAL(mset.size(), sizeof(order13) / sizeof(Xapian::docid));
+    for (i = 0; i < sizeof(order13) / sizeof(Xapian::docid); ++i) {
+        TEST_EQUAL(*mset[i], order13[i]);
+    }
+
     enquire.set_sort_by_value_then_relevance(1);
     enquire.set_docid_order(Xapian::Enquire::DESCENDING);
 

xapian-core/tests/harness/index_utils.cc

@@ -78 +78 @@
     for (Xapian::valueno i = min(para.length(), size_t(10)); i >= 1; --i) {
         doc.add_value(i, para.substr(i, 1));
     }
+    if (para.length() > 1 && para[0] == 'V') {
+        doc.add_value(100, Xapian::IntegerMatchCmp::int_to_value(atoi(para.c_str() + 1)));
+    }
 
     Xapian::termcount pos = 0;
     string::const_iterator end = para.begin();

xapian-core/tests/testdata/apitest_sortrel.txt

@@ -12 +12 @@
 
 V1 woman
 
-V1 man woman fish
+V10 man woman fish
 
 V1 man woman

xapian-core/include/xapian/enquire.h

@@ -31 +31 @@
 #include <xapian/types.h>
 #include <xapian/termiterator.h>
 #include <xapian/visibility.h>
+#include <xapian/document.h>
 
 namespace Xapian {
 
@@ -40 +41 @@
 class MSetIterator;
 class Query;
 class Weight;
+class MatchCmp;
 
+namespace Internal {
+    class MSetItem;
+}
+
 /** A match set (MSet).
  *  This class represents (a portion of) the results of a query.
  */
@@ -812 +818 @@
         void set_sort_by_relevance_then_value(Xapian::valueno sort_key,
                                               bool ascending = true);
 
+        /** Set a comparison functor to be used when comparing matches.
+         *
+         *  This overrides the sort options set by set_sort_by_relevance(),
+         *  set_sort_by_value(), set_sort_by_relevance_then_value() and
+         *  set_sort_by_value_then_relevance().  For matches which the
+         *  comparison functor considers equal (ie, for which the functor
+         *  returns 0 when comparing them), the matches will be compared in
+         *  document ID order, as set by set_docid_order();
+         *
+         *  The comparison function will be called many times in the process of
+         *  performing the search.  Therefore, it is important that it is
+         *  implemented as efficiently as possible.
+         *
+         *  @param cmpfn_ The comparion function to use to compare matches.
+         */
+        void set_sort_by_cmpfn(const MatchCmp &cmpfn_);
+
         /** Get (a portion of) the match set for the current query.
          *
          *  @param first     the first item in the result set to return.
@@ -1220 +1243 @@
         bool get_sumpart_needs_doclength() const;
 };
 
+/// An item matching a search.
+class XAPIAN_VISIBILITY_DEFAULT MatchItem {
+    private:
+        const Xapian::Internal::MSetItem & item;
+
+    public:
+        MatchItem(const Xapian::Internal::MSetItem & item_)
+            : item(item_) {}
+
+        /** Get the weight calculated. */
+        Xapian::weight get_wt() const;
+
+        /** Get the document id. */
+        Xapian::docid get_docid() const;
+
+        /** Get the document object, for access to document data, values,
+         *  and terms. */
+        Xapian::Document get_document() const;
+};
+
+/// Abstract base class for match comparison functors.
+class XAPIAN_VISIBILITY_DEFAULT MatchCmp {
+    friend class Enquire; // So Enquire can clone us
+    friend class ::RemoteServer; // So RemoteServer can clone us - FIXME
+    protected:
+        MatchCmp(const MatchCmp &);
+    private:
+        void operator=(MatchCmp &);
+
+        /** Return a new MatchCmp object of this type.
+         *
+         * A subclass called FooMatchCmp taking parameters param1 and param2
+         * should implement this as:
+         *
+         * virtual FooMatchCmp * clone() const {
+         *     return new FooMatchCmp(param1, param2);
+         * }
+         */
+        virtual MatchCmp * clone() const = 0;
+
+    public:
+        MatchCmp() { }
+        virtual ~MatchCmp() { }
+
+        /** Name of the comparison functor.
+         *
+         *  If the subclass is called FooMatchCmp, this should return "Foo".
+         */
+        virtual std::string name() const = 0;
+
+        /// Serialise object parameters into a string.
+        virtual std::string serialise() const = 0;
+
+        /** Create object given string serialisation returned by serialise().
+         *
+         *  If the string is not a valid serialised form, this should raise a
+         *  Xapian::NetworkError;
+         */
+        virtual MatchCmp * unserialise(const std::string &s) const = 0;
+
+        /** Compare a potential Mset entry a to an entry b.
+         *
+         *  This function should return an integer less than 0 if MatchItem a
+         *  should be ranked below MatchItem b, and greater than 0 if MatchItem
+         *  a should be ranked above MatchItem b.  The function may return 0 if
+         *  it considers the two match items to be equal.
+         *
+         *  The function will never be called with items which have identical
+         *  document IDs.
+         *
+         *  The comparison function must be well behaved, such that:
+         *
+         *   - it must produce the same result if called a second time on a
+         *     particular pair of documents
+         *   - if cmp(x,y) is less than 0 and cmp(y,z) is less than 0, cmp(x,z)
+         *     must be less than 0.
+         *   - if cmp(x,y) is less than 0, cmp(y,x) must be greater than 0.
+         *
+         *  Bear in mind that the comparison will be performed very frequently
+         *  when performing a search, so must be as fast as possible.  In
+         *  particular, although the MatchItem provides access to the document
+         *  object, it is recommended that only the value items stored in the
+         *  document object are accessed.
+         */
+        virtual int cmp(const MatchItem & a,
+                        const MatchItem & b) const = 0;
+};
+
+/** Comparison function for comparing by value as integer.
+ */
+class XAPIAN_VISIBILITY_DEFAULT IntegerMatchCmp : public MatchCmp {
+        Xapian::valueno sort_key;
+        int direction;
+    public:
+        /** Return a new IntegerMatchCmp with the same parameters as this one.
+         */
+        IntegerMatchCmp * clone() const {
+            return new IntegerMatchCmp(sort_key, direction == 1);
+        }
+        IntegerMatchCmp(Xapian::valueno sort_key_,
+                        bool ascending = true)
+                : sort_key(sort_key_),
+                  direction(ascending ? 1 : -1) { }
+        IntegerMatchCmp()
+                : sort_key(0),
+                  direction(1) { }
+        ~IntegerMatchCmp() { }
+        std::string name() const { return "Integer"; }
+
+        std::string serialise() const;
+        IntegerMatchCmp * unserialise(const std::string &s) const;
+
+        int cmp(const MatchItem & a,
+                const MatchItem & b) const;
+
+        /** Convert an integer to a string, suitable for comparing with this
+         *  comparison functor.
+         */
+        static std::string int_to_value(int num);
+
+        /** Convert a string suitable for comparison with this comparison
+         *  functor to an integer.
+         */
+        static int value_to_int(const std::string &s);
+};
+
+/** Comparison function for comparing by value as a double.
+ */
+class XAPIAN_VISIBILITY_DEFAULT DoubleMatchCmp : public MatchCmp {
+        Xapian::valueno sort_key;
+        int direction;
+    public:
+        /** Return a new DoubleMatchCmp with the same parameters as this one.
+         */
+        DoubleMatchCmp * clone() const {
+            return new DoubleMatchCmp(sort_key, direction == 1);
+        }
+        DoubleMatchCmp(Xapian::valueno sort_key_,
+                       bool ascending = true)
+                : sort_key(sort_key_),
+                  direction(ascending ? 1 : -1) { }
+        DoubleMatchCmp()
+                : sort_key(0),
+                  direction(1) { }
+        ~DoubleMatchCmp() { }
+        std::string name() const { return "Double"; }
+
+        std::string serialise() const;
+        DoubleMatchCmp * unserialise(const std::string &s) const;
+
+        int cmp(const MatchItem & a,
+                const MatchItem & b) const;
+
+        /** Convert a double to a string, suitable for comparing with this
+         *  comparison functor.
+         */
+        static std::string double_to_value(double num);
+
+        /** Convert a string suitable for comparison with this comparison
+         *  functor to a double.
+         */
+        static double value_to_double(const std::string &s);
+};
+
 }
 
 #endif /* XAPIAN_INCLUDED_ENQUIRE_H */

xapian-core/net/serialise.cc

@@ -223 +223 @@
 }
 
 Xapian::MSet
-unserialise_mset(const string &s)
+unserialise_mset(const string &s,
+                 Xapian::Internal::RefCntPtr<Xapian::Database::Internal> internal_db)
 {
     const char * p = s.data();
     const char * p_end = p + s.size();
@@ -242 +243 @@
         size_t len = decode_length(&p, p_end, true);
         string key(p, len);
         p += len;
-        items.push_back(Xapian::Internal::MSetItem(wt, did, key,
+        items.push_back(Xapian::Internal::MSetItem(wt, did,
+                                                   internal_db,
+                                                   did,
+                                                   key,
                                                    decode_length(&p, p_end, false)));
     }
 

xapian-core/net/remoteserver.cc

@@ -90 +90 @@
     wtschemes[weight->name()] = weight;
     weight = new Xapian::TradWeight();
     wtschemes[weight->name()] = weight;
+
+    // Register match comparison functions.
+    Xapian::MatchCmp * match_cmp;
+    match_cmp = new Xapian::IntegerMatchCmp();
+    match_cmps[match_cmp->name()] = match_cmp;
+    match_cmp = new Xapian::DoubleMatchCmp();
+    match_cmps[match_cmp->name()] = match_cmp;
 }
 
 RemoteServer::~RemoteServer()
@@ -98 +105 @@
     for (i = wtschemes.begin(); i != wtschemes.end(); ++i) {
         delete i->second;
     }
+    map<string, Xapian::MatchCmp*>::const_iterator j;
+    for (j = match_cmps.begin(); j != match_cmps.end(); ++j) {
+        delete j->second;
+    }
 }
 
 message_type
@@ -322 +333 @@
 
     Xapian::valueno sort_key = decode_length(&p, p_end, false);
 
-    if (*p < '0' || *p > '3') {
+    if (*p < '0' || *p > '4') {
         throw Xapian::NetworkError("bad message (sort_by)");
     }
     Xapian::Enquire::Internal::sort_setting sort_by;
@@ -356 +367 @@
     AutoPtr<Xapian::Weight> wt(i->second->unserialise(string(p, len)));
     p += len;
 
+    // Unserialise the MatchCmp object
+    len = decode_length(&p, p_end, true);
+    AutoPtr<Xapian::MatchCmp> mcmp;
+    if (len > 0) {
+        map<string, Xapian::MatchCmp *>::const_iterator j;
+        j = match_cmps.find(string(p, len));
+        if (j == match_cmps.end()) {
+            throw Xapian::InvalidArgumentError("MatchCmp type " + string(p, len) + " not registered");
+        }
+        p += len;
+
+        len = decode_length(&p, p_end, true);
+        mcmp = j->second->unserialise(string(p, len));
+        p += len;
+    }
+
     // Unserialise the RSet object.
     Xapian::RSet rset = unserialise_rset(string(p, p_end - p));
 
@@ -364 +391 @@
     MultiMatch match(*db, query.get(), qlen, rset, collapse_key,
                      percent_cutoff, weight_cutoff, order,
                      sort_key, sort_by, sort_value_forward,
-                     NULL, gatherer, wt.get());
+                     NULL, gatherer, wt.get(), mcmp.get());
 
     send_message(REPLY_STATS, serialise_stats(gatherer->get_local_stats()));
 

xapian-core/common/remote-database.h

@@ -124 +124 @@
      * @param percent_cutoff            Percentage cutoff.
      * @param weight_cutoff             Weight cutoff.
      * @param wtscheme                  Weighting scheme.
+     * @param sort_cmpfn                Comparison functor for sorting.
      * @param omrset                    The rset.
      */
     void set_query(const Xapian::Query::Internal *query,
@@ -135 +136 @@
                    bool sort_value_forward,
                    int percent_cutoff, Xapian::weight weight_cutoff,
                    const Xapian::Weight *wtscheme,
+                   const Xapian::MatchCmp *sort_cmpfn,
                    const Xapian::RSet &omrset);
 
     /** Get the Stats from the remote server.

xapian-core/common/omenquireinternal.h

@@ -31 +31 @@
 #include <math.h>
 #include <map>
 #include <set>
+#include "document.h"
 
 using namespace std;
 
@@ -64 +65 @@
 };
 
 /** An item resulting from a query.
+ *
  *  This item contains the document id, and the weight calculated for
  *  the document.
+ *
+ *  It also provides access to the underlying document,
+ *  and stores information needed to support collapse operations.
  */
 class MSetItem {
     public:
         MSetItem(Xapian::weight wt_, Xapian::docid did_)
-                : wt(wt_), did(did_), collapse_count(0) {}
+                : wt(wt_),
+                  did(did_),
+                  collapse_count(0),
+                  internal_docid(0) {}
 
-        MSetItem(Xapian::weight wt_, Xapian::docid did_, const string &key_)
-                : wt(wt_), did(did_), collapse_key(key_), collapse_count(0) {}
+        MSetItem(Xapian::weight wt_,
+                 Xapian::docid did_,
+                 const Xapian::Internal::RefCntPtr<Xapian::Database::Internal> & internal_db_,
+                 Xapian::docid internal_docid_)
+                : wt(wt_),
+                  did(did_),
+                  collapse_count(0),
+                  internal_db(internal_db_),
+                  internal_docid(internal_docid_) {}
 
-        MSetItem(Xapian::weight wt_, Xapian::docid did_, const string &key_,
+        MSetItem(Xapian::weight wt_,
+                 Xapian::docid did_,
+                 const Xapian::Internal::RefCntPtr<Xapian::Database::Internal> & internal_db_,
+                 Xapian::docid internal_docid_,
+                 const string &collapse_key_,
                  Xapian::doccount collapse_count_)
-                : wt(wt_), did(did_), collapse_key(key_),
-                  collapse_count(collapse_count_) {}
+                : wt(wt_),
+                  did(did_),
+                  collapse_key(collapse_key_),
+                  collapse_count(collapse_count_),
+                  internal_db(internal_db_),
+                  internal_docid(internal_docid_) {}
 
         /** Weight calculated. */
         Xapian::weight wt;
 
-        /** Document id. */
+        /** Document id (as publically displayed). */
         Xapian::docid did;
 
         /** Value which was used to collapse upon.
@@ -96 +119 @@
          *  for this item, the value will be a null string.  Only one instance
          *  of each key value (apart from the null string) will be present in
          *  the items in the returned Xapian::MSet.
+         *
+         *  FIXME - just use doc.
          */
         string collapse_key;
 
-        /** Count of collapses done on collapse_key so far
+        /** Count of collapses done on collapse_key so far.
          *
-         * This is normally 0, and goes up for each collapse done
-         * It is not neccessarily an indication of how many collapses
-         * might be done if an exhaustive match was done
+         *  This is normally 0, and goes up for each collapse done
+         *  It is not neccessarily an indication of how many collapses
+         *  might be done if an exhaustive match was done
          */
         Xapian::doccount collapse_count;
 
-        /** Used when sorting by value. */
-        /* FIXME: why not just cache the Xapian::Document here!?! */
-        string sort_key;
+        /** Get the document.
+         *
+         *  This will cache the document from previous requests, and will
+         *  return a pointer to NULL if the MSetItem was not initialised with
+         *  and internal_db and internal_docid.
+         */
+        const Xapian::Internal::RefCntPtr<Xapian::Document::Internal> &
+        get_document() const
+        {
+            if (doc.get() == NULL)
+            {
+                if (internal_db.get() == NULL)
+                {
+                    Xapian::Internal::RefCntPtr<Xapian::Document::Internal> temp(new Xapian::Document::Internal());
+                    doc = temp;
+                } else {
+                    Xapian::Internal::RefCntPtr<Xapian::Document::Internal> temp(internal_db->open_document(internal_docid, true));
+                    doc = temp;
+                }
+            }
+            return doc;
+        }
 
         /** Returns a string representing the mset item.
          *  Introspection method.
          */
         string get_description() const;
+
+    private:
+        /** Database containing the document. */
+        Xapian::Internal::RefCntPtr<Xapian::Database::Internal> internal_db;
+
+        /** Document id in database containing the document. */
+        Xapian::docid internal_docid;
+
+        /** Document object.  Points to NULL before the document has
+         *  been requested (with get_document()). */
+        mutable Xapian::Internal::RefCntPtr<Xapian::Document::Internal> doc;
 };
 
 }
@@ -140 +195 @@
         void operator=(const Internal &);
 
     public:
-        typedef enum { REL, VAL, VAL_REL, REL_VAL } sort_setting;
+        typedef enum { REL, VAL, VAL_REL, REL_VAL, USER } sort_setting;
 
         Xapian::valueno collapse_key;
 
@@ -153 +208 @@
         Xapian::valueno sort_key;
         sort_setting sort_by;
         bool sort_value_forward;
+        MatchCmp * sort_cmpfn;
 
         /** The error handler, if set.  (0 if not set).
          */

xapian-core/common/multimatch.h

@@ -64 +64 @@
         /// Weighting scheme
         const Xapian::Weight * weight;
 
+        /// User supplied match comparison function
+        const Xapian::MatchCmp * match_cmp;
+
         /** Internal flag to note that w_max needs to be recalculated
          *  while query is running.
          */
@@ -74 +77 @@
 
         /// get the collapse key
         string get_collapse_key(PostList *pl,
-                                Xapian::docid did, Xapian::valueno keyno,
-                                Xapian::Internal::RefCntPtr<Xapian::Document::Internal> &doc);
+                                Xapian::valueno keyno,
+                                const Xapian::Internal::MSetItem & item);
 
         /** get the maxweight that the postlist pl may return, calling
          *  recalc_maxweight if recalculate_w_max is set, and unsetting it.
@@ -115 +118 @@
                    bool sort_value_forward_,
                    Xapian::ErrorHandler * errorhandler,
                    StatsGatherer * gatherer_,
-                   const Xapian::Weight *wtscheme);
+                   const Xapian::Weight *wtscheme,
+                   const Xapian::MatchCmp * match_cmp_);
 
         void get_mset(Xapian::doccount first,
                       Xapian::doccount maxitems,

xapian-core/common/remoteprotocol.h

@@ -30 +30 @@
 // 25: Support for delete_document and replace_document with unique term
 // 26: Tweak delete_document with unique term; delta encode rset and termpos
 // 27: Support for postlists (always passes the whole list across)
-#define XAPIAN_REMOTE_PROTOCOL_VERSION 27
+// 28: MatchCmp serialisation
+#define XAPIAN_REMOTE_PROTOCOL_VERSION 28
 
 /// Message types (client -> server).
 enum message_type {

xapian-core/common/remoteserver.h

@@ -68 +68 @@
     /// Registered weighting schemes.
     map<string, Xapian::Weight *> wtschemes;
 
+    /// Registered match cmpfns.
+    map<string, Xapian::MatchCmp *> match_cmps;
+
     /// Initialisation code needed by both ctors.
     void initialise();
 
@@ -186 +189 @@
     void register_weighting_scheme(const Xapian::Weight &wt) {
         wtschemes[wt.name()] = wt.clone();
     }
+
+    /// Register a user-defined match comparision class.
+    void register_match_cmp(const Xapian::MatchCmp &match_cmp) {
+        match_cmps[match_cmp.name()] = match_cmp.clone();
+    }
 };
 
 #endif // XAPIAN_INCLUDED_REMOTESERVER_H

xapian-core/common/serialise.h

@@ -24 +24 @@
 #include <xapian/visibility.h>
 
 #include <string>
+#include <xapian/database.h>
 
 // Forward class declarations:
 
@@ -115 +116 @@
  *
  *  @return     The unserialised Xapian::MSet object.
  */
-Xapian::MSet unserialise_mset(const std::string &s);
+Xapian::MSet unserialise_mset(const std::string &s,
+                              Xapian::Internal::RefCntPtr<Xapian::Database::Internal> internal_db);
 
 /** Serialise a Xapian::RSet object.
  *

xapian-core/api/omenquire.cc

@@ -644 +644 @@
   : db(db_), query(), collapse_key(Xapian::BAD_VALUENO),
     order(Enquire::ASCENDING), percent_cutoff(0), weight_cutoff(0),
     sort_key(Xapian::BAD_VALUENO), sort_by(REL), sort_value_forward(true),
-    errorhandler(errorhandler_), weight(0)
+    sort_cmpfn(0), errorhandler(errorhandler_), weight(0)
 {
 }
 
 Enquire::Internal::~Internal()
 {
     delete weight;
+    delete sort_cmpfn;
     weight = 0;
 }
 
@@ -685 +686 @@
         ::MultiMatch match(db, query.internal.get(), qlen, RSet(), collapse_key,
                        percent_cutoff, weight_cutoff,
                        order, sort_key, sort_by, sort_value_forward,
-                       errorhandler, new LocalStatsGatherer(), weight);
+                       errorhandler, new LocalStatsGatherer(), weight, sort_cmpfn);
         // Run query and put results into supplied Xapian::MSet object.
         match.get_mset(first, maxitems, check_at_least, retval, mdecider);
     } else {
         ::MultiMatch match(db, query.internal.get(), qlen, *rset, collapse_key,
                        percent_cutoff, weight_cutoff,
                        order, sort_key, sort_by, sort_value_forward,
-                       errorhandler, new LocalStatsGatherer(), weight);
+                       errorhandler, new LocalStatsGatherer(), weight, sort_cmpfn);
         // Run query and put results into supplied Xapian::MSet object.
         match.get_mset(first, maxitems, check_at_least, retval, mdecider);
     }
@@ -909 +910 @@
 {
     DEBUGAPICALL(void, "Xapian::Enquire::set_weighting_scheme", "[Weight]");
     delete internal->weight;
+    internal->weight = NULL; // Set to NULL before cloning, to avoid double free if clone() throws an exception.
     internal->weight = weight_.clone();
 }
 
@@ -935 +937 @@
 Enquire::set_sort_by_relevance()
 {
     internal->sort_by = Internal::REL;
+    delete internal->sort_cmpfn;
+    internal->sort_cmpfn = NULL;
 }
 
 void
@@ -943 +947 @@
     internal->sort_key = sort_key;
     internal->sort_by = Internal::VAL;
     internal->sort_value_forward = ascending;
+    delete internal->sort_cmpfn;
+    internal->sort_cmpfn = NULL;
 }
 
 void
@@ -952 +958 @@
     internal->sort_key = sort_key;
     internal->sort_by = Internal::VAL_REL;
     internal->sort_value_forward = ascending;
+    delete internal->sort_cmpfn;
+    internal->sort_cmpfn = NULL;
 }
 
 void
@@ -961 +969 @@
     internal->sort_key = sort_key;
     internal->sort_by = Internal::REL_VAL;
     internal->sort_value_forward = ascending;
+    delete internal->sort_cmpfn;
+    internal->sort_cmpfn = NULL;
 }
 
+void
+Enquire::set_sort_by_cmpfn(const MatchCmp &cmpfn_)
+{
+    DEBUGAPICALL(void, "Xapian::Enquire::set_sort_by_cmpfn", "[cmpfn:" << cmpfn_.name() << "(" << cmpfn_.serialise() << ")]");
+    delete internal->sort_cmpfn;
+    internal->sort_by = Internal::USER;
+    internal->sort_cmpfn = NULL; // Set to NULL before cloning, to avoid double free if clone() throws an exception.
+    internal->sort_cmpfn = cmpfn_.clone();
+}
+
 MSet
 Enquire::get_mset(Xapian::doccount first, Xapian::doccount maxitems,
                   Xapian::doccount check_at_least, const RSet *rset,

xapian-core/backends/remote/remote-database.cc

@@ -409 +409 @@
                          bool sort_value_forward,
                          int percent_cutoff, Xapian::weight weight_cutoff,
                          const Xapian::Weight *wtscheme,
+                         const Xapian::MatchCmp *sort_cmpfn,
                          const Xapian::RSet &omrset)
 {
     string tmp = query->serialise();
@@ -425 +426 @@
     message += char(percent_cutoff);
     message += serialise_double(weight_cutoff);
 
+    // Serialise the weight scheme
     tmp = wtscheme->name();
     message += encode_length(tmp.size());
     message += tmp;
-
     tmp = wtscheme->serialise();
     message += encode_length(tmp.size());
     message += tmp;
 
+    // Serialise the match cmp object
+    if (sort_cmpfn == NULL) {
+        message += encode_length(0);
+    } else {
+        tmp = sort_cmpfn->name();
+        message += encode_length(tmp.size());
+        message += tmp;
+        tmp = sort_cmpfn->serialise();
+        message += encode_length(tmp.size());
+        message += tmp;
+    }
+
     message += serialise_rset(omrset);
 
     send_message(MSG_QUERY, message);
@@ -466 +479 @@
 {
     string message;
     get_message(message, REPLY_RESULTS);
-    mset = unserialise_mset(message);
+    mset = unserialise_mset(message, this);
 }
 
 void

xapian-bindings/csharp/Makefile.am

@@ -23 +23 @@
         ExpandDecider.cs \
         Flint.cs \
         InMemory.cs \
+        MatchCmp.cs \
+        MatchItem.cs \
         MatchDecider.cs \
         MSet.cs \
         MSetIterator.cs \
@@ -35 +37 @@
         RSet.cs \
         SWIGTYPE_p_std__vectorTstd__string_t.cs \
         SWIGTYPE_p_std__vectorTXapian__Query_t.cs \
+        SWIGTYPE_p_Xapian__Internal__MSetItem.cs \
         SimpleStopper.cs \
         Stem.cs \
         Stopper.cs \

xapian-bindings/xapian.i

@@ -156 +156 @@
 int xapian_revision();
 
 class Weight;
+class MatchCmp;
 class Stopper;
 
 // from xapian/positioniterator.h
@@ -482 +483 @@
                                           bool ascending = true);
     void set_sort_by_relevance_then_value(Xapian::valueno sort_key,
                                           bool ascending = true);
+    void set_sort_by_cmpfn(const MatchCmp& cmpfn);
 
 #ifdef XAPIAN_SWIG_DIRECTORS
     MSet get_mset(doccount first,
@@ -635 +637 @@
         bool get_sumpart_needs_doclength() const;
 };
 
+class MatchItem {
+    public:
+        MatchItem(const Xapian::Internal::MSetItem & item_);
+        Xapian::weight get_wt() const;
+        Xapian::docid get_docid() const;
+        Xapian::Document get_document() const;
+};
+
+class MatchCmp {
+/* SWIG doesn't handle this:
+    private:
+        virtual MatchCmp * clone() const = 0; */
+    public:
+        virtual ~MatchCmp();
+
+        virtual std::string name() const = 0;
+        virtual std::string serialise() const = 0;
+        virtual MatchCmp * unserialise(const std::string &s) const = 0;
+
+        virtual int cmp(const MatchItem & a,
+                        const MatchItem & b) const = 0;
+};
+
+%warnfilter(842) IntegerMatchCmp::unserialise;
+class IntegerMatchCmp : public MatchCmp {
+    public:
+        IntegerMatchCmp * clone() const;
+        IntegerMatchCmp(Xapian::valueno sort_key_,
+                        bool ascending = true);
+        IntegerMatchCmp();
+        ~IntegerMatchCmp();
+
+        std::string name() const;
+        std::string serialise() const;
+        IntegerMatchCmp * unserialise(const std::string &s) const;
+
+        int cmp(const MatchItem & a,
+                const MatchItem & b) const;
+
+        static std::string int_to_value(int num);
+        static int value_to_int(const std::string &s);
+};
+
+%warnfilter(842) DoubleMatchCmp::unserialise;
+class DoubleMatchCmp : public MatchCmp {
+    public:
+        DoubleMatchCmp * clone() const;
+        DoubleMatchCmp(Xapian::valueno sort_key_,
+                        bool ascending = true);
+        DoubleMatchCmp();
+        ~DoubleMatchCmp();
+
+        std::string name() const;
+        std::string serialise() const;
+        DoubleMatchCmp * unserialise(const std::string &s) const;
+
+        int cmp(const MatchItem & a,
+                const MatchItem & b) const;
+
+        static std::string double_to_value(double num);
+        static double value_to_double(const std::string &s);
+};
+
+
 // xapian/database.h
 
 class Database {