Index: xapian-maintainer-tools/win32msvc/win32_matcher.mak =================================================================== --- xapian-maintainer-tools/win32msvc/win32_matcher.mak (revision 8312) +++ xapian-maintainer-tools/win32msvc/win32_matcher.mak (working copy) @@ -32,6 +32,7 @@ $(INTDIR)\multimatch.obj \ $(INTDIR)\expand.obj \ $(INTDIR)\stats.obj \ + $(INTDIR)\matchcmp.obj \ $(INTDIR)\mergepostlist.obj \ $(INTDIR)\msetpostlist.obj \ $(INTDIR)\msetcmp.obj \ @@ -166,6 +167,10 @@ $(CPP_PROJ) $** << +"$(INTDIR)\matchcmp.obj" : ".\matchcmp.cc" + $(CPP) @<< + $(CPP_PROJ) $** +<< "$(INTDIR)\mergepostlist.obj" : ".\mergepostlist.cc" $(CPP) @<< Index: xapian-core/matcher/Makefile.mk =================================================================== --- xapian-core/matcher/Makefile.mk (revision 8312) +++ xapian-core/matcher/Makefile.mk (working copy) @@ -43,6 +43,7 @@ matcher/expandweight.cc\ matcher/filterpostlist.cc\ matcher/localmatch.cc\ + matcher/matchcmp.cc\ matcher/mergepostlist.cc\ matcher/msetcmp.cc\ matcher/msetpostlist.cc\ Index: xapian-core/matcher/multimatch.cc =================================================================== --- xapian-core/matcher/multimatch.cc (revision 8312) +++ xapian-core/matcher/multimatch.cc (working copy) @@ -86,13 +86,15 @@ bool sort_value_forward_, Xapian::ErrorHandler * errorhandler_, StatsGatherer * gatherer_, - const Xapian::Weight * weight_) + const Xapian::Weight * weight_, + const Xapian::MatchCmp * match_cmp_) : gatherer(gatherer_), db(db_), query(query_), collapse_key(collapse_key_), percent_cutoff(percent_cutoff_), weight_cutoff(weight_cutoff_), order(order_), sort_key(sort_key_), sort_by(sort_by_), sort_value_forward(sort_value_forward_), errorhandler(errorhandler_), weight(weight_), + match_cmp(match_cmp_), is_remote(db.internal.size()) { DEBUGCALL(MATCH, void, "MultiMatch", db_ << ", " << query_ << ", " << @@ -139,7 +141,8 @@ is_remote[i] = true; rem_db->set_query(query, qlen, collapse_key, order, sort_key, sort_by, sort_value_forward, percent_cutoff, - weight_cutoff, weight, subrsets[i]); + weight_cutoff, weight, match_cmp, + subrsets[i]); bool decreasing_relevance = (sort_by == REL || sort_by == REL_VAL); smatch = new RemoteSubMatch(rem_db, gatherer.get(), @@ -193,22 +196,21 @@ } string -MultiMatch::get_collapse_key(PostList *pl, Xapian::docid did, - Xapian::valueno keyno, Xapian::Internal::RefCntPtr &doc) +MultiMatch::get_collapse_key(PostList *pl, + Xapian::valueno keyno, + const Xapian::Internal::MSetItem & item) { - DEBUGCALL(MATCH, string, "MultiMatch::get_collapse_key", pl << ", " << did << ", " << keyno << ", [doc]"); + DEBUGCALL(MATCH, string, "MultiMatch::get_collapse_key", pl << ", " << item.did << ", " << keyno << ", [doc]"); + + // FIXME - this gets the collapse key from the postlist. This only + // actually happens for Mset postlists (or mergepostlists containing an + // mset postlist), since this is the only case where the collapse key has + // been read already. Instead, we should use the document object stored in + // the mset item. + const string *key = pl->get_collapse_key(); if (key) RETURN(*key); - if (doc.get() == 0) { - unsigned int multiplier = db.internal.size(); - Assert(multiplier != 0); - Xapian::doccount n = (did - 1) % multiplier; // which actual database - Xapian::docid m = (did - 1) / multiplier + 1; // real docid in that database - - Xapian::Internal::RefCntPtr temp(db.internal[n]->open_document(m, true)); - doc = temp; - } - RETURN(doc->get_value(keyno)); + RETURN(item.get_document()->get_value(keyno)); } Xapian::weight @@ -231,7 +233,8 @@ void MultiMatch::get_mset(Xapian::doccount first, Xapian::doccount maxitems, Xapian::doccount check_at_least, - Xapian::MSet & mset, const Xapian::MatchDecider *mdecider) + Xapian::MSet & mset, + const Xapian::MatchDecider *mdecider) { DEBUGCALL(MATCH, void, "MultiMatch::get_mset", first << ", " << maxitems << ", " << check_at_least << ", ..."); @@ -362,8 +365,13 @@ /// Comparison functor for sorting MSet bool sort_forward = (order != Xapian::Enquire::DESCENDING); - MSetCmp mcmp(get_msetcmp_function(sort_by, sort_forward, sort_value_forward)); + mset_cmp cmpfn = get_msetcmp_function(sort_by, sort_forward, sort_value_forward); + MSetCmp mcmp(cmpfn, sort_key, match_cmp); + /// Number of databases making up the database we're using. + const unsigned int db_internal_size = db.internal.size(); + Assert(db_internal_size != 0); + // Perform query // We form the mset in two stages. In the first we fill up our working @@ -410,23 +418,15 @@ if (min_item.wt > 0.0) wt = pl->get_weight(); DEBUGLINE(MATCH, "Candidate document id " << did << " wt " << wt); - Xapian::Internal::MSetItem new_item(wt, did); - if (sort_by != REL) { - const unsigned int multiplier = db.internal.size(); - Assert(multiplier != 0); - Xapian::doccount n = (new_item.did - 1) % multiplier; // which actual database - Xapian::docid m = (new_item.did - 1) / multiplier + 1; // real docid in that database - Xapian::Internal::RefCntPtr doc(db.internal[n]->open_document(m, true)); - new_item.sort_key = doc->get_value(sort_key); - } + Xapian::Internal::RefCntPtr internal_db = db.internal[((did - 1) % db_internal_size)]; // database holding document + Xapian::docid internal_docid = (did - 1) / db_internal_size + 1; // docid in internal_db + Xapian::Internal::MSetItem new_item(wt, did, internal_db, internal_docid); // Test if item has high enough weight (or sort key) to get into // proto-mset. if (sort_by != REL || min_item.wt > 0.0) if (!mcmp(new_item, min_item)) continue; - Xapian::Internal::RefCntPtr doc; - // Use the decision functor if any. if (mdecider != NULL) { const unsigned int multiplier = db.internal.size(); @@ -435,13 +435,7 @@ // If the results are from a remote database, then the functor will // already have been applied there so we can skip this step. if (!is_remote[n]) { - if (doc.get() == 0) { - Xapian::docid m = (did - 1) / multiplier + 1; // real docid in that database - - Xapian::Internal::RefCntPtr temp(db.internal[n]->open_document(m, true)); - doc = temp; - } - Xapian::Document mydoc(doc.get()); + Xapian::Document mydoc(new_item.get_document().get()); if (!mdecider->operator()(mydoc)) continue; } } @@ -457,8 +451,7 @@ // Perform collapsing on key if requested. if (collapse_key != Xapian::BAD_VALUENO) { - new_item.collapse_key = get_collapse_key(pl, did, collapse_key, - doc); + new_item.collapse_key = get_collapse_key(pl, collapse_key, new_item); // Don't collapse on null key if (!new_item.collapse_key.empty()) { Index: xapian-core/matcher/msetcmp.h =================================================================== --- xapian-core/matcher/msetcmp.h (revision 8312) +++ xapian-core/matcher/msetcmp.h (working copy) @@ -20,9 +20,12 @@ #include "omenquireinternal.h" +class MSetCmp; + // typedef for MSetItem comparison function. typedef bool (* mset_cmp)(const Xapian::Internal::MSetItem &, - const Xapian::Internal::MSetItem &); + const Xapian::Internal::MSetItem &, + const MSetCmp *); /// Select the appropriate msetcmp function. mset_cmp get_msetcmp_function(Xapian::Enquire::Internal::sort_setting sort_by, bool sort_forward, bool sort_value_forward); @@ -31,10 +34,19 @@ class MSetCmp { mset_cmp fn; public: - MSetCmp(mset_cmp fn_) : fn(fn_) { } + Xapian::valueno sort_key; + const Xapian::MatchCmp * cmp; + + /// Construct an instance of MSetCmp using a built-in comparison function. + MSetCmp(mset_cmp fn_, + Xapian::valueno sort_key_, + const Xapian::MatchCmp * cmp_) + : fn(fn_), sort_key(sort_key_), cmp(cmp_) { } + /// Return true if MSetItem a should be ranked above MSetItem b. bool operator()(const Xapian::Internal::MSetItem &a, - const Xapian::Internal::MSetItem &b) const { - return fn(a, b); + const Xapian::Internal::MSetItem &b) const { + return fn(a, b, this); } }; + Index: xapian-core/matcher/msetcmp.cc =================================================================== --- xapian-core/matcher/msetcmp.cc (revision 8312) +++ xapian-core/matcher/msetcmp.cc (working copy) @@ -20,6 +20,7 @@ #include #include "msetcmp.h" +#include /* We use templates to generate the 14 different comparison functions * which we need. This avoids having to write them all out by hand. @@ -46,7 +47,8 @@ // Order by relevance, then docid. template bool msetcmp_by_relevance(const Xapian::Internal::MSetItem &a, - const Xapian::Internal::MSetItem &b) + const Xapian::Internal::MSetItem &b, + const MSetCmp *) { if (a.wt > b.wt) return true; if (a.wt < b.wt) return false; @@ -56,30 +58,36 @@ // Order by value, then docid. template bool msetcmp_by_value(const Xapian::Internal::MSetItem &a, - const Xapian::Internal::MSetItem &b) + const Xapian::Internal::MSetItem &b, + const MSetCmp * cmp) { if (!FORWARD_VALUE) { // We want dummy did 0 to compare worse than any other. if (a.did == 0) return false; if (b.did == 0) return true; } - if (a.sort_key > b.sort_key) return FORWARD_VALUE; - if (a.sort_key < b.sort_key) return !FORWARD_VALUE; + std::string sort_key_a = a.get_document()->get_value(cmp->sort_key); + std::string sort_key_b = b.get_document()->get_value(cmp->sort_key); + if (sort_key_a > sort_key_b) return FORWARD_VALUE; + if (sort_key_a < sort_key_b) return !FORWARD_VALUE; return msetcmp_by_did(a, b); } // Order by value, then relevance, then docid. template bool msetcmp_by_value_then_relevance(const Xapian::Internal::MSetItem &a, - const Xapian::Internal::MSetItem &b) + const Xapian::Internal::MSetItem &b, + const MSetCmp * cmp) { if (!FORWARD_VALUE) { // two special cases to make min_item compares work when did == 0 if (a.did == 0) return false; if (b.did == 0) return true; } - if (a.sort_key > b.sort_key) return FORWARD_VALUE; - if (a.sort_key < b.sort_key) return !FORWARD_VALUE; + std::string sort_key_a = a.get_document()->get_value(cmp->sort_key); + std::string sort_key_b = b.get_document()->get_value(cmp->sort_key); + if (sort_key_a > sort_key_b) return FORWARD_VALUE; + if (sort_key_a < sort_key_b) return !FORWARD_VALUE; if (a.wt > b.wt) return true; if (a.wt < b.wt) return false; return msetcmp_by_did(a, b); @@ -88,7 +96,8 @@ // Order by relevance, then value, then docid. template bool msetcmp_by_relevance_then_value(const Xapian::Internal::MSetItem &a, - const Xapian::Internal::MSetItem &b) + const Xapian::Internal::MSetItem &b, + const MSetCmp * cmp) { if (!FORWARD_VALUE) { // two special cases to make min_item compares work when did == 0 @@ -97,11 +106,37 @@ } if (a.wt > b.wt) return true; if (a.wt < b.wt) return false; - if (a.sort_key > b.sort_key) return FORWARD_VALUE; - if (a.sort_key < b.sort_key) return !FORWARD_VALUE; + std::string sort_key_a = a.get_document()->get_value(cmp->sort_key); + std::string sort_key_b = b.get_document()->get_value(cmp->sort_key); + if (sort_key_a > sort_key_b) return FORWARD_VALUE; + if (sort_key_a < sort_key_b) return !FORWARD_VALUE; return msetcmp_by_did(a, b); } +template bool +msetcmp_by_user_cmpfn(const Xapian::Internal::MSetItem &a, + const Xapian::Internal::MSetItem &b, + const MSetCmp * cmp) +{ + // We want dummy did 0 to compare worse than any other. + if (a.did == 0) return false; + if (b.did == 0) return true; + + if (a.did == b.did) return false; // FIXME - can this ever happen? If not, convert this to Assert(a.did != b.did). + Xapian::MatchItem item_a(a); + Xapian::MatchItem item_b(b); + int c = (cmp->cmp->cmp)(item_a, item_b); + if (c < 0) + return true; + else if (c > 0) + return false; + if (FORWARD_DID) { + return (a.did < b.did); + } else { + return (a.did > b.did); + } +} + static mset_cmp mset_cmp_table[] = { // Xapian::Enquire::Internal::REL msetcmp_by_relevance, @@ -122,7 +157,12 @@ msetcmp_by_relevance_then_value, msetcmp_by_relevance_then_value, msetcmp_by_relevance_then_value, - msetcmp_by_relevance_then_value + msetcmp_by_relevance_then_value, + // Xapian::Enquire::Internal::USER + msetcmp_by_user_cmpfn, + msetcmp_by_user_cmpfn, + msetcmp_by_user_cmpfn, + msetcmp_by_user_cmpfn }; mset_cmp get_msetcmp_function(Xapian::Enquire::Internal::sort_setting sort_by, bool sort_forward, bool sort_value_forward) { Index: xapian-core/tests/internaltest.cc =================================================================== --- xapian-core/tests/internaltest.cc (revision 8312) +++ xapian-core/tests/internaltest.cc (working copy) @@ -385,6 +385,8 @@ 255.5, 256.125, 257.03125, + 0.12268031290495594321l, + 0.12268031290495592933l, }; check_double_serialisation(0.0); Index: xapian-core/tests/api_db.cc =================================================================== --- xapian-core/tests/api_db.cc (revision 8312) +++ xapian-core/tests/api_db.cc (working copy) @@ -1264,6 +1264,7 @@ // set_sort_by_value // set_sort_by_value_then_relevance // set_sort_by_relevance_then_value +// User comparison functions static bool test_sortrel1() { Xapian::Enquire enquire(get_database("apitest_sortrel")); @@ -1279,6 +1280,10 @@ const Xapian::docid order7[] = { 7,9,8,6,5,4,2,1,3 }; const Xapian::docid order8[] = { 7,6,2,9,5,1,8,4,3 }; const Xapian::docid order9[] = { 2,6,7,1,5,9,3,4,8 }; + const Xapian::docid order10[] = { 7,9,4,5,6,1,2,3,8 }; + const Xapian::docid order11[] = { 8,1,2,3,4,5,6,7,9 }; + const Xapian::docid order12[] = { 9,7,6,5,4,3,2,1,8 }; + const Xapian::docid order13[] = { 8,3,2,1,6,5,4,9,7 }; Xapian::MSet mset; size_t i; @@ -1289,6 +1294,30 @@ TEST_EQUAL(*mset[i], order1[i]); } + enquire.set_sort_by_cmpfn(Xapian::IntegerMatchCmp(100)); + + mset = enquire.get_mset(0, 10); + TEST_EQUAL(mset.size(), sizeof(order10) / sizeof(Xapian::docid)); + for (i = 0; i < sizeof(order10) / sizeof(Xapian::docid); ++i) { + TEST_EQUAL(*mset[i], order10[i]); + } + + enquire.set_sort_by_cmpfn(Xapian::IntegerMatchCmp(100, true)); + + mset = enquire.get_mset(0, 10); + TEST_EQUAL(mset.size(), sizeof(order10) / sizeof(Xapian::docid)); + for (i = 0; i < sizeof(order10) / sizeof(Xapian::docid); ++i) { + TEST_EQUAL(*mset[i], order10[i]); + } + + enquire.set_sort_by_cmpfn(Xapian::IntegerMatchCmp(100, false)); + + mset = enquire.get_mset(0, 10); + TEST_EQUAL(mset.size(), sizeof(order11) / sizeof(Xapian::docid)); + for (i = 0; i < sizeof(order11) / sizeof(Xapian::docid); ++i) { + TEST_EQUAL(*mset[i], order11[i]); + } + enquire.set_sort_by_value_then_relevance(1); mset = enquire.get_mset(0, 10); @@ -1305,6 +1334,33 @@ TEST_EQUAL(*mset[i], order1[i]); } + enquire.set_sort_by_cmpfn(Xapian::IntegerMatchCmp(100)); + enquire.set_docid_order(Xapian::Enquire::DESCENDING); + + mset = enquire.get_mset(0, 10); + TEST_EQUAL(mset.size(), sizeof(order12) / sizeof(Xapian::docid)); + for (i = 0; i < sizeof(order12) / sizeof(Xapian::docid); ++i) { + TEST_EQUAL(*mset[i], order12[i]); + } + + enquire.set_sort_by_cmpfn(Xapian::IntegerMatchCmp(100, true)); + enquire.set_docid_order(Xapian::Enquire::DESCENDING); + + mset = enquire.get_mset(0, 10); + TEST_EQUAL(mset.size(), sizeof(order12) / sizeof(Xapian::docid)); + for (i = 0; i < sizeof(order12) / sizeof(Xapian::docid); ++i) { + TEST_EQUAL(*mset[i], order12[i]); + } + + enquire.set_sort_by_cmpfn(Xapian::IntegerMatchCmp(100, false)); + enquire.set_docid_order(Xapian::Enquire::DESCENDING); + + mset = enquire.get_mset(0, 10); + TEST_EQUAL(mset.size(), sizeof(order13) / sizeof(Xapian::docid)); + for (i = 0; i < sizeof(order13) / sizeof(Xapian::docid); ++i) { + TEST_EQUAL(*mset[i], order13[i]); + } + enquire.set_sort_by_value_then_relevance(1); enquire.set_docid_order(Xapian::Enquire::DESCENDING); Index: xapian-core/tests/harness/index_utils.cc =================================================================== --- xapian-core/tests/harness/index_utils.cc (revision 8312) +++ xapian-core/tests/harness/index_utils.cc (working copy) @@ -78,6 +78,9 @@ for (Xapian::valueno i = min(para.length(), size_t(10)); i >= 1; --i) { doc.add_value(i, para.substr(i, 1)); } + if (para.length() > 1 && para[0] == 'V') { + doc.add_value(100, Xapian::IntegerMatchCmp::int_to_value(atoi(para.c_str() + 1))); + } Xapian::termcount pos = 0; string::const_iterator end = para.begin(); Index: xapian-core/tests/testdata/apitest_sortrel.txt =================================================================== --- xapian-core/tests/testdata/apitest_sortrel.txt (revision 8312) +++ xapian-core/tests/testdata/apitest_sortrel.txt (working copy) @@ -12,6 +12,6 @@ V1 woman -V1 man woman fish +V10 man woman fish V1 man woman Index: xapian-core/include/xapian/enquire.h =================================================================== --- xapian-core/include/xapian/enquire.h (revision 8312) +++ xapian-core/include/xapian/enquire.h (working copy) @@ -31,6 +31,7 @@ #include #include #include +#include namespace Xapian { @@ -40,7 +41,12 @@ class MSetIterator; class Query; class Weight; +class MatchCmp; +namespace Internal { + class MSetItem; +} + /** A match set (MSet). * This class represents (a portion of) the results of a query. */ @@ -812,6 +818,23 @@ void set_sort_by_relevance_then_value(Xapian::valueno sort_key, bool ascending = true); + /** Set a comparison functor to be used when comparing matches. + * + * This overrides the sort options set by set_sort_by_relevance(), + * set_sort_by_value(), set_sort_by_relevance_then_value() and + * set_sort_by_value_then_relevance(). For matches which the + * comparison functor considers equal (ie, for which the functor + * returns 0 when comparing them), the matches will be compared in + * document ID order, as set by set_docid_order(); + * + * The comparison function will be called many times in the process of + * performing the search. Therefore, it is important that it is + * implemented as efficiently as possible. + * + * @param cmpfn_ The comparion function to use to compare matches. + */ + void set_sort_by_cmpfn(const MatchCmp &cmpfn_); + /** Get (a portion of) the match set for the current query. * * @param first the first item in the result set to return. @@ -1220,6 +1243,170 @@ bool get_sumpart_needs_doclength() const; }; +/// An item matching a search. +class XAPIAN_VISIBILITY_DEFAULT MatchItem { + private: + const Xapian::Internal::MSetItem & item; + + public: + MatchItem(const Xapian::Internal::MSetItem & item_) + : item(item_) {} + + /** Get the weight calculated. */ + Xapian::weight get_wt() const; + + /** Get the document id. */ + Xapian::docid get_docid() const; + + /** Get the document object, for access to document data, values, + * and terms. */ + Xapian::Document get_document() const; +}; + +/// Abstract base class for match comparison functors. +class XAPIAN_VISIBILITY_DEFAULT MatchCmp { + friend class Enquire; // So Enquire can clone us + friend class ::RemoteServer; // So RemoteServer can clone us - FIXME + protected: + MatchCmp(const MatchCmp &); + private: + void operator=(MatchCmp &); + + /** Return a new MatchCmp object of this type. + * + * A subclass called FooMatchCmp taking parameters param1 and param2 + * should implement this as: + * + * virtual FooMatchCmp * clone() const { + * return new FooMatchCmp(param1, param2); + * } + */ + virtual MatchCmp * clone() const = 0; + + public: + MatchCmp() { } + virtual ~MatchCmp() { } + + /** Name of the comparison functor. + * + * If the subclass is called FooMatchCmp, this should return "Foo". + */ + virtual std::string name() const = 0; + + /// Serialise object parameters into a string. + virtual std::string serialise() const = 0; + + /** Create object given string serialisation returned by serialise(). + * + * If the string is not a valid serialised form, this should raise a + * Xapian::NetworkError; + */ + virtual MatchCmp * unserialise(const std::string &s) const = 0; + + /** Compare a potential Mset entry a to an entry b. + * + * This function should return an integer less than 0 if MatchItem a + * should be ranked below MatchItem b, and greater than 0 if MatchItem + * a should be ranked above MatchItem b. The function may return 0 if + * it considers the two match items to be equal. + * + * The function will never be called with items which have identical + * document IDs. + * + * The comparison function must be well behaved, such that: + * + * - it must produce the same result if called a second time on a + * particular pair of documents + * - if cmp(x,y) is less than 0 and cmp(y,z) is less than 0, cmp(x,z) + * must be less than 0. + * - if cmp(x,y) is less than 0, cmp(y,x) must be greater than 0. + * + * Bear in mind that the comparison will be performed very frequently + * when performing a search, so must be as fast as possible. In + * particular, although the MatchItem provides access to the document + * object, it is recommended that only the value items stored in the + * document object are accessed. + */ + virtual int cmp(const MatchItem & a, + const MatchItem & b) const = 0; +}; + +/** Comparison function for comparing by value as integer. + */ +class XAPIAN_VISIBILITY_DEFAULT IntegerMatchCmp : public MatchCmp { + Xapian::valueno sort_key; + int direction; + public: + /** Return a new IntegerMatchCmp with the same parameters as this one. + */ + IntegerMatchCmp * clone() const { + return new IntegerMatchCmp(sort_key, direction == 1); + } + IntegerMatchCmp(Xapian::valueno sort_key_, + bool ascending = true) + : sort_key(sort_key_), + direction(ascending ? 1 : -1) { } + IntegerMatchCmp() + : sort_key(0), + direction(1) { } + ~IntegerMatchCmp() { } + std::string name() const { return "Integer"; } + + std::string serialise() const; + IntegerMatchCmp * unserialise(const std::string &s) const; + + int cmp(const MatchItem & a, + const MatchItem & b) const; + + /** Convert an integer to a string, suitable for comparing with this + * comparison functor. + */ + static std::string int_to_value(int num); + + /** Convert a string suitable for comparison with this comparison + * functor to an integer. + */ + static int value_to_int(const std::string &s); +}; + +/** Comparison function for comparing by value as a double. + */ +class XAPIAN_VISIBILITY_DEFAULT DoubleMatchCmp : public MatchCmp { + Xapian::valueno sort_key; + int direction; + public: + /** Return a new DoubleMatchCmp with the same parameters as this one. + */ + DoubleMatchCmp * clone() const { + return new DoubleMatchCmp(sort_key, direction == 1); + } + DoubleMatchCmp(Xapian::valueno sort_key_, + bool ascending = true) + : sort_key(sort_key_), + direction(ascending ? 1 : -1) { } + DoubleMatchCmp() + : sort_key(0), + direction(1) { } + ~DoubleMatchCmp() { } + std::string name() const { return "Double"; } + + std::string serialise() const; + DoubleMatchCmp * unserialise(const std::string &s) const; + + int cmp(const MatchItem & a, + const MatchItem & b) const; + + /** Convert a double to a string, suitable for comparing with this + * comparison functor. + */ + static std::string double_to_value(double num); + + /** Convert a string suitable for comparison with this comparison + * functor to a double. + */ + static double value_to_double(const std::string &s); +}; + } #endif /* XAPIAN_INCLUDED_ENQUIRE_H */ Index: xapian-core/net/serialise.cc =================================================================== --- xapian-core/net/serialise.cc (revision 8312) +++ xapian-core/net/serialise.cc (working copy) @@ -223,7 +223,8 @@ } Xapian::MSet -unserialise_mset(const string &s) +unserialise_mset(const string &s, + Xapian::Internal::RefCntPtr internal_db) { const char * p = s.data(); const char * p_end = p + s.size(); @@ -242,7 +243,10 @@ size_t len = decode_length(&p, p_end, true); string key(p, len); p += len; - items.push_back(Xapian::Internal::MSetItem(wt, did, key, + items.push_back(Xapian::Internal::MSetItem(wt, did, + internal_db, + did, + key, decode_length(&p, p_end, false))); } Index: xapian-core/net/remoteserver.cc =================================================================== --- xapian-core/net/remoteserver.cc (revision 8312) +++ xapian-core/net/remoteserver.cc (working copy) @@ -90,6 +90,13 @@ wtschemes[weight->name()] = weight; weight = new Xapian::TradWeight(); wtschemes[weight->name()] = weight; + + // Register match comparison functions. + Xapian::MatchCmp * match_cmp; + match_cmp = new Xapian::IntegerMatchCmp(); + match_cmps[match_cmp->name()] = match_cmp; + match_cmp = new Xapian::DoubleMatchCmp(); + match_cmps[match_cmp->name()] = match_cmp; } RemoteServer::~RemoteServer() @@ -98,6 +105,10 @@ for (i = wtschemes.begin(); i != wtschemes.end(); ++i) { delete i->second; } + map::const_iterator j; + for (j = match_cmps.begin(); j != match_cmps.end(); ++j) { + delete j->second; + } } message_type @@ -322,7 +333,7 @@ Xapian::valueno sort_key = decode_length(&p, p_end, false); - if (*p < '0' || *p > '3') { + if (*p < '0' || *p > '4') { throw Xapian::NetworkError("bad message (sort_by)"); } Xapian::Enquire::Internal::sort_setting sort_by; @@ -356,6 +367,22 @@ AutoPtr wt(i->second->unserialise(string(p, len))); p += len; + // Unserialise the MatchCmp object + len = decode_length(&p, p_end, true); + AutoPtr mcmp; + if (len > 0) { + map::const_iterator j; + j = match_cmps.find(string(p, len)); + if (j == match_cmps.end()) { + throw Xapian::InvalidArgumentError("MatchCmp type " + string(p, len) + " not registered"); + } + p += len; + + len = decode_length(&p, p_end, true); + mcmp = j->second->unserialise(string(p, len)); + p += len; + } + // Unserialise the RSet object. Xapian::RSet rset = unserialise_rset(string(p, p_end - p)); @@ -364,7 +391,7 @@ MultiMatch match(*db, query.get(), qlen, rset, collapse_key, percent_cutoff, weight_cutoff, order, sort_key, sort_by, sort_value_forward, - NULL, gatherer, wt.get()); + NULL, gatherer, wt.get(), mcmp.get()); send_message(REPLY_STATS, serialise_stats(gatherer->get_local_stats())); Index: xapian-core/common/remote-database.h =================================================================== --- xapian-core/common/remote-database.h (revision 8312) +++ xapian-core/common/remote-database.h (working copy) @@ -124,6 +124,7 @@ * @param percent_cutoff Percentage cutoff. * @param weight_cutoff Weight cutoff. * @param wtscheme Weighting scheme. + * @param sort_cmpfn Comparison functor for sorting. * @param omrset The rset. */ void set_query(const Xapian::Query::Internal *query, @@ -135,6 +136,7 @@ bool sort_value_forward, int percent_cutoff, Xapian::weight weight_cutoff, const Xapian::Weight *wtscheme, + const Xapian::MatchCmp *sort_cmpfn, const Xapian::RSet &omrset); /** Get the Stats from the remote server. Index: xapian-core/common/omenquireinternal.h =================================================================== --- xapian-core/common/omenquireinternal.h (revision 8312) +++ xapian-core/common/omenquireinternal.h (working copy) @@ -31,6 +31,7 @@ #include #include #include +#include "document.h" using namespace std; @@ -64,26 +65,48 @@ }; /** An item resulting from a query. + * * This item contains the document id, and the weight calculated for * the document. + * + * It also provides access to the underlying document, + * and stores information needed to support collapse operations. */ class MSetItem { public: MSetItem(Xapian::weight wt_, Xapian::docid did_) - : wt(wt_), did(did_), collapse_count(0) {} + : wt(wt_), + did(did_), + collapse_count(0), + internal_docid(0) {} - MSetItem(Xapian::weight wt_, Xapian::docid did_, const string &key_) - : wt(wt_), did(did_), collapse_key(key_), collapse_count(0) {} + MSetItem(Xapian::weight wt_, + Xapian::docid did_, + const Xapian::Internal::RefCntPtr & internal_db_, + Xapian::docid internal_docid_) + : wt(wt_), + did(did_), + collapse_count(0), + internal_db(internal_db_), + internal_docid(internal_docid_) {} - MSetItem(Xapian::weight wt_, Xapian::docid did_, const string &key_, + MSetItem(Xapian::weight wt_, + Xapian::docid did_, + const Xapian::Internal::RefCntPtr & internal_db_, + Xapian::docid internal_docid_, + const string &collapse_key_, Xapian::doccount collapse_count_) - : wt(wt_), did(did_), collapse_key(key_), - collapse_count(collapse_count_) {} + : wt(wt_), + did(did_), + collapse_key(collapse_key_), + collapse_count(collapse_count_), + internal_db(internal_db_), + internal_docid(internal_docid_) {} /** Weight calculated. */ Xapian::weight wt; - /** Document id. */ + /** Document id (as publically displayed). */ Xapian::docid did; /** Value which was used to collapse upon. @@ -96,25 +119,57 @@ * for this item, the value will be a null string. Only one instance * of each key value (apart from the null string) will be present in * the items in the returned Xapian::MSet. + * + * FIXME - just use doc. */ string collapse_key; - /** Count of collapses done on collapse_key so far + /** Count of collapses done on collapse_key so far. * - * This is normally 0, and goes up for each collapse done - * It is not neccessarily an indication of how many collapses - * might be done if an exhaustive match was done + * This is normally 0, and goes up for each collapse done + * It is not neccessarily an indication of how many collapses + * might be done if an exhaustive match was done */ Xapian::doccount collapse_count; - /** Used when sorting by value. */ - /* FIXME: why not just cache the Xapian::Document here!?! */ - string sort_key; + /** Get the document. + * + * This will cache the document from previous requests, and will + * return a pointer to NULL if the MSetItem was not initialised with + * and internal_db and internal_docid. + */ + const Xapian::Internal::RefCntPtr & + get_document() const + { + if (doc.get() == NULL) + { + if (internal_db.get() == NULL) + { + Xapian::Internal::RefCntPtr temp(new Xapian::Document::Internal()); + doc = temp; + } else { + Xapian::Internal::RefCntPtr temp(internal_db->open_document(internal_docid, true)); + doc = temp; + } + } + return doc; + } /** Returns a string representing the mset item. * Introspection method. */ string get_description() const; + + private: + /** Database containing the document. */ + Xapian::Internal::RefCntPtr internal_db; + + /** Document id in database containing the document. */ + Xapian::docid internal_docid; + + /** Document object. Points to NULL before the document has + * been requested (with get_document()). */ + mutable Xapian::Internal::RefCntPtr doc; }; } @@ -140,7 +195,7 @@ void operator=(const Internal &); public: - typedef enum { REL, VAL, VAL_REL, REL_VAL } sort_setting; + typedef enum { REL, VAL, VAL_REL, REL_VAL, USER } sort_setting; Xapian::valueno collapse_key; @@ -153,6 +208,7 @@ Xapian::valueno sort_key; sort_setting sort_by; bool sort_value_forward; + MatchCmp * sort_cmpfn; /** The error handler, if set. (0 if not set). */ Index: xapian-core/common/multimatch.h =================================================================== --- xapian-core/common/multimatch.h (revision 8312) +++ xapian-core/common/multimatch.h (working copy) @@ -64,6 +64,9 @@ /// Weighting scheme const Xapian::Weight * weight; + /// User supplied match comparison function + const Xapian::MatchCmp * match_cmp; + /** Internal flag to note that w_max needs to be recalculated * while query is running. */ @@ -74,8 +77,8 @@ /// get the collapse key string get_collapse_key(PostList *pl, - Xapian::docid did, Xapian::valueno keyno, - Xapian::Internal::RefCntPtr &doc); + Xapian::valueno keyno, + const Xapian::Internal::MSetItem & item); /** get the maxweight that the postlist pl may return, calling * recalc_maxweight if recalculate_w_max is set, and unsetting it. @@ -115,7 +118,8 @@ bool sort_value_forward_, Xapian::ErrorHandler * errorhandler, StatsGatherer * gatherer_, - const Xapian::Weight *wtscheme); + const Xapian::Weight *wtscheme, + const Xapian::MatchCmp * match_cmp_); void get_mset(Xapian::doccount first, Xapian::doccount maxitems, Index: xapian-core/common/remoteprotocol.h =================================================================== --- xapian-core/common/remoteprotocol.h (revision 8312) +++ xapian-core/common/remoteprotocol.h (working copy) @@ -30,7 +30,8 @@ // 25: Support for delete_document and replace_document with unique term // 26: Tweak delete_document with unique term; delta encode rset and termpos // 27: Support for postlists (always passes the whole list across) -#define XAPIAN_REMOTE_PROTOCOL_VERSION 27 +// 28: MatchCmp serialisation +#define XAPIAN_REMOTE_PROTOCOL_VERSION 28 /// Message types (client -> server). enum message_type { Index: xapian-core/common/remoteserver.h =================================================================== --- xapian-core/common/remoteserver.h (revision 8312) +++ xapian-core/common/remoteserver.h (working copy) @@ -68,6 +68,9 @@ /// Registered weighting schemes. map wtschemes; + /// Registered match cmpfns. + map match_cmps; + /// Initialisation code needed by both ctors. void initialise(); @@ -186,6 +189,11 @@ void register_weighting_scheme(const Xapian::Weight &wt) { wtschemes[wt.name()] = wt.clone(); } + + /// Register a user-defined match comparision class. + void register_match_cmp(const Xapian::MatchCmp &match_cmp) { + match_cmps[match_cmp.name()] = match_cmp.clone(); + } }; #endif // XAPIAN_INCLUDED_REMOTESERVER_H Index: xapian-core/common/serialise.h =================================================================== --- xapian-core/common/serialise.h (revision 8312) +++ xapian-core/common/serialise.h (working copy) @@ -24,6 +24,7 @@ #include #include +#include // Forward class declarations: @@ -115,7 +116,8 @@ * * @return The unserialised Xapian::MSet object. */ -Xapian::MSet unserialise_mset(const std::string &s); +Xapian::MSet unserialise_mset(const std::string &s, + Xapian::Internal::RefCntPtr internal_db); /** Serialise a Xapian::RSet object. * Index: xapian-core/api/omenquire.cc =================================================================== --- xapian-core/api/omenquire.cc (revision 8312) +++ xapian-core/api/omenquire.cc (working copy) @@ -644,13 +644,14 @@ : db(db_), query(), collapse_key(Xapian::BAD_VALUENO), order(Enquire::ASCENDING), percent_cutoff(0), weight_cutoff(0), sort_key(Xapian::BAD_VALUENO), sort_by(REL), sort_value_forward(true), - errorhandler(errorhandler_), weight(0) + sort_cmpfn(0), errorhandler(errorhandler_), weight(0) { } Enquire::Internal::~Internal() { delete weight; + delete sort_cmpfn; weight = 0; } @@ -685,14 +686,14 @@ ::MultiMatch match(db, query.internal.get(), qlen, RSet(), collapse_key, percent_cutoff, weight_cutoff, order, sort_key, sort_by, sort_value_forward, - errorhandler, new LocalStatsGatherer(), weight); + errorhandler, new LocalStatsGatherer(), weight, sort_cmpfn); // Run query and put results into supplied Xapian::MSet object. match.get_mset(first, maxitems, check_at_least, retval, mdecider); } else { ::MultiMatch match(db, query.internal.get(), qlen, *rset, collapse_key, percent_cutoff, weight_cutoff, order, sort_key, sort_by, sort_value_forward, - errorhandler, new LocalStatsGatherer(), weight); + errorhandler, new LocalStatsGatherer(), weight, sort_cmpfn); // Run query and put results into supplied Xapian::MSet object. match.get_mset(first, maxitems, check_at_least, retval, mdecider); } @@ -909,6 +910,7 @@ { DEBUGAPICALL(void, "Xapian::Enquire::set_weighting_scheme", "[Weight]"); delete internal->weight; + internal->weight = NULL; // Set to NULL before cloning, to avoid double free if clone() throws an exception. internal->weight = weight_.clone(); } @@ -935,6 +937,8 @@ Enquire::set_sort_by_relevance() { internal->sort_by = Internal::REL; + delete internal->sort_cmpfn; + internal->sort_cmpfn = NULL; } void @@ -943,6 +947,8 @@ internal->sort_key = sort_key; internal->sort_by = Internal::VAL; internal->sort_value_forward = ascending; + delete internal->sort_cmpfn; + internal->sort_cmpfn = NULL; } void @@ -952,6 +958,8 @@ internal->sort_key = sort_key; internal->sort_by = Internal::VAL_REL; internal->sort_value_forward = ascending; + delete internal->sort_cmpfn; + internal->sort_cmpfn = NULL; } void @@ -961,8 +969,20 @@ internal->sort_key = sort_key; internal->sort_by = Internal::REL_VAL; internal->sort_value_forward = ascending; + delete internal->sort_cmpfn; + internal->sort_cmpfn = NULL; } +void +Enquire::set_sort_by_cmpfn(const MatchCmp &cmpfn_) +{ + DEBUGAPICALL(void, "Xapian::Enquire::set_sort_by_cmpfn", "[cmpfn:" << cmpfn_.name() << "(" << cmpfn_.serialise() << ")]"); + delete internal->sort_cmpfn; + internal->sort_by = Internal::USER; + internal->sort_cmpfn = NULL; // Set to NULL before cloning, to avoid double free if clone() throws an exception. + internal->sort_cmpfn = cmpfn_.clone(); +} + MSet Enquire::get_mset(Xapian::doccount first, Xapian::doccount maxitems, Xapian::doccount check_at_least, const RSet *rset, Index: xapian-core/backends/remote/remote-database.cc =================================================================== --- xapian-core/backends/remote/remote-database.cc (revision 8312) +++ xapian-core/backends/remote/remote-database.cc (working copy) @@ -409,6 +409,7 @@ bool sort_value_forward, int percent_cutoff, Xapian::weight weight_cutoff, const Xapian::Weight *wtscheme, + const Xapian::MatchCmp *sort_cmpfn, const Xapian::RSet &omrset) { string tmp = query->serialise(); @@ -425,14 +426,26 @@ message += char(percent_cutoff); message += serialise_double(weight_cutoff); + // Serialise the weight scheme tmp = wtscheme->name(); message += encode_length(tmp.size()); message += tmp; - tmp = wtscheme->serialise(); message += encode_length(tmp.size()); message += tmp; + // Serialise the match cmp object + if (sort_cmpfn == NULL) { + message += encode_length(0); + } else { + tmp = sort_cmpfn->name(); + message += encode_length(tmp.size()); + message += tmp; + tmp = sort_cmpfn->serialise(); + message += encode_length(tmp.size()); + message += tmp; + } + message += serialise_rset(omrset); send_message(MSG_QUERY, message); @@ -466,7 +479,7 @@ { string message; get_message(message, REPLY_RESULTS); - mset = unserialise_mset(message); + mset = unserialise_mset(message, this); } void Index: xapian-bindings/csharp/Makefile.am =================================================================== --- xapian-bindings/csharp/Makefile.am (revision 8312) +++ xapian-bindings/csharp/Makefile.am (working copy) @@ -23,6 +23,8 @@ ExpandDecider.cs \ Flint.cs \ InMemory.cs \ + MatchCmp.cs \ + MatchItem.cs \ MatchDecider.cs \ MSet.cs \ MSetIterator.cs \ @@ -35,6 +37,7 @@ RSet.cs \ SWIGTYPE_p_std__vectorTstd__string_t.cs \ SWIGTYPE_p_std__vectorTXapian__Query_t.cs \ + SWIGTYPE_p_Xapian__Internal__MSetItem.cs \ SimpleStopper.cs \ Stem.cs \ Stopper.cs \ Index: xapian-bindings/xapian.i =================================================================== --- xapian-bindings/xapian.i (revision 8312) +++ xapian-bindings/xapian.i (working copy) @@ -156,6 +156,7 @@ int xapian_revision(); class Weight; +class MatchCmp; class Stopper; // from xapian/positioniterator.h @@ -482,6 +483,7 @@ bool ascending = true); void set_sort_by_relevance_then_value(Xapian::valueno sort_key, bool ascending = true); + void set_sort_by_cmpfn(const MatchCmp& cmpfn); #ifdef XAPIAN_SWIG_DIRECTORS MSet get_mset(doccount first, @@ -635,6 +637,70 @@ bool get_sumpart_needs_doclength() const; }; +class MatchItem { + public: + MatchItem(const Xapian::Internal::MSetItem & item_); + Xapian::weight get_wt() const; + Xapian::docid get_docid() const; + Xapian::Document get_document() const; +}; + +class MatchCmp { +/* SWIG doesn't handle this: + private: + virtual MatchCmp * clone() const = 0; */ + public: + virtual ~MatchCmp(); + + virtual std::string name() const = 0; + virtual std::string serialise() const = 0; + virtual MatchCmp * unserialise(const std::string &s) const = 0; + + virtual int cmp(const MatchItem & a, + const MatchItem & b) const = 0; +}; + +%warnfilter(842) IntegerMatchCmp::unserialise; +class IntegerMatchCmp : public MatchCmp { + public: + IntegerMatchCmp * clone() const; + IntegerMatchCmp(Xapian::valueno sort_key_, + bool ascending = true); + IntegerMatchCmp(); + ~IntegerMatchCmp(); + + std::string name() const; + std::string serialise() const; + IntegerMatchCmp * unserialise(const std::string &s) const; + + int cmp(const MatchItem & a, + const MatchItem & b) const; + + static std::string int_to_value(int num); + static int value_to_int(const std::string &s); +}; + +%warnfilter(842) DoubleMatchCmp::unserialise; +class DoubleMatchCmp : public MatchCmp { + public: + DoubleMatchCmp * clone() const; + DoubleMatchCmp(Xapian::valueno sort_key_, + bool ascending = true); + DoubleMatchCmp(); + ~DoubleMatchCmp(); + + std::string name() const; + std::string serialise() const; + DoubleMatchCmp * unserialise(const std::string &s) const; + + int cmp(const MatchItem & a, + const MatchItem & b) const; + + static std::string double_to_value(double num); + static double value_to_double(const std::string &s); +}; + + // xapian/database.h class Database {