Index: xapian-core/matcher/multweightpostlist.cc =================================================================== --- xapian-core/matcher/multweightpostlist.cc (revision 0) +++ xapian-core/matcher/multweightpostlist.cc (revision 0) @@ -0,0 +1,121 @@ +/** @file multweightpostlist.cc + * @brief Return documents from a subquery with weights multiplied by a double. + */ +/* Copyright 2007 Lemur Consulting Ltd + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include + +#include "omassert.h" +#include "multweightpostlist.h" + +// Include branchpostlist for utility functions "next_handling_prune" and +// "skip_to_handling_prune" +#include "branchpostlist.h" + +Xapian::doccount +MultWeightPostList::get_termfreq_min() const +{ + return source->get_termfreq_min(); +} + +Xapian::doccount +MultWeightPostList::get_termfreq_est() const +{ + return source->get_termfreq_est(); +} + +Xapian::doccount +MultWeightPostList::get_termfreq_max() const +{ + return source->get_termfreq_max(); +} + +Xapian::weight +MultWeightPostList::get_maxweight() const +{ + return source->get_maxweight() * multiplier; +} + +Xapian::docid +MultWeightPostList::get_docid() const +{ + return source->get_docid(); +} + +Xapian::weight +MultWeightPostList::get_weight() const +{ + return source->get_weight() * multiplier; +} + +Xapian::doclength +MultWeightPostList::get_doclength() const +{ + return source->get_doclength(); +} + +Xapian::weight +MultWeightPostList::recalc_maxweight() +{ + return source->recalc_maxweight() * multiplier; +} + +PositionList * +MultWeightPostList::read_position_list() +{ + return source->read_position_list(); +} + +PositionList * +MultWeightPostList::open_position_list() const +{ + return source->open_position_list(); +} + +PostList * +MultWeightPostList::next(Xapian::weight w_min) +{ + AssertNe(multiplier, 0); + next_handling_prune(source, w_min / multiplier, matcher); + return NULL; +} + +PostList * +MultWeightPostList::skip_to(Xapian::docid did, Xapian::weight w_min) +{ + AssertNe(multiplier, 0); + skip_to_handling_prune(source, did, w_min / multiplier, matcher); + return NULL; +} + +bool +MultWeightPostList::at_end() const +{ + return source->at_end(); +} + +string +MultWeightPostList::get_description() const +{ + string desc = "MultWeightPostList("; + desc += source->get_description(); + desc += " * "; + desc += om_tostring(multiplier); + desc += ")"; + return desc; +} Property changes on: xapian-core/matcher/multweightpostlist.cc ___________________________________________________________________ Name: svn:eol-style + native Index: xapian-core/matcher/multweightpostlist.h =================================================================== --- xapian-core/matcher/multweightpostlist.h (revision 0) +++ xapian-core/matcher/multweightpostlist.h (revision 0) @@ -0,0 +1,71 @@ +/** @file multweightpostlist.h + * @brief Return documents from a subquery with weights multiplied by a double. + */ +/* Copyright 2007 Lemur Consulting Ltd + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef XAPIAN_INCLUDED_MULTWEIGHTPOSTLIST_H +#define XAPIAN_INCLUDED_MULTWEIGHTPOSTLIST_H + +#include "database.h" +#include "postlist.h" + +class MultiMatch; + +class MultWeightPostList : public PostList { + /** The sub-postlist. */ + PostList *source; + + /** The multiplier to apply to the weights. */ + double multiplier; + + /** The object which is using this postlist to perform + * a match. This object needs to be notified when the + * tree changes such that the maximum weights need to be + * recalculated. + */ + MultiMatch *matcher; + + /// Disallow copying. + MultWeightPostList(const MultWeightPostList &); + + /// Disallow assignment. + void operator=(const MultWeightPostList &); + + public: + MultWeightPostList(PostList *source_, double multiplier_, + MultiMatch *matcher_) + : source(source_), multiplier(multiplier_), matcher(matcher_) {} + ~MultWeightPostList() { delete source; } + + Xapian::doccount get_termfreq_min() const; + Xapian::doccount get_termfreq_est() const; + Xapian::doccount get_termfreq_max() const; + Xapian::weight get_maxweight() const; + Xapian::docid get_docid() const; + Xapian::weight get_weight() const; + Xapian::doclength get_doclength() const; + Xapian::weight recalc_maxweight(); + PositionList * read_position_list(); + PositionList * open_position_list() const; + PostList * next(Xapian::weight w_min); + PostList * skip_to(Xapian::docid did, Xapian::weight w_min); + bool at_end() const; + string get_description() const; +}; + +#endif /* XAPIAN_INCLUDED_MULTWEIGHTPOSTLIST_H */ Property changes on: xapian-core/matcher/multweightpostlist.h ___________________________________________________________________ Name: svn:eol-style + native Index: xapian-core/matcher/Makefile.mk =================================================================== --- xapian-core/matcher/Makefile.mk (revision 9345) +++ xapian-core/matcher/Makefile.mk (working copy) @@ -11,6 +11,7 @@ matcher/mergepostlist.h\ matcher/msetcmp.h\ matcher/msetpostlist.h\ + matcher/multweightpostlist.h\ matcher/orpostlist.h\ matcher/phrasepostlist.h\ matcher/remotesubmatch.h\ @@ -44,6 +45,7 @@ matcher/msetcmp.cc\ matcher/msetpostlist.cc\ matcher/multimatch.cc\ + matcher/multweightpostlist.cc\ matcher/orpostlist.cc\ matcher/phrasepostlist.cc\ matcher/rset.cc\ Index: xapian-core/matcher/localmatch.cc =================================================================== --- xapian-core/matcher/localmatch.cc (revision 9345) +++ xapian-core/matcher/localmatch.cc (working copy) @@ -3,6 +3,7 @@ * Copyright 1999,2000,2001 BrightStation PLC * Copyright 2002 Ananova Ltd * Copyright 2002,2003,2004,2005,2006,2007 Olly Betts + * Copyright 2007 Lemur Consulting Ltd * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as @@ -38,12 +39,14 @@ #include "mergepostlist.h" #include "extraweightpostlist.h" #include "valuerangepostlist.h" +#include "multweightpostlist.h" #include "omqueryinternal.h" #include #include "autoptr.h" #include +#include ///////////////////////////////////////////// // Comparison operators which we will need // @@ -476,6 +479,15 @@ case Xapian::Query::OP_VALUE_RANGE: RETURN(new ValueRangePostList(db, Xapian::valueno(query->parameter), query->tname, query->str_parameter)); + case Xapian::Query::OP_MULT_WEIGHT: + Assert(query->subqs.size() == 1); + if (is_bool || query->dbl_parameter < DBL_EPSILON) { + // Return as a boolean query. + RETURN(postlist_from_query(query->subqs[0], matcher, true)); + } else { + RETURN(new MultWeightPostList(postlist_from_query(query->subqs[0], matcher, is_bool), + query->dbl_parameter, matcher)); + } } Assert(false); RETURN(NULL); Index: xapian-core/tests/harness/testsuite.h =================================================================== --- xapian-core/tests/harness/testsuite.h (revision 9345) +++ xapian-core/tests/harness/testsuite.h (working copy) @@ -246,6 +246,11 @@ "Expected `"STRINGIZE(a)"' and `"STRINGIZE(b)"' to be (nearly) equal:" \ " were " << setprecision(DBL_DIG) << (a) << " and " << (b) << ")" << setprecision(6)) +/// Test two doubles for non-near-equality. +#define TEST_NOT_EQUAL_DOUBLE(a, b) TEST_AND_EXPLAIN(!TEST_EQUAL_DOUBLE_((a), (b)), \ + "Expected `"STRINGIZE(a)"' and `"STRINGIZE(b)"' not to be (nearly) equal:" \ + " were " << setprecision(DBL_DIG) << (a) << " and " << (b) << ")" << setprecision(6)) + /// Test for non-equality of two things. #define TEST_NOT_EQUAL(a, b) TEST_AND_EXPLAIN(((a) != (b)), \ "Expected `"STRINGIZE(a)"' and `"STRINGIZE(b)"' not to be equal:" \ Index: xapian-core/tests/harness/testsuite.cc =================================================================== --- xapian-core/tests/harness/testsuite.cc (revision 9345) +++ xapian-core/tests/harness/testsuite.cc (working copy) @@ -673,5 +673,6 @@ bool TEST_EQUAL_DOUBLE_(double a, double b) { + if (a == b) return true; return (ceil(log10(max(fabs(a), fabs(b)))) - log10(fabs(a - b)) > DBL_DIG); } Index: xapian-core/tests/api_anydb.cc =================================================================== --- xapian-core/tests/api_anydb.cc (revision 9345) +++ xapian-core/tests/api_anydb.cc (working copy) @@ -1809,6 +1809,69 @@ return true; } +// Feature test for Query::OP_MULT_WEIGHT. +static bool test_multweight1() { + Xapian::Database db(get_database("apitest_phrase")); + Xapian::Enquire enq(db); + Xapian::QueryParser qp; + + static const char * queries[] = { + "pad", + "milk fridge", + "leave milk on fridge", + "ordered milk operator", + "ordered phrase operator", + "leave \"milk on fridge\"", + "notpresent", + "leave \"milk notpresent\"", + NULL + }; + static double multipliers[] = { + -1000000, -2.5, -1, -0.5, 0, 0.5, 1, 2.5, 1000000, + 0, 0 + }; + + for (const char **qstr = queries; *qstr; ++qstr) { + Xapian::Query query1 = qp.parse_query(*qstr); + tout << "query1: " << query1.get_description() << endl; + for (double *multp = multipliers; multp[0] != multp[1]; ++multp) { + double mult = *multp; + Xapian::Query query2(Xapian::Query::OP_MULT_WEIGHT, query1, mult); + tout << "query2: " << query2.get_description() << endl; + + enq.set_query(query1); + Xapian::MSet mset1 = enq.get_mset(0, 20); + enq.set_query(query2); + Xapian::MSet mset2 = enq.get_mset(0, 20); + + TEST_EQUAL(mset1.size(), mset2.size()); + + Xapian::MSetIterator i1, i2; + if (mult > 0) { + for (i1 = mset1.begin(), i2 = mset2.begin(); + i1 != mset1.end() && i2 != mset2.end(); ++i1, ++i2) { + TEST_EQUAL_DOUBLE(i1.get_weight() * mult, i2.get_weight()); + TEST_EQUAL(*i1, *i2); + } + } else { + // Weights in mset2 are 0; so it should be sorted by docid. + vector ids1; + vector ids2; + for (i1 = mset1.begin(), i2 = mset2.begin(); + i1 != mset1.end() && i2 != mset2.end(); ++i1, ++i2) { + TEST_NOT_EQUAL_DOUBLE(i1.get_weight(), 0); + TEST_EQUAL_DOUBLE(i2.get_weight(), 0); + ids1.push_back(*i1); + ids2.push_back(*i2); + } + sort(ids1.begin(), ids1.end()); + TEST_EQUAL(ids1, ids2); + } + } + } + return true; +} + // ####################################################################### // # End of test cases: now we list the tests to run. @@ -1875,5 +1938,6 @@ {"allpostlist1", test_allpostlist1}, {"emptyterm1", test_emptyterm1}, {"valuerange1", test_valuerange1}, + {"multweight1", test_multweight1}, {0, 0} }; Index: xapian-core/include/xapian/query.h =================================================================== --- xapian-core/include/xapian/query.h (revision 9345) +++ xapian-core/include/xapian/query.h (working copy) @@ -4,7 +4,7 @@ /* Copyright 1999,2000,2001 BrightStation PLC * Copyright 2002 Ananova Ltd * Copyright 2003,2004,2005,2006,2007 Olly Betts - * Copyright 2006 Lemur Consulting Ltd + * Copyright 2006,2007 Lemur Consulting Ltd * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as @@ -99,7 +99,12 @@ /** Select an elite set from the subqueries, and perform * a query with these combined as an OR query. */ - OP_ELITE_SET = 10 + OP_ELITE_SET = 10, + + /** Multiply the weight of a subquery by a multiplier. Only + * positive multiplers are supported - negative multipliers will be + * clipped to 0. */ + OP_MULT_WEIGHT } op; /** Copy constructor. */ @@ -154,6 +159,9 @@ /** Apply the specified operator to a single Xapian::Query object. */ Query(Query::op op_, Xapian::Query q); + /** Apply the specified operator to a single Xapian::Query object, with a parameter. */ + Query(Query::op op_, Xapian::Query q, double parameter); + /** Construct a value range query on a document value. * * A value range query matches those documents which have a value @@ -281,6 +289,9 @@ /// Within query frequency of this term - leaf node only Xapian::termcount wqf; + /// Used to store a multiplier for subquery weights. + double dbl_parameter; + /** swap the contents of this with another Xapian::Query::Internal, * in a way which is guaranteed not to throw. This is * used with the assignment operator to make it exception @@ -360,6 +371,10 @@ */ void add_subquery(const Query::Internal * subq); + void set_dbl_parameter(double dbl_parameter_) { + dbl_parameter = dbl_parameter_; + } + /** Finish off the construction. */ Query::Internal * end_construction(); Index: xapian-core/api/omqueryinternal.cc =================================================================== --- xapian-core/api/omqueryinternal.cc (revision 9345) +++ xapian-core/api/omqueryinternal.cc (working copy) @@ -3,7 +3,7 @@ * Copyright 1999,2000,2001 BrightStation PLC * Copyright 2002 Ananova Ltd * Copyright 2002,2003,2004,2005,2006,2007 Olly Betts - * Copyright 2006 Lemur Consulting Ltd + * Copyright 2006,2007 Lemur Consulting Ltd * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as @@ -28,6 +28,7 @@ #include "omqueryinternal.h" #include "utils.h" #include "serialise.h" +#include "serialise-double.h" #include #include @@ -40,6 +41,7 @@ #include #include #include +#include using namespace std; @@ -58,6 +60,8 @@ case Xapian::Query::OP_ELITE_SET: case Xapian::Query::OP_VALUE_RANGE: return 0; + case Xapian::Query::OP_MULT_WEIGHT: + return 1; case Xapian::Query::OP_FILTER: case Xapian::Query::OP_AND_MAYBE: case Xapian::Query::OP_AND_NOT: @@ -75,6 +79,8 @@ case Xapian::Query::Internal::OP_LEAF: case Xapian::Query::OP_VALUE_RANGE: return 0; + case Xapian::Query::OP_MULT_WEIGHT: + return 1; case Xapian::Query::OP_FILTER: case Xapian::Query::OP_AND_MAYBE: case Xapian::Query::OP_AND_NOT: @@ -177,6 +183,10 @@ result += str_parameter; result += om_tostring(parameter); break; + case Xapian::Query::OP_MULT_WEIGHT: + result += "."; + result += serialise_double(dbl_parameter); + break; } } return result; @@ -202,6 +212,7 @@ case Xapian::Query::OP_PHRASE: name = "PHRASE"; break; case Xapian::Query::OP_ELITE_SET: name = "ELITE_SET"; break; case Xapian::Query::OP_VALUE_RANGE: name = "VALUE_RANGE"; break; + case Xapian::Query::OP_MULT_WEIGHT: name = "MULT_WEIGHT"; break; } return name; } @@ -248,6 +259,12 @@ if (!description.empty()) description += opstr; description += (**i).get_description(); } + + if (op == Xapian::Query::OP_MULT_WEIGHT) { + description += " * "; + description += om_tostring(dbl_parameter); + } + return "(" + description + ")"; } @@ -451,6 +468,12 @@ return new Xapian::Query::Internal(Xapian::Query::OP_VALUE_RANGE, valno, start, stop); } + case '.': { + AutoPtr result( + qint_from_vector(Xapian::Query::OP_MULT_WEIGHT, subqs)); + result->set_dbl_parameter(unserialise_double(&p, end)); + return result.release(); + } default: DEBUGLINE(UNKNOWN, "Can't parse remainder `" << p - 1 << "'"); throw Xapian::InvalidArgumentError("Invalid query string"); @@ -498,6 +521,7 @@ std::swap(str_parameter, other.str_parameter); std::swap(term_pos, other.term_pos); std::swap(wqf, other.wqf); + std::swap(dbl_parameter, other.dbl_parameter); } Xapian::Query::Internal::Internal(const Xapian::Query::Internal ©me) @@ -508,7 +532,8 @@ tname(copyme.tname), str_parameter(copyme.str_parameter), term_pos(copyme.term_pos), - wqf(copyme.wqf) + wqf(copyme.wqf), + dbl_parameter(copyme.dbl_parameter) { for (subquery_list::const_iterator i = copyme.subqs.begin(); i != copyme.subqs.end(); @@ -672,6 +697,11 @@ // match anything. if (tname > str_parameter) return 0; return this; + case OP_MULT_WEIGHT: + if (fabs(dbl_parameter - 1.0) > DBL_EPSILON) return this; + // If the multiplier is 1, this node doesn't actually do anything, + // so we leave it to be removed. + break; case OP_PHRASE: case OP_NEAR: // Default to the number of subqueries. if (!parameter) parameter = subqs.size(); Index: xapian-core/api/omquery.cc =================================================================== --- xapian-core/api/omquery.cc (revision 9345) +++ xapian-core/api/omquery.cc (working copy) @@ -3,7 +3,7 @@ * Copyright 1999,2000,2001 BrightStation PLC * Copyright 2001,2002 Ananova Ltd * Copyright 2003,2004,2005,2006,2007 Olly Betts - * Copyright 2006 Lemur Consulting Ltd + * Copyright 2006,2007 Lemur Consulting Ltd * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as @@ -132,6 +132,21 @@ } } +Query::Query(Query::op op_, Xapian::Query q, double parameter) +{ + DEBUGAPICALL(void, "Xapian::Query::Query", + op_ << ", " << q << ", " << parameter); + try { + start_construction(op_, 0); + internal->set_dbl_parameter(parameter); + add_subquery(q); + end_construction(); + } catch (...) { + abort_construction(); + throw; + } +} + Query::Query(Query::op op_, Xapian::valueno valno, const string &begin, const string &end) : internal(new Query::Internal(op_, valno, begin, end)) Index: xapian-bindings/python/pythontest.py =================================================================== --- xapian-bindings/python/pythontest.py (revision 9345) +++ xapian-bindings/python/pythontest.py (working copy) @@ -817,6 +817,78 @@ del dbr shutil.rmtree(dbpath) +def test_mult_weight(): + """Test query OP_MULT_WEIGHT feature. + + """ + db = setup_database() + for mult in (-2.5, -1, 0, 1, 2.5): + context("checking queries with OP_MULT_WEIGHT with a multipler of %r" % + mult) + query1 = xapian.Query("it") + query2 = xapian.Query(xapian.Query.OP_MULT_WEIGHT, query1, mult) + + enquire = xapian.Enquire(db) + enquire.set_query(query1) + mset1 = enquire.get_mset(0, 10) + enquire.set_query(query2) + mset2 = enquire.get_mset(0, 10) + if mult <= 0: + expected = [(0, item.docid) for item in mset1] + expected.sort() + else: + expected = [(item.weight * mult, item.docid) for item in mset1] + expect([(item.weight, item.docid) for item in mset2], expected) + + +def test_weight_normalise(): + """Test normalising of query weights using the OP_MULT_WEIGHT feature. + + This test first runs a search (asking for no results) to get the maximum + possible weight for a query, and then checks that the results of + MSet.get_max_possible() match this. + + This tests that the get_max_possible() value is correct (though it isn't + guaranteed to be at a tight bound), and that the mult-weight query can + compensate correctly. + + """ + db = setup_database() + for query in ( + "it", + "was", + "it was", + "it was four", + "it was four five", + "\"was it warm\" four notpresent", + "notpresent", + ): + context("checking query %r using OP_MULT_WEIGHT to normalise the weights" % query) + qp = xapian.QueryParser() + query1 = qp.parse_query(query) + enquire = xapian.Enquire(db) + enquire.set_query(query1) + mset1 = enquire.get_mset(0, 0) + + # Check the max_attained value is 0 - this gives us some reassurance + # that the match didn't actually do the work of calculating any + # results. + expect(mset1.get_max_attained(), 0) + + max_possible = mset1.get_max_possible() + mult = 1.0 / max_possible + query2 = xapian.Query(xapian.Query.OP_MULT_WEIGHT, query1, mult) + + enquire = xapian.Enquire(db) + enquire.set_query(query2) + mset2 = enquire.get_mset(0, 10) + # max_possible should be 1 (excluding rounding errors) for mset2 + expect(int(mset2.get_max_possible() * 1000000.0 + 0.5), 1000000) + for item in mset2: + expect(item.weight > 0, True) + expect(item.weight <= 1, True) + + # The legacy sequence API is only supported for Python >= 2.3 so don't try # testing it for Python 2.2. vinfo = sys.version_info Index: xapian-bindings/xapian.i =================================================================== --- xapian-bindings/xapian.i (revision 9345) +++ xapian-bindings/xapian.i (working copy) @@ -7,6 +7,7 @@ * Copyright 2001,2002 Ananova Ltd * Copyright 2002,2003,2005 James Aylett * Copyright 2002,2003,2004,2005,2006,2007 Olly Betts + * Copyright 2007 Lemur Consulting Ltd * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as @@ -839,7 +840,8 @@ OP_NEAR, OP_PHRASE, OP_VALUE_RANGE, - OP_ELITE_SET = 10 + OP_ELITE_SET = 10, + OP_MULT_WEIGHT }; // FIXME wrap optional arguments in PHP? Query(const string &tname, termcount wqf = 1, termpos term_pos = 0); @@ -868,6 +870,9 @@ /** Apply the specified operator to a single Xapian::Query object. */ Query(Query::op op_, Xapian::Query q); + /** Apply the specified operator to a single Xapian::Query object, with a parameter. */ + Query(Query::op op_, Xapian::Query q, double parameter); + /** Constructs a new empty query object */ Query();