Ticket #537: eset_min_wt.diff

xapian-core/api/omenquire.cc

@@ -4 +4 @@
  * Copyright 2001,2002 Ananova Ltd
  * Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2010 Olly Betts
  * Copyright 2007,2009 Lemur Consulting Ltd
+ * Copyright 2011 Dan Colish
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -690 +691 @@
 ESet
 Enquire::Internal::get_eset(Xapian::termcount maxitems,
                     const RSet & rset, int flags, double k,
-                    const ExpandDecider * edecider) const
+                    const ExpandDecider * edecider, Xapian::weight min_wt) const
 {
-    LOGCALL(MATCH, ESet, "Enquire::Internal::get_eset", maxitems | rset | flags | k | edecider);
+    LOGCALL(MATCH, ESet, "Enquire::Internal::get_eset", maxitems | rset | flags | k | edecider | min_wt);
 
     if (maxitems == 0 || rset.empty()) {
         // Either we were asked for no results, or wouldn't produce any
@@ -728 +729 @@
     ExpandWeight eweight(db, rset.size(), use_exact_termfreq, k);
 
     Xapian::ESet eset;
-    eset.internal->expand(maxitems, db, rset, edecider, eweight);
+    eset.internal->expand(maxitems, db, rset, edecider, eweight, min_wt);
     RETURN(eset);
 }
 
@@ -1043 +1044 @@
     LOGCALL(API, Xapian::ESet, "Xapian::Enquire::get_eset", maxitems | rset | flags | k | edecider);
 
     try {
-        RETURN(internal->get_eset(maxitems, rset, flags, k, edecider));
+        RETURN(internal->get_eset(maxitems, rset, flags, k, edecider, 0));
+    } catch (Error & e) {
+        if (internal->errorhandler) (*internal->errorhandler)(e);
+        throw;
+    }
+}
+
+ESet
+Enquire::get_eset(Xapian::termcount maxitems, const RSet & rset, int flags,
+                  double k, const ExpandDecider * edecider, Xapian::weight min_wt) const
+{
+    LOGCALL(API, Xapian::ESet, "Xapian::Enquire::get_eset", maxitems | rset | flags | k | edecider | min_wt);
+
+    try {
+        RETURN(internal->get_eset(maxitems, rset, flags, k, edecider, min_wt));
     } catch (Error & e) {
         if (internal->errorhandler) (*internal->errorhandler)(e);
         throw;

xapian-core/common/esetinternal.h

@@ -2 +2 @@
  * @brief Xapian::ESet::Internal class
  */
 /* Copyright (C) 2008,2010 Olly Betts
+ * Copyright (C) 2011 Dan Colish
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -101 +102 @@
                 const Xapian::Database & db,
                 const Xapian::RSet & rset,
                 const Xapian::ExpandDecider * edecider,
-                const Xapian::Internal::ExpandWeight & eweight);
+                const Xapian::Internal::ExpandWeight & eweight,
+                Xapian::weight min_wt);
 
     /// Return a string describing this object.
     std::string get_description() const;

xapian-core/common/omenquireinternal.h

@@ -4 +4 @@
  * Copyright 2001,2002 Ananova Ltd
  * Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2010 Olly Betts
  * Copyright 2009 Lemur Consulting Ltd
+ * Copyright 2011 Dan Colish
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -179 +180 @@
                       const RSet *omrset,
                       const MatchDecider *mdecider,
                       const MatchDecider *matchspy_legacy) const;
+
         ESet get_eset(Xapian::termcount maxitems, const RSet & omrset, int flags,
-                      double k, const ExpandDecider *edecider) const;
+                      double k, const ExpandDecider *edecider, Xapian::weight min_wt) const;
 
         TermIterator get_matching_terms(Xapian::docid did) const;
         TermIterator get_matching_terms(const Xapian::MSetIterator &it) const;

xapian-core/expand/esetinternal.cc

@@ -2 +2 @@
  * @brief Xapian::ESet::Internal class
  */
 /* Copyright (C) 2008,2010 Olly Betts
+ * Copyright (C) 2011 Dan Colish
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -138 +139 @@
                        const Xapian::Database & db,
                        const RSet & rset,
                        const Xapian::ExpandDecider * edecider,
-                       const Xapian::Internal::ExpandWeight & eweight)
+                       const Xapian::Internal::ExpandWeight & eweight,
+                       Xapian::weight min_wt)
 {
     LOGCALL_VOID(EXPAND, "ESet::Internal::expand", max_esize | db | rset | edecider | eweight);
     // These two cases are handled by our caller.
@@ -149 +151 @@
     Assert(ebound == 0);
     Assert(items.empty());
 
-    Xapian::weight min_wt = 0;
-
     AutoPtr<TermList> tree(build_termlist_tree(db, rset));
     Assert(tree.get());
 

xapian-core/expand/expandweight.cc

@@ -2 +2 @@
  * @brief Calculate term weights for the ESet.
  */
 /* Copyright (C) 2007,2008 Olly Betts
- *
+ * Copyright (C) 2011 Dan Colish
+ * 
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
  * published by the Free Software Foundation; either version 2 of the
@@ -103 +104 @@
     denom = (termfreq - rtermfreq + 0.5) * (reldocs_without_term + 0.5);
     AssertRel(denom,>,0);
 
-    // If the returned termweight would be negative, clamp it to 0.0.
-    if (rare(num <= denom)) RETURN(0.0);
-
     Xapian::weight tw = log(num / denom);
     LOGVALUE(EXPAND, tw);
     LOGVALUE(EXPAND, stats.multiplier);

xapian-core/include/xapian/enquire.h

@@ -5 +5 @@
  * Copyright 2001,2002 Ananova Ltd
  * Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2011 Olly Betts
  * Copyright 2009 Lemur Consulting Ltd
+ * Copyright 2011 Dan Colish
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -1005 +1006 @@
          *  @param edecider  a decision functor to use to decide whether a
          *                   given term should be put in the ESet
          *
+         *  @param min_wt    the minimum weight for included terms
+         *
          *  @return          An ESet object containing the results of the
          *                   expand.
          *
@@ -1015 +1018 @@
                         int flags = 0,
                         double k = 1.0,
                         const Xapian::ExpandDecider * edecider = 0) const;
-
         /** Get the expand set for the given rset.
          *
          *  @param maxitems  the maximum number of items to return.
@@ -1034 +1036 @@
             return get_eset(maxitems, omrset, 0, 1.0, edecider);
         }
 
+        /** Get the expand set for the given rset.
+         *
+         *  @param maxitems  the maximum number of items to return.
+         *  @param omrset    the relevance set to use when performing
+         *                   the expand operation.
+         *  @param flags     zero or more of these values |-ed together:
+         *                    - Xapian::Enquire::INCLUDE_QUERY_TERMS query
+         *                      terms may be returned from expand
+         *                    - Xapian::Enquire::USE_EXACT_TERMFREQ for multi
+         *                      dbs, calculate the exact termfreq; otherwise an
+         *                      approximation is used which can greatly improve
+         *                      efficiency, but still returns good results.
+         *  @param k         the parameter k in the query expansion algorithm
+         *                   (default is 1.0)
+         *  @param edecider  a decision functor to use to decide whether a
+         *                   given term should be put in the ESet
+         *
+         *  @param min_wt    the minimum weight for included terms
+         *
+         *  @return          An ESet object containing the results of the
+         *                   expand.
+         *
+         *  @exception Xapian::InvalidArgumentError  See class documentation.
+         */
+        ESet get_eset(Xapian::termcount maxitems,
+                        const RSet & omrset,
+                        int flags,
+                        double k,
+                        const Xapian::ExpandDecider * edecider,
+                        Xapian::weight min_wt) const;
+
         /** Get terms which match a given document, by document id.
          *
          *  This method returns the terms in the current query which match

xapian-core/tests/api_anydb.cc

@@ -4 +4 @@
  * Copyright 2002 Ananova Ltd
  * Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2010,2011 Olly Betts
  * Copyright 2006,2008 Lemur Consulting Ltd
+ * Copyright 2011 Dan Colish
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -421 +422 @@
     return true;
 }
 
+// tests the returned weights are as expected (regression test for remote
+// backend which was using the average weight rather than the actual document
+// weight for computing weights - fixed in 1.0.0).
+DEFINE_TESTCASE(expandweights3, backend) {
+    Xapian::Enquire enquire(get_database("apitest_simpledata"));
+    enquire.set_query(Xapian::Query("this"));
+
+    Xapian::MSet mymset = enquire.get_mset(0, 10);
+
+    Xapian::RSet myrset;
+    Xapian::MSetIterator i = mymset.begin();
+    myrset.add_document(*i);
+    myrset.add_document(*(++i));
+
+    // Set min_wt to 0
+    Xapian::ESet eset = enquire.get_eset(50, myrset, 0, 1.0, 0, 0);
+    if (!startswith(get_dbtype(), "multi")) {
+        // For a single database, the weights should be the same with or
+        // without USE_EXACT_TERMFREQ.
+        TEST_EQUAL_DOUBLE(eset[0].get_weight(), 6.08904001099445);
+        TEST_EQUAL_DOUBLE(eset[1].get_weight(), 6.08904001099445);
+        TEST_EQUAL_DOUBLE(eset[2].get_weight(), 4.73383620844021);
+        TEST_EQUAL(eset[49].get_weight() < 0, false);
+    } else {
+        // For multiple databases, we expect that using USE_EXACT_TERMFREQ
+        // will result in different weights in some cases.
+        TEST_NOT_EQUAL_DOUBLE(eset[0].get_weight(), 6.08904001099445);
+        TEST_EQUAL_DOUBLE(eset[1].get_weight(), 6.08904001099445);
+        TEST_NOT_EQUAL_DOUBLE(eset[2].get_weight(), 4.73383620844021);
+        TEST_EQUAL(eset[49].get_weight() < 0, false);
+    }
+
+    return true;
+}
+
+
+// tests that negative weights are returned
+DEFINE_TESTCASE(expandweights4, backend) {
+    Xapian::Enquire enquire(get_database("apitest_simpledata"));
+    enquire.set_query(Xapian::Query("paragraph"));
+
+    Xapian::MSet mymset = enquire.get_mset(0, 20);
+
+    Xapian::RSet myrset;
+    Xapian::MSetIterator i = mymset.begin();
+    myrset.add_document(*i);
+    myrset.add_document(*(++i));
+
+    Xapian::ESet eset = enquire.get_eset(37, myrset, 0, 1.0, 0, -100);
+    // Now include negative weights
+    // Multi backends need bigger sets
+    if (!startswith(get_dbtype(), "multi")) {
+        TEST_EQUAL(eset.size(), 37);
+        TEST_EQUAL(eset[36].get_weight() < 0, true);
+    } else {
+        TEST_EQUAL(eset.size(), 37);
+        TEST_EQUAL(eset[36].get_weight() < 0, true);
+    }
+    return true;
+}
+
+
 // tests that when specifying maxitems to get_eset, no more than
 // that are returned.
 DEFINE_TESTCASE(expandmaxitems1, backend) {