Ticket #153: xapian-allprefixedterms-wildcard.patch

xapian-core/queryparser/queryparser.lemony

@@ -295 +295 @@
 Term::as_wildcarded_query(State * state_) const
 {
     Database db = state_->get_database();
-    TermIterator t = db.allterms_begin();
     string root = prefix;
     root += name;
-    t.skip_to(root);
+    TermIterator t = db.allterms_begin(root);
     QpQuery q;
-    while (t != db.allterms_end() && (*t).substr(0, root.size()) == root) {
+    while (t != db.allterms_end(root)) {
         add_to_query(q, Query::OP_OR, QpQuery(*t, 1, pos));
         ++t;
     }
@@ -316 +315 @@
 Term::as_partial_query(State * state_) const
 {
     Database db = state_->get_database();
-    TermIterator t = db.allterms_begin();
     string root = prefix;
     root += name;
-    t.skip_to(root);
+    TermIterator t = db.allterms_begin(root);
     QpQuery q;
-    while (t != db.allterms_end() && (*t).substr(0, root.size()) == root) {
+    while (t != db.allterms_end(root)) {
         add_to_query(q, Query::OP_OR, QpQuery(*t, 1, pos));
         ++t;
     }

xapian-core/tests/api_db.cc

@@ -707 +707 @@
     return true;
 }
 
+// test allterms iterators with prefixes
+static bool test_allterms6()
+{
+    Xapian::Database db;
+    db.add_database(get_database("apitest_allterms"));
+    db.add_database(get_database("apitest_allterms2"));
+
+    Xapian::TermIterator ati = db.allterms_begin("three");
+    TEST(ati != db.allterms_end("three"));
+    TEST_EQUAL(*ati, "three");
+    ati.skip_to("three");
+    TEST(ati != db.allterms_end("three"));
+    TEST_EQUAL(*ati, "three");
+    ati++;
+    TEST(ati == db.allterms_end("three"));
+
+    ati = db.allterms_begin("thre");
+    TEST(ati != db.allterms_end("thre"));
+    TEST_EQUAL(*ati, "three");
+    ati.skip_to("three");
+    TEST(ati != db.allterms_end("thre"));
+    TEST_EQUAL(*ati, "three");
+    ati++;
+    TEST(ati == db.allterms_end("thre"));
+
+    ati = db.allterms_begin("f");
+    TEST(ati != db.allterms_end("f"));
+    TEST_EQUAL(*ati, "five");
+    TEST(ati != db.allterms_end("f"));
+    ati.skip_to("three");
+    TEST(ati == db.allterms_end("f"));
+
+    ati = db.allterms_begin("f");
+    TEST(ati != db.allterms_end("f"));
+    TEST_EQUAL(*ati, "five");
+    ati++;
+    TEST(ati != db.allterms_end("f"));
+    TEST_EQUAL(*ati, "four");
+    ati++;
+    TEST(ati == db.allterms_end("f"));
+
+    return true;
+}
+
 // test that searching for a term with a special characters in it works
 static bool test_specialterms1()
 {
@@ -1632 +1676 @@
     {"allterms3",          test_allterms3},
     {"allterms4",          test_allterms4},
     {"allterms5",          test_allterms5},
+    {"allterms6",          test_allterms6},
     {"specialterms2",      test_specialterms2},
     {0, 0}
 };

xapian-core/include/xapian/database.h

@@ -163 +163 @@
             return TermIterator(NULL);
         }
 
+        /** An iterator which runs across all terms with a given prefix.
+         *
+         *  This is functionally similar to getting an iterator with
+         *  allterms_begin() and then calling skip_to(prefix) on that iterator
+         *  to move to the start of the prefix, but is more convenient (because
+         *  it detects the end of the prefixed terms), and may be more
+         *  efficient than simply calling skip_to() after opening the iterator,
+         *  particularly for network databases.
+         *
+         *  @param prefix The prefix to restrict the returned terms to.
+         */
+        TermIterator allterms_begin(const std::string & prefix) const;
+
+        /** Corresponding end iterator to allterms_begin(prefix).
+         */
+        TermIterator allterms_end(const std::string &) const {
+            return TermIterator(NULL);
+        }
+
         /// Get the number of documents in the database.
         Xapian::doccount get_doccount() const;
 

xapian-core/net/remoteserver.cc

@@ -237 +237 @@
 }
 
 void
-RemoteServer::msg_allterms(const string &)
+RemoteServer::msg_allterms(const string &message)
 {
-    const Xapian::TermIterator end = db->allterms_end();
-    for (Xapian::TermIterator t = db->allterms_begin(); t != end; ++t) {
+    const char *p = message.data();
+    const char *p_end = p + message.size();
+    string prefix(p, p_end - p);
+
+    const Xapian::TermIterator end = db->allterms_end(prefix);
+    for (Xapian::TermIterator t = db->allterms_begin(prefix); t != end; ++t) {
         string item = encode_length(t.get_termfreq());
         item += *t;
         send_message(REPLY_ALLTERMS, item);

xapian-core/common/remote-database.h

@@ -150 +150 @@
     TermList * open_term_list(Xapian::docid did) const;
 
     /// Iterate all terms.
-    TermList * open_allterms() const;
+    TermList * open_allterms(const string & prefix) const;
 
     bool has_positions() const;
 

xapian-core/common/remoteprotocol.h

@@ -32 +32 @@
 // 27: Support for postlists (always passes the whole list across)
 // 28: Pass document length in reply to MSG_TERMLIST
 // 29: Serialisation of Xapian::Error includes error_string.
-#define XAPIAN_REMOTE_PROTOCOL_VERSION 29
+// 30: Pass the prefix parameter for MSG_ALLTERMS, and use it.
+#define XAPIAN_REMOTE_PROTOCOL_VERSION 30
 
 /// Message types (client -> server).
 enum message_type {

xapian-core/common/database.h

@@ -197 +197 @@
          *
          *  This is a list of all the terms in the database
          *
+         *  @param prefix The prefix to restrict the terms to.
          *  @return       A pointer to the newly created allterms list.
          *                This object must be deleted by the caller after
          *                use.
          */
-        virtual TermList * open_allterms() const = 0;
+        virtual TermList * open_allterms(const string & prefix) const = 0;
 
         /** Open a position list for the given term in the given document.
          *

xapian-core/api/omdatabase.cc

@@ -151 +151 @@
 TermIterator
 Database::allterms_begin() const
 {
+    return allterms_begin("");
+}
+
+TermIterator
+Database::allterms_begin(const std::string & prefix) const
+{
     DEBUGAPICALL(TermIterator, "Database::allterms_begin", "");
     if (internal.empty()) RETURN(TermIterator(NULL));
 
     if (internal.size() == 1)
-        RETURN(TermIterator(internal[0]->open_allterms()));
+        RETURN(TermIterator(internal[0]->open_allterms(prefix)));
  
     vector<TermList *> lists;
 
     vector<Xapian::Internal::RefCntPtr<Database::Internal> >::const_iterator i;
     for (i = internal.begin(); i != internal.end(); ++i) {
-        lists.push_back((*i)->open_allterms());
+        lists.push_back((*i)->open_allterms(prefix));
     }
 
     RETURN(TermIterator(new MultiAllTermsList(lists)));

xapian-core/backends/inmemory/inmemory_alltermslist.cc

@@ -25 +25 @@
 #include "inmemory_alltermslist.h"
 
 InMemoryAllTermsList::InMemoryAllTermsList(const std::map<string, InMemoryTerm> *tmap_,
-                                           Xapian::Internal::RefCntPtr<const InMemoryDatabase> database_)
-        : tmap(tmap_), it(tmap->begin()), database(database_), started(false)
+                                           Xapian::Internal::RefCntPtr<const InMemoryDatabase> database_,
+                                           const string & prefix_)
+        : tmap(tmap_), it(tmap->begin()), database(database_), started(false), prefix(prefix_)
 {
-    while (it != tmap->end() && it->second.term_freq == 0) ++it;
+    while (it != tmap->end() &&
+           (it->second.term_freq == 0 ||
+            it->first.substr(0, prefix.size()) != prefix))
+        ++it;
 }
 
 InMemoryAllTermsList::~InMemoryAllTermsList()
@@ -73 +77 @@
     // FIXME: might skip backwards - is this a problem?
     it = tmap->lower_bound(tname);
     while (it != tmap->end() && it->second.term_freq == 0) ++it;
+    if (it != tmap->end() && it->first.substr(0, prefix.size()) != prefix)
+        it = tmap->end();
     return NULL;
 }
 
@@ -86 +92 @@
         ++it;
         while (it != tmap->end() && it->second.term_freq == 0) ++it;
     }
+    if (it != tmap->end() && it->first.substr(0, prefix.size()) != prefix)
+        it = tmap->end();
     return NULL;
 }
 

xapian-core/backends/inmemory/inmemory_alltermslist.h

@@ -44 +44 @@
         Xapian::Internal::RefCntPtr<const InMemoryDatabase> database;
 
         bool started;
+
+        string prefix;
     public:
         /// Standard constructor for base class.
         InMemoryAllTermsList(const std::map<string, InMemoryTerm> *tmap_,
-                             Xapian::Internal::RefCntPtr<const InMemoryDatabase> database_);
+                             Xapian::Internal::RefCntPtr<const InMemoryDatabase> database_,
+                             const string & prefix);
 
         /// Standard destructor for base class.
         ~InMemoryAllTermsList();

xapian-core/backends/inmemory/inmemory_database.cc

@@ -707 +707 @@
 }
 
 TermList *
-InMemoryDatabase::open_allterms() const
+InMemoryDatabase::open_allterms(const string & prefix) const
 {
     return new InMemoryAllTermsList(&postlists,
-                                    Xapian::Internal::RefCntPtr<const InMemoryDatabase>(this));
+                                    Xapian::Internal::RefCntPtr<const InMemoryDatabase>(this),
+                                    prefix);
 }

xapian-core/backends/inmemory/inmemory_database.h

@@ -306 +306 @@
         Xapian::Document::Internal * open_document(Xapian::docid did, bool lazy = false) const;
         PositionList * open_position_list(Xapian::docid did,
                                           const string & tname) const;
-        TermList * open_allterms() const;
+        TermList * open_allterms(const string & prefix) const;
 };
 
 #endif /* OM_HGUARD_INMEMORY_DATABASE_H */

xapian-core/backends/quartz/quartz_database.h

@@ -220 +220 @@
         Xapian::Document::Internal * open_document(Xapian::docid did, bool lazy = false) const;
         PositionList * open_position_list(Xapian::docid did,
                                           const string & tname) const;
-        TermList * open_allterms() const;
+        TermList * open_allterms(const string & prefix) const;
         //@}
 };
 
@@ -317 +317 @@
         Xapian::Document::Internal * open_document(Xapian::docid did, bool lazy = false) const;
         PositionList * open_position_list(Xapian::docid did,
                                           const string & tname) const;
-        TermList * open_allterms() const;
+        TermList * open_allterms(const string & prefix) const;
         //@}
 };
 

xapian-core/backends/quartz/quartz_alltermslist.cc

@@ -29 +29 @@
 
 QuartzAllTermsList::QuartzAllTermsList(Xapian::Internal::RefCntPtr<const Xapian::Database::Internal> database_,
                                        AutoPtr<Bcursor> pl_cursor_,
-                                       quartz_tablesize_t size_)
+                                       quartz_tablesize_t size_,
+                                       const string & prefix_)
         : database(database_), pl_cursor(pl_cursor_), size(size_), 
-          started(false)
+          started(false), prefix(prefix_)
 {
     DEBUGCALL(DB, void, "QuartzAllTermsList", "[database_], [pl_cursor_]");
-    /* Seek to the first term */
-    pl_cursor->find_entry(string());
+    if (prefix.empty()) {
+        /* Seek to the first term */
+        pl_cursor->find_entry(string());
 
-    if (pl_cursor->current_key.empty()) {
-        pl_cursor->next();
+        if (pl_cursor->current_key.empty()) {
+            pl_cursor->next();
+        }
+    } else {
+        // Seek to the first key before one with the desired prefix.
+        if (!pl_cursor->find_entry(pack_string_preserving_sort(prefix))) {
+            // Found a a key which is before the prefix - advance to the
+            // first one following the prefix.
+            pl_cursor->next();
+        }
     }
 
     is_at_end = pl_cursor->after_end();
@@ -50 +60 @@
         }
     }
 
+    if (current_term.substr(0, prefix.size()) != prefix)
+        is_at_end = true;
+
     have_stats = false;
 }
 
@@ -128 +141 @@
             is_at_end = true;
         } else {
             next();
+            // next() checks the prefix, so we don't need to do that here.
         }
     } else {
         // This assertion isn't true if key contains zero bytes.
         // Assert(key == pl_cursor->current_key);
         current_term = tname;
+
+        // Check that we haven't gone past the prefix.
+        if (current_term.substr(0, prefix.size()) != prefix)
+            is_at_end = true;
     }
     RETURN(NULL);
 }
@@ -156 +174 @@
             if (!unpack_string_preserving_sort(&start, end, current_term)) {
                 throw Xapian::DatabaseCorruptError("Failed to read the key field from a Bcursor's key");
             }
+            if (current_term.substr(0, prefix.size()) != prefix) {
+                is_at_end = true;
+                break;
+            }
             // Check if this is the first chunk of a postlist, skip otherwise
             if (start == end) break;
         }

xapian-core/backends/quartz/quartz_alltermslist.h

@@ -55 +55 @@
         /// Cached termname
         string current_term;
 
+        /// The prefix to restrict the terms to.
+        string prefix;
+
         /// Cached statistics
         mutable bool have_stats;
         mutable Xapian::termcount termfreq;
@@ -65 +68 @@
         /// Standard constructor for base class.
         QuartzAllTermsList(Xapian::Internal::RefCntPtr<const Xapian::Database::Internal> database_,
                            AutoPtr<Bcursor> pl_cursor_,
-                           quartz_tablesize_t size_);
+                           quartz_tablesize_t size_,
+                           const string & prefix_);
 
         /// Standard destructor for base class.
         ~QuartzAllTermsList();

xapian-core/backends/quartz/quartz_database.cc

@@ -663 +663 @@
 }
 
 TermList *
-QuartzDatabase::open_allterms() const
+QuartzDatabase::open_allterms(const string & prefix) const
 {
     DEBUGCALL(DB, TermList *, "QuartzDatabase::open_allterms", "");
     AutoPtr<Bcursor> pl_cursor(postlist_table.cursor_get());
     RETURN(new QuartzAllTermsList(Xapian::Internal::RefCntPtr<const QuartzDatabase>(this),
-                                  pl_cursor, postlist_table.get_entry_count()));
+                                  pl_cursor, postlist_table.get_entry_count(), prefix));
 }
 
 size_t QuartzWritableDatabase::flush_threshold = 0;
@@ -1195 +1195 @@
 }
 
 TermList *
-QuartzWritableDatabase::open_allterms() const
+QuartzWritableDatabase::open_allterms(const string & prefix) const
 {
     DEBUGCALL(DB, TermList *, "QuartzWritableDatabase::open_allterms", "");
     if (transaction_active())
@@ -1205 +1205 @@
     QuartzPostListTable *t = &database_ro.postlist_table;
     AutoPtr<Bcursor> pl_cursor(t->cursor_get());
     RETURN(new QuartzAllTermsList(Xapian::Internal::RefCntPtr<const QuartzWritableDatabase>(this),
-                                  pl_cursor, t->get_entry_count()));
+                                  pl_cursor, t->get_entry_count(), prefix));
 }
 
 void

xapian-core/backends/remote/remote-database.cc

@@ -152 +152 @@
 }
 
 TermList *
-RemoteDatabase::open_allterms() const {
+RemoteDatabase::open_allterms(const string & prefix) const {
     // Ensure that avlength and doccount are up-to-date.
     if (!cached_stats_valid) update_stats();
 
-    send_message(MSG_ALLTERMS, "");
+    send_message(MSG_ALLTERMS, prefix);
 
     AutoPtr<NetworkTermList> tlist;
     tlist = new NetworkTermList(0.0, doccount,

xapian-core/backends/flint/flint_database.h

@@ -213 +213 @@
         Xapian::Document::Internal * open_document(Xapian::docid did, bool lazy = false) const;
         PositionList * open_position_list(Xapian::docid did,
                                           const string & tname) const;
-        TermList * open_allterms() const;
+        TermList * open_allterms(const string & prefix) const;
         //@}
 };
 
@@ -310 +310 @@
         Xapian::Document::Internal * open_document(Xapian::docid did, bool lazy = false) const;
         PositionList * open_position_list(Xapian::docid did,
                                           const string & tname) const;
-        TermList * open_allterms() const;
+        TermList * open_allterms(const string & prefix) const;
         //@}
 };
 

xapian-core/backends/flint/flint_alltermslist.cc

@@ -103 +103 @@
             throw Xapian::DatabaseCorruptError("PostList table key has unexpected format");
         }
 
+        if (current_term.substr(0, prefix.size()) != prefix) {
+            // We've reached the end of the end of the prefixed terms.
+            cursor->to_end();
+            current_term = "";
+            break;
+        }
+
         // If this key is for the first chunk of a postlist, we're done.  Otherwise we need
         // to skip past continuation chunks until we find the first chunk of the next postlist.
         if (p == pend) break;
@@ -114 +121 @@
 FlintAllTermsList::skip_to(const string &tname)
 {
     DEBUGCALL(DB, TermList *, "FlintAllTermsList::skip_to", tname);
+    if (at_end()) abort();
     Assert(!at_end());
     // Set termfreq to 0 to indicate no value has been read for the current term.
     termfreq = 0;

xapian-core/backends/flint/flint_cursor.h

@@ -199 +199 @@
          */
         bool find_entry(const string &key);
 
+        /** Set the cursor to be off the end of the table.
+         */
+        void to_end() { is_after_end = true; }
+
         /** Determine whether cursor is off the end of table.
          *
          *  @return true if the cursor has been moved off the end of the

xapian-core/backends/flint/flint_alltermslist.h

@@ -48 +48 @@
     /// The termname at the current position.
     string current_term;
 
+    /// The prefix to restrict the terms to.
+    string prefix;
+
     /** The term frequency of the term at the current position.
      *
      *  If this value is zero, then we haven't read the term frequency or
@@ -64 +67 @@
 
   public:
     FlintAllTermsList(Xapian::Internal::RefCntPtr<const Xapian::Database::Internal> database_,
-                      const FlintPostListTable * pltab) : database(database_), termfreq(0) {
+                      const FlintPostListTable * pltab,
+                      const string & prefix_)
+            : database(database_), prefix(prefix_), termfreq(0) {
         // The number of entries in the postlist table will be the number of
         // terms, probably plus some extra entries for chunked posting lists,
         // plus 1 for the metainfo key (unless the table is completely empty).
@@ -77 +82 @@
         approx_size = pltab->get_entry_count();
         if (approx_size) --approx_size;
 
-        // Seek to the metainfo key, so the first next will advance us to the
-        // first real key.
         cursor = pltab->cursor_get();
-        cursor->find_entry(string("", 1));
+        if (prefix.empty()) {
+            // Seek to the metainfo key, so the first next() will advance us to the
+            // first real key.
+            cursor->find_entry(string("", 1));
+        } else {
+            // Seek to the first key before one with the desired prefix.
+            if (cursor->find_entry(pack_string_preserving_sort(prefix))) {
+                // Found a key which is exactly the prefix - move back, so that
+                // next() moves to it.
+                cursor->prev();
+            }
+        }
     }
 
     /// Destructor.

xapian-core/backends/flint/flint_database.cc

@@ -516 +516 @@
 }
 
 TermList *
-FlintDatabase::open_allterms() const
+FlintDatabase::open_allterms(const string & prefix) const
 {
     DEBUGCALL(DB, TermList *, "FlintDatabase::open_allterms", "");
     RETURN(new FlintAllTermsList(Xapian::Internal::RefCntPtr<const FlintDatabase>(this),
-                                 &postlist_table));
+                                 &postlist_table, prefix));
 }
 
 size_t FlintWritableDatabase::flush_threshold = 0;
@@ -1053 +1053 @@
 }
 
 TermList *
-FlintWritableDatabase::open_allterms() const
+FlintWritableDatabase::open_allterms(const string & prefix) const
 {
     DEBUGCALL(DB, TermList *, "FlintWritableDatabase::open_allterms", "");
     if (transaction_active())
@@ -1062 +1062 @@
     // need to flush.
     if (changes_made) do_flush_const();
     RETURN(new FlintAllTermsList(Xapian::Internal::RefCntPtr<const FlintWritableDatabase>(this),
-                                 &database_ro.postlist_table));
+                                 &database_ro.postlist_table, prefix));
 }
 
 void

xapian-bindings/python/extra.i

@@ -536 +536 @@
 Query.__iter__ = _query_gen_iter
 
 # Modify Database to add an "__iter__()" method and an "allterms()" method.
-def _database_gen_allterms_iter(self):
+def _database_gen_allterms_iter(self, prefix=None):
     """Get an iterator over all the terms in the database.
 
     The iterator will return TermListItem objects, but these will not support
@@ -545 +545 @@
     Access to term frequency information is only available until the iterator
     has moved on.
 
+    If prefix is supplied, only terms which start with that prefix will be
+    returned.
+
     """
-    return TermIter(self.allterms_begin(), self.allterms_end(),
-                    has_termfreq=TermIter.LAZY)
+    if prefix is None:
+        return TermIter(self.allterms_begin(), self.allterms_end(),
+                        has_termfreq=TermIter.LAZY)
+    else:
+        return TermIter(self.allterms_begin(prefix), self.allterms_end(prefix),
+                        has_termfreq=TermIter.LAZY)
 Database.__iter__ = _database_gen_allterms_iter
 Database.allterms = _database_gen_allterms_iter
 

xapian-bindings/xapian.i

@@ -657 +657 @@
         PositionIterator positionlist_end(docid did, const std::string& tname) const;
         TermIterator allterms_begin() const;
         TermIterator allterms_end() const;
+        TermIterator allterms_begin(const std::string &prefix) const;
+        TermIterator allterms_end(const std::string &prefix) const;
 
         doccount get_doccount() const;
         docid get_lastdocid() const;