Changeset 10716

Timestamp:

2008-06-23 00:47:29 (7 months ago)

Author:

richard

Message:

backends/chert/: Change ChertAllDocsPostList? to inherit from
ChertPostList? and use the doclen list from the posting table
rather than using the termlist table. This helps towards making
the termlist table optional, reduces the amount of data read in
the process of iterating through an alldocs postlist, and can
make a massive difference in performance: I've measured the time
to iterate through all the documents in a 1000000 document
database, and the patch speeds this operation up by a factor of
6.
Add keep_reference parameter to the ChertPostList? constructor,
instead of not keeping a reference if the term is empty.
ChertAllDocsPostList? uses this to keep a reference to the
database while using an empty term.
Add ChertAllDocsModifiedPostList? class, inspired by
ChertModifiedPostList?, (with corresponding new source files) to
handle alldocs postlist with modifications: this wasn't needed
before since the termlist is updated immediately after changes.

Location:

trunk/xapian-core

Files:

• ChangeLog (modified) (1 diff)
• backends/chert/Makefile.mk (modified) (2 diffs)
• backends/chert/chert_alldocsmodifiedpostlist.cc (added)
• backends/chert/chert_alldocsmodifiedpostlist.h (added)
• backends/chert/chert_alldocspostlist.cc (modified) (6 diffs)
• backends/chert/chert_alldocspostlist.h (modified) (4 diffs)
• backends/chert/chert_database.cc (modified) (4 diffs)
• backends/chert/chert_modifiedpostlist.h (modified) (1 diff)
• backends/chert/chert_postlist.cc (modified) (5 diffs)
• backends/chert/chert_postlist.h (modified) (1 diff)

trunk/xapian-core/ChangeLog

@@ +1 @@
+Sun Jun 22 23:44:46 GMT 2008  Richard Boulton <richard@lemurconsulting.com>
+
+        * backends/chert/: Change ChertAllDocsPostList to inherit from
+          ChertPostList and use the doclen list from the posting table
+          rather than using the termlist table.  This helps towards making
+          the termlist table optional, reduces the amount of data read in
+          the process of iterating through an alldocs postlist, and can
+          make a massive difference in performance: I've measured the time
+          to iterate through all the documents in a 1000000 document
+          database, and the patch speeds this operation up by a factor of
+          6.
+
+          Add `keep_reference` parameter to the ChertPostList constructor,
+          instead of not keeping a reference if the term is empty.
+          ChertAllDocsPostList uses this to keep a reference to the
+          database while using an empty term.
+
+          Add ChertAllDocsModifiedPostList class, inspired by
+          ChertModifiedPostList, (with corresponding new source files) to
+          handle alldocs postlist with modifications: this wasn't needed
+          before since the termlist is updated immediately after changes.
+
 Sun Jun 22 21:37:00 GMT 2008  Richard Boulton <richard@lemurconsulting.com>
 

trunk/xapian-core/backends/chert/Makefile.mk

@@ -5 +5 @@
 if BUILD_BACKEND_CHERT
 noinst_HEADERS +=\
+        backends/chert/chert_alldocsmodifiedpostlist.h\
         backends/chert/chert_alldocspostlist.h\
         backends/chert/chert_alltermslist.h\
@@ -31 +32 @@
 
 libxapian_la_SOURCES +=\
+        backends/chert/chert_alldocsmodifiedpostlist.cc\
         backends/chert/chert_alldocspostlist.cc\
         backends/chert/chert_alltermslist.cc\

trunk/xapian-core/backends/chert/chert_alldocspostlist.cc

@@ -3 +3 @@
  */
 /* Copyright (C) 2006,2007,2008 Olly Betts
+ * Copyright (C) 2008 Lemur Consulting Ltd
  *
  * This program is free software; you can redistribute it and/or modify
@@ -23 +24 @@
 #include <string>
 
+#include "chert_alldocspostlist.h"
 #include "chert_database.h"
-#include "chert_alldocspostlist.h"
 
 #include "utils.h"
@@ -30 +31 @@
 using namespace std;
 
+ChertAllDocsPostList::ChertAllDocsPostList(Xapian::Internal::RefCntPtr<const ChertDatabase> db_,
+                                           Xapian::doccount doccount_)
+        : ChertPostList(db_, string(), true),
+          doccount(doccount_)
+{
+    DEBUGCALL(DB, void, "ChertAllDocsPostList::ChertAllDocsPostList", db_.get() << ", " << doccount_);
+}
+
 Xapian::doccount
 ChertAllDocsPostList::get_termfreq() const
 {
-    return doccount;
-}
-
-Xapian::docid
-ChertAllDocsPostList::get_docid() const
-{
-    return current_did;
+    DEBUGCALL(DB, Xapian::doccount, "ChertAllDocsPostList::get_termfreq", "");
+    RETURN(doccount);
 }
 
@@ -46 +50 @@
 {
     DEBUGCALL(DB, Xapian::doclength, "ChertAllDocsPostList::get_doclength", "");
-    Assert(current_did);
 
-    cursor->read_tag();
-
-    if (cursor->current_tag.empty()) RETURN(0);
-
-    const char * pos = cursor->current_tag.data();
-    const char * end = pos + cursor->current_tag.size();
-
-    chert_doclen_t doclen;
-    if (!unpack_uint(&pos, end, &doclen)) {
-        const char *msg;
-        if (pos == 0) {
-            msg = "Too little data for doclen in termlist";
-        } else {
-            msg = "Overflowed value for doclen in termlist";
-        }
-        throw Xapian::DatabaseCorruptError(msg);
-    }
-
-    RETURN(doclen);
+    RETURN(ChertPostList::get_wdf());
 }
 
@@ -73 +58 @@
 {
     DEBUGCALL(DB, Xapian::termcount, "ChertAllDocsPostList::get_wdf", "");
-    Assert(current_did);
+    AssertParanoid(!at_end());
     RETURN(1);
 }
 
-PostList *
-ChertAllDocsPostList::read_did_from_current_key()
+PositionList *
+ChertAllDocsPostList::read_position_list()
 {
-    DEBUGCALL(DB, PostList *, "ChertAllDocsPostList::read_did_from_current_key",
-              "");
-    const string & key = cursor->current_key;
-    const char * pos = key.data();
-    const char * end = pos + key.size();
-    if (!unpack_uint_preserving_sort(&pos, end, &current_did)) {
-        const char *msg;
-        if (pos == 0) {
-            msg = "Too little data in termlist key";
-        } else {
-            msg = "Overflowed value in termlist key";
-        }
-        throw Xapian::DatabaseCorruptError(msg);
-    }
-
-    // Return NULL to help the compiler tail-call optimise our callers.
-    RETURN(NULL);
+    DEBUGCALL(DB, Xapian::termcount, "ChertAllDocsPostList::read_position_list", "");
+    throw Xapian::InvalidOperationError("ChertAllDocsPostList::read_position_list() not meaningful");
 }
 
-PostList *
-ChertAllDocsPostList::next(Xapian::weight /*w_min*/)
+PositionList *
+ChertAllDocsPostList::open_position_list() const
 {
-    DEBUGCALL(DB, PostList *, "ChertAllDocsPostList::next", "/*w_min*/");
-    Assert(!at_end());
-    if (!cursor->next()) RETURN(NULL);
-    RETURN(read_did_from_current_key());
-}
-
-PostList *
-ChertAllDocsPostList::skip_to(Xapian::docid did, Xapian::weight /*w_min*/)
-{
-    DEBUGCALL(DB, PostList *, "ChertAllDocsPostList::skip_to",
-              did << ", /*w_min*/");
-
-    if (did <= current_did || at_end()) RETURN(NULL);
-
-    if (cursor->find_entry_ge(pack_uint_preserving_sort(did))) {
-        // The exact docid that was asked for exists.
-        current_did = did;
-        RETURN(NULL);
-    }
-    if (cursor->after_end()) RETURN(NULL);
-
-    RETURN(read_did_from_current_key());
-}
-
-bool
-ChertAllDocsPostList::at_end() const {
-    DEBUGCALL(DB, bool, "ChertAllDocsPostList::at_end", "");
-    RETURN(cursor->after_end());
+    DEBUGCALL(DB, Xapian::termcount, "ChertAllDocsPostList::open_position_list", "");
+    throw Xapian::InvalidOperationError("ChertAllDocsPostList::open_position_list() not meaningful");
 }
 
@@ -136 +80 @@
 {
     string desc = "ChertAllDocsPostList(did=";
-    desc += om_tostring(current_did);
+    desc += om_tostring(get_docid());
     desc += ",doccount=";
     desc += om_tostring(doccount);

trunk/xapian-core/backends/chert/chert_alldocspostlist.h

@@ -3 +3 @@
  */
 /* Copyright (C) 2006,2007,2008 Olly Betts
+ * Copyright (C) 2008 Lemur Consulting Ltd
  *
  * This program is free software; you can redistribute it and/or modify
@@ -25 +26 @@
 
 #include "leafpostlist.h"
+#include "chert_postlist.h"
 
-class ChertAllDocsPostList : public LeafPostList {
+class ChertAllDocsPostList : public ChertPostList {
     /// Don't allow assignment.
     void operator=(const ChertAllDocsPostList &);
@@ -33 +35 @@
     ChertAllDocsPostList(const ChertAllDocsPostList &);
 
-    /// Set @a current_did from @a cursor->current_key.
-    PostList * read_did_from_current_key();
-
-    /// The database we're iterating over.
-    Xapian::Internal::RefCntPtr<const ChertDatabase> db;
-
     /// The number of documents in the database.
     Xapian::doccount doccount;
 
-    /// Cursor running over termlist table keys.
-    AutoPtr<ChertCursor> cursor;
-
-    /// The current document id.
-    Xapian::docid current_did;
-
   public:
     ChertAllDocsPostList(Xapian::Internal::RefCntPtr<const ChertDatabase> db_,
-                         Xapian::doccount doccount_)
-      : db(db_), doccount(doccount_), cursor(db->termlist_table.cursor_get()),
-        current_did(0)
-    {
-        cursor->find_entry("");
-    }
+                         Xapian::doccount doccount_);
 
     Xapian::doccount get_termfreq() const;
-
-    Xapian::docid get_docid() const;
 
     Xapian::doclength get_doclength() const;
@@ -65 +48 @@
     Xapian::termcount get_wdf() const;
 
-    PostList * next(Xapian::weight w_min);
+    PositionList *read_position_list();
 
-    PostList * skip_to(Xapian::docid desired_did, Xapian::weight w_min);
-
-    bool at_end() const;
+    PositionList *open_position_list() const;
 
     std::string get_description() const;

trunk/xapian-core/backends/chert/chert_database.cc

@@ -32 +32 @@
 
 #include "contiguousalldocspostlist.h"
+#include "chert_alldocsmodifiedpostlist.h"
 #include "chert_alldocspostlist.h"
 #include "chert_alltermslist.h"
@@ -915 +916 @@
     }
 
-    RETURN(new ChertPostList(ptrtothis, term));
+    RETURN(new ChertPostList(ptrtothis, term, true));
 }
 
@@ -1804 +1805 @@
             RETURN(new ContiguousAllDocsPostList(ptrtothis, doccount));
         }
-        RETURN(new ChertAllDocsPostList(ptrtothis, doccount));
+        if (doclens.empty()) {
+            RETURN(new ChertAllDocsPostList(ptrtothis, doccount));
+        } else {
+            RETURN(new ChertAllDocsModifiedPostList(ptrtothis, doccount, doclens));
+        }
     }
 
@@ -1815 +1820 @@
     }
 
-    RETURN(new ChertPostList(ptrtothis, tname));
+    RETURN(new ChertPostList(ptrtothis, tname, true));
 }
 

trunk/xapian-core/backends/chert/chert_modifiedpostlist.h

@@ -47 +47 @@
                           const string & tname_,
                           const map<Xapian::docid, pair<char, Xapian::termcount> > & mods_)
-        : ChertPostList(this_db_, tname_),
+        : ChertPostList(this_db_, tname_, true),
           mods(mods_), it(mods.begin()), poslist(0)
     { }

trunk/xapian-core/backends/chert/chert_postlist.cc

@@ -3 +3 @@
  * Copyright 1999,2000,2001 BrightStation PLC
  * Copyright 2002,2003,2004,2005,2007,2008 Olly Betts
- * Copyright 2007 Lemur Consulting Ltd
+ * Copyright 2007,2008 Lemur Consulting Ltd
  *
  * This program is free software; you can redistribute it and/or
@@ -62 +62 @@
                                   Xapian::Internal::RefCntPtr<const ChertDatabase> db) const {
     if (!doclen_pl.get()) {
-        doclen_pl.reset(new ChertPostList(db, string()));
+        // Don't keep a reference back to the database, since this
+        // would make a reference loop.
+        doclen_pl.reset(new ChertPostList(db, string(), false));
     }
     if (!doclen_pl->jump_to(did))
@@ -656 +658 @@
  */
 ChertPostList::ChertPostList(Xapian::Internal::RefCntPtr<const ChertDatabase> this_db_,
-                             const string & tname_)
-        : this_db(tname_.empty() ? NULL : this_db_), // Don't keep a reference if this is a "doclen postlist".
+                             const string & tname_,
+                             bool keep_reference)
+        : this_db(keep_reference ? this_db_ : NULL),
           tname(tname_),
           have_started(false),
@@ -664 +667 @@
 {
     DEBUGCALL(DB, void, "ChertPostList::ChertPostList",
-              this_db_.get() << ", " << tname_);
+              this_db_.get() << ", " << tname_ << ", " << keep_reference);
     string key = ChertPostListTable::make_key(tname);
     int found = cursor->find_entry(key);
     if (!found) {
+        DEBUGLINE(DB, "postlist for term not found");
         number_of_entries = 0;
         is_at_end = true;
@@ -685 +689 @@
                                             &is_last_chunk);
     read_wdf(&pos, end, &wdf);
+    DEBUGLINE(DB, "Initial docid " << did);
 }
 

trunk/xapian-core/backends/chert/chert_postlist.h

@@ -224 +224 @@
         /// Default constructor.
         ChertPostList(Xapian::Internal::RefCntPtr<const ChertDatabase> this_db_,
-                      const string & tname);
+                      const string & tname,
+                      bool keep_reference);
 
         /// Destructor.