Ticket #326: small_doclen_chunks.patch

chert_postlist.h

@@ -81 +81 @@
 
         Xapian::docid get_chunk(const string &tname,
                 Xapian::docid did, bool adding,
-                PostlistChunkReader ** from, PostlistChunkWriter **to);
+                PostlistChunkReader ** from, PostlistChunkWriter **to,
+                unsigned int chunksize);
 
         /// Compose a key from a termname and docid.
         static string make_key(const string & term, Xapian::docid did) {

chert_postlist.cc

@@ -76 +76 @@
 // maximise how well blocks are used.  Or performance.
 // Or indexing speed.  Or something...
 const unsigned int CHUNKSIZE = 2000;
+const unsigned int DOCLEN_CHUNKSIZE = 70;
 
 /** PostlistChunkWriter is a wrapper which acts roughly as an
  *  output iterator on a postlist chunk, taking care of the
@@ -88 +89 @@
         PostlistChunkWriter(const string &orig_key_,
                             bool is_first_chunk_,
                             const string &tname_,
-                            bool is_last_chunk_);
+                            bool is_last_chunk_,
+                            unsigned int chunksize_);
 
         /// Append an entry to this chunk.
         void append(ChertTable * table, Xapian::docid did,
@@ -118 +120 @@
         bool is_first_chunk;
         bool is_last_chunk;
         bool started;
+        unsigned int chunksize;
 
         Xapian::docid first_did;
         Xapian::docid current_did;
@@ -302 +305 @@
 PostlistChunkWriter::PostlistChunkWriter(const string &orig_key_,
                                          bool is_first_chunk_,
                                          const string &tname_,
-                                         bool is_last_chunk_)
+                                         bool is_last_chunk_,
+                                         unsigned int chunksize_)
         : orig_key(orig_key_),
           tname(tname_), is_first_chunk(is_first_chunk_),
           is_last_chunk(is_last_chunk_),
-          started(false)
+          started(false),
+          chunksize(chunksize_)
 {
     DEBUGCALL(DB, void, "PostlistChunkWriter::PostlistChunkWriter",
               orig_key_ << ", " << is_first_chunk_ << ", " << tname_ << ", " <<
-              is_last_chunk_);
+              is_last_chunk_ << ", " << chunksize_);
 }
 
 void
@@ -323 +328 @@
     } else {
         Assert(did > current_did);
         // Start a new chunk if this one has grown to the threshold.
-        if (chunk.size() >= CHUNKSIZE) {
+        if (chunk.size() >= chunksize) {
             bool save_is_last_chunk = is_last_chunk;
             is_last_chunk = false;
             flush(table);
@@ -961 +966 @@
 Xapian::docid
 ChertPostListTable::get_chunk(const string &tname,
           Xapian::docid did, bool adding,
-          PostlistChunkReader ** from, PostlistChunkWriter **to)
+          PostlistChunkReader ** from, PostlistChunkWriter **to,
+          unsigned int chunksize)
 {
-    DEBUGCALL(DB, Xapian::docid, "ChertPostListTable::get_chunk", tname << ", " << did << ", " << adding << ", [from], [to]");
+    DEBUGCALL(DB, Xapian::docid, "ChertPostListTable::get_chunk", tname << ", " << did << ", " << adding << ", [from], [to], " << chunksize);
     // Get chunk containing entry
     string key = make_key(tname, did);
 
@@ -982 +988 @@
             throw Xapian::DatabaseCorruptError("Attempted to delete or modify an entry in a non-existent posting list for " + tname);
 
         *from = NULL;
-        *to = new PostlistChunkWriter(string(), true, tname, true);
+        *to = new PostlistChunkWriter(string(), true, tname, true, chunksize);
         RETURN(Xapian::docid(-1));
     }
 
@@ -1007 +1013 @@
     Xapian::docid last_did_in_chunk;
     last_did_in_chunk = read_start_of_chunk(&pos, end, first_did_in_chunk, &is_last_chunk);
     *to = new PostlistChunkWriter(cursor->current_key, is_first_chunk, tname,
-                                  is_last_chunk);
+                                  is_last_chunk, chunksize);
     if (did > last_did_in_chunk) {
         // This is the shortcut.  Not very pretty, but I'll leave refactoring
         // until I've a clearer picture of everything which needs to be done.
@@ -1065 +1071 @@
         Xapian::docid max_did;
         PostlistChunkReader *from;
         PostlistChunkWriter *to;
-        max_did = get_chunk(string(), j->first, true, &from, &to);
+        max_did = get_chunk(string(), j->first, true, &from, &to, DOCLEN_CHUNKSIZE);
         LOGVALUE(DB, max_did);
         for ( ; j != doclens.end(); ++j) {
             Xapian::docid did = j->first;
@@ -1085 +1091 @@
                 delete from;
                 to->flush(this);
                 delete to;
-                max_did = get_chunk(string(), did, false, &from, &to);
+                max_did = get_chunk(string(), did, false, &from, &to, DOCLEN_CHUNKSIZE);
                 goto next_doclen_chunk;
             }
 
@@ -1180 +1186 @@
         PostlistChunkReader *from;
         PostlistChunkWriter *to;
         max_did = get_chunk(tname, j->first, j->second.first == 'A',
-                            &from, &to);
+                            &from, &to, CHUNKSIZE);
         for ( ; j != i->second.end(); ++j) {
             Xapian::docid did = j->first;
 
@@ -1202 +1208 @@
                 delete from;
                 to->flush(this);
                 delete to;
-                max_did = get_chunk(tname, did, false, &from, &to);
+                max_did = get_chunk(tname, did, false, &from, &to, CHUNKSIZE);
                 goto next_chunk;
             }