Ticket #326: chunktypes.patch

chert_postlist.h

@@ -122 +122 @@
 /** A postlist in a chert database.
  */
 class ChertPostList : public LeafPostList {
+    public:
+        /// Types of chunks.
+        typedef enum {
+            /// Items just consist of wdfs - all docids in range are present.
+            DENSE,
+
+            /// Items consist of docid increments and wdfs.
+            SPARSE
+        } chunk_type;
+
     protected: // ChertModifiedPostList needs to access these.
         /** The database we are searching.  This pointer is held so that the
          *  database doesn't get deleted before us, and also to give us access
@@ -157 +167 @@
         /// Pointer to byte after end of current chunk.
         const char * end;
 
+        /// The type of the current chunk.
+        chunk_type current_type;
+
         /// Document id we're currently at.
         Xapian::docid did;
 
@@ -179 +192 @@
          *  If already at the end of the chunk, returns false.
          */
         bool next_in_chunk();
+        bool next_in_chunk_dense();
+        bool next_in_chunk_sparse();
 
         /** Move to the next chunk.
          *

chert_postlist.cc

@@ -96 +96 @@
 
         /// Append a block of raw entries to this chunk.
         void raw_append(Xapian::docid first_did_, Xapian::docid current_did_,
-                        const string & s) {
+                        const string & s, ChertPostList::chunk_type type_) {
             Assert(!started);
             first_did = first_did_;
             current_did = current_did_;
             if (!s.empty()) {
                 chunk.append(s);
                 started = true;
+                current_type = type_;
             }
         }
 
@@ -121 +122 @@
 
         Xapian::docid first_did;
         Xapian::docid current_did;
+        ChertPostList::chunk_type current_type;
 
         string chunk;
+        void start_new_chunk(ChertTable * table,
+                             Xapian::docid new_first_did);
 };
 
 // Static functions
@@ -222 +226 @@
 read_start_of_chunk(const char ** posptr,
                     const char * end,
                     Xapian::docid first_did_in_chunk,
-                    bool * is_last_chunk_ptr)
+                    bool * is_last_chunk_ptr,
+                    ChertPostList::chunk_type * new_type)
 {
     DEBUGCALL_STATIC(DB, Xapian::docid, "read_start_of_chunk",
                      reinterpret_cast<const void*>(posptr) << ", " <<
                      reinterpret_cast<const void*>(end) << ", " <<
                      first_did_in_chunk << ", " <<
-                     reinterpret_cast<const void*>(is_last_chunk_ptr));
+                     reinterpret_cast<const void*>(is_last_chunk_ptr) <<
+                     reinterpret_cast<const void*>(new_type));
 
     // Read whether this is the last chunk
     if (!unpack_bool(posptr, end, is_last_chunk_ptr))
@@ -241 +247 @@
     if (!unpack_uint(posptr, end, &increase_to_last))
         report_read_error(*posptr);
     Xapian::docid last_did_in_chunk = first_did_in_chunk + increase_to_last;
+
+    // Read the chunk format.
+    unsigned tmp;
+    if (!unpack_uint(posptr, end, &tmp))
+        report_read_error(*posptr);
+    *new_type = ChertPostList::chunk_type(tmp);
+
     LOGVALUE(DB, last_did_in_chunk);
     RETURN(last_did_in_chunk);
 }
@@ -260 +273 @@
     Xapian::docid did;
     Xapian::termcount wdf;
 
+    ChertPostList::chunk_type current_type;
+
   public:
     /** Initialise the postlist chunk reader.
      *
      *  @param first_did  First document id in this chunk.
      *  @param data       The tag string with the header removed.
      */
-    PostlistChunkReader(Xapian::docid first_did, const string & data_)
-        : data(data_), pos(data.data()), end(pos + data.length()), at_end(data.empty()), did(first_did)
+    PostlistChunkReader(Xapian::docid first_did, const string & data_,
+                        ChertPostList::chunk_type current_type_)
+        : data(data_), pos(data.data()), end(pos + data.length()),
+          at_end(data.empty()), did(first_did), current_type(current_type_)
     {
         if (!at_end) read_wdf(&pos, end, &wdf);
     }
@@ -294 +311 @@
     if (pos == end) {
         at_end = true;
     } else {
-        read_did_increase(&pos, end, &did);
+        switch (current_type) {
+            case ChertPostList::DENSE:
+                ++did;
+                break;
+            case ChertPostList::SPARSE:
+                read_did_increase(&pos, end, &did);
+                break;
+        }
         read_wdf(&pos, end, &wdf);
     }
 }
@@ -306 +330 @@
         : orig_key(orig_key_),
           tname(tname_), is_first_chunk(is_first_chunk_),
           is_last_chunk(is_last_chunk_),
-          started(false)
+          started(false),
+          current_type(ChertPostList::DENSE)
 {
     DEBUGCALL(DB, void, "PostlistChunkWriter::PostlistChunkWriter",
               orig_key_ << ", " << is_first_chunk_ << ", " << tname_ << ", " <<
@@ -314 +339 @@
 }
 
 void
+PostlistChunkWriter::start_new_chunk(ChertTable * table,
+                                     Xapian::docid new_first_did)
+{
+    bool save_is_last_chunk = is_last_chunk;
+    is_last_chunk = false;
+    flush(table);
+    is_last_chunk = save_is_last_chunk;
+    is_first_chunk = false;
+    first_did = new_first_did;
+    chunk.resize(0);
+    orig_key = ChertPostListTable::make_key(tname, first_did);
+    current_type = ChertPostList::DENSE;
+}
+
+void
 PostlistChunkWriter::append(ChertTable * table, Xapian::docid did,
                             Xapian::termcount wdf)
 {
@@ -324 +364 @@
         Assert(did > current_did);
         // Start a new chunk if this one has grown to the threshold.
         if (chunk.size() >= CHUNKSIZE) {
-            bool save_is_last_chunk = is_last_chunk;
-            is_last_chunk = false;
-            flush(table);
-            is_last_chunk = save_is_last_chunk;
-            is_first_chunk = false;
-            first_did = did;
-            chunk.resize(0);
-            orig_key = ChertPostListTable::make_key(tname, first_did);
+            start_new_chunk(table, did);
+            goto append_packed_wdf;
         } else {
-            chunk.append(pack_uint(did - current_did - 1));
+            if (current_type == ChertPostList::DENSE) {
+                if (current_did + 1 != did) {
+                    if (current_did == first_did) {
+                        // Only have one entry - can just change type.
+                        current_type = ChertPostList::SPARSE;
+                    } else {
+                        start_new_chunk(table, did);
+                        goto append_packed_wdf;
+                    }
+                }
+            }
+            if (current_type == ChertPostList::SPARSE) {
+                chunk.append(pack_uint(did - current_did - 1));
+            }
         }
     }
+append_packed_wdf:
     current_did = did;
     chunk.append(pack_uint(wdf));
 }
@@ -355 +403 @@
 static inline string
 make_start_of_chunk(bool new_is_last_chunk,
                     Xapian::docid new_first_did,
-                    Xapian::docid new_final_did)
+                    Xapian::docid new_final_did,
+                    ChertPostList::chunk_type new_type)
 {
     Assert(new_final_did >= new_first_did);
     return pack_bool(new_is_last_chunk) +
-            pack_uint(new_final_did - new_first_did);
+            pack_uint(new_final_did - new_first_did) +
+            pack_uint(static_cast<unsigned>(new_type));
 }
 
 static void
@@ -368 +418 @@
                      unsigned int end_of_chunk_header,
                      bool is_last_chunk,
                      Xapian::docid first_did_in_chunk,
-                     Xapian::docid last_did_in_chunk)
+                     Xapian::docid last_did_in_chunk,
+                     ChertPostList::chunk_type new_type)
 {
     Assert((size_t)(end_of_chunk_header - start_of_chunk_header) <= chunk.size());
 
     chunk.replace(start_of_chunk_header,
                   end_of_chunk_header - start_of_chunk_header,
                   make_start_of_chunk(is_last_chunk, first_did_in_chunk,
-                                      last_did_in_chunk));
+                                      last_did_in_chunk, new_type));
 }
 
 void
@@ -465 +516 @@
 
             // Read the chunk header
             bool new_is_last_chunk;
+            ChertPostList::chunk_type new_type;
             Xapian::docid new_last_did_in_chunk =
                 read_start_of_chunk(&tagpos, tagend, new_first_did,
-                                    &new_is_last_chunk);
+                                    &new_is_last_chunk, &new_type);
 
             string chunk_data(tagpos, tagend);
 
@@ -478 +530 @@
             string tag;
             tag = make_start_of_first_chunk(num_ent, coll_freq, new_first_did);
             tag += make_start_of_chunk(new_is_last_chunk,
-                                              new_first_did,
-                                              new_last_did_in_chunk);
+                                       new_first_did,
+                                       new_last_did_in_chunk,
+                                       new_type);
             tag += chunk_data;
             table->add(orig_key, tag);
             return;
@@ -527 +580 @@
                     report_read_error(keypos);
             }
             bool wrong_is_last_chunk;
+            ChertPostList::chunk_type new_type;
             string::size_type start_of_chunk_header = tagpos - tag.data();
             Xapian::docid last_did_in_chunk =
                 read_start_of_chunk(&tagpos, tagend, first_did_in_chunk,
-                                    &wrong_is_last_chunk);
+                                    &wrong_is_last_chunk, &new_type);
             string::size_type end_of_chunk_header = tagpos - tag.data();
 
             // write new is_last flag
@@ -539 +593 @@
                                  end_of_chunk_header,
                                  true, // is_last_chunk
                                  first_did_in_chunk,
-                                 last_did_in_chunk);
+                                 last_did_in_chunk,
+                                 new_type);
             table->add(cursor->current_key, tag);
         }
     } else {
@@ -576 +631 @@
 
             tag = make_start_of_first_chunk(num_ent, coll_freq, first_did);
 
-            tag += make_start_of_chunk(is_last_chunk, first_did, current_did);
+            tag += make_start_of_chunk(is_last_chunk, first_did, current_did, current_type);
             tag += chunk;
             table->add(key, tag);
             return;
@@ -616 +671 @@
         }
 
         // ...and write the start of this chunk.
-        tag = make_start_of_chunk(is_last_chunk, first_did, current_did);
+        tag = make_start_of_chunk(is_last_chunk, first_did, current_did, current_type);
 
         tag += chunk;
         table->add(new_key, tag);
@@ -687 +742 @@
     did = read_start_of_first_chunk(&pos, end, &number_of_entries, NULL);
     first_did_in_chunk = did;
     last_did_in_chunk = read_start_of_chunk(&pos, end, first_did_in_chunk,
-                                            &is_last_chunk);
+                                            &is_last_chunk, &current_type);
     read_wdf(&pos, end, &wdf);
     LOGLINE(DB, "Initial docid " << did);
 }
@@ -712 +767 @@
     DEBUGCALL(DB, bool, "ChertPostList::next_in_chunk", "");
     if (pos == end) RETURN(false);
 
+    switch (current_type) {
+        case ChertPostList::DENSE:
+            ++did;
+            break;
+        case ChertPostList::SPARSE:
+            read_did_increase(&pos, end, &did);
+            break;
+    }
+    read_wdf(&pos, end, &wdf);
+
+    // Either not at last doc in chunk, or pos == end, but not both.
+    Assert(did <= last_did_in_chunk);
+    Assert(did < last_did_in_chunk || pos == end);
+    Assert(pos != end || did == last_did_in_chunk);
+
+    RETURN(true);
+}
+
+bool
+ChertPostList::next_in_chunk_dense()
+{
+    DEBUGCALL(DB, bool, "ChertPostList::next_in_chunk_dense", "");
+    if (pos == end) RETURN(false);
+    Assert(current_type == ChertPostList::DENSE);
+
+    ++did;
+    read_wdf(&pos, end, &wdf);
+
+    // Either not at last doc in chunk, or pos == end, but not both.
+    Assert(did <= last_did_in_chunk);
+    Assert(did < last_did_in_chunk || pos == end);
+    Assert(pos != end || did == last_did_in_chunk);
+
+    RETURN(true);
+}
+
+bool
+ChertPostList::next_in_chunk_sparse()
+{
+    DEBUGCALL(DB, bool, "ChertPostList::next_in_chunk_sparse", "");
+    if (pos == end) RETURN(false);
+    Assert(current_type == ChertPostList::SPARSE);
+
     read_did_increase(&pos, end, &did);
     read_wdf(&pos, end, &wdf);
 
@@ -765 +863 @@
 
     first_did_in_chunk = did;
     last_did_in_chunk = read_start_of_chunk(&pos, end, first_did_in_chunk,
-                                            &is_last_chunk);
+                                            &is_last_chunk, &current_type);
     read_wdf(&pos, end, &wdf);
 }
 
@@ -859 +957 @@
 
     first_did_in_chunk = did;
     last_did_in_chunk = read_start_of_chunk(&pos, end, first_did_in_chunk,
-                                            &is_last_chunk);
+                                            &is_last_chunk, &current_type);
     read_wdf(&pos, end, &wdf);
 
     // Possible, since desired_did might be after end of this chunk and before
@@ -872 +970 @@
 {
     DEBUGCALL(DB, bool,
               "ChertPostList::move_forward_in_chunk_to_at_least", desired_did);
+    Assert(did <= desired_did);
     if (desired_did > last_did_in_chunk) {
         pos = end;
         RETURN(false);
     }
-    while (did < desired_did) {
-        bool at_end_of_chunk = !next_in_chunk();
-        // If we hit the end of the chunk then last_did_in_chunk must be
-        // wrong.
-        Assert(!at_end_of_chunk);
-        if (at_end_of_chunk) RETURN(false);
+    switch (current_type) {
+        case ChertPostList::DENSE: {
+            unsigned count = desired_did - did;
+            if (count) {
+                if (count > 1) {
+                    count -= 1;
+                    while (pos != end) {
+                        om_byte part = static_cast<om_byte>(*pos);
+                        ++pos;
+                        if ((part & 0x80) == 0) {
+                            if ((--count) == 0)
+                                break;
+                        }
+                    }
+                    if (pos == end)
+                        throw Xapian::DatabaseCorruptError("Insufficient entries in posting list chunk");
+                }
+                read_wdf(&pos, end, &wdf);
+                did = desired_did;
+            }
+            break;
+        }
+        case ChertPostList::SPARSE:
+            while (did < desired_did) {
+                bool at_end_of_chunk = !next_in_chunk_sparse();
+                // If we hit the end of the chunk then last_did_in_chunk must be
+                // wrong.
+                Assert(!at_end_of_chunk);
+                if (at_end_of_chunk) RETURN(false);
+            }
+            break;
     }
     RETURN(true);
 }
@@ -1004 +1128 @@
     }
 
     bool is_last_chunk;
+    ChertPostList::chunk_type current_type;
     Xapian::docid last_did_in_chunk;
-    last_did_in_chunk = read_start_of_chunk(&pos, end, first_did_in_chunk, &is_last_chunk);
+    last_did_in_chunk = read_start_of_chunk(&pos, end, first_did_in_chunk, &is_last_chunk, &current_type);
     *to = new PostlistChunkWriter(cursor->current_key, is_first_chunk, tname,
                                   is_last_chunk);
     if (did > last_did_in_chunk) {
@@ -1014 +1139 @@
         // (FIXME)
         *from = NULL;
         (*to)->raw_append(first_did_in_chunk, last_did_in_chunk,
-                          string(pos, end));
+                          string(pos, end), current_type);
     } else {
-        *from = new PostlistChunkReader(first_did_in_chunk, string(pos, end));
+        *from = new PostlistChunkReader(first_did_in_chunk, string(pos, end),
+                                        current_type);
     }
     if (is_last_chunk) RETURN(Xapian::docid(-1));
 
@@ -1054 +1180 @@
         if (!key_exists(current_key)) {
             LOGLINE(DB, "Adding dummy first chunk");
             string newtag = make_start_of_first_chunk(0, 0, 0);
-            newtag += make_start_of_chunk(true, 0, 0);
+            newtag += make_start_of_chunk(true, 0, 0, ChertPostList::DENSE);
             add(current_key, newtag);
         }
 
@@ -1126 +1252 @@
             Xapian::termcount termfreq, collfreq;
             Xapian::docid firstdid, lastdid;
             bool islast;
+            ChertPostList::chunk_type current_type;
             if (pos == end) {
                 termfreq = 0;
                 collfreq = 0;
@@ -1136 +1263 @@
                 firstdid = read_start_of_first_chunk(&pos, end,
                                                      &termfreq, &collfreq);
                 // Handle the generic start of chunk header.
-                lastdid = read_start_of_chunk(&pos, end, firstdid, &islast);
+                lastdid = read_start_of_chunk(&pos, end, firstdid, &islast, &current_type);
             }
 
             termfreq += deltas->second.first;
@@ -1163 +1290 @@
 
             // Rewrite start of first chunk to update termfreq and collfreq.
             string newhdr = make_start_of_first_chunk(termfreq, collfreq, firstdid);
-            newhdr += make_start_of_chunk(islast, firstdid, lastdid);
+            newhdr += make_start_of_chunk(islast, firstdid, lastdid, current_type);
             if (pos == end) {
                 add(current_key, newhdr);
             } else {