Ticket #671: postlist_record_readahead.patch

xapian-core/backends/chert/chert_database.cc

@@ -472 +472 @@
     }
 }
 
+void
+ChertDatabase::request_document(Xapian::docid did) const
+{
+    record_table.readahead_for_record(did);
+}
+
+void
+ChertDatabase::readahead_for_query(const Xapian::Query &query)
+{
+    Xapian::TermIterator t;
+    for (t = query.get_terms_begin(); t != Xapian::TermIterator(); ++t) {
+        const string & term = *t;
+        postlist_table.readahead_key(term);
+    }
+}
+
 bool
 ChertDatabase::reopen()
 {

xapian-core/backends/chert/chert_database.h

@@ -219 +219 @@
                                      chert_revision_number_t * startrev,
                                      chert_revision_number_t * endrev) const;
     public:
+
+    void request_document(Xapian::docid /*did*/) const;
+    void readahead_for_query(const Xapian::Query &query);
         /** Create and open a chert database.
          *
          *  @exception Xapian::DatabaseCorruptError is thrown if there is no

xapian-core/backends/chert/chert_record.cc

@@ -66 +66 @@
 }
 
 void
+ChertRecordTable::readahead_for_record(Xapian::docid did) const
+{
+    readahead_key(make_key(did));
+}
+
+void
 ChertRecordTable::replace_record(const string & data, Xapian::docid did)
 {
     LOGCALL_VOID(DB, "ChertRecordTable::replace_record", data | did);

xapian-core/backends/chert/chert_record.h

@@ -65 +65 @@
          */
         void replace_record(const string & data, Xapian::docid did);
 
+    void readahead_for_record(Xapian::docid did) const;
+
         /** Delete a record from the table.
          */
         void delete_record(Xapian::docid did);

xapian-core/backends/chert/chert_table.cc

@@ -167 +167 @@
 
 #define BYTE_PAIR_RANGE (1 << 2 * CHAR_BIT)
 
+void
+ChertTable::readahead_block(uint4 n) const
+{
+    if (rare(handle == -2))
+        ChertTable::throw_database_closed();
+
+    /* Use the base bit_map_size not the bitmap's size, because
+     * the latter is uninitialised in readonly mode.
+     */
+    Assert(n / CHAR_BIT < base.get_bit_map_size());
+
+    io_readahead_block(handle, block_size, n);
+}
+
 /// read_block(n, p) reads block n of the DB file to address p.
 void
 ChertTable::read_block(uint4 n, byte * p) const
@@ -1134 +1148 @@
 }
 
 bool
+ChertTable::readahead_key(const string &key) const
+{
+    LOGCALL(DB, bool, "ChertTable::readahead_key", key);
+    Assert(!key.empty());
+
+    form_key(key);
+
+    const byte * p;
+    int c;
+
+    Key ktkey = kt.key();
+
+    // We'll only readahead the first level, since descending the B-tree would
+    // require actual reads that would likely hurt performance more than help.
+    p = C[level].p;
+    c = find_in_block(p, ktkey, false, C[level].c);
+    uint4 n = Item(p, c).block_given_by();
+
+    readahead_block(n);
+    RETURN(true);
+}
+
+bool
 ChertTable::get_exact_entry(const string &key, string & tag) const
 {
     LOGCALL(DB, bool, "ChertTable::get_exact_entry", key | tag);

xapian-core/backends/chert/chert_table.h

@@ -334 +334 @@
          */
         void close(bool permanent=false);
 
+    bool readahead_key(const string &key) const;
+
         /** Determine whether the btree exists on disk.
          */
         bool exists() const;
@@ -627 +629 @@
         bool find(Cursor *) const;
         int delete_kt();
         void read_block(uint4 n, byte *p) const;
+    void readahead_block(uint4 n) const;
         void write_block(uint4 n, const byte *p) const;
         XAPIAN_NORETURN(void set_overwritten() const);
         void block_to_cursor(Cursor *C_, int j, uint4 n) const;

xapian-core/backends/database.cc

@@ -50 +50 @@
 }
 
 
+void
+Database::Internal::readahead_for_query (const Xapian::Query & query)
+{
+    (void)query;
+    throw Xapian::UnimplementedError("This backend doesn't support readaheads");
+}
+
 Xapian::doccount
 Database::Internal::get_value_freq(Xapian::valueno) const
 {

xapian-core/backends/database.h

@@ -105 +105 @@
          */
         virtual void keep_alive();
 
+    virtual void readahead_for_query (const Xapian::Query & query);
+
         //////////////////////////////////////////////////////////////////
         // Database statistics:
         // ====================

xapian-core/common/io_utils.cc

@@ -105 +105 @@
 }
 
 void
+io_readahead_block(int fd, size_t n, off_t b)
+{
+    off_t o = b * n;
+    posix_fadvise(fd, o, n, POSIX_FADV_WILLNEED);
+}
+
+void
 io_read_block(int fd, char * p, size_t n, off_t b)
 {
     off_t o = b * n;

xapian-core/common/io_utils.h

@@ -82 +82 @@
     io_write(fd, reinterpret_cast<const char *>(p), n);
 }
 
+/// Readahead block b size n bytes from file descriptor fd
+void io_readahead_block(int fd, size_t n, off_t b);
+
 /// Read block b size n bytes into buffer p from file descriptor fd.
 void io_read_block(int fd, char * p, size_t n, off_t b);
 

xapian-core/include/xapian/database.h

@@ -36 +36 @@
 #include <xapian/attributes.h>
 #include <xapian/intrusive_ptr.h>
 #include <xapian/types.h>
+#include <xapian/query.h>
 #include <xapian/positioniterator.h>
 #include <xapian/postingiterator.h>
 #include <xapian/termiterator.h>

xapian-core/matcher/multimatch.cc

@@ -332 +332 @@
                 is_remote[i] = true;
             } else {
                 smatch = new LocalSubMatch(subdb, query, qlen, subrsets[i], weight);
+        subdb->readahead_for_query(query);
             }
 #else
             // Avoid unused parameter warnings.