Ticket #52: revpostlist.patch

backends/quartz/quartz_database.cc

@@ -585 +585 @@
                               tname));
 }
 
+LeafPostList *
+QuartzDatabase::do_open_post_list_rev(const string& tname) const
+{
+    DEBUGCALL(DB, LeafPostList *, "QuartzDatabase::do_open_post_list_rev",
+              tname);
+    Assert(!tname.empty());
+
+    Xapian::Internal::RefCntPtr<const QuartzDatabase> ptrtothis(this);
+    return(new QuartzRevPostList(ptrtothis,
+                                 &postlist_table,
+                                 &positionlist_table,
+                                 tname));
+}
+
 LeafTermList *
 QuartzDatabase::open_term_list(Xapian::docid did) const
 {
@@ -1102 +1116 @@
                               &database_ro.postlist_table,
                               &database_ro.positionlist_table,
                               tname));
+}
+
+LeafPostList *
+QuartzWritableDatabase::do_open_post_list_rev(const string& tname) const
+{
+    DEBUGCALL(DB, LeafPostList *, "QuartzWritableDatabase::do_open_post_list_rev", tname);
+    Assert(!tname.empty());
+
+    // Need to flush iff we've got buffered changes to this term's postlist.
+    map<string, map<docid, pair<char, termcount> > >::const_iterator j;
+    j = mod_plists.find(tname);
+    if (j != mod_plists.end()) do_flush_const();
+
+    Xapian::Internal::RefCntPtr<const QuartzWritableDatabase> ptrtothis(this);
+    return(new QuartzRevPostList(ptrtothis,
+                                 &database_ro.postlist_table,
+                                 &database_ro.positionlist_table,
+                                 tname));
 }
 
 LeafTermList *

backends/quartz/quartz_database.h

@@ -210 +210 @@
         bool term_exists(const string & tname) const;
 
         LeafPostList * do_open_post_list(const string & tname) const;
+        LeafPostList * do_open_post_list_rev(const string & tname) const;
         LeafTermList * open_term_list(Xapian::docid did) const;
         Xapian::Document::Internal * open_document(Xapian::docid did, bool lazy = false) const;
         PositionList * open_position_list(Xapian::docid did,
@@ -289 +290 @@
         bool term_exists(const string & tname) const;
 
         LeafPostList * do_open_post_list(const string & tname) const;
+        LeafPostList * do_open_post_list_rev(const string & tname) const;
         LeafTermList * open_term_list(Xapian::docid did) const;
         Xapian::Document::Internal * open_document(Xapian::docid did, bool lazy = false) const;
         PositionList * open_position_list(Xapian::docid did,

backends/quartz/quartz_postlist.cc

@@ -604 +604 @@
     }
 }
 
+/*****************************************************************************/
+
 /** Read the number of entries in the posting list.
  *  This must only be called when *posptr is pointing to the start of
  *  the first chunk of the posting list.
@@ -913 +915 @@
 {
     return tname + ":" + om_tostring(number_of_entries);
 }
+
+/*****************************************************************************/
+
+QuartzRevPostList::QuartzRevPostList(Xapian::Internal::RefCntPtr<const Xapian::Database::Internal> this_db_,
+                               const Btree * table_,
+                               const Btree * positiontable_,
+                               const string & tname_)
+        : this_db(this_db_),
+          table(table_),
+          positiontable(positiontable_),
+          tname(tname_),
+          cursor(table->cursor_get()),
+          is_at_end(false),
+          have_started(false)
+{
+    DEBUGCALL(DB, void, "QuartzRevPostList::QuartzRevPostList",
+              this_db_.get() << ", " << table_ << ", " <<
+              positiontable_ << ", " << tname_);
+    string key;
+    make_key(tname, key);
+    int found = cursor->find_entry(key);
+    if (!found) {
+        number_of_entries = 0;
+        collection_freq = 0;
+        is_at_end = true;
+        pos = 0;
+        end = 0;
+        first_did_in_chunk = 0;
+        last_did_in_chunk = 0;
+        return;
+    }
+    cursor->read_tag();
+    pos = cursor->current_tag.data();
+    end = pos + cursor->current_tag.size();
+
+    did = read_start_of_first_chunk(&pos, end,
+                              &number_of_entries, &collection_freq);
+    first_did_in_chunk = did;
+    last_did_in_chunk = read_start_of_chunk(&pos, end, first_did_in_chunk,
+                                            &is_last_chunk);
+    read_wdf_and_length(&pos, end, &wdf, &doclength);
+    if (!is_last_chunk) {
+        make_key(tname, (Xapian::docid)-1, key);
+        (void) cursor->find_entry(key);
+        const char * keypos = cursor->current_key.data();
+        const char * keyend = keypos + cursor->current_key.size();
+        if (!check_tname_in_key_lite(&keypos, keyend, tname)) {
+            throw Xapian::DatabaseCorruptError("Chunk for wrong term");
+        }
+
+        if (!unpack_uint_preserving_sort(&keypos, keyend, &first_did_in_chunk)) {
+            report_read_error(keypos);
+        }
+
+        cursor->read_tag();
+        pos = cursor->current_tag.data();
+        end = pos + cursor->current_tag.size();
+
+        last_did_in_chunk = read_start_of_chunk(&pos, end, first_did_in_chunk,
+                                                &is_last_chunk);
+        read_wdf_and_length(&pos, end, &wdf, &doclength);
+    }
+
+    buffer.clear();
+    did = first_did_in_chunk;
+    while (pos != end) {
+        buffer.push_back(PLEntry(did, wdf, doclength));
+        read_did_increase(&pos, end, &did);
+        read_wdf_and_length(&pos, end, &wdf, &doclength);
+        // Either not at last doc in chunk, or pos == end, but not both.
+        Assert(did <= last_did_in_chunk);
+        Assert(did < last_did_in_chunk || pos == end);
+        Assert(pos != end || did == last_did_in_chunk);
+    }
+}
+
+QuartzRevPostList::~QuartzRevPostList()
+{
+    DEBUGCALL(DB, void, "QuartzRevPostList::~QuartzRevPostList", "");
+}
+
+bool
+QuartzRevPostList::next_in_chunk()
+{
+    DEBUGCALL(DB, bool, "QuartzRevPostList::next_in_chunk", "");
+    if (buffer.empty()) RETURN(false);
+
+    did = buffer.back().did;
+    wdf = buffer.back().wdf;
+    doclength = buffer.back().doclength;
+    buffer.pop_back();
+
+    RETURN(true);
+}
+
+void
+QuartzRevPostList::next_chunk()
+{
+    DEBUGCALL(DB, void, "QuartzRevPostList::next_chunk", "");
+    if (is_last_chunk) {
+        is_at_end = true;
+        return;
+    }
+    
+    cursor->prev();
+    const char * keypos = cursor->current_key.data();
+    const char * keyend = keypos + cursor->current_key.size();
+    if (!check_tname_in_key_lite(&keypos, keyend, tname)) {
+        throw Xapian::DatabaseCorruptError("Chunk for wrong term");
+    }
+
+    if (keypos == keyend) {
+        is_last_chunk = true;
+        first_did_in_chunk = read_start_of_first_chunk(&pos, end,
+                &number_of_entries, &collection_freq);
+    } else {
+        if (!unpack_uint_preserving_sort(&keypos, keyend, &first_did_in_chunk)) {
+            report_read_error(keypos);
+        }
+
+        cursor->read_tag();
+        pos = cursor->current_tag.data();
+        end = pos + cursor->current_tag.size();
+
+        last_did_in_chunk = read_start_of_chunk(&pos, end, first_did_in_chunk,
+                                                &is_last_chunk);
+    }
+    read_wdf_and_length(&pos, end, &wdf, &doclength);
+    buffer.clear();
+    did = first_did_in_chunk;
+    while (pos != end) {
+        buffer.push_back(PLEntry(did, wdf, doclength));
+        read_did_increase(&pos, end, &did);
+        read_wdf_and_length(&pos, end, &wdf, &doclength);
+        // Either not at last doc in chunk, or pos == end, but not both.
+        Assert(did <= last_did_in_chunk);
+        Assert(did < last_did_in_chunk || pos == end);
+        Assert(pos != end || did == last_did_in_chunk);
+    }
+}
+
+PositionList *
+QuartzRevPostList::read_position_list()
+{
+    DEBUGCALL(DB, PositionList *, "QuartzRevPostList::read_position_list", "");
+
+    positionlist.read_data(positiontable, did, tname);
+
+    RETURN(&positionlist);
+}
+
+PositionList *
+QuartzRevPostList::open_position_list() const
+{
+    DEBUGCALL(DB, PositionList *, "QuartzRevPostList::open_position_list", "");
+
+    AutoPtr<QuartzPositionList> poslist(new QuartzPositionList());
+    poslist->read_data(positiontable, did, tname);
+
+    RETURN(poslist.release());
+}
+
+PostList *
+QuartzRevPostList::next(Xapian::weight w_min)
+{
+    DEBUGCALL(DB, PostList *, "QuartzRevPostList::next", w_min);
+    (void)w_min; // no warning
+
+    if (!have_started) {
+        have_started = true;
+    } else {
+        if (!next_in_chunk()) next_chunk();
+    }
+
+    DEBUGLINE(DB, string("Moved to ") <<
+              (is_at_end ? string("end.") : string("docid, wdf, doclength = ") +
+               om_tostring(did) + ", " + om_tostring(wdf) + ", " +
+               om_tostring(doclength)));
+
+    RETURN(NULL);
+}
+
+bool
+QuartzRevPostList::current_chunk_contains(Xapian::docid desired_did)
+{
+    DEBUGCALL(DB, bool, "QuartzRevPostList::current_chunk_contains", desired_did);
+    if (desired_did >= first_did_in_chunk &&
+        desired_did <= last_did_in_chunk) {
+        RETURN(true);
+    }
+    RETURN(false);
+}
+
+void
+QuartzRevPostList::move_to_chunk_containing(Xapian::docid desired_did)
+{
+    DEBUGCALL(DB, void,
+              "QuartzRevPostList::move_to_chunk_containing", desired_did);
+    string key;
+    make_key(tname, desired_did, key);
+    (void) cursor->find_entry(key);
+    Assert(!cursor->after_end());
+
+    const char * keypos = cursor->current_key.data();
+    const char * keyend = keypos + cursor->current_key.size();
+    // Check we're still in same postlist
+    if (!check_tname_in_key_lite(&keypos, keyend, tname)) {
+        // This should only happen if the postlist doesn't exist at all.
+        is_at_end = true;
+        is_last_chunk = true;
+        return;
+    }
+    is_at_end = false;
+
+    cursor->read_tag();
+    pos = cursor->current_tag.data();
+    end = pos + cursor->current_tag.size();
+
+    if (keypos == keyend) {
+        // In first chunk
+#ifdef XAPIAN_DEBUG
+        Xapian::termcount old_number_of_entries = number_of_entries;
+        Xapian::termcount old_collection_freq = collection_freq;
+        first_did_in_chunk = read_start_of_first_chunk(&pos, end, &number_of_entries,
+                                        &collection_freq);
+        Assert(old_number_of_entries == number_of_entries);
+        Assert(old_collection_freq == collection_freq);
+#else
+        first_did_in_chunk = read_start_of_first_chunk(&pos, end, 0, 0);
+#endif
+    } else {
+        // In normal chunk
+        if (!unpack_uint_preserving_sort(&keypos, keyend, &first_did_in_chunk)) {
+            report_read_error(keypos);
+        }
+    }
+
+    last_did_in_chunk = read_start_of_chunk(&pos, end, first_did_in_chunk,
+                                            &is_last_chunk);
+    read_wdf_and_length(&pos, end, &wdf, &doclength);
+    buffer.clear();
+    did = first_did_in_chunk;
+    while (pos != end) {
+        buffer.push_back(PLEntry(did, wdf, doclength));
+        read_did_increase(&pos, end, &did);
+        read_wdf_and_length(&pos, end, &wdf, &doclength);
+        // Either not at last doc in chunk, or pos == end, but not both.
+        Assert(did <= last_did_in_chunk);
+        Assert(did < last_did_in_chunk || pos == end);
+        Assert(pos != end || did == last_did_in_chunk);
+    }
+}
+
+bool
+QuartzRevPostList::move_forward_in_chunk_to_at_least(Xapian::docid desired_did)
+{
+    DEBUGCALL(DB, bool,
+              "QuartzRevPostList::move_forward_in_chunk_to_at_least", desired_did);
+    if (desired_did > last_did_in_chunk) {
+        pos = end;
+        RETURN(false);
+    }
+    while (did < desired_did) {
+        // FIXME: perhaps we don't need to decode the wdf and document length
+        // for documents we're skipping past.
+        bool at_end_of_chunk = !next_in_chunk();
+        if (at_end_of_chunk) RETURN(false);
+    }
+    RETURN(true);
+}
+
+PostList *
+QuartzRevPostList::skip_to(Xapian::docid desired_did, Xapian::weight w_min)
+{
+    DEBUGCALL(DB, PostList *,
+              "QuartzRevPostList::skip_to", desired_did << ", " << w_min);
+    (void)w_min; // no warning
+    // We've started now - if we hadn't already, we're already positioned
+    // at start so there's no need to actually do anything.
+    have_started = true;
+
+    // Don't skip back, and don't need to do anything if already there.
+    if (desired_did <= did) RETURN(NULL);
+
+    // Move to correct chunk
+    if (!current_chunk_contains(desired_did)) {
+        move_to_chunk_containing(desired_did);
+        // Might be at_end now - this is why we need the test before moving
+        // forward in chunk.
+    }
+
+    // Move to correct position in chunk
+    if (!is_at_end) {
+#ifdef XAPIAN_DEBUG
+        bool have_document =
+#else
+                (void)
+#endif
+                move_forward_in_chunk_to_at_least(desired_did);
+        Assert(have_document);
+    }
+
+    DEBUGLINE(DB, string("Skipped to ") <<
+              (is_at_end ? string("end.") : string("docid, wdf, doclength = ") +
+               om_tostring(did) + ", " + om_tostring(wdf) + ", " +
+               om_tostring(doclength) + "."));
+
+    RETURN(NULL);
+}
+
+string
+QuartzRevPostList::get_description() const
+{
+    return tname + ":" + om_tostring(number_of_entries);
+}
+
+/*****************************************************************************/
 
 // Returns the last did to allow in this chunk.
 Xapian::docid

backends/quartz/quartz_postlist.h

@@ -247 +247 @@
                                            Xapian::termcount * collection_freq_ptr);
 };
 
+/** A postlist in a quartz database.
+ */
+class QuartzRevPostList : public LeafPostList {
+    private:
+        /** The database we are searching.  This pointer is held so that the
+         *  database doesn't get deleted before us.
+         */
+        Xapian::Internal::RefCntPtr<const Xapian::Database::Internal> this_db;
+
+        /// The table containing the postlist.
+        const Btree * table;
+
+        /// The table containing positionlists.
+        const Btree * positiontable;
+
+        /// The termname for this postlist.
+        string tname;
+
+        /// Cursor pointing to current chunk of postlist.
+        AutoPtr<Bcursor> cursor;
+
+        /// True if this is the last chunk.
+        bool is_last_chunk;
+
+        /// The first document id in this chunk.
+        Xapian::docid first_did_in_chunk;
+
+        /// The last document id in this chunk.
+        Xapian::docid last_did_in_chunk;
+
+        /// Position of iteration through current chunk.
+        const char * pos;
+
+        /// Pointer to byte after end of current chunk.
+        const char * end;
+
+        /// Document id we're currently at.
+        Xapian::docid did;
+
+        /// The (absolute) length of the current document.
+        quartz_doclen_t doclength;
+
+        /// The wdf of the current document.
+        Xapian::termcount wdf;
+
+        /// Whether we've run off the end of the list yet.
+        bool is_at_end;
+
+        /// Whether we've started reading the list yet.
+        bool have_started;
+
+        /// The number of entries in the posting list.
+        Xapian::doccount number_of_entries;
+
+        /// The number of occurrences of the term in the posting list.
+        Xapian::termcount collection_freq;
+
+        /// The position list object for this posting list.
+        QuartzPositionList positionlist;
+
+        struct PLEntry {
+            Xapian::docid did;
+            Xapian::doccount wdf;
+            Xapian::termcount doclength;
+            PLEntry(Xapian::docid did_, Xapian::termcount wdf_, quartz_doclen_t doclength_) : did(did_), wdf(wdf_), doclength(doclength_) { }
+        };
+
+        vector<PLEntry> buffer;
+
+        /// Copying is not allowed.
+        QuartzRevPostList(const QuartzRevPostList &);
+
+        /// Assignment is not allowed.
+        void operator=(const QuartzRevPostList &);
+
+        /** Move to the next item in the chunk, if possible.
+         *  If already at the end of the chunk, returns false.
+         */
+        bool next_in_chunk();
+
+        /** Move to the next chunk.
+         *
+         *  If there are no more chunks in this postlist, this will set
+         *  is_at_end to true.
+         */
+        void next_chunk();
+
+        /** Return true if the given document ID lies in the range covered
+         *  by the current chunk.  This does not say whether the document ID
+         *  is actually present.  It will return false if the document ID
+         *  is greater than the last document ID in the chunk, even if it is
+         *  less than the first document ID in the next chunk: it is possible
+         *  for no chunk to contain a particular document ID.
+         */
+        bool current_chunk_contains(Xapian::docid desired_did);
+
+        /** Move to chunk containing the specified document ID.
+         *
+         *  This moves to the chunk whose starting document ID is
+         *  <= desired_did, but such that the next chunk's starting
+         *  document ID is > desired_did.
+         *
+         *  It is thus possible that current_chunk_contains(desired_did)
+         *  will return false after this call, since the document ID
+         *  might lie after the end of this chunk, but before the start
+         *  of the next chunk.
+         */
+        void move_to_chunk_containing(Xapian::docid desired_did);
+
+        /** Scan forward in the current chunk for the specified document ID.
+         *
+         *  This is particularly efficient if the desired document ID is
+         *  greater than the last in the chunk - it then skips straight
+         *  to the end.
+         *
+         *  @return true if we moved to a valid document,
+         *          false if we reached the end of the chunk.
+         */
+        bool move_forward_in_chunk_to_at_least(Xapian::docid desired_did);
+
+    public:
+        /// Default constructor.
+        QuartzRevPostList(Xapian::Internal::RefCntPtr<const Xapian::Database::Internal> this_db_,
+                       const Btree * table_,
+                       const Btree * positiontable_,
+                       const string & tname);
+
+        /// Destructor.
+        ~QuartzRevPostList();
+
+        /** Returns number of docs indexed by this term.
+         *
+         *  This is the length of the postlist.
+         */
+        Xapian::doccount get_termfreq() const { return number_of_entries; }
+
+        /** Returns the number of occurrences of the term in the database.
+         *
+         *  This is the sum of the wdfs in the postlist.
+         */
+        Xapian::termcount get_collection_freq() const { return collection_freq; }
+
+        /// Returns the current docid.
+        Xapian::docid get_docid() const { Assert(have_started); return did; }
+
+        /// Returns the length of current document.
+        Xapian::doclength get_doclength() const {
+            Assert(have_started);
+            return static_cast<Xapian::doclength>(doclength);
+        }
+
+        /** Returns the Within Document Frequency of the term in the current
+         *  document.
+         */
+        Xapian::termcount get_wdf() const { Assert(have_started); return wdf; }
+
+        /** Get the list of positions of the term in the current document.
+         */
+        PositionList *read_position_list();
+
+        /** Get the list of positions of the term in the current document.
+         */
+        PositionList * open_position_list() const;
+
+        /// Move to the next document.
+        PostList * next(Xapian::weight w_min);
+
+        /// Skip to next document with docid >= docid.
+        PostList * skip_to(Xapian::docid desired_did, Xapian::weight w_min);
+
+        /// Return true if and only if we're off the end of the list.
+        bool at_end() const { return is_at_end; }
+
+        /// Get a description of the document.
+        std::string get_description() const;
+};
+
 #endif /* OM_HGUARD_QUARTZ_POSTLIST_H */

common/database.h

@@ -89 +89 @@
          *                use.
          */
         virtual LeafPostList * do_open_post_list(const string & tname) const = 0;
+        virtual LeafPostList * do_open_post_list_rev(const string & tname) const = 0;
 
     public:
         /** Destroy the database.
@@ -189 +190 @@
          *                This object must be deleted by the caller after
          *                use.
          */
-        LeafPostList * open_post_list(const string & tname) const {
+        LeafPostList * open_post_list(const string & tname, bool rev = false) const {
             if (!term_exists(tname)) {
                 DEBUGLINE(MATCH, tname + " is not in database.");
                 // Term doesn't exist in this database.  However, we create
@@ -197 +198 @@
                 // cleaner (term might exist in other databases).
                 return new EmptyPostList();
             }
-            return do_open_post_list(tname);
+            return rev ? do_open_post_list_rev(tname) : do_open_post_list(tname);
         }
 
         /** Open a term list.

common/multimatch.h

@@ -143 +143 @@
          *  and the maxweight now possible is smaller.
          */
         void recalc_maxweight();
+
+        /** Called by LocalSubMatch to find out the sort direction so it
+         * can decide whether to use reverse postlists or not.
+         */
+        bool get_sort_forward() const { return sort_forward; }
 };
 
 #endif /* OM_HGUARD_MULTIMATCH_H */

matcher/localmatch.cc

@@ -410 +410 @@
             }
 
             // MULTI
-            LeafPostList * pl = db->open_post_list(query->tname);
+            LeafPostList * pl;
+            pl = db->open_post_list(query->tname, !matcher->get_sort_forward());
             pl->set_termweight(wt);
             RETURN(pl);
         }

matcher/multimatch.cc

@@ -738 +738 @@
                 if (!sort_bands && items.size() == max_msize) {
                     // We're done if this is a forward boolean match
                     // (bodgetastic, FIXME better if we can)
-                    if (max_weight == 0 && sort_forward) break;
+                    if (max_weight == 0) break;
                 }
             }
         }