Ticket #38: xapian-remote-postlist.patch

net/remoteserver.cc

@@ -139 +139 @@
 {
     while (true) {
         try {
+            /* This list needs to be kept in the same order as the list of
+             * message types in "remoteprotocol.h". Note that messages at the
+             * end of the list in "remoteprotocol.h" can be omitted if they
+             * don't correspond to dispatch actions.
+             */
             static const dispatch_func dispatch[] = {
                 &RemoteServer::msg_allterms,
                 &RemoteServer::msg_collfreq,
@@ -150 +155 @@
                 &RemoteServer::msg_query,
                 &RemoteServer::msg_termlist,
                 &RemoteServer::msg_positionlist,
+                &RemoteServer::msg_postlist,
                 &RemoteServer::msg_reopen,
                 &RemoteServer::msg_update,
                 &RemoteServer::msg_adddocument,
@@ -223 +229 @@
     Xapian::docid did = decode_length(&p, p_end, false);
     string term(p, p_end - p);
 
-    int N = 0;
     Xapian::termpos lastpos = static_cast<Xapian::termpos>(-1);
     const Xapian::PositionIterator end = db->positionlist_end(did, term);
     for (Xapian::PositionIterator i = db->positionlist_begin(did, term);
@@ -231 +236 @@
         Xapian::termpos pos = *i;
         send_message(REPLY_POSITIONLIST, encode_length(pos - lastpos - 1));
         lastpos = pos;
-        ++N;
     }
 
     send_message(REPLY_DONE, "");
 }
 
 void
+RemoteServer::msg_postlist(const string &message)
+{
+    const char *p = message.data();
+    const char *p_end = p + message.size();
+    string term(p, p_end - p);
+
+    Xapian::doccount termfreq = db->get_termfreq(term);
+    send_message(REPLY_POSTLISTSTART, encode_length(termfreq));
+
+    Xapian::docid lastdocid = static_cast<Xapian::termpos>(0);
+    const Xapian::PostingIterator end = db->postlist_end(term);
+    for (Xapian::PostingIterator i = db->postlist_begin(term);
+         i != end; ++i) {
+
+        Xapian::docid newdocid = *i;
+        string reply = encode_length(newdocid - lastdocid - 1);
+        reply += encode_length(i.get_wdf());
+        // FIXME: get_doclength should always return an integer, but
+        // Xapian::doclength is a double...
+        reply += serialise_double(i.get_doclength());
+
+        send_message(REPLY_POSTLISTITEM, reply);
+        lastdocid = newdocid;
+    }
+
+    send_message(REPLY_DONE, "");
+}
+
+void
 RemoteServer::msg_reopen(const string & msg)
 {
     db->reopen();

common/remote-database.h

@@ -33 +33 @@
 }
 
 class Stats;
+class NetworkPostList;
 
 /** RemoteDatabase is the baseclass for remote database implementations.
  *
@@ -161 +162 @@
 
     LeafPostList * do_open_post_list(const string & tname) const;
 
+    void read_post_list(const string &term, NetworkPostList & pl) const;
+
     PositionList * open_position_list(Xapian::docid did,
                                       const string & tname) const;
 

common/remoteprotocol.h

@@ -28 +28 @@
 // 24: Support for OP_VALUE_RANGE in query serialisation
 // 25: Support for delete_document and replace_document with unique term
 // 26: Tweak delete_document with unique term; delta encode rset and termpos
-#define XAPIAN_REMOTE_PROTOCOL_VERSION 26
+// 27: Support for postlists (always passes the whole list across)
+#define XAPIAN_REMOTE_PROTOCOL_VERSION 27
 
 /// Message types (client -> server).
 enum message_type {
@@ -42 +43 @@
     MSG_QUERY,                  // Run Query
     MSG_TERMLIST,               // Get TermList
     MSG_POSITIONLIST,           // Get PositionList
+    MSG_POSTLIST,               // Get PostList
     MSG_REOPEN,                 // Reopen
     MSG_UPDATE,                 // Get Updated DocCount and AvLength
     MSG_ADDDOCUMENT,            // Add Document
@@ -72 +74 @@
     REPLY_STATS,                // Stats
     REPLY_TERMLIST,             // Get Termlist
     REPLY_POSITIONLIST,         // Get PositionList
+    REPLY_POSTLISTSTART,        // Start of a postlist
+    REPLY_POSTLISTITEM,         // Item in body of a postlist
     REPLY_UPDATE,               // Get Updated DocCount and AvLength
     REPLY_VALUE,                // Document Value
     REPLY_ADDDOCUMENT,          // Add Document

common/remoteserver.h

@@ -103 +103 @@
     // get termlist
     void msg_termlist(const std::string & message);
 
+    // get postlist
+    void msg_postlist(const std::string & message);
+
     // get positionlist
     void msg_positionlist(const std::string &message);
 

backends/remote/remote-database.cc

@@ -26 +26 @@
 #include <xapian/error.h>
 
 #include "inmemory_positionlist.h"
+#include "net_postlist.h"
 #include "net_termlist.h"
 #include "net_document.h"
 #include "omassert.h"
@@ -173 +174 @@
 }
 
 LeafPostList *
-RemoteDatabase::do_open_post_list(const string &) const
+RemoteDatabase::do_open_post_list(const string &term) const
 {
-    throw Xapian::UnimplementedError("RemoteDatabase::do_open_post_list not implemented");
+    AutoPtr<NetworkPostList> plist;
+    return new NetworkPostList(Xapian::Internal::RefCntPtr<const RemoteDatabase>(this), term);
 }
 
+void
+RemoteDatabase::read_post_list(const string &term, NetworkPostList & pl) const
+{
+    send_message(MSG_POSTLIST, term);
+
+    string message;
+    char type;
+    get_message(message, REPLY_POSTLISTSTART);
+
+    const char * p = message.data();
+    const char * p_end = p + message.size();
+    pl.initialise(decode_length(&p, p_end, false));
+
+    while ((type = get_message(message)) == REPLY_POSTLISTITEM) {
+        pl.append_posting(message);
+    }
+    if (type != REPLY_DONE) {
+        throw Xapian::NetworkError("Bad message received", context);
+    }
+}
+
 PositionList *
 RemoteDatabase::open_position_list(Xapian::docid did, const string &term) const
 {

backends/remote/Makefile.mk

@@ -5 +5 @@
 if BUILD_BACKEND_REMOTE
 noinst_HEADERS +=\
         backends/remote/net_document.h\
+        backends/remote/net_postlist.h\
         backends/remote/net_termlist.h
 
 libxapian_la_SOURCES +=\

backends/remote/net_postlist.h

@@ +1 @@
+/* net_postlist.h
+ *
+ * Copyright 2007 Lemur Consulting Ltd
+ *
+ * This program is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU General Public License as
+ * published by the Free Software Foundation; either version 2 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
+ * USA
+ */
+
+#ifndef OM_HGUARD_NET_POSTLIST_H
+#define OM_HGUARD_NET_POSTLIST_H
+
+#include <string>
+
+#include "leafpostlist.h"
+#include "remote-database.h"
+#include "serialise.h"
+#include "serialise-double.h"
+
+using namespace std;
+
+/** A postlist in a remote database.
+ */
+class NetworkPostList : public LeafPostList {
+    private:
+    friend class RemoteDatabase;
+        Xapian::Internal::RefCntPtr<const RemoteDatabase> db;
+        string term;
+
+        string postings;
+        bool started;
+        const char * pos;
+        const char * pos_end;
+
+        Xapian::docid lastdocid;
+        Xapian::termcount lastwdf;
+        Xapian::doclength lastdoclen;
+        Xapian::Internal::RefCntPtr<PositionList> lastposlist;
+
+        Xapian::doccount termfreq;
+
+        void initialise(Xapian::doccount termfreq_) {
+            termfreq = termfreq_;
+        }
+
+        void append_posting(const string & serialised)
+        {
+            Assert(pos == NULL);
+            Assert(!started);
+            postings.append(serialised);
+        }
+
+    public:
+        NetworkPostList(Xapian::Internal::RefCntPtr<const RemoteDatabase> db_,
+                        const string & term_)
+                : db(db_),
+                  term(term_),
+                  started(false),
+                  pos(NULL),
+                  pos_end(NULL),
+                  lastdocid(0),
+                  lastwdf(0),
+                  lastdoclen(0),
+                  termfreq(0)
+        {
+            db->read_post_list(term, *this);
+        }
+
+        Xapian::doccount get_termfreq() const
+        {
+            return termfreq;
+        }
+
+        Xapian::docid get_docid() const
+        {
+            return lastdocid;
+        }
+
+        Xapian::doclength get_doclength() const
+        {
+            return lastdoclen;
+        }
+
+        Xapian::termcount get_wdf() const
+        {
+            return lastwdf;
+        }
+
+        PositionList * read_position_list()
+        {
+            lastposlist = db->open_position_list(lastdocid, term);
+            return lastposlist.get();
+        }
+
+        PositionList * open_position_list() const
+        {
+            return db->open_position_list(lastdocid, term);
+        }
+
+        PostList * next(Xapian::weight)
+        {
+            if (!started) {
+                started = true;
+                pos = postings.data();
+                pos_end = pos + postings.size();
+                lastdocid = 0;
+            }
+
+            if (pos == pos_end) {
+                pos = NULL;
+            } else {
+                lastdocid += decode_length(&pos, pos_end, false) + 1;
+                lastwdf = decode_length(&pos, pos_end, false);
+                lastdoclen = unserialise_double(&pos, pos_end);
+            }
+
+            return NULL;
+        }
+
+        PostList *skip_to(Xapian::docid did, Xapian::weight weight)
+        {
+            if (pos == NULL)
+                next(weight);
+            while (pos != pos_end && lastdocid < did)
+                next(weight);
+            return NULL;
+        }
+
+        bool at_end() const
+        {
+            return (pos == NULL && started);
+        }
+
+        string get_description() const
+        {
+            return "NetworkPostList(" + term + ")";
+        }
+};
+
+#endif /* OM_HGUARD_NET_POSTLIST_H */