Ticket #178: metadata_and_mvcms_register_and_spelling.patch

xapian-core/tests/api_metadata.cc

@@ -122 +122 @@
 }
 
 // Test metadata iterators.
-DEFINE_TESTCASE(metadata5, writable) {
+DEFINE_TESTCASE(metadata5, writable && !remote) {
     Xapian::WritableDatabase db = get_writable_database();
 
     // Check that iterator on empty database returns nothing.

xapian-core/tests/harness/testrunner.cc

@@ -58 +58 @@
                "synonyms,replicas,flint" },
     { "multi_flint", "backend,positional,multi" },
     { "multi_chert", "backend,positional,valuestats,multi" },
-    { "remoteprog_flint", "backend,remote,transactions,positional,writable" },
-    { "remotetcp_flint", "backend,remote,transactions,positional,writable" },
-    { "remoteprog_chert", "backend,remote,transactions,positional,valuestats,writable" },
-    { "remotetcp_chert", "backend,remote,transactions,positional,valuestats,writable" },
+    { "remoteprog_flint", "backend,remote,transactions,positional,writable,"
+      "metadata,spelling" },
+    { "remotetcp_flint", "backend,remote,transactions,positional,writable,"
+      "metadata,spelling" },
+    { "remoteprog_chert", "backend,remote,transactions,positional,valuestats,writable," 
+      "metadata,spelling" },
+    { "remotetcp_chert", "backend,remote,transactions,positional,valuestats,writable," 
+      "metadata,spelling" },
     { NULL, NULL }
 };
 

xapian-core/net/remoteserver.cc

@@ -36 +36 @@
 #include "multimatch.h"
 #include "omassert.h"
 #include "omtime.h"
+#include "ortermlist.h"
+#include "termlist.h"
 #include "serialise.h"
 #include "serialise-double.h"
 #include "utils.h"
 #include "weightinternal.h"
+#include "database.h" // so we can use internal in msg_openspellingtermlist
 
 /// Class to throw when we receive the connection closing message.
 struct ConnectionClosed { };
@@ -185 +188 @@
                 &RemoteServer::msg_replacedocumentterm,
                 &RemoteServer::msg_deletedocument,
                 &RemoteServer::msg_writeaccess,
+                &RemoteServer::msg_getmetadata,
+                &RemoteServer::msg_setmetadata,
+                &RemoteServer::msg_addspelling,
+                &RemoteServer::msg_removespelling,
+                &RemoteServer::msg_getspellingsuggestion,
+                &RemoteServer::msg_openspellingtermlist,
+                &RemoteServer::msg_openspellingwordlist,
+                &RemoteServer::msg_getspellingfreq
                 // MSG_GETMSET - used during a conversation.
                 // MSG_SHUTDOWN - handled by get_message().
             };
@@ -658 +669 @@
 
     send_message(REPLY_ADDDOCUMENT, encode_length(did));
 }
+
+
+void
+RemoteServer::msg_getmetadata(const string & message)
+{
+  send_message(REPLY_METADATA, db->get_metadata(message));
+}
+
+void
+RemoteServer::msg_setmetadata(const string & message)
+{               
+    if (!wdb)
+        throw Xapian::InvalidOperationError("Server is read-only");
+    const char *p = message.data();
+    const char *p_end = p + message.size();
+    size_t keylen = decode_length(&p, p_end, false);
+    string key = string(p, keylen);
+    p += keylen;
+    string val = string(p);
+    wdb->set_metadata(key, val);
+}
+
+void 
+RemoteServer::msg_addspelling(const string & message)
+{
+    if (!wdb)
+        throw Xapian::InvalidOperationError("Server is read-only");
+    const char *p = message.data();
+    const char *p_end = p + message.size();
+    Xapian::termcount freq = decode_length(&p, p_end, false);
+    string term = string(p, p_end - p);
+    wdb->add_spelling(term, freq);
+}
+
+void 
+RemoteServer::msg_removespelling(const string & message)
+{
+    if (!wdb)
+        throw Xapian::InvalidOperationError("Server is read-only");
+    const char *p = message.data();
+    const char *p_end = p + message.size();
+    Xapian::termcount freqdec = decode_length(&p, p_end, false);
+    wdb->remove_spelling(string(p, p_end - p), freqdec);
+}
+
+void
+RemoteServer::msg_getspellingsuggestion(const string & message)
+{
+    const char *p = message.data();
+    const char *p_end = p + message.size();
+    size_t dist = decode_length(&p, p_end, false);
+    string word = string(p);
+    send_message(REPLY_SPELLINGSUGGESTION, db->get_spelling_suggestion(word, dist));
+}
+
+void 
+RemoteServer::msg_openspellingwordlist(const std::string& message)
+
+{
+    // FIXME - we should possibly not be sending the words in one go here?
+    spelling_term_or_wordlist(message, true);
+}
+
+void
+RemoteServer::msg_openspellingtermlist(const std::string& message)
+{
+    spelling_term_or_wordlist(message, false);
+}
+
+void
+RemoteServer::spelling_term_or_wordlist(const std::string& message, bool words)
+{
+
+    // this code is basically cut and paste from
+    // Database::get_spelling_suggestion - if I knew where to put stuff
+    // I'd factor out the common code.
+    
+    // n.b. this uses internal. Comments up the code suggest this is a
+    // bad thing - but I don't see how we're going to get hold of the
+    // spellings otherwise.
+    
+    AutoPtr<TermList> merger;
+    for (size_t i = 0; i < db->internal.size(); ++i) {
+        TermList * tl = (words?
+                         db->internal[i]->open_spelling_wordlist() :
+                         db->internal[i]->open_spelling_termlist(message)) ;
+        if (tl) {
+            if (merger.get()) {
+                if (words) 
+                    merger.reset(new OrTermList(merger.release(), tl));
+                else
+                    merger.reset(new FreqAdderOrTermList(merger.release(), tl));
+            } else {
+                merger.reset(tl);
+            }
+        }
+    }
+
+    string data;
+    int count = 0;
+    // is it OK to send these in one shot?
+    if (merger.get()) {
+        while (true) {
+            TermList * t = merger->next();
+            if (t) merger.reset(t);
+            if (merger->at_end()) break;
+            string termname = merger->get_termname();
+            data += encode_length(termname.length());
+            data += termname;
+            Xapian::termcount score = (words?
+                                       merger->get_termfreq():
+                                       merger->get_wdf());
+            data += encode_length(score);
+            ++count;
+        }
+    }
+  
+    send_message(REPLY_SPELLINGTERMS, encode_length(count)+data) ;
+}
+
+
+void
+RemoteServer::msg_getspellingfreq(const string & message)
+{
+    Xapian::doccount freq = 0;
+    for (size_t j = 0; j < db->internal.size(); ++j)
+        freq += db->internal[j]->get_spelling_frequency(message);
+    send_message(REPLY_SPELLINGFREQ, encode_length(freq));
+}

xapian-core/common/remote-database.h

@@ -89 +89 @@
 
     void update_stats(message_type msg_code = MSG_UPDATE) const;
 
+    // get spelling word or termlist;
+    TermList * receive_spelling_list(bool) const;
+
   protected:
     /** Constructor.  The constructor is protected so that raw instances
      *  can't be created - a derived class must be instantiated which
@@ -241 +244 @@
                                    const Xapian::Document & document);
 
     std::string get_uuid() const;
+
+    string get_metadata(const string & key) const;
+    void set_metadata(const string & key, const string & value);
+    string get_spelling_suggestion(const string &word,
+                                   unsigned max_edit_distance);
+    TermList * open_spelling_termlist(const string& word) const;
+    TermList * open_spelling_wordlist() const;
+
+    void add_spelling(const std::string&, Xapian::termcount) const;
+    void remove_spelling(const std::string&,  Xapian::termcount freqdec) const;
+    Xapian::doccount get_spelling_frequency(const string & word) const;
 };
-
 #endif // XAPIAN_INCLUDED_REMOTE_DATABASE_H

xapian-core/common/remoteprotocol.h

@@ -73 +73 @@
     MSG_REPLACEDOCUMENTTERM,    // Replace Document by term
     MSG_DELETEDOCUMENT,         // Delete Document
     MSG_WRITEACCESS,            // Upgrade to WritableDatabase
+    MSG_GETMETADATA,            // get metadata
+    MSG_SETMETADATA,            // set metadata
+    MSG_ADDSPELLING,            // add a spelling
+    MSG_REMOVESPELLING,            // add a spelling
+    MSG_GETSPELLINGSUGGESTION,  // get a spelling suggestion
+    MSG_OPENSPELLINGTERMLIST,   // get spelling termlist
+    MSG_OPENSPELLINGWORDLIST,   // get spelling wordlist
+    MSG_GETSPELLINGFREQ,        // get spelling frequency
     MSG_GETMSET,                // Get MSet
     MSG_SHUTDOWN,               // Shutdown
     MSG_MAX
@@ -100 +108 @@
     REPLY_VALUE,                // Document Value
     REPLY_ADDDOCUMENT,          // Add Document
     REPLY_RESULTS,              // Results (MSet)
+    REPLY_METADATA,             // Metadata
+    REPLY_SPELLINGSUGGESTION,   // Get Spelling
+    REPLY_SPELLINGTERMS,        // Spelling terms
+    REPLY_SPELLINGFREQ,         // Spelling frequency
     REPLY_MAX
 };
 

xapian-core/common/remoteserver.h

@@ -69 +69 @@
     /// The registry, which allows unserialisation of user subclasses.
     Xapian::Registry reg;
 
+    /// construct spelling termlist or wordlist
+    void spelling_term_or_wordlist (const std::string& message, bool words);
+
     /// Accept a message from the client.
     message_type get_message(Xapian::timeout timeout, std::string & result,
                              message_type required_type = MSG_MAX);
@@ -142 +145 @@
     // replace document with unique term
     void msg_replacedocumentterm(const std::string & message);
 
+    // get metadata
+    void msg_getmetadata(const std::string & message);
+
+    // set metadata
+    void msg_setmetadata(const std::string & message);
+
+    // add a spelling
+    void msg_addspelling(const std::string & message);
+
+    // remove a spelling
+    void msg_removespelling(const std::string & message);
+
+    // get spellings
+    void msg_getspellingsuggestion(const std::string & message);
+
+    // open spelling termlist
+    void msg_openspellingtermlist(const std::string&);
+
+    // open spelling wordlist
+    void msg_openspellingwordlist(const std::string&);
+
+    // get spelling frequency
+    void msg_getspellingfreq(const std::string&);
+
   public:
     /** Construct a RemoteServer.
      *

xapian-core/api/registry.cc

@@ -194 +194 @@
     Xapian::MatchSpy * spy;
     spy = new Xapian::ValueCountMatchSpy();
     matchspies[spy->name()] = spy;
+
+    Xapian::MatchSpy * mvspy;
+    mvspy = new Xapian::MultiValueCountMatchSpy();
+    matchspies[mvspy->name()] = mvspy;
 }
 
 void

xapian-core/backends/remote/remote-database.cc

@@ -723 +723 @@
 {
     return uuid;
 }
+
+string
+RemoteDatabase::get_metadata(const string & key) const
+{
+    send_message(MSG_GETMETADATA, key);
+    string metadata;
+    get_message(metadata, REPLY_METADATA);
+    return metadata;
+}
+
+void
+RemoteDatabase::set_metadata(const string & key, const string & value)
+{
+    string data = encode_length(key.size());
+    data += key;
+    data += value;
+    send_message(MSG_SETMETADATA, data);
+}
+
+void
+RemoteDatabase::add_spelling(const std::string & word,
+                             Xapian::termcount freqinc) const
+{
+    string data = encode_length(freqinc);
+    data += word;
+    send_message(MSG_ADDSPELLING, data);
+}
+
+void
+RemoteDatabase::remove_spelling(const std::string & word,
+                                Xapian::termcount freqdec) const
+{
+    string data = encode_length(freqdec);
+    data += word;
+    send_message(MSG_REMOVESPELLING, data);
+}
+
+string 
+RemoteDatabase::get_spelling_suggestion(const string & word,
+                                        unsigned max_edit_distance)
+{
+    string data = encode_length(max_edit_distance);
+    data += word;
+    send_message(MSG_GETSPELLINGSUGGESTION, data);
+    
+    string message;
+    get_message(message, REPLY_SPELLINGSUGGESTION);
+    return message;
+}
+
+TermList *
+RemoteDatabase::open_spelling_wordlist() const{
+    send_message(MSG_OPENSPELLINGWORDLIST, "");
+    return receive_spelling_list(true);
+}
+
+TermList *
+RemoteDatabase::open_spelling_termlist(const string& word) const {
+    send_message(MSG_OPENSPELLINGTERMLIST, word);
+    // FIXME - gets them all - probably not what's wanted?
+    return receive_spelling_list(false);
+}
+
+TermList *
+RemoteDatabase::receive_spelling_list(bool words) const { 
+    string message;
+    get_message(message, REPLY_SPELLINGTERMS);
+    const char * p = message.data();
+    const char * p_end = p + message.size();
+    int item_count = decode_length(&p, p_end, false);
+    AutoPtr<NetworkTermList> tlist(
+          new NetworkTermList(0, item_count,
+                              Xapian::Internal::RefCntPtr<const RemoteDatabase>(this),
+                              0));
+    vector<NetworkTermListItem> & items = tlist->items;
+
+    while (p < p_end) {
+        size_t tlen = decode_length(&p, p_end, false);
+        NetworkTermListItem item;
+        item.tname.assign(p, tlen);
+        p += tlen;
+        Xapian::termcount score = decode_length(&p, p_end, false);
+        if (words)
+            item.termfreq = score;
+        else
+            item.wdf = score;
+        items.push_back(item);
+    }
+    tlist->current_position = tlist->items.begin();
+    return tlist.release();
+}
+
+Xapian::doccount
+RemoteDatabase::get_spelling_frequency(const string & word) const 
+{
+    send_message(MSG_GETSPELLINGFREQ, word);
+    string message;
+    get_message(message, REPLY_SPELLINGFREQ);
+    const char * p = message.data();
+    const char * p_end = p + message.size();
+    Xapian::doccount freq = decode_length(&p, p_end, false);
+    return freq;
+}