Ticket #178: remote-spell-remainder.patch

xapian-core/tests/api_spelling.cc

@@ -182 +182 @@
 }
 
 // Test spelling correction with multi databases
-DEFINE_TESTCASE(spell3, spelling) {
+DEFINE_TESTCASE(spell3, spelling && !remote) {
+    // Spelling iterator not implemented for remote databases.
     Xapian::WritableDatabase db1 = get_writable_database();
     // We can't just call get_writable_database() since it would delete db1
     // which doesn't work at all under __WIN32__ and will go wrong elsewhere if

xapian-core/tests/harness/testrunner.cc

@@ -62 +62 @@
     { "multi_brass", "backend,positional,valuestats,multi" },
     { "multi_chert", "backend,positional,valuestats,multi" },
     { "multi_flint", "backend,positional,multi" },
-    { "remoteprog_brass", "backend,remote,transactions,positional,valuestats,writable,metadata" },
-    { "remotetcp_brass", "backend,remote,transactions,positional,valuestats,writable,metadata" },
-    { "remoteprog_chert", "backend,remote,transactions,positional,valuestats,writable,metadata" },
-    { "remotetcp_chert", "backend,remote,transactions,positional,valuestats,writable,metadata" },
-    { "remoteprog_flint", "backend,remote,transactions,positional,writable,metadata" },
-    { "remotetcp_flint", "backend,remote,transactions,positional,writable,metadata" },
+    { "remoteprog_brass", "backend,remote,transactions,positional,valuestats,writable,metadata,spelling" },
+    { "remotetcp_brass", "backend,remote,transactions,positional,valuestats,writable,metadata,spelling" },
+    { "remoteprog_chert", "backend,remote,transactions,positional,valuestats,writable,metadata,spelling" },
+    { "remotetcp_chert", "backend,remote,transactions,positional,valuestats,writable,metadata,spelling" },
+    { "remoteprog_flint", "backend,remote,transactions,positional,writable,metadata,spelling" },
+    { "remotetcp_flint", "backend,remote,transactions,positional,writable,metadata,spelling" },
     { NULL, NULL }
 };
 

xapian-core/net/remoteserver.cc

@@ -40 +40 @@
 #include "serialise-double.h"
 #include "utils.h"
 #include "weightinternal.h"
+#include "database.h" // so we can use internal in msg_openspellingtermlist
 
 /// Class to throw when we receive the connection closing message.
 struct ConnectionClosed { };
@@ -189 +190 @@
                 &RemoteServer::msg_setmetadata,
                 &RemoteServer::msg_addspelling,
                 &RemoteServer::msg_removespelling,
+                &RemoteServer::msg_getspellingsuggestion,
+                &RemoteServer::msg_getspellingfreq
                 // MSG_GETMSET - used during a conversation.
                 // MSG_SHUTDOWN - handled by get_message().
             };
@@ -704 +707 @@
     Xapian::termcount freqdec = decode_length(&p, p_end, false);
     wdb->remove_spelling(string(p, p_end - p), freqdec);
 }
+
+void
+RemoteServer::msg_getspellingsuggestion(const string & message)
+{
+    const char *p = message.data();
+    const char *p_end = p + message.size();
+    size_t dist = decode_length(&p, p_end, false);
+    string word(p, p_end - p);
+    send_message(REPLY_SPELLINGSUGGESTION, db->get_spelling_suggestion(word, dist));
+}
+
+void
+RemoteServer::msg_getspellingfreq(const string & message)
+{
+    Xapian::doccount freq = 0;
+    for (size_t j = 0; j < db->internal.size(); ++j)
+        freq += db->internal[j]->get_spelling_frequency(message);
+    send_message(REPLY_SPELLINGFREQ, encode_length(freq));
+}

xapian-core/common/remote-database.h

@@ -249 +249 @@
     void add_spelling(const std::string&, Xapian::termcount) const;
 
     void remove_spelling(const std::string&,  Xapian::termcount freqdec) const;
+
+    std::string get_spelling_suggestion(const std::string &word,
+                                        unsigned max_edit_distance);
+
+    Xapian::doccount get_spelling_frequency(const std::string & word) const;
 };
 
 #endif // XAPIAN_INCLUDED_REMOTE_DATABASE_H

xapian-core/common/remoteprotocol.h

@@ -44 +44 @@
 // 33: 1.1.3 Support for passing matchspies over the remote connection.
 // 34: 1.1.4 Support for metadata over with remote databases.
 // 35: 1.1.5 Support for add_spelling() and remove_spelling().
-#define XAPIAN_REMOTE_PROTOCOL_MAJOR_VERSION 35
+// 36: ????? Full remote spelling support.
+#define XAPIAN_REMOTE_PROTOCOL_MAJOR_VERSION 36
 #define XAPIAN_REMOTE_PROTOCOL_MINOR_VERSION 0
 
 /** Message types (client -> server).
@@ -79 +80 @@
     MSG_SETMETADATA,            // Set metadata
     MSG_ADDSPELLING,            // Add a spelling
     MSG_REMOVESPELLING,         // Remove a spelling
+    MSG_GETSPELLINGSUGGESTION,  // Get a spelling suggestion
+    MSG_GETSPELLINGFREQ,        // Get spelling frequency
     MSG_GETMSET,                // Get MSet
     MSG_SHUTDOWN,               // Shutdown
     MSG_MAX
@@ -107 +110 @@
     REPLY_ADDDOCUMENT,          // Add Document
     REPLY_RESULTS,              // Results (MSet)
     REPLY_METADATA,             // Metadata
+    REPLY_SPELLINGSUGGESTION,   // Get Spelling Suggestion
+    REPLY_SPELLINGFREQ,         // Spelling frequency
     REPLY_MAX
 };
 

xapian-core/common/remoteserver.h

@@ -154 +154 @@
     // remove a spelling
     void msg_removespelling(const std::string & message);
 
+    // get spellings
+    void msg_getspellingsuggestion(const std::string & message);
+
+    // get spelling frequency
+    void msg_getspellingfreq(const std::string & message);
+
   public:
     /** Construct a RemoteServer.
      *

xapian-core/backends/remote/remote-database.cc

@@ -759 +759 @@
     data += word;
     send_message(MSG_REMOVESPELLING, data);
 }
+
+string
+RemoteDatabase::get_spelling_suggestion(const string & word,
+                                        unsigned max_edit_distance)
+{
+    string data = encode_length(max_edit_distance);
+    data += word;
+    send_message(MSG_GETSPELLINGSUGGESTION, data);
+
+    string message;
+    get_message(message, REPLY_SPELLINGSUGGESTION);
+    return message;
+}
+
+Xapian::doccount
+RemoteDatabase::get_spelling_frequency(const string & word) const
+{
+    send_message(MSG_GETSPELLINGFREQ, word);
+    string message;
+    get_message(message, REPLY_SPELLINGFREQ);
+    const char * p = message.data();
+    const char * p_end = p + message.size();
+    return decode_length(&p, p_end, false);
+}

xapian-core/api/omdatabase.cc

@@ -42 +42 @@
 #include "editdistance.h"
 #include "ortermlist.h"
 #include "noreturn.h"
+#include "remote-database.h"
 
 #include <cstdlib> // For abs().
 
@@ -498 +499 @@
     return "Database()";
 }
 
-// We sum the character frequency histogram absolute differences to compute a
-// lower bound on the edit distance.  Rather than counting each Unicode code
-// point uniquely, we use an array with VEC_SIZE elements and tally code points
-// modulo VEC_SIZE which can only reduce the bound we calculate.
-//
-// There will be a trade-off between how good the bound is and how large and
-// array is used (a larger array takes more time to clear and sum over).  The
-// value 64 is somewhat arbitrary - it works as well as 128 for the testsuite
-// but that may not reflect real world performance.  FIXME: profile and tune.
-
-#define VEC_SIZE 64
-
-static int
-freq_edit_lower_bound(const vector<unsigned> & a, const vector<unsigned> & b)
-{
-    int vec[VEC_SIZE];
-    memset(vec, 0, sizeof(vec));
-    vector<unsigned>::const_iterator i;
-    for (i = a.begin(); i != a.end(); ++i) {
-        ++vec[(*i) % VEC_SIZE];
-    }
-    for (i = b.begin(); i != b.end(); ++i) {
-        --vec[(*i) % VEC_SIZE];
-    }
-    unsigned int total = 0;
-    for (size_t j = 0; j < VEC_SIZE; ++j) {
-        total += abs(vec[j]);
-    }
-    // Each insertion or deletion adds at most 1 to total.  Each transposition
-    // doesn't change it at all.  But each substitution can change it by 2 so
-    // we need to divide it by 2.  Rounding up is OK, since the odd change must
-    // be due to an actual edit.
-    return (total + 1) / 2;
-}
-
 // Word must have a trigram score at least this close to the best score seen
 // so far.
 #define TRIGRAM_SCORE_THRESHOLD 2
@@ -545 +511 @@
                  word << ", " << max_edit_distance);
     if (word.size() <= 1) return string();
     AutoPtr<TermList> merger;
+    bool got_remote = false;
     for (size_t i = 0; i < internal.size(); ++i) {
-        TermList * tl = internal[i]->open_spelling_termlist(word);
-        LOGLINE(SPELLING, "Sub db " << i << " tl = " << (void*)tl);
-        if (tl) {
-            if (merger.get()) {
-                merger.reset(new OrTermList(merger.release(), tl));
-            } else {
-                merger.reset(tl);
-            }
-        }
+        RemoteDatabase *rdb = internal[i].get()->as_remotedatabase();
+        if (!rdb) {
+            TermList * tl = internal[i]->open_spelling_termlist(word);
+            LOGLINE(SPELLING, "Sub db " << i << " tl = " << (void*)tl);
+            if (tl) {
+                if (merger.get()) {
+                    merger.reset(new OrTermList(merger.release(), tl));
+                } else {
+                    merger.reset(tl);
+                }
+            }
+        } else 
+            got_remote = true;
     }
-    if (!merger.get()) RETURN(string());
 
-    // Convert word to UTF-32.
-#ifdef __SUNPRO_CC
-    vector<unsigned> utf32_word;
-    for (Utf8Iterator sunpro_it(word); sunpro_it != Utf8Iterator(); ++sunpro_it) {
-        utf32_word.push_back(*sunpro_it);
+    if (! (merger.get() or got_remote)) {
+      RETURN(string());
     }
-#else
-    // Extra brackets needed to avoid this being misparsed as a function
-    // prototype.
-    vector<unsigned> utf32_word((Utf8Iterator(word)), Utf8Iterator());
-#endif
+    
+    EditDistance edcomp(word);
 
-    vector<unsigned> utf32_term;
-
     Xapian::termcount best = 1;
     string result;
     int edist_best = max_edit_distance;
     Xapian::doccount freq_best = 0;
-    while (true) {
-        TermList *ret = merger->next();
-        if (ret) merger.reset(ret);
+    if (merger.get()) {
+        while (true) {
+            TermList *ret = merger->next();
+            if (ret) merger.reset(ret);
 
-        if (merger->at_end()) break;
+            if (merger->at_end()) break;
 
-        string term = merger->get_termname();
-        Xapian::termcount score = merger->get_wdf();
+            string term = merger->get_termname();
+            Xapian::termcount score = merger->get_wdf();
 
-        LOGLINE(SPELLING, "Term \"" << term << "\" ngram score " << score);
-        if (score + TRIGRAM_SCORE_THRESHOLD >= best) {
-            if (score > best) best = score;
+            LOGLINE(SPELLING, "Term \"" << term << "\" ngram score " << score);
+            if (score + TRIGRAM_SCORE_THRESHOLD >= best) {
+                if (score > best) best = score;
 
-            // There's no point considering a word where the difference
-            // in length is greater than the smallest number of edits we've
-            // found so far.
+                // There's no point considering a word where the difference
+                // in length is greater than the smallest number of edits we've
+                // found so far.
+            
+                int edist = edcomp.distance(term, edist_best);
 
-            // First check the length of the encoded UTF-8 version of term.
-            // Each UTF-32 character is 1-4 bytes in UTF-8.
-            if (abs(long(term.size()) - long(word.size())) > edist_best * 4) {
-                LOGLINE(SPELLING, "Lengths much too different");
-                continue;
-            }
+                if (edist > edist_best) continue;
 
-            // Now convert to UTF-32, and compare the true lengths more
-            // strictly.
-            utf32_term.assign(Utf8Iterator(term), Utf8Iterator());
-
-            if (abs(long(utf32_term.size()) - long(utf32_word.size()))
-                    > edist_best) {
-                LOGLINE(SPELLING, "Lengths too different");
-                continue;
+                LOGLINE(SPELLING, "Edit distance " << edist);
+                // If we have an exact match, return an empty string since there's
+                // no correction required.
+                if (edist == 0) RETURN(string());
+                if (edist <= edist_best) {
+                    Xapian::doccount freq = 0;
+                    for (size_t j = 0; j < internal.size(); ++j)
+                        // this includes calls on get_spelling_frequency
+                        // for remote databases
+                        freq += internal[j]->get_spelling_frequency(term);
+                    LOGLINE(SPELLING, "Freq " << freq << " best " << freq_best);
+                    if (edist < edist_best || freq > freq_best) {
+                        LOGLINE(SPELLING, "Best so far: \"" << term <<
+                                "\" edist " << edist << " freq " << freq);
+                        result = term;
+                        edist_best = edist;
+                        freq_best = freq;
+                    }
+                }
             }
+        }
+    }
 
-            if (freq_edit_lower_bound(utf32_term, utf32_word) > edist_best) {
-                LOGLINE(SPELLING, "Rejected by character frequency test");
-                continue;
-            }
-
-            int edist = edit_distance_unsigned(&utf32_term[0],
-                                               int(utf32_term.size()),
-                                               &utf32_word[0],
-                                               int(utf32_word.size()),
-                                               edist_best);
-            LOGLINE(SPELLING, "Edit distance " << edist);
-            // If we have an exact match, return an empty string since there's
-            // no correction required.
-            if (edist == 0) RETURN(string());
-
-            if (edist <= edist_best) {
-                Xapian::doccount freq = 0;
-                for (size_t j = 0; j < internal.size(); ++j)
-                    freq += internal[j]->get_spelling_frequency(term);
-
-                LOGLINE(SPELLING, "Freq " << freq << " best " << freq_best);
-                if (edist < edist_best || freq > freq_best) {
-                    LOGLINE(SPELLING, "Best so far: \"" << term <<
-                                      "\" edist " << edist << " freq " << freq);
-                    result = term;
-                    edist_best = edist;
-                    freq_best = freq;
+    // we've got the best we can locally which - it there's no
+    // correction need means we won't need to check remote
+    // databases. If we get this far we try corrections from remote
+    // databases.
+    for (size_t i = 0; i < internal.size(); ++i) {
+        RemoteDatabase *rdb = internal[i].get()->as_remotedatabase();
+        if (rdb) {
+            string rterm = rdb->get_spelling_suggestion(word, edist_best);
+            if (rterm != "") {
+                // FIXME: repetitious of stuff in the loop above - should be factored out.
+                int edist = edcomp.distance(rterm, edist_best);
+                if (edist == 0) RETURN(string());
+                if (edist <= edist_best) {
+                    Xapian::doccount freq = 0;
+                    for (size_t j = 0; j < internal.size(); ++j)
+                        freq += internal[j]->get_spelling_frequency(rterm);
+                    LOGLINE(SPELLING, "Freq " << freq << " best " << freq_best);
+                    if (edist < edist_best || freq > freq_best) {
+                        LOGLINE(SPELLING, "Best so far: \"" << term <<
+                                "\" edist " << edist << " freq " << freq);
+                        result = rterm;
+                        edist_best = edist;
+                        freq_best = freq;
+                    }
                 }
             }
         }
     }
     RETURN(result);
 }

xapian-core/api/editdistance.cc

@@ -32 +32 @@
 #include "editdistance.h"
 
 #include "omassert.h"
+#include "debuglog.h"
 
+#include <xapian/unicode.h>
+
 #include <algorithm>
 #include <cstdlib>
-
+#include <cstring>
 using namespace std;
 
 template<class CHR>
@@ -213 +216 @@
 {
     return seqcmp_editdist<unsigned>(ptr1, len1, ptr2, len2, max_distance);
 }
+
+using namespace Xapian;
+
+EditDistance::EditDistance(const string& word_) 
+#ifdef __SUNPRO_CC
+ {
+     for (Utf8Iterator sunpro_it(word); sunpro_it != Utf8Iterator(); ++sunpro_it) {
+         utf32_word.push_back(*sunpro_it);
+     }
+ }
+#else
+ 
+  :word(word_), utf32_word(Utf8Iterator(word_), Utf8Iterator()) {}
+
+#endif
+
+
+// We sum the character frequency histogram absolute differences to compute a
+// lower bound on the edit distance.  Rather than counting each Unicode code
+// point uniquely, we use an array with VEC_SIZE elements and tally code points
+// modulo VEC_SIZE which can only reduce the bound we calculate.
+//
+// There will be a trade-off between how good the bound is and how large and
+// array is used (a larger array takes more time to clear and sum over).  The
+// value 64 is somewhat arbitrary - it works as well as 128 for the testsuite
+// but that may not reflect real world performance.  FIXME: profile and tune.
+
+#define VEC_SIZE 64
+
+static int
+freq_edit_lower_bound(const vector<unsigned> & a, const vector<unsigned> & b)
+{
+    int vec[VEC_SIZE];
+    memset(vec, 0, sizeof(vec));
+    vector<unsigned>::const_iterator i;
+    for (i = a.begin(); i != a.end(); ++i) {
+        ++vec[(*i) % VEC_SIZE];
+    }
+    for (i = b.begin(); i != b.end(); ++i) {
+        --vec[(*i) % VEC_SIZE];
+    }
+    unsigned int total = 0;
+    for (size_t j = 0; j < VEC_SIZE; ++j) {
+        total += abs(vec[j]);
+    }
+    // Each insertion or deletion adds at most 1 to total.  Each transposition
+    // doesn't change it at all.  But each substitution can change it by 2 so
+    // we need to divide it by 2.  Rounding up is OK, since the odd change must
+    // be due to an actual edit.
+    return (total + 1) / 2;
+}
+
+int 
+EditDistance::distance(const string& term,int limit) {
+    vector<unsigned> utf32_term;
+    // we use this to signal that the distance is clearly large and that we're 
+    // not going to work it out exactly
+    const int big_distance = 1000;
+
+    // First check the length of the encoded UTF-8 version of term.
+    // Each UTF-32 character is 1-4 bytes in UTF-8.
+    if (abs(long(term.size()) - long(word.size())) > limit * 4) {
+        LOGLINE(SPELLING, "Lengths much too different");
+        return big_distance;
+    }
+
+    // Now convert to UTF-32, and compare the true lengths more
+    // strictly.
+    utf32_term.assign(Utf8Iterator(term), Utf8Iterator());
+
+    if (abs(long(utf32_term.size()) - long(utf32_word.size()))
+        > limit) {
+        LOGLINE(SPELLING, "Lengths too different");
+        return big_distance;
+    }
+
+    if (freq_edit_lower_bound(utf32_term, utf32_word) > limit) {
+        LOGLINE(SPELLING, "Rejected by character frequency test");
+        return big_distance;
+    }
+
+    return edit_distance_unsigned(&utf32_term[0],
+                                  int(utf32_term.size()),
+                                  &utf32_word[0],
+                                  int(utf32_word.size()),
+                                  limit);
+}

xapian-core/api/editdistance.h

@@ -22 +22 @@
 #ifndef XAPIAN_INCLUDED_EDITDISTANCE_H
 #define XAPIAN_INCLUDED_EDITDISTANCE_H
 
+#include<string>
+#include<vector>
 /** Calculate the edit distance between two sequences.
  *
  *  Edit distance is defined as the minimum number of edit operations
@@ -49 +51 @@
                            const unsigned* ptr2, int len2,
                            int max_distance);
 
+
+class EditDistance {
+    const std::string word;
+    const std::vector<unsigned> utf32_word;
+
+ public:
+    EditDistance(const std::string& word_) ;
+    int distance(const std::string& other, int limit);
+};
 #endif // XAPIAN_INCLUDED_EDITDISTANCE_H