Ticket #225: spelling_frequency_updated.patch

api/omdatabase.cc

@@ -43 +43 @@
 #include "ortermlist.h"
 #include "noreturn.h"
 
+#include <cmath> // For pow().
 #include <cstdlib> // For abs().
 
 #include <cstring>
@@ -574 +575 @@
     string result;
     int edist_best = max_edit_distance;
     Xapian::doccount freq_best = 0;
+    Xapian::doccount freq_best_edist = 0;
     while (true) {
         TermList *ret = merger->next();
         if (ret) merger.reset(ret);
@@ -593 +595 @@
 
             // First check the length of the encoded UTF-8 version of term.
             // Each UTF-32 character is 1-4 bytes in UTF-8.
-            if (abs(long(term.size()) - long(word.size())) > edist_best * 4) {
+            if (abs(long(term.size()) - long(word.size())) > max_edit_distance * 4) {
                 LOGLINE(SPELLING, "Lengths much too different");
                 continue;
             }
@@ -603 +605 @@
             utf32_term.assign(Utf8Iterator(term), Utf8Iterator());
 
             if (abs(long(utf32_term.size()) - long(utf32_word.size()))
-                    > edist_best) {
+                    > max_edit_distance) {
                 LOGLINE(SPELLING, "Lengths too different");
                 continue;
             }
@@ -613 +615 @@
                 continue;
             }
 
+            if ((long)utf32_term.size() <= TRIGRAM_SCORE_THRESHOLD + 1) {
+                LOGLINE(SPELLING, "Term too short");
+                continue;
+            }
+
             int edist = edit_distance_unsigned(&utf32_term[0],
                                                int(utf32_term.size()),
                                                &utf32_word[0],
                                                int(utf32_word.size()),
                                                edist_best);
             LOGLINE(SPELLING, "Edit distance " << edist);
-            // If we have an exact match, return an empty string since there's
-            // no correction required.
-            if (edist == 0) RETURN(string());
 
-            if (edist <= edist_best) {
+            {
                 Xapian::doccount freq = 0;
                 for (size_t j = 0; j < internal.size(); ++j)
                     freq += internal[j]->get_spelling_frequency(term);
 
                 LOGLINE(SPELLING, "Freq " << freq << " best " << freq_best);
-                if (edist < edist_best || freq > freq_best) {
+                if (edist <= edist_best && (freq > (max_edit_distance - edist_best)*freq_best_edist)) {
                     LOGLINE(SPELLING, "Best so far: \"" << term <<
                                       "\" edist " << edist << " freq " << freq);
                     result = term;
                     edist_best = edist;
                     freq_best = freq;
+                    freq_best_edist = freq;
+                }
+                else if ((freq > pow(freq_best_edist, edist+2)) &&
+                         ((edist - 2) < edist_best) &&
+                         (freq > freq_best)) { 
+                    LOGLINE(SPELLING, "Best so far: \"" << term <<
+                                        " freq " <<
+                                        freq);
+                    result = term;
+                    freq_best = freq;
                 }
             }
         }