Ticket #225: spelling_frequency.diff

api/omdatabase.cc

@@ -43 +43 @@
 #include "noreturn.h"
 
 #include <stdlib.h> // For abs().
+#include <math.h> // For pow().
 
 #include <vector>
 
@@ -384 +385 @@
     string result;
     int edist_best = max_edit_distance;
     Xapian::doccount freq_best = 0;
+    Xapian::doccount freq_best_edist = 0;
     while (true) {
         TermList *ret = merger->next();
         if (ret) merger = ret;
@@ -403 +405 @@
 
             // First check the length of the encoded UTF-8 version of term.
             // Each UTF-32 character is 1-4 bytes in UTF-8.
-            if (abs((long)term.size() - (long)word.size()) > edist_best * 4) {
+            if (abs((long)term.size() - (long)word.size()) > max_edit_distance* 4) {
                 DEBUGLINE(SPELLING, "Lengths much too different");
                 continue;
             }
@@ -413 +415 @@
             utf32_term.assign(Utf8Iterator(term), Utf8Iterator());
 
             if (abs((long)utf32_term.size() - (long)utf32_word.size())
-                    > edist_best) {
+                    > max_edit_distance) {
                 DEBUGLINE(SPELLING, "Lengths too different");
                 continue;
             }
 
+            if ((long)utf32_term.size() <= TRIGRAM_SCORE_THRESHOLD + 1) {
+                DEBUGLINE(SPELLING, "Term too short");
+                continue;
+            }
+
             int edist = edit_distance_unsigned(&utf32_term[0],
                                                utf32_term.size(),
                                                &utf32_word[0],
                                                utf32_word.size());
             DEBUGLINE(SPELLING, "Edit distance " << edist);
-            // If we have an exact match, return an empty string since there's
-            // no correction required.
-            if (edist == 0) return string();
 
-            if (edist <= edist_best) {
                 Xapian::doccount freq = 0;
                 for (size_t j = 0; j < internal.size(); ++j)
                     freq += internal[j]->get_spelling_frequency(term);
 
                 DEBUGLINE(SPELLING, "Freq " << freq << " best " << freq_best);
-                if (edist < edist_best || freq > freq_best) {
+                if (edist <= edist_best && (freq > (max_edit_distance - edist_best)*freq_best_edist)) {
                     DEBUGLINE(SPELLING, "Best so far: \"" << term <<
                                         "\" edist " << edist << " freq " <<
                                         freq);
                     result = term;
                     edist_best = edist;
                     freq_best = freq;
+                    freq_best_edist = freq;
                 }
+                else if ((freq > pow(freq_best_edist, edist+2)) &&
+                         ((edist - 2) < edist_best) &&
+                         (freq > freq_best)) { 
+                    DEBUGLINE(SPELLING, "Best so far: \"" << term <<
+                                        " freq " <<
+                                        freq);
+                    result = term;
+                    freq_best = freq;
             }
         }
     }