Ticket #562: modified.patch

xapian-core/include/xapian/queryparser.h

@@ -409 +409 @@
         FLAG_DEFAULT = FLAG_PHRASE|FLAG_BOOLEAN|FLAG_LOVEHATE
     } feature_flag;
 
-    typedef enum { STEM_NONE, STEM_SOME, STEM_ALL } stem_strategy;
+    typedef enum { STEM_NONE, STEM_SOME, STEM_ALL, STEM_ALL_Z } stem_strategy;
 
     /// Copy constructor.
     QueryParser(const QueryParser & o);
@@ -452 +452 @@
      *                terms are prefixed with 'Z'.  (default in Xapian > 1.3.1)
      *   - STEM_ALL:  Search for stemmed forms of all words (note: no 'Z'
      *                prefix is added).
+     *   - STEM_ALL_Z:  Search for stemmed forms of all words (note: 'Z'
+     *                prefix is added).
      */
     void set_stemming_strategy(stem_strategy strategy);
 

xapian-core/include/xapian/termgenerator.h

@@ -88 +88 @@
         FLAG_SPELLING = 128 // Value matches QueryParser flag.
     };
 
+    /** Modes for indexing text 
+     * Following option provides matching modes for QueryParser and TermGenerator.
+     */
+    typedef enum { STEM_NONE, STEM_SOME, STEM_ALL, STEM_ALL_Z } stem_strategy;
+
     /** Set flags.
      *
      *  The new value of flags is: (flags & mask) ^ toggle
@@ -102 +107 @@
      */
     flags set_flags(flags toggle, flags mask = flags(0));
 
+    /** Set the mode for indexing. (default is STEM_SOME)
+     *
+     *  @param strategy The strategy to use - possible values are:
+     *  - STEM_NONE: Don't index any stemmed word
+     *  - STEM_SOME: Index both stemmed as well as full (non-stemmed) words.(note:
+     *                prefix "Z" is present in front of the stemmed words). 
+     *  - STEM_ALL:  Index only stemmed words.(note: stemmed words DONT have "Z" prefix).
+     *  - STEM_ALL_Z:  Index only stemmed words. (note: stemmed words have "Z" prefix).
+     */
+    void set_stemming_strategy(stem_strategy strategy);
+
     /** Index some text.
      *
      * @param itor      Utf8Iterator pointing to the text to index.

xapian-core/queryparser/queryparser.lemony

@@ -291 +291 @@
 Term::make_term(const string & prefix) const
 {
     string term;
-    if (stem == QueryParser::STEM_SOME) term += 'Z';
+    if (stem == QueryParser::STEM_SOME || stem == QueryParser::STEM_ALL_Z) term += 'Z';
     if (!prefix.empty()) {
         term += prefix;
         if (prefix_needs_colon(prefix, name[0])) term += ':';

xapian-core/queryparser/termgenerator.cc

@@ -83 +83 @@
 }
 
 void
+TermGenerator::set_stemming_strategy(stem_strategy strategy)
+{
+    internal->stem_action = strategy;
+}
+
+void
 TermGenerator::index_text(const Xapian::Utf8Iterator & itor,
                           Xapian::termcount weight,
                           const string & prefix)

xapian-core/queryparser/termgenerator_internal.cc

@@ -177 +177 @@
                     if (stop_mode == STOPWORDS_IGNORE && (*stopper)(cjk_token))
                         continue;
 
-                    if (with_positions && tk.get_length() == 1) {
-                        doc.add_posting(prefix + cjk_token, ++termpos, wdf_inc);
-                    } else {
-                        doc.add_term(prefix + cjk_token, wdf_inc);
+                    if(stem_action == TermGenerator::STEM_SOME || stem_action == TermGenerator::STEM_NONE) {
+                        if (with_positions && tk.get_length() == 1) {
+                            doc.add_posting(prefix + cjk_token, ++termpos, wdf_inc);
+                        } else {
+                            doc.add_term(prefix + cjk_token, wdf_inc);
+                        }
                     }
+                    
                     if ((flags & FLAG_SPELLING) && prefix.empty())
                         db.add_spelling(cjk_token);
 
@@ -196 +199 @@
                     if (!should_stem(cjk_token)) continue;
 
                     // Add stemmed form without positional information.
-                    string stem("Z");
-                    stem += prefix;
-                    stem += stemmer(cjk_token);
-                    doc.add_term(stem, wdf_inc);
+                    if (stem_action != TermGenerator::STEM_NONE) {
+                        string stem("");
+                        if(stem_action != TermGenerator::STEM_ALL) {    
+                                stem += "Z";
+                        } 
+                        stem += prefix;
+                        stem += stemmer(cjk_token);
+                        doc.add_term(stem, wdf_inc);
+                   }
                 }
                 while (true) {
                     if (itor == Utf8Iterator()) return;
@@ -258 +266 @@
 
         if (stop_mode == STOPWORDS_IGNORE && (*stopper)(term)) continue;
 
-        if (with_positions) {
-            doc.add_posting(prefix + term, ++termpos, wdf_inc);
-        } else {
-            doc.add_term(prefix + term, wdf_inc);
+        if(stem_action == TermGenerator::STEM_SOME || stem_action == TermGenerator::STEM_NONE) {
+                if (with_positions) {
+                    doc.add_posting(prefix + term, ++termpos, wdf_inc);
+                } else {
+                    doc.add_term(prefix + term, wdf_inc);
+                }
         }
         if ((flags & FLAG_SPELLING) && prefix.empty()) db.add_spelling(term);
 
@@ -275 +285 @@
         if (!should_stem(term)) continue;
 
         // Add stemmed form without positional information.
-        string stem("Z");
-        stem += prefix;
-        stem += stemmer(term);
-        doc.add_term(stem, wdf_inc);
+        if (stem_action != TermGenerator::STEM_NONE) {
+                string stem("");
+                if(stem_action != TermGenerator::STEM_ALL) {    
+                        stem += "Z";
+                } 
+                stem += prefix;
+                stem += stemmer(term);
+                doc.add_term(stem, wdf_inc);
+        }
     }
 }
 

xapian-core/queryparser/termgenerator_internal.h

@@ -34 +34 @@
 class TermGenerator::Internal : public Xapian::Internal::intrusive_base {
     friend class TermGenerator;
     Stem stemmer;
+    stem_strategy stem_action;
     const Stopper * stopper;
     Document doc;
     termcount termpos;
@@ -41 +42 @@
     WritableDatabase db;
 
   public:
-    Internal() : stopper(NULL), termpos(0),
+    Internal() : stem_action(STEM_SOME), stopper(NULL), termpos(0),
         flags(TermGenerator::flags(0)) { }
     void index_text(Utf8Iterator itor,
                     termcount weight,