Ticket #157: patch1.2

Index: queryparser/queryparser.lemony
===================================================================
--- queryparser/queryparser.lemony      (revision 8676)
+++ queryparser/queryparser.lemony      (working copy)
@@ -25,6 +25,7 @@
 #include "queryparser_internal.h"
 #include <xapian/unicode.h>
 #include "utils.h"
+#include "autoptr.h"

 // Include the list of token values lemon generates.
 #include "queryparser_token.h"
@@ -177,21 +178,25 @@
     bool stem;
     termpos pos;

-    std::string make_term() const;
-
   public:
     Term(const string &name_, termpos pos_) : name(name_), stem(false), pos(pos_) { }
     Term(const string &name_) : name(name_), stem(false), pos(0) { }
+    Term(const string &name_, const string &prefix_)
+       : name(name_), prefix(prefix_), stem(false), pos(0) { }
     Term(termpos pos_) : stem(false), pos(pos_) { }
     Term(State * state_, const string &name_, const string &prefix_,
         const string &unstemmed_, bool stem_, termpos pos_)
        : state(state_), name(name_), prefix(prefix_), unstemmed(unstemmed_),
          stem(stem_), pos(pos_) { }

+    std::string make_term() const;
+
     void dont_stem() { stem = false; }

     termpos get_termpos() const { return pos; }

+    string get_prefix() const { return prefix; }
+
     QpQuery * as_query() const { return new QpQuery(make_term(), 1, pos); }

     QpQuery * as_wildcarded_query(State * state) const;
@@ -589,17 +594,14 @@
                        // until the next space or ')' as part of the boolean
                        // term.
                        it = p;
-                       if (prefix_needs_colon(prefix, *it))
-                           prefix += ':';
-                       string term;
+                       string name;
                        while (it != end && *it > ' ' && *it != ')')
-                           Unicode::append_utf8(term, *it++);
-                       prefix += term;
+                           Unicode::append_utf8(name, *it++);
+                       AutoPtr<Term> term(new Term(name, prefix));
                        field += ':';
-                       field += term;
-                       unstem.insert(make_pair(prefix, field));
-                       Parse(pParser, BOOLEAN_FILTER, new Term(prefix),
-                             &state);
+                       field += name;
+                       unstem.insert(make_pair(term->make_term(), field));
+                       Parse(pParser, BOOLEAN_FILTER, term.release(), &state);
                        continue;
                    }

@@ -858,7 +860,21 @@
     QpQuery query;
     QpQuery love;
     QpQuery hate;
-    QpQuery filter;
+
+    // filter is a map from prefix to a query for that prefix.  Queries with
+    // the same prefix are combined with OR, and the results of this are
+    // combined with AND to get the full filter.
+    map<string, QpQuery> filter;
+
+    QpQuery merge_filters() const {
+       QpQuery q;
+        for (map<string, QpQuery>::const_iterator i = filter.begin();
+            i != filter.end(); ++i)
+       {
+           add_to_query(q, Query::OP_AND, i->second.get());
+       }
+       return q;
+    }
 };

 class TermList {
@@ -1078,10 +1094,10 @@
     // Handle any boolean filters.
     if (!P->filter.empty()) {
        if (E->empty()) {
-           *E = P->filter;
+           *E = P->merge_filters();
            // FIXME and make the query boolean somehow...
        } else {
-           *E = QpQuery(Query::OP_FILTER, *E, P->filter);
+           *E = QpQuery(Query::OP_FILTER, *E, P->merge_filters());
        }
     }
     // FIXME what if E->empty() (all terms are stopwords)?
@@ -1101,6 +1117,8 @@
 %destructor prob {delete $$;}

 prob(P) ::= RANGE_START(A) RANGE_END(B). {
+    string prefix = A->get_prefix();
+    Assert(prefix == B->get_prefix());
     QpQuery * range = state->value_range(A, B);
     if (range == NULL) {
        // Already deleted, so stop yy_parse_failed deleting them again.
@@ -1109,11 +1127,13 @@
        return;
     }
     P = new ProbQuery;
-    P->filter = *range;
+    P->filter[prefix] = *range;
     delete range;
 }

 prob(P) ::= stop_prob(Q) RANGE_START(A) RANGE_END(B). {
+    string prefix = A->get_prefix();
+    Assert(prefix == B->get_prefix());
     QpQuery * range = state->value_range(A, B);
     if (range == NULL) {
        // Already deleted, so stop yy_parse_failed deleting them again.
@@ -1122,7 +1142,7 @@
        return;
     }
     P = Q;
-    add_to_query(P->filter, Query::OP_AND, *range);
+    add_to_query(P->filter[prefix], Query::OP_AND, *range);
     delete range;
 }

@@ -1192,29 +1212,29 @@

 prob(P) ::= BOOLEAN_FILTER(T). {
     P = new ProbQuery;
-    P->filter = T->as_query_object();
+    P->filter[T->get_prefix()] = T->as_query_object();
     delete T;
 }

 prob(P) ::= stop_prob(Q) BOOLEAN_FILTER(T). {
     P = Q;
-    // FIXME we should OR filters with the same prefix...
-    add_to_query(P->filter, Query::OP_AND, T->as_query_object());
+    // We OR filters with the same prefix...
+    add_to_query(P->filter[T->get_prefix()], Query::OP_OR, T->as_query_object());
     delete T;
 }

 prob(P) ::= LOVE BOOLEAN_FILTER(T). {
     // LOVE BOOLEAN_FILTER(T) is just the same as BOOLEAN_FILTER
     P = new ProbQuery;
-    P->filter = T->as_query_object();
+    P->filter[T->get_prefix()] = T->as_query_object();
     delete T;
 }

 prob(P) ::= stop_prob(Q) LOVE BOOLEAN_FILTER(T). {
     // LOVE BOOLEAN_FILTER(T) is just the same as BOOLEAN_FILTER
     P = Q;
-    // FIXME we should OR filters with the same prefix...
-    add_to_query(P->filter, Query::OP_AND, T->as_query_object());
+    // We OR filters with the same prefix...
+    add_to_query(P->filter[T->get_prefix()], Query::OP_OR, T->as_query_object());
     delete T;
 }

Index: tests/queryparsertest.cc
===================================================================
--- tests/queryparsertest.cc    (revision 8676)
+++ tests/queryparsertest.cc    (working copy)
@@ -532,6 +532,24 @@
     { "- NEAR 12V voeding", "(near:(pos=1) OR 12v:(pos=2) OR Zvoed:(pos=3))" },
     { "waarom \"~\" in directorynaam", "(Zwaarom:(pos=1) OR Zin:(pos=2) OR Zdirectorynaam:(pos=3))" },
     { "cd'r NEAR toebehoren", "(cd'r:(pos=1) NEAR 11 toebehoren:(pos=2))" },
+    { "site:1 site:2", "(H1 OR H2)" },
+    { "site:1 site2:2", "(H1 AND J2)" },
+    { "site:1 site:2 site2:2", "((H1 OR H2) AND J2)" },
+    { "site:1 OR site:2", "(H1 OR H2)" },
+    { "site:1 AND site:2", "(H1 AND H2)" },
+#if 0
+    { "A site:1 site:2", "(a FILTER (H1 OR H2))" },
+    { "A (site:1 OR site:2)", "(a FILTER (H1 OR H2))" },
+    { "A (site:1 OR site:2)", "(a FILTER (H1 OR H2))" },
+    { "A site:1 site2:2", "(a FILTER (H1 AND J2))" },
+    { "A site:1 site:2 site2:2", "(a FILTER ((H1 OR H2) AND J2))" },
+    { "A site:1 OR site:2", "(a FILTER (H1 OR H2))" },
+    { "A site:1 AND site:2", "(a FILTER (H1 AND H2))" },
+#endif
+    { "site:xapian.org OR site:www.xapian.org", "(Hxapian.org OR Hwww.xapian.org)" },
+    { "site:xapian.org site:www.xapian.org", "(Hxapian.org OR Hwww.xapian.org)" },
+    { "site:xapian.org AND site:www.xapian.org", "(Hxapian.org AND Hwww.xapian.org)" },
+    { "Xapian site:xapian.org site:www.xapian.org", "(xapian:(pos=1) FILTER (Hxapian.org OR Hwww.xapian.org))" },
     { NULL, NULL }
 };

@@ -576,6 +594,7 @@
     queryparser.add_prefix("title", "XT");
     queryparser.add_prefix("subject", "XT");
     queryparser.add_boolean_prefix("site", "H");
+    queryparser.add_boolean_prefix("site2", "J");
     for (test *p = test_or_queries; p->query; ++p) {
        string expect, parsed;
        if (p->expect)