Index: matcher/Makefile.mk =================================================================== --- matcher/Makefile.mk (revision 13285) +++ matcher/Makefile.mk (working copy) @@ -3,6 +3,7 @@ matcher/andnotpostlist.h\ matcher/branchpostlist.h\ matcher/collapser.h\ + matcher/exactphrasecheck.h\ matcher/exactphrasepostlist.h\ matcher/externalpostlist.h\ matcher/extraweightpostlist.h\ @@ -38,6 +39,7 @@ matcher/andnotpostlist.cc\ matcher/branchpostlist.cc\ matcher/collapser.cc\ + matcher/exactphrasecheck.cc\ matcher/exactphrasepostlist.cc\ matcher/externalpostlist.cc\ matcher/localmatch.cc\ Index: matcher/exactphrasecheck.cc =================================================================== --- matcher/exactphrasecheck.cc (revision 0) +++ matcher/exactphrasecheck.cc (revision 0) @@ -0,0 +1,170 @@ +/** @file exactphrasecheck.cc + * @brief Check if terms form a particular exact phrase. + */ +/* Copyright (C) 2006,2007,2009 Olly Betts + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +// FIXME: this could probably share code with ExactPhrasePostList. + +#include + +#include "exactphrasecheck.h" +#include "positionlist.h" +#include "postlist.h" +#include "omassert.h" +#include "omdebug.h" + +#include +#include + +using namespace std; + +namespace { + +class TermCompare { + const Xapian::Database & db; + vector & terms; + + public: + TermCompare(const Xapian::Database & db_, + vector & terms_) + : db(db_), terms(terms_) { } + + bool operator()(unsigned a, unsigned b) const { + return db.get_collection_freq(terms[a]) < db.get_collection_freq(terms[b]); + } +}; + +} + +ExactPhraseCheck::ExactPhraseCheck(const Xapian::Database & db_, + const vector &terms_) + : db(db_), terms(terms_) +{ + if (terms.empty()) { + poslists = NULL; + order = NULL; + return; + } + + AssertRel(terms.size(),>,1); + size_t n = terms_.size(); + poslists = new PositionList*[n]; + try { + order = new unsigned[n]; + } catch (...) { + delete [] poslists; + throw; + } + for (size_t i = 0; i < n; ++i) { + poslists[i] = NULL; + order[i] = unsigned(i); + } + + // We often don't need to read all the position lists, so rather than using + // the shortest position lists first, we approximate by using the terms + // with the lowest collection freq first. Overall this should give a + // similar order. + sort(order, order + terms.size(), TermCompare(db, terms)); +} + +ExactPhraseCheck::~ExactPhraseCheck() +{ + delete [] poslists; + delete [] order; +} + +bool +ExactPhraseCheck::start_position_list(unsigned i, Xapian::docid did) +{ + AssertRel(i,<,terms.size()); + unsigned index = order[i]; + // FIXME: nasty hacking around with internals and refcount - we should + // just add a new Databse::Internal method to do what we want. + Xapian::PositionIterator p = db.positionlist_begin(did, terms[index]); + PositionList * tmp = p.internal.get(); + if (!tmp) + return false; + ++tmp->ref_count; + p.internal = poslists[i]; + poslists[i] = tmp; + poslists[i]->index = index; + return true; +} + +bool +ExactPhraseCheck::operator()(Xapian::docid did) +{ + DEBUGCALL(MATCH, bool, "ExactPhraseCheck::operator()", did); + + AssertRel(terms.size(),>,1); + + bool result = false; + // If the first term we check only occurs too close to the start of the + // document, we only need to read one term's positions. E.g. search for + // "ripe mango" when the only occurrence of 'mango' in the current document + // is at position 0. + if (!start_position_list(0, did)) + goto done; + poslists[0]->skip_to(poslists[0]->index); + if (poslists[0]->at_end()) goto done; + + // If we get here, we'll need to read the positionlists for at least two + // terms, so check the true positionlist length for the two terms with the + // lowest wdf and if necessary swap them so the true shorter one is first. + if (!start_position_list(1, did)) + goto done; + if (poslists[0]->get_size() < poslists[1]->get_size()) { + poslists[1]->skip_to(poslists[1]->index); + if (poslists[1]->at_end()) goto done; + swap(poslists[0], poslists[1]); + } + + { + unsigned read_hwm = 1; + Xapian::termpos idx0 = poslists[0]->index; + do { + Xapian::termpos base = poslists[0]->get_position() - idx0; + unsigned i = 1; + while (true) { + if (i > read_hwm) { + read_hwm = i; + if (!start_position_list(i, did)) + goto done; + // FIXME: consider comparing with poslist[0] and swapping + // if less common. Should we allow for the number of positions + // we've read from poslist[0] already? + } + Xapian::termpos required = base + poslists[i]->index; + poslists[i]->skip_to(required); + if (poslists[i]->at_end()) goto done; + if (poslists[i]->get_position() != required) break; + if (++i == terms.size()) { + result = true; + goto done; + } + } + poslists[0]->next(); + } while (!poslists[0]->at_end()); + } +done: + for (size_t i = 0; i < terms.size(); ++i) { + delete poslists[i]; + poslists[i] = NULL; + } + RETURN(result); +} Index: matcher/multimatch.cc =================================================================== --- matcher/multimatch.cc (revision 13285) +++ matcher/multimatch.cc (working copy) @@ -46,6 +46,8 @@ #include "weightinternal.h" +#include "exactphrasecheck.h" + #include #include #include // For XAPIAN_HAS_REMOTE_BACKEND @@ -356,6 +358,7 @@ map * termfreqandwts_ptr; termfreqandwts_ptr = &termfreqandwts; + vector pool_terms; Xapian::termcount total_subqs = 0; // Keep a count of matches which we know exist, but we won't see. This // occurs when a submatch is remote, and returns a lower bound on the @@ -365,9 +368,11 @@ for (size_t i = 0; i != leaves.size(); ++i) { PostList *pl; try { + if (!is_remote[i]) pool_terms.clear(); pl = leaves[i]->get_postlist_and_term_info(this, termfreqandwts_ptr, - &total_subqs); + &total_subqs, + pool_terms); if (termfreqandwts_ptr && !termfreqandwts.empty()) termfreqandwts_ptr = NULL; if (is_remote[i]) { @@ -522,6 +527,16 @@ // Is the mset a valid heap? bool is_heap = false; + size_t SETTLING_POND_SIZE = 0; + if (!pool_terms.empty()) { + const char * sps = getenv("POND_SIZE"); + SETTLING_POND_SIZE = sps ? atoi(sps) : 100000; + } + ExactPhraseCheck phrase_check(db, pool_terms); + // FIXME: a min/max heap is probably a better choice here (notably more + // compact) but the STL doesn't provide one so we'd have to find an + // implementation or write one. + multimap settling_pond; while (true) { bool pushback; @@ -649,6 +664,27 @@ new_item.wt = wt; } + if (SETTLING_POND_SIZE) { + if (items.size() >= max_msize) { + // Settling pond handling... + multimap::iterator it; + it = settling_pond.upper_bound(-min_weight); + settling_pond.erase(it, settling_pond.end()); + + settling_pond.insert(make_pair(-new_item.wt, new_item)); + if (settling_pond.size() < SETTLING_POND_SIZE) { + continue; + } + + // Take the last item off the heap, which will have a reasonably + // high weight in general. + it = settling_pond.begin(); + swap(new_item, it->second); + settling_pond.erase(it); + } + if (!phrase_check(new_item.did)) continue; + } + pushback = true; // Perform collapsing on key if requested. @@ -811,6 +847,117 @@ } } + multimap::iterator it; + for (it = settling_pond.begin(); it != settling_pond.end(); ++it) { + const Xapian::Internal::MSetItem & new_item = it->second; + if (new_item.wt < min_weight) break; + if (!phrase_check(new_item.did)) continue; + + { + ++docs_matched; + if (items.size() >= max_msize) { + items.push_back(new_item); + if (!is_heap) { + is_heap = true; + make_heap(items.begin(), items.end(), mcmp); + } else { + push_heap::iterator, + MSetCmp>(items.begin(), items.end(), mcmp); + } + pop_heap::iterator, + MSetCmp>(items.begin(), items.end(), mcmp); + items.pop_back(); + + min_item = items.front(); + if (sort_by == REL || sort_by == REL_VAL) { + if (docs_matched >= check_at_least) { + if (sort_by == REL) { + // We're done if this is a forward boolean match + // with only one database (bodgetastic, FIXME + // better if we can!) + if (rare(max_possible == 0 && sort_forward)) { + // In the multi database case, MergePostList + // currently processes each database + // sequentially (which actually may well be + // more efficient) so the docids in general + // won't arrive in order. + // FIXME: is this still good here: + // if (leaves.size() == 1) break; + } + } + if (min_item.wt > min_weight) { + LOGLINE(MATCH, "Setting min_weight to " << + min_item.wt << " from " << min_weight); + min_weight = min_item.wt; + } + } + } + } else { + items.push_back(new_item); + is_heap = false; + if (sort_by == REL && items.size() == max_msize) { + if (docs_matched >= check_at_least) { + // We're done if this is a forward boolean match + // with only one database (bodgetastic, FIXME + // better if we can!) + if (rare(max_possible == 0 && sort_forward)) { + // In the multi database case, MergePostList + // currently processes each database + // sequentially (which actually may well be + // more efficient) so the docids in general + // won't arrive in order. + // FIXME: if (leaves.size() == 1) break; + } + } + } + } + } + + // Keep a track of the greatest weight we've seen. + if (new_item.wt > greatest_wt) { + greatest_wt = new_item.wt; +#ifdef XAPIAN_HAS_REMOTE_BACKEND + const unsigned int multiplier = db.internal.size(); + unsigned int db_num = (new_item.did - 1) % multiplier; + if (is_remote[db_num]) { + // Note that the greatest weighted document came from a remote + // database, and which one. + greatest_wt_subqs_db_num = db_num; + } else +#endif + { + greatest_wt_subqs_matched = pl->count_matching_subqs(); +#ifdef XAPIAN_HAS_REMOTE_BACKEND + greatest_wt_subqs_db_num = UINT_MAX; +#endif + } + if (percent_cutoff) { + Xapian::weight w = new_item.wt * percent_cutoff_factor; + if (w > min_weight) { + min_weight = w; + if (!is_heap) { + is_heap = true; + make_heap::iterator, + MSetCmp>(items.begin(), items.end(), mcmp); + } + while (!items.empty() && items.front().wt < min_weight) { + pop_heap::iterator, + MSetCmp>(items.begin(), items.end(), mcmp); + Assert(items.back().wt < min_weight); + items.pop_back(); + } +#ifdef XAPIAN_ASSERTIONS_PARANOID + vector::const_iterator i; + for (i = items.begin(); i != items.end(); ++i) { + Assert(i->wt >= min_weight); + } +#endif + } + } + } + } + + // done with posting list tree delete pl; Index: matcher/localmatch.cc =================================================================== --- matcher/localmatch.cc (revision 13285) +++ matcher/localmatch.cc (working copy) @@ -89,7 +89,8 @@ PostList * LocalSubMatch::get_postlist_and_term_info(MultiMatch * matcher, map * termfreqandwts, - Xapian::termcount * total_subqs_ptr) + Xapian::termcount * total_subqs_ptr, + std::vector & pool_terms) { DEBUGCALL(MATCH, PostList *, "LocalSubMatch::get_postlist_and_term_info", matcher << ", [termfreqandwts], [total_subqs_ptr]"); @@ -98,7 +99,7 @@ // Build the postlist tree for the query. This calls // LocalSubMatch::postlist_from_op_leaf_query() for each term in the query, // which builds term_info as a side effect. - QueryOptimiser opt(*db, *this, matcher); + QueryOptimiser opt(*db, *this, matcher, pool_terms); PostList * pl = opt.optimise_query(&orig_query); *total_subqs_ptr = opt.get_total_subqueries(); Index: matcher/exactphrasecheck.h =================================================================== --- matcher/exactphrasecheck.h (revision 0) +++ matcher/exactphrasecheck.h (revision 0) @@ -0,0 +1,59 @@ +/** @file exactphrasecheck.cc + * @brief Check if terms form a particular exact phrase. + */ +/* Copyright (C) 2006 Olly Betts + * Copyright (C) 2009 Lemur Consulting Ltd + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#ifndef XAPIAN_INCLUDED_EXACTPHRASEPOSTLIST_H +#define XAPIAN_INCLUDED_EXACTPHRASEPOSTLIST_H + +#include "xapian/database.h" + +#include +#include + +typedef Xapian::PositionIterator::Internal PositionList; + +/** Check for an exact phrase using positional information. + * + * Tests if the terms occur somewhere in the document in the order given + * and at adjacent term positions. + */ +class ExactPhraseCheck { + Xapian::Database db; + + std::vector terms; + + PositionList ** poslists; + + unsigned * order; + + /// Start reading from the i-th position list. + bool start_position_list(unsigned i, Xapian::docid did); + + public: + ExactPhraseCheck(const Xapian::Database & db_, + const std::vector &terms_); + + ~ExactPhraseCheck(); + + /// Test if the specified document contains the terms as an exact phrase. + bool operator()(Xapian::docid did); +}; + +#endif Index: matcher/localmatch.h =================================================================== --- matcher/localmatch.h (revision 13285) +++ matcher/localmatch.h (working copy) @@ -81,7 +81,8 @@ /// Get PostList and term info. PostList * get_postlist_and_term_info(MultiMatch *matcher, std::map *termfreqandwts, - Xapian::termcount * total_subqs_ptr); + Xapian::termcount * total_subqs_ptr, + std::vector & pool_terms); /** Convert a postlist into a synonym postlist. */ Index: matcher/remotesubmatch.cc =================================================================== --- matcher/remotesubmatch.cc (revision 13285) +++ matcher/remotesubmatch.cc (working copy) @@ -64,7 +64,8 @@ PostList * RemoteSubMatch::get_postlist_and_term_info(MultiMatch *, map * termfreqandwts, - Xapian::termcount * total_subqs_ptr) + Xapian::termcount * total_subqs_ptr, + std::vector &) { DEBUGCALL(MATCH, PostList *, "RemoteSubMatch::get_postlist_and_term_info", "[matcher], " << (void*)termfreqandwts << ", " << (void*)total_subqs_ptr); Index: matcher/queryoptimiser.cc =================================================================== --- matcher/queryoptimiser.cc (revision 13285) +++ matcher/queryoptimiser.cc (working copy) @@ -29,6 +29,7 @@ #include "emptypostlist.h" #include "exactphrasepostlist.h" #include "externalpostlist.h" +#include "leafpostlist.h" #include "multiandpostlist.h" #include "multimatch.h" #include "omassert.h" @@ -49,7 +50,8 @@ using namespace std; PostList * -QueryOptimiser::do_subquery(const Xapian::Query::Internal * query, double factor) +QueryOptimiser::do_subquery(const Xapian::Query::Internal * query, double factor, + bool top_and) { DEBUGCALL(MATCH, PostList *, "QueryOptimiser::do_subquery", query << ", " << factor); @@ -79,7 +81,7 @@ case Xapian::Query::OP_FILTER: case Xapian::Query::OP_NEAR: case Xapian::Query::OP_PHRASE: - RETURN(do_and_like(query, factor)); + RETURN(do_and_like(query, factor, top_and)); case Xapian::Query::OP_OR: case Xapian::Query::OP_XOR: @@ -99,15 +101,15 @@ case Xapian::Query::OP_AND_NOT: { AssertEq(query->subqs.size(), 2); - PostList * l = do_subquery(query->subqs[0], factor); - PostList * r = do_subquery(query->subqs[1], 0.0); + PostList * l = do_subquery(query->subqs[0], factor, top_and); + PostList * r = do_subquery(query->subqs[1], 0.0, false); RETURN(new AndNotPostList(l, r, matcher, db_size)); } case Xapian::Query::OP_AND_MAYBE: { AssertEq(query->subqs.size(), 2); - PostList * l = do_subquery(query->subqs[0], factor); - PostList * r = do_subquery(query->subqs[1], factor); + PostList * l = do_subquery(query->subqs[0], factor, top_and); + PostList * r = do_subquery(query->subqs[1], factor, false); RETURN(new AndMaybePostList(l, r, matcher, db_size)); } @@ -140,7 +142,7 @@ AssertEq(query->subqs.size(), 1); double sub_factor = factor; if (sub_factor != 0.0) sub_factor *= query->get_dbl_parameter(); - RETURN(do_subquery(query->subqs[0], sub_factor)); + RETURN(do_subquery(query->subqs[0], sub_factor, top_and)); } default: @@ -163,7 +165,8 @@ }; PostList * -QueryOptimiser::do_and_like(const Xapian::Query::Internal *query, double factor) +QueryOptimiser::do_and_like(const Xapian::Query::Internal *query, double factor, + bool top_and) { DEBUGCALL(MATCH, PostList *, "QueryOptimiser::do_and_like", query << ", " << factor); @@ -195,7 +198,19 @@ pl = new NearPostList(pl, window, terms); } else if (window == filter.end - filter.begin) { AssertEq(filter.op, Xapian::Query::OP_PHRASE); - pl = new ExactPhrasePostList(pl, terms); + if (top_and) { + vector::const_iterator j; + for (j = terms.begin(); j != terms.end(); ++j) { + // FIXME: avoid dynamic_cast<> here. + LeafPostList * lpl = dynamic_cast(*j); + if (!lpl || lpl->term.empty()) goto cannot_pool; + pool_terms.push_back(lpl->term); + } + top_and = false; + } else { +cannot_pool: + pl = new ExactPhrasePostList(pl, terms); + } } else { AssertEq(filter.op, Xapian::Query::OP_PHRASE); pl = new PhrasePostList(pl, window, terms); @@ -244,7 +259,7 @@ if (is_and_like(subq->op)) { do_and_like(subq, factor, and_plists, pos_filters); } else { - PostList * pl = do_subquery(subq, factor); + PostList * pl = do_subquery(subq, factor, false); and_plists.push_back(pl); } } @@ -255,6 +270,8 @@ size_t begin = end - queries.size(); Xapian::termcount window = query->parameter; + if (window == queries.size()) { + } pos_filters.push_back(PosFilter(op, begin, end, window)); } } @@ -335,7 +352,7 @@ Xapian::Query::Internal::subquery_list::const_iterator q; for (q = queries.begin(); q != queries.end(); ++q) { - postlists.push_back(do_subquery(*q, factor)); + postlists.push_back(do_subquery(*q, factor, false)); } if (op == Xapian::Query::OP_ELITE_SET) { Index: matcher/queryoptimiser.h =================================================================== --- matcher/queryoptimiser.h (revision 13285) +++ matcher/queryoptimiser.h (working copy) @@ -44,6 +44,8 @@ MultiMatch * matcher; + std::vector & pool_terms; + /** How many leaf subqueries there are. * * Used for scaling percentages when the highest weighted document doesn't @@ -59,7 +61,7 @@ * @return A PostList subtree. */ PostList * do_subquery(const Xapian::Query::Internal * query, - double factor); + double factor, bool top_and); /** Optimise an AND-like Xapian::Query::Internal subtree into a PostList * subtree. @@ -69,7 +71,8 @@ * * @return A PostList subtree. */ - PostList * do_and_like(const Xapian::Query::Internal *query, double factor); + PostList * do_and_like(const Xapian::Query::Internal *query, double factor, + bool top_and); /** Optimise an AND-like Xapian::Query::Internal subtree into a PostList * subtree. @@ -107,12 +110,13 @@ public: QueryOptimiser(const Xapian::Database::Internal & db_, LocalSubMatch & localsubmatch_, - MultiMatch * matcher_) + MultiMatch * matcher_, + std::vector & pool_terms_) : db(db_), db_size(db.get_doccount()), localsubmatch(localsubmatch_), - matcher(matcher_), total_subqs(0) { } + matcher(matcher_), pool_terms(pool_terms_), total_subqs(0) { } PostList * optimise_query(Xapian::Query::Internal * query) { - return do_subquery(query, 1.0); + return do_subquery(query, 1.0, true); } Xapian::termcount get_total_subqueries() const { return total_subqs; } Index: matcher/remotesubmatch.h =================================================================== --- matcher/remotesubmatch.h (revision 13285) +++ matcher/remotesubmatch.h (working copy) @@ -72,7 +72,8 @@ PostList * get_postlist_and_term_info(MultiMatch *matcher, std::map *termfreqandwts, - Xapian::termcount * total_subqs_ptr); + Xapian::termcount * total_subqs_ptr, + std::vector & pool_terms); /// Get percentage factor - only valid after get_postlist_and_term_info(). double get_percent_factor() const { return percent_factor; } Index: common/leafpostlist.h =================================================================== --- common/leafpostlist.h (revision 13285) +++ common/leafpostlist.h (working copy) @@ -47,9 +47,11 @@ bool need_doclength; + public: // FIXME: avoid having to make term public. /// The term name for this postlist ("" for an alldocs postlist). std::string term; + protected: /// Only constructable as a base class for derived classes. LeafPostList(const std::string & term_) : weight(0), need_doclength(false), term(term_) { } Index: common/submatch.h =================================================================== --- common/submatch.h (revision 13285) +++ common/submatch.h (working copy) @@ -76,7 +76,8 @@ virtual PostList * get_postlist_and_term_info(MultiMatch *matcher, std::map *termfreqandwts, - Xapian::termcount * total_subqs_ptr) + Xapian::termcount * total_subqs_ptr, + std::vector & pool_terms) = 0; };