root / tags / 1.0.8 / xapian-core / tests / api_anydb.cc

/* api_anydb.cc: tests which work with any backend
 *
 * Copyright 1999,2000,2001 BrightStation PLC
 * Copyright 2002 Ananova Ltd
 * Copyright 2002,2003,2004,2005,2006,2007 Olly Betts
 * Copyright 2006,2008 Lemur Consulting Ltd
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301
 * USA
 */

#include <config.h>

#include "api_anydb.h"

#include <algorithm>
#include <iomanip>
#include <string>

#include <xapian.h>
#include "backendmanager_local.h"
#include "testsuite.h"
#include "testutils.h"
#include "utils.h"

#include "apitest.h"

#include <list>

using namespace std;

static void
print_mset_weights(const Xapian::MSet &mset)
{
    Xapian::MSetIterator i = mset.begin();
    for ( ; i != mset.end(); ++i) {
        tout << " " << i.get_weight();
    }
}

static void
print_mset_percentages(const Xapian::MSet &mset)
{
    Xapian::MSetIterator i = mset.begin();
    for ( ; i != mset.end(); ++i) {
        tout << " " << mset.convert_to_percent(i);
    }
}

static Xapian::Query
query(Xapian::Query::op op, string t1 = "", string t2 = "",
      string t3 = "", string t4 = "", string t5 = "",
      string t6 = "", string t7 = "", string t8 = "",
      string t9 = "", string t10 = "")
{
    vector<string> v;
    Xapian::Stem stemmer("english");
    if (!t1.empty()) v.push_back(stemmer(t1));
    if (!t2.empty()) v.push_back(stemmer(t2));
    if (!t3.empty()) v.push_back(stemmer(t3));
    if (!t4.empty()) v.push_back(stemmer(t4));
    if (!t5.empty()) v.push_back(stemmer(t5));
    if (!t6.empty()) v.push_back(stemmer(t6));
    if (!t7.empty()) v.push_back(stemmer(t7));
    if (!t8.empty()) v.push_back(stemmer(t8));
    if (!t9.empty()) v.push_back(stemmer(t9));
    if (!t10.empty()) v.push_back(stemmer(t10));
    return Xapian::Query(op, v.begin(), v.end());
}

static Xapian::Query
query(Xapian::Query::op op, Xapian::termcount parameter,
      string t1 = "", string t2 = "",
      string t3 = "", string t4 = "", string t5 = "",
      string t6 = "", string t7 = "", string t8 = "",
      string t9 = "", string t10 = "")
{
    vector<string> v;
    Xapian::Stem stemmer("english");
    if (!t1.empty()) v.push_back(stemmer(t1));
    if (!t2.empty()) v.push_back(stemmer(t2));
    if (!t3.empty()) v.push_back(stemmer(t3));
    if (!t4.empty()) v.push_back(stemmer(t4));
    if (!t5.empty()) v.push_back(stemmer(t5));
    if (!t6.empty()) v.push_back(stemmer(t6));
    if (!t7.empty()) v.push_back(stemmer(t7));
    if (!t8.empty()) v.push_back(stemmer(t8));
    if (!t9.empty()) v.push_back(stemmer(t9));
    if (!t10.empty()) v.push_back(stemmer(t10));
    return Xapian::Query(op, v.begin(), v.end(), parameter);
}

static Xapian::Query
query(const string &t)
{
    return Xapian::Query(Xapian::Stem("english")(t));
}

// #######################################################################
// # Tests start here

// tests that the backend doesn't return zero docids
DEFINE_TESTCASE(zerodocid1, backend) {
    // open the database (in this case a simple text file
    // we prepared earlier)

    Xapian::Database mydb(get_database("apitest_onedoc"));

    Xapian::Enquire enquire(mydb);

    // make a simple query, with one word in it - "word".
    enquire.set_query(Xapian::Query("word"));

    // retrieve the top ten results (we only expect one)
    Xapian::MSet mymset = enquire.get_mset(0, 10);

    // We've done the query, now check that the result is what
    // we expect (1 document, with non-zero docid)
    TEST_MSET_SIZE(mymset, 1);

    TEST_AND_EXPLAIN(*(mymset.begin()) != 0,
                     "A query on a database returned a zero docid");

    return true;
}

// tests that an empty query returns no matches
DEFINE_TESTCASE(emptyquery1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));

    enquire.set_query(Xapian::Query());
    Xapian::MSet mymset = enquire.get_mset(0, 10);
    TEST_MSET_SIZE(mymset, 0);
    TEST_EQUAL(mymset.get_matches_lower_bound(), 0);
    TEST_EQUAL(mymset.get_matches_upper_bound(), 0);
    TEST_EQUAL(mymset.get_matches_estimated(), 0);

    vector<Xapian::Query> v;
    enquire.set_query(Xapian::Query(Xapian::Query::OP_AND, v.begin(), v.end()));
    mymset = enquire.get_mset(0, 10);
    TEST_MSET_SIZE(mymset, 0);
    TEST_EQUAL(mymset.get_matches_lower_bound(), 0);
    TEST_EQUAL(mymset.get_matches_upper_bound(), 0);
    TEST_EQUAL(mymset.get_matches_estimated(), 0);

    return true;
}

// tests the document count for a simple query
DEFINE_TESTCASE(simplequery1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(Xapian::Query("word"));
    Xapian::MSet mymset = enquire.get_mset(0, 10);
    TEST_MSET_SIZE(mymset, 2);
    return true;
}

// tests for the right documents and weights returned with simple query
DEFINE_TESTCASE(simplequery2, backend) {
    // open the database (in this case a simple text file
    // we prepared earlier)
    Xapian::Database db = get_database("apitest_simpledata");
    Xapian::Enquire enquire(db);
    enquire.set_query(Xapian::Query("word"));

    // retrieve the top results
    Xapian::MSet mymset = enquire.get_mset(0, 10);

    // We've done the query, now check that the result is what
    // we expect (documents 2 and 4)
    mset_expect_order(mymset, 2, 4);

    // Check the weights
    Xapian::MSetIterator i = mymset.begin();
    // These weights are for BM25Weight(1,0,1,0.5,0.5)
    TEST_EQUAL_DOUBLE(i.get_weight(), 1.04648168717725);
    i++;
    TEST_EQUAL_DOUBLE(i.get_weight(), 0.640987686595914);

    return true;
}

// tests for the right document count for another simple query
DEFINE_TESTCASE(simplequery3, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(query("this"));
    Xapian::MSet mymset = enquire.get_mset(0, 10);

    // Check that 6 documents were returned.
    TEST_MSET_SIZE(mymset, 6);

    return true;
}

// tests for the right document count for a wildcard query
// FIXME: move this to querytest (and just use an InMemory DB).
DEFINE_TESTCASE(wildquery1, backend) {
    Xapian::QueryParser queryparser;
    unsigned flags = Xapian::QueryParser::FLAG_WILDCARD |
                     Xapian::QueryParser::FLAG_LOVEHATE;
    queryparser.set_stemmer(Xapian::Stem("english"));
    queryparser.set_stemming_strategy(Xapian::QueryParser::STEM_ALL);
    Xapian::Database db = get_database("apitest_simpledata");
    queryparser.set_database(db);
    Xapian::Enquire enquire(db);

    Xapian::Query qobj = queryparser.parse_query("th*", flags);
    tout << qobj.get_description() << endl;
    enquire.set_query(qobj);
    Xapian::MSet mymset = enquire.get_mset(0, 10);
    // Check that 6 documents were returned.
    TEST_MSET_SIZE(mymset, 6);

    qobj = queryparser.parse_query("notindb* \"this\"", flags);
    tout << qobj.get_description() << endl;
    enquire.set_query(qobj);
    mymset = enquire.get_mset(0, 10);
    // Check that 6 documents were returned.
    TEST_MSET_SIZE(mymset, 6);

    qobj = queryparser.parse_query("+notindb* \"this\"", flags);
    tout << qobj.get_description() << endl;
    enquire.set_query(qobj);
    mymset = enquire.get_mset(0, 10);
    // Check that 0 documents were returned.
    TEST_MSET_SIZE(mymset, 0);

    return true;
}

// tests a query across multiple databases
DEFINE_TESTCASE(multidb1, backend) {
    Xapian::Database mydb1(get_database("apitest_simpledata", "apitest_simpledata2"));
    Xapian::Enquire enquire1(mydb1);

    Xapian::Database mydb2(get_database("apitest_simpledata"));
    mydb2.add_database(get_database("apitest_simpledata2"));
    Xapian::Enquire enquire2(mydb2);

    // make a simple query, with one word in it - "word".
    Xapian::Query myquery("word");
    enquire1.set_query(myquery);
    enquire2.set_query(myquery);

    // retrieve the top ten results from each method of accessing
    // multiple text files
    Xapian::MSet mymset1 = enquire1.get_mset(0, 10);
    Xapian::MSet mymset2 = enquire2.get_mset(0, 10);

    TEST_EQUAL(mymset1.size(), mymset2.size());
    TEST(mset_range_is_same_weights(mymset1, 0, mymset2, 0, mymset1.size()));
    return true;
}

// tests a query across multiple databases with terms only
// in one of the two databases
DEFINE_TESTCASE(multidb2, backend && !multi) {
    Xapian::Database mydb1(get_database("apitest_simpledata",
                                  "apitest_simpledata2"));
    Xapian::Enquire enquire1(mydb1);

    Xapian::Database mydb2(get_database("apitest_simpledata"));
    mydb2.add_database(get_database("apitest_simpledata2"));
    Xapian::Enquire enquire2(mydb2);

    // make a simple query
    Xapian::Query myquery = query(Xapian::Query::OP_OR, "inmemory", "word");
    enquire1.set_query(myquery);
    enquire2.set_query(myquery);

    // retrieve the top ten results from each method of accessing
    // multiple text files
    Xapian::MSet mymset1 = enquire1.get_mset(0, 10);
    Xapian::MSet mymset2 = enquire2.get_mset(0, 10);

    TEST_EQUAL(mymset1.size(), mymset2.size());
    TEST(mset_range_is_same_weights(mymset1, 0, mymset2, 0, mymset1.size()));
    return true;
}

// test that a multidb with 2 dbs query returns correct docids
DEFINE_TESTCASE(multidb3, backend && !multi) {
    Xapian::Database mydb2(get_database("apitest_simpledata"));
    mydb2.add_database(get_database("apitest_simpledata2"));
    Xapian::Enquire enquire(mydb2);

    // make a query
    Xapian::Query myquery = query(Xapian::Query::OP_OR, "inmemory", "word");
    enquire.set_weighting_scheme(Xapian::BoolWeight());
    enquire.set_query(myquery);

    // retrieve the top ten results
    Xapian::MSet mymset = enquire.get_mset(0, 10);
    mset_expect_order(mymset, 2, 3, 7);

    return true;
}

// test that a multidb with 3 dbs query returns correct docids
DEFINE_TESTCASE(multidb4, backend && !multi) {
    Xapian::Database mydb2(get_database("apitest_simpledata"));
    mydb2.add_database(get_database("apitest_simpledata2"));
    mydb2.add_database(get_database("apitest_termorder"));
    Xapian::Enquire enquire(mydb2);

    // make a query
    Xapian::Query myquery = query(Xapian::Query::OP_OR, "inmemory", "word");
    enquire.set_weighting_scheme(Xapian::BoolWeight());
    enquire.set_query(myquery);

    // retrieve the top ten results
    Xapian::MSet mymset = enquire.get_mset(0, 10);
    mset_expect_order(mymset, 2, 3, 4, 10);

    return true;
}

// tests MultiPostList::skip_to().
DEFINE_TESTCASE(multidb5, backend && !multi) {
    Xapian::Database mydb2(get_database("apitest_simpledata"));
    mydb2.add_database(get_database("apitest_simpledata2"));
    Xapian::Enquire enquire(mydb2);

    // make a query
    Xapian::Query myquery = query(Xapian::Query::OP_AND, "inmemory", "word");
    enquire.set_weighting_scheme(Xapian::BoolWeight());
    enquire.set_query(myquery);

    // retrieve the top ten results
    Xapian::MSet mymset = enquire.get_mset(0, 10);
    mset_expect_order(mymset, 2);

    return true;
}

// tests that when specifying maxitems to get_mset, no more than
// that are returned.
DEFINE_TESTCASE(msetmaxitems1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(query("this"));
    Xapian::MSet mymset = enquire.get_mset(0, 1);
    TEST_MSET_SIZE(mymset, 1);

    mymset = enquire.get_mset(0, 5);
    TEST_MSET_SIZE(mymset, 5);

    return true;
}

// tests the returned weights are as expected (regression test for remote
// backend which was using the average weight rather than the actual document
// weight for computing weights - fixed in 1.0.0).
DEFINE_TESTCASE(expandweights1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(Xapian::Query("this"));

    Xapian::MSet mymset = enquire.get_mset(0, 10);

    Xapian::RSet myrset;
    Xapian::MSetIterator i = mymset.begin();
    myrset.add_document(*i);
    myrset.add_document(*(++i));

    Xapian::ESet eset = enquire.get_eset(3, myrset, enquire.USE_EXACT_TERMFREQ);
    TEST_EQUAL(eset.size(), 3);
    TEST_EQUAL_DOUBLE(eset[0].get_weight(), 6.08904001099445);
    TEST_EQUAL_DOUBLE(eset[1].get_weight(), 6.08904001099445);
    TEST_EQUAL_DOUBLE(eset[2].get_weight(), 4.73383620844021);

    return true;
}

// Just like test_expandweights1 but without USE_EXACT_TERMFREQ.
DEFINE_TESTCASE(expandweights2, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(Xapian::Query("this"));

    Xapian::MSet mymset = enquire.get_mset(0, 10);

    Xapian::RSet myrset;
    Xapian::MSetIterator i = mymset.begin();
    myrset.add_document(*i);
    myrset.add_document(*(++i));

    Xapian::ESet eset = enquire.get_eset(3, myrset);
    TEST_EQUAL(eset.size(), 3);
    if (strcmp(get_dbtype(), "multi") != 0) {
        // For a single database, the weights should be the same with or
        // without USE_EXACT_TERMFREQ.
        TEST_EQUAL_DOUBLE(eset[0].get_weight(), 6.08904001099445);
        TEST_EQUAL_DOUBLE(eset[1].get_weight(), 6.08904001099445);
        TEST_EQUAL_DOUBLE(eset[2].get_weight(), 4.73383620844021);
    } else {
        // For multiple databases, we expect that using USE_EXACT_TERMFREQ
        // will result in different weights in some cases.
        TEST_NOT_EQUAL_DOUBLE(eset[0].get_weight(), 6.08904001099445);
        TEST_EQUAL_DOUBLE(eset[1].get_weight(), 6.08904001099445);
        TEST_NOT_EQUAL_DOUBLE(eset[2].get_weight(), 4.73383620844021);
    }

    return true;
}

// tests that when specifying maxitems to get_eset, no more than
// that are returned.
DEFINE_TESTCASE(expandmaxitems1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(Xapian::Query("this"));

    Xapian::MSet mymset = enquire.get_mset(0, 10);
    tout << "mymset.size() = " << mymset.size() << endl;
    TEST(mymset.size() >= 2);

    Xapian::RSet myrset;
    Xapian::MSetIterator i = mymset.begin();
    myrset.add_document(*i);
    myrset.add_document(*(++i));

    Xapian::ESet myeset = enquire.get_eset(1, myrset);
    TEST_EQUAL(myeset.size(), 1);

    return true;
}

// tests that a pure boolean query has all weights set to 0
DEFINE_TESTCASE(boolquery1, backend) {
    Xapian::Query myboolquery(query("this"));

    // open the database (in this case a simple text file
    // we prepared earlier)
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(myboolquery);
    enquire.set_weighting_scheme(Xapian::BoolWeight());

    // retrieve the top results
    Xapian::MSet mymset = enquire.get_mset(0, 10);

    TEST_NOT_EQUAL(mymset.size(), 0);
    TEST_EQUAL(mymset.get_max_possible(), 0);
    for (Xapian::MSetIterator i = mymset.begin(); i != mymset.end(); ++i) {
        TEST_EQUAL(i.get_weight(), 0);
    }
    return true;
}

// tests that get_mset() specifying "this" works as expected
DEFINE_TESTCASE(msetfirst1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(query("this"));
    Xapian::MSet mymset1 = enquire.get_mset(0, 6);
    Xapian::MSet mymset2 = enquire.get_mset(3, 3);
    TEST(mset_range_is_same(mymset1, 3, mymset2, 0, 3));

    // Regression test - we weren't adjusting the index into items[] by
    // firstitem in api/omenquire.cc.
    TEST_EQUAL(mymset1[5].get_document().get_data(),
               mymset2[2].get_document().get_data());
    return true;
}

// tests the converting-to-percent functions
DEFINE_TESTCASE(topercent1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(query("this"));
    Xapian::MSet mymset = enquire.get_mset(0, 20);

    int last_pct = 100;
    Xapian::MSetIterator i = mymset.begin();
    for ( ; i != mymset.end(); ++i) {
        int pct = mymset.convert_to_percent(i);
        TEST_AND_EXPLAIN(pct == i.get_percent(),
                         "convert_to_%(msetitor) != convert_to_%(wt)");
        TEST_AND_EXPLAIN(pct == mymset.convert_to_percent(i.get_weight()),
                         "convert_to_%(msetitor) != convert_to_%(wt)");
        TEST_AND_EXPLAIN(pct >= 0 && pct <= 100,
                         "percentage out of range: " << pct);
        TEST_AND_EXPLAIN(pct <= last_pct, "percentage increased down mset");
        last_pct = pct;
    }
    return true;
}

// tests the percentage values returned
DEFINE_TESTCASE(topercent2, backend) {
    BackendManagerLocal local_manager;
    local_manager.set_datadir(test_driver::get_srcdir() + "/testdata/");
    Xapian::Enquire localenq(local_manager.get_database("apitest_simpledata"));
    Xapian::Enquire enquire(get_database("apitest_simpledata"));

    int pct;

    // First, test a search in which the top document scores 100%.
    enquire.set_query(query("this"));
    localenq.set_query(query("this"));
    Xapian::MSet mymset = enquire.get_mset(0, 20);
    Xapian::MSet localmset = localenq.get_mset(0, 20);

    Xapian::MSetIterator i = mymset.begin();
    TEST(i != mymset.end());
    pct = mymset.convert_to_percent(i);
    TEST_EQUAL(pct, 100);

    TEST_EQUAL(mymset, localmset);
    TEST(mset_range_is_same_percents(mymset, 0, localmset, 0, mymset.size()));

    // A search in which the top document doesn't have 100%
    Xapian::Query q = query(Xapian::Query::OP_OR,
                            "this", "line", "paragraph", "rubbish");
    enquire.set_query(q);
    localenq.set_query(q);
    mymset = enquire.get_mset(0, 20);
    localmset = localenq.get_mset(0, 20);

    i = mymset.begin();
    TEST(i != mymset.end());
    pct = mymset.convert_to_percent(i);
    TEST_GREATER(pct, 65);
    TEST_LESSER(pct, 75);

    ++i;

    TEST(i != mymset.end());
    pct = mymset.convert_to_percent(i);
    TEST_GREATER(pct, 40);
    TEST_LESSER(pct, 50);

    TEST_EQUAL(mymset, localmset);
    TEST(mset_range_is_same_percents(mymset, 0, localmset, 0, mymset.size()));

    return true;
}

class myExpandFunctor : public Xapian::ExpandDecider {
    public:
        bool operator()(const string & tname) const {
            unsigned long sum = 0;
            for (string::const_iterator i=tname.begin(); i!=tname.end(); ++i) {
                sum += *i;
            }
//          if (verbose) {
//              tout << tname << "==> " << sum << "\n";
//          }
            return (sum % 2) == 0;
        }
};

// tests the expand decision functor
DEFINE_TESTCASE(expandfunctor1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(Xapian::Query("this"));

    Xapian::MSet mymset = enquire.get_mset(0, 10);
    TEST(mymset.size() >= 2);

    Xapian::RSet myrset;
    Xapian::MSetIterator i = mymset.begin();
    myrset.add_document(*i);
    myrset.add_document(*(++i));

    myExpandFunctor myfunctor;

    Xapian::ESet myeset_orig = enquire.get_eset(1000, myrset);
    unsigned int neweset_size = 0;
    Xapian::ESetIterator j = myeset_orig.begin();
    for ( ; j != myeset_orig.end(); ++j) {
        if (myfunctor(*j)) neweset_size++;
    }
    Xapian::ESet myeset = enquire.get_eset(neweset_size, myrset, &myfunctor);

#if 0
    // Compare myeset with the hand-filtered version of myeset_orig.
    if (verbose) {
        tout << "orig_eset: ";
        copy(myeset_orig.begin(), myeset_orig.end(),
             ostream_iterator<Xapian::ESetItem>(tout, " "));
        tout << "\n";

        tout << "new_eset: ";
        copy(myeset.begin(), myeset.end(),
             ostream_iterator<Xapian::ESetItem>(tout, " "));
        tout << "\n";
    }
#endif
    Xapian::ESetIterator orig = myeset_orig.begin();
    Xapian::ESetIterator filt = myeset.begin();
    for (; orig != myeset_orig.end() && filt != myeset.end(); ++orig, ++filt) {
        // skip over items that shouldn't be in myeset
        while (orig != myeset_orig.end() && !myfunctor(*orig)) {
            ++orig;
        }

        TEST_AND_EXPLAIN(*orig == *filt &&
                         orig.get_weight() == filt.get_weight(),
                         "Mismatch in items " << *orig << " vs. " << *filt
                         << " after filtering");
    }

    while (orig != myeset_orig.end() && !myfunctor(*orig)) {
        ++orig;
    }

    TEST_EQUAL(orig, myeset_orig.end());
    TEST_AND_EXPLAIN(filt == myeset.end(),
                     "Extra items in the filtered eset.");
    return true;
}

// tests the percent cutoff option
DEFINE_TESTCASE(pctcutoff1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(query(Xapian::Query::OP_OR,
                            "this", "line", "paragraph", "rubbish"));
    Xapian::MSet mymset1 = enquire.get_mset(0, 100);

    if (verbose) {
        tout << "Original mset pcts:";
        print_mset_percentages(mymset1);
        tout << "\n";
    }

    unsigned int num_items = 0;
    int my_pct = 100;
    int changes = 0;
    Xapian::MSetIterator i = mymset1.begin();
    int c = 0;
    for ( ; i != mymset1.end(); ++i, ++c) {
        int new_pct = mymset1.convert_to_percent(i);
        if (new_pct != my_pct) {
            changes++;
            if (changes > 3) break;
            num_items = c;
            my_pct = new_pct;
        }
    }

    TEST_AND_EXPLAIN(changes > 3, "MSet not varied enough to test");
    if (verbose) {
        tout << "Cutoff percent: " << my_pct << "\n";
    }

    enquire.set_cutoff(my_pct);
    Xapian::MSet mymset2 = enquire.get_mset(0, 100);

    if (verbose) {
        tout << "Percentages after cutoff:";
        print_mset_percentages(mymset2);
        tout << "\n";
    }

    TEST_AND_EXPLAIN(mymset2.size() >= num_items,
                     "Match with % cutoff lost too many items");

    TEST_AND_EXPLAIN(mymset2.size() == num_items ||
                     (mymset2.convert_to_percent(mymset2[num_items]) == my_pct &&
                      mymset2.convert_to_percent(mymset2.back()) == my_pct),
                     "Match with % cutoff returned too many items");

    return true;
}

// Tests the percent cutoff option combined with collapsing
DEFINE_TESTCASE(pctcutoff2, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(Xapian::Query("this"));
    enquire.set_query(Xapian::Query(Xapian::Query::OP_AND_NOT, Xapian::Query("this"), Xapian::Query("banana")));
    Xapian::MSet mset = enquire.get_mset(0, 100);

    if (verbose) {
        tout << "Original mset pcts:";
        print_mset_percentages(mset);
        tout << "\n";
    }

    TEST(mset.size() >= 2);
    TEST(mset[0].get_percent() - mset[1].get_percent() >= 2);

    Xapian::percent cutoff = mset[0].get_percent() + mset[1].get_percent();
    cutoff /= 2;

    enquire.set_cutoff(cutoff);
    enquire.set_collapse_key(1234); // Value which is always empty.

    mset = enquire.get_mset(0, 1);
    TEST_EQUAL(mset.size(), 1);
    TEST_EQUAL(mset.get_matches_lower_bound(), 1);

    return true;
}

// tests the cutoff option
DEFINE_TESTCASE(cutoff1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(query(Xapian::Query::OP_OR,
                            "this", "line", "paragraph", "rubbish"));
    Xapian::MSet mymset1 = enquire.get_mset(0, 100);

    if (verbose) {
        tout << "Original mset weights:";
        print_mset_weights(mymset1);
        tout << "\n";
    }

    unsigned int num_items = 0;
    Xapian::weight my_wt = -100;
    int changes = 0;
    Xapian::MSetIterator i = mymset1.begin();
    int c = 0;
    for ( ; i != mymset1.end(); ++i, ++c) {
        Xapian::weight new_wt = i.get_weight();
        if (new_wt != my_wt) {
            changes++;
            if (changes > 3) break;
            num_items = c;
            my_wt = new_wt;
        }
    }

    TEST_AND_EXPLAIN(changes > 3, "MSet not varied enough to test");
    if (verbose) {
        tout << "Cutoff weight: " << my_wt << "\n";
    }

    enquire.set_cutoff(0, my_wt);
    Xapian::MSet mymset2 = enquire.get_mset(0, 100);

    if (verbose) {
        tout << "Weights after cutoff:";
        print_mset_weights(mymset2);
        tout << "\n";
    }

    TEST_AND_EXPLAIN(mymset2.size() >= num_items,
                     "Match with cutoff lost too many items");

    TEST_AND_EXPLAIN(mymset2.size() == num_items ||
                     (mymset2[num_items].get_weight() == my_wt &&
                      mymset2.back().get_weight() == my_wt),
                     "Match with cutoff returned too many items");

    return true;
}

// tests the allow query terms expand option
DEFINE_TESTCASE(allowqterms1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(Xapian::Query("this"));

    Xapian::MSet mymset = enquire.get_mset(0, 10);
    TEST(mymset.size() >= 2);

    Xapian::RSet myrset;
    Xapian::MSetIterator i = mymset.begin();
    myrset.add_document(*i);
    myrset.add_document(*(++i));

    Xapian::ESet myeset = enquire.get_eset(1000, myrset);
    Xapian::ESetIterator j = myeset.begin();
    for ( ; j != myeset.end(); ++j) {
        TEST_NOT_EQUAL(*j, "this");
    }

    Xapian::ESet myeset2 = enquire.get_eset(1000, myrset, Xapian::Enquire::INCLUDE_QUERY_TERMS);
    j = myeset2.begin();
    for ( ; j != myeset2.end(); ++j) {
        if (*j == "this") break;
    }
    TEST(j != myeset2.end());
    return true;
}

// tests that the MSet max_attained works
DEFINE_TESTCASE(maxattain1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(query("this"));
    Xapian::MSet mymset = enquire.get_mset(0, 100);

    Xapian::weight mymax = 0;
    Xapian::MSetIterator i = mymset.begin();
    for ( ; i != mymset.end(); ++i) {
        if (i.get_weight() > mymax) mymax = i.get_weight();
    }
    TEST_EQUAL(mymax, mymset.get_max_attained());

    return true;
}

// tests a reversed boolean query
DEFINE_TESTCASE(reversebool1, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(Xapian::Query("this"));
    enquire.set_weighting_scheme(Xapian::BoolWeight());

    Xapian::MSet mymset1 = enquire.get_mset(0, 100);
    TEST_AND_EXPLAIN(mymset1.size() > 1,
                     "Mset was too small to test properly");

    enquire.set_docid_order(Xapian::Enquire::ASCENDING);
    Xapian::MSet mymset2 = enquire.get_mset(0, 100);
    enquire.set_docid_order(Xapian::Enquire::DESCENDING);
    Xapian::MSet mymset3 = enquire.get_mset(0, 100);

    // mymset1 and mymset2 should be identical
    TEST_EQUAL(mymset1.size(), mymset2.size());

    {
        Xapian::MSetIterator i = mymset1.begin();
        Xapian::MSetIterator j = mymset2.begin();
        for ( ; i != mymset1.end(), j != mymset2.end(); ++i, j++) {
            // if this fails, then setting match_sort_forward=true was not
            // the same as the default.
            TEST_EQUAL(*i, *j);
        }
    }

    // mymset1 and mymset3 should be same but reversed
    TEST_EQUAL(mymset1.size(), mymset3.size());

    {
        Xapian::MSetIterator i = mymset1.begin();
        vector<Xapian::docid> rev(mymset3.begin(), mymset3.end());
        // Next iterator not const because of compiler brokenness (egcs 1.1.2)
        vector<Xapian::docid>::reverse_iterator j = rev.rbegin();
        for ( ; i != mymset1.end(); ++i, j++) {
            // if this fails, then setting match_sort_forward=false didn't
            // reverse the results.
            TEST_EQUAL(*i, *j);
        }
    }

    return true;
}

// tests a reversed boolean query, where the full mset isn't returned
DEFINE_TESTCASE(reversebool2, backend) {
    Xapian::Enquire enquire(get_database("apitest_simpledata"));
    enquire.set_query(Xapian::Query("this"));
    enquire.set_weighting_scheme(Xapian::BoolWeight());

    Xapian::MSet mymset1 = enquire.get_mset(0, 100);

    TEST_AND_EXPLAIN(mymset1.size() > 1,
                     "Mset was too small to test properly");

    enquire.set_docid_order(Xapian::Enquire::ASCENDING);
    Xapian::doccount msize = mymset1.size() / 2;
    Xapian::MSet mymset2 = enquire.get_mset(0, msize);
    enquire.set_docid_order(Xapian::Enquire::DESCENDING);
    Xapian::MSet mymset3 = enquire.get_mset(0, msize);

    // mymset2 should be first msize items of mymset1
    TEST_EQUAL(msize, mymset2.size());
    {
        Xapian::MSetIterator i = mymset1.begin();
        Xapian::MSetIterator j = mymset2.begin();
        for ( ; i != mymset1.end(), j != mymset2.end(); ++i, j++) {
            // if this fails, then setting match_sort_forward=true was not
            // the same as the default.
            TEST_EQUAL(*i, *j);
        }
    }

    // mymset3 should be last msize items of mymset1, in reverse order
    TEST_EQUAL(msize, mymset3.size());
    {
        vector<Xapian::docid> rev(mymset1.begin(), mymset1.end());
        // Next iterator not const because of compiler brokenness (egcs 1.1.2)
        vector<Xapian::docid>::reverse_iterator i = rev.rbegin();
        Xapian::MSetIterator j = mymset3.begin();
        for ( ; j != mymset3.end(); ++i, j++) {
            // if this fails, then setting match_sort_forward=false didn't
            // reverse the results.
            TEST_EQUAL(*i, *j);
        }
    }

    return true;
}

// tests that get_matching_terms() returns the terms in the right order
DEFINE_TESTCASE(getmterms1, backend) {
    list<string> answers_list;
    answers_list.push_back("one");
    answers_list.push_back("two");
    answers_list.push_back("three");
    answers_list.push_back("four");

    Xapian::Database mydb(get_database("apitest_termorder"));
    Xapian::Enquire enquire(mydb);

    Xapian::Query myquery(Xapian::Query::OP_OR,
            Xapian::Query(Xapian::Query::OP_AND,
                    Xapian::Query("one", 1, 1),
                    Xapian::Query("three", 1, 3)),
            Xapian::Query(Xapian::Query::OP_OR,
                    Xapian::Query("four", 1, 4),
                    Xapian::Query("two", 1, 2)));

    enquire.set_query(myquery);

    Xapian::MSet mymset = enquire.get_mset(0, 10);

    TEST_MSET_SIZE(mymset, 1);
    list<string> list(enquire.get_matching_terms_begin(mymset.begin()),
                          enquire.get_matching_terms_end(mymset.begin()));
    TEST(list == answers_list);

    return true;
}

// tests that get_matchi