Index: docs/quartzdesign.html =================================================================== --- docs/quartzdesign.html (revision 7333) +++ docs/quartzdesign.html (working copy) @@ -243,6 +243,23 @@ It is quite possible that the termlists and position lists would benefit from being split into chunks in this way. +

All document lists

+ +It is possible to use the Xapian API to obtain a list of all documents in the +database. This is done by creating a special postinglist. This functionality +was added after the file structure in use by Quartz was frozen, and it is +unfortunately impossible to implement efficiently for Quartz. + +The problem is that it is not possible to read the list of documents in sorted +order direct from disk - instead, the list is read into memory to be sorted. +For databases which do not have sparse document IDs, this should not use much +memory since the list is kept in memory in a range-compressed form (but does +require an iteration over the entirety of one of the tables of the Quartz +database - no skipping can be done in this case. This is unlikely to be fixed, +since we don't believe it can be without changing Quartz's structure. In any +case, it is not a priority since Quartz is due to be replaced by Flint as the +default backend soon. +

Btree implementation

The tables are currently all implemented as B-trees (actually a form of Index: tests/apitest.cc =================================================================== --- tests/apitest.cc (revision 7333) +++ tests/apitest.cc (working copy) @@ -3,6 +3,7 @@ * Copyright 1999,2000,2001 BrightStation PLC * Copyright 2002 Ananova Ltd * Copyright 2003,2004,2006 Olly Betts + * Copyright 2006 Richard Boulton * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as @@ -45,6 +46,13 @@ BackendManager backendmanager; Xapian::Database +get_database() +{ + vector dbnames; + return backendmanager.get_database(dbnames); +} + +Xapian::Database get_database(const string &dbname) { return backendmanager.get_database(dbname); @@ -101,6 +109,7 @@ RUNTESTS("inmemory", anydb); RUNTESTS("inmemory", specchar); RUNTESTS("inmemory", writabledb); + RUNTESTS("inmemory", writablelocaldb); RUNTESTS("inmemory", localdb); RUNTESTS("inmemory", positionaldb); RUNTESTS("inmemory", localpositionaldb); @@ -114,6 +123,7 @@ RUNTESTS("flint", anydb); RUNTESTS("flint", specchar); RUNTESTS("flint", writabledb); + RUNTESTS("flint", writablelocaldb); RUNTESTS("flint", localdb); RUNTESTS("flint", positionaldb); RUNTESTS("flint", localpositionaldb); @@ -129,6 +139,7 @@ RUNTESTS("quartz", anydb); RUNTESTS("quartz", specchar); RUNTESTS("quartz", writabledb); + RUNTESTS("quartz", writablelocaldb); RUNTESTS("quartz", localdb); RUNTESTS("quartz", positionaldb); RUNTESTS("quartz", localpositionaldb); Index: tests/apitest.h =================================================================== --- tests/apitest.h (revision 7333) +++ tests/apitest.h (working copy) @@ -3,6 +3,7 @@ * ----START-LICENCE---- * Copyright 1999,2000,2001 BrightStation PLC * Copyright 2003,2004 Olly Betts + * Copyright 2006 Richard Boulton * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as @@ -26,6 +27,7 @@ #include +Xapian::Database get_database(); Xapian::Database get_database(const std::string &dbname); Xapian::Database get_database(const std::string &dbname, const std::string &dbname2); Index: tests/api_db.cc =================================================================== --- tests/api_db.cc (revision 7333) +++ tests/api_db.cc (working copy) @@ -3,6 +3,7 @@ * Copyright 1999,2000,2001 BrightStation PLC * Copyright 2002 Ananova Ltd * Copyright 2002,2003,2004,2005,2006 Olly Betts + * Copyright 2006 Richard Boulton * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as @@ -895,8 +896,6 @@ { Xapian::Database db(get_database("apitest_simpledata")); - TEST_EXCEPTION(Xapian::InvalidArgumentError, db.postlist_begin("")); - TEST_EQUAL(db.postlist_begin("rosebud"), db.postlist_end("rosebud")); string s = "let_us_see_if_we_can_break_it_with_a_really_really_long_term."; @@ -1011,6 +1010,62 @@ return true; } +// tests all document postlists +static bool test_allpostlist1() +{ + Xapian::Database db(get_database("apitest_manydocs")); + Xapian::PostingIterator i = db.postlist_begin(""); + unsigned int j = 1; + while (i != db.postlist_end("")) { + TEST_EQUAL(*i, j); + i++; + j++; + } + TEST_EQUAL(j, 513); + + i = db.postlist_begin(""); + j = 1; + while (i != db.postlist_end("")) { + TEST_EQUAL(*i, j); + i++; + j++; + if (j == 50) { + j += 10; + i.skip_to(j); + } + } + TEST_EQUAL(j, 513); + + return true; +} + +static void test_emptyterm1_helper(Xapian::Database & db) +{ + // Don't bother with postlist_begin() because allpostlist tests cover that. + TEST_EXCEPTION(Xapian::InvalidArgumentError, db.positionlist_begin(1, "")); + TEST_EQUAL(db.get_doccount(), db.get_termfreq("")); + TEST_EQUAL(db.get_doccount() != 0, db.term_exists("")); + TEST_EQUAL(db.get_doccount(), db.get_collection_freq("")); +} + +// tests results of passing an empty term to various methods +static bool test_emptyterm1() +{ + Xapian::Database db(get_database("apitest_manydocs")); + TEST_EQUAL(db.get_doccount(), 512); + test_emptyterm1_helper(db); + + db = get_database("apitest_onedoc"); + TEST_EQUAL(db.get_doccount(), 1); + test_emptyterm1_helper(db); + + db = get_database(); + TEST_EQUAL(db.get_doccount(), 0); + test_emptyterm1_helper(db); + + return true; +} + // tests collection frequency static bool test_collfreq1() { @@ -1379,6 +1434,8 @@ {"postlist4", test_postlist4}, {"postlist5", test_postlist5}, {"postlist6", test_postlist6}, + {"allpostlist1", test_allpostlist1}, + {"emptyterm1", test_emptyterm1}, {"termstats", test_termstats}, {"sortvalue1", test_sortvalue1}, // consistency1 will run on the remote backend, but it's particularly slow Index: tests/api_wrdb.cc =================================================================== --- tests/api_wrdb.cc (revision 7333) +++ tests/api_wrdb.cc (working copy) @@ -4,6 +4,7 @@ * Copyright 2001 Hein Ragas * Copyright 2002 Ananova Ltd * Copyright 2002,2003,2004,2005,2006 Olly Betts + * Copyright 2006 Richard Boulton * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as @@ -909,6 +910,90 @@ return true; } +// tests all document postlists +static bool test_allpostlist2() +{ + Xapian::WritableDatabase db(get_writable_database("apitest_manydocs")); + Xapian::PostingIterator i = db.postlist_begin(""); + unsigned int j = 1; + while (i != db.postlist_end("")) { + TEST_EQUAL(*i, j); + i++; + j++; + } + TEST_EQUAL(j, 513); + + db.delete_document(1); + db.delete_document(50); + db.delete_document(512); + + i = db.postlist_begin(""); + j = 2; + while (i != db.postlist_end("")) { + TEST_EQUAL(*i, j); + i++; + j++; + if (j == 50) j++; + } + TEST_EQUAL(j, 512); + + i = db.postlist_begin(""); + j = 2; + while (i != db.postlist_end("")) { + TEST_EQUAL(*i, j); + i++; + j++; + if (j == 40) { + j += 10; + i.skip_to(j); + j++; + } + } + TEST_EQUAL(j, 512); + + return true; +} + +static void test_emptyterm2_helper(Xapian::WritableDatabase & db) +{ + // Don't bother with postlist_begin() because allpostlist tests cover that. + TEST_EXCEPTION(Xapian::InvalidArgumentError, db.positionlist_begin(1, "")); + TEST_EQUAL(db.get_doccount(), db.get_termfreq("")); + TEST_EQUAL(db.get_doccount() != 0, db.term_exists("")); + TEST_EQUAL(db.get_doccount(), db.get_collection_freq("")); +} + +// tests results of passing an empty term to various methods +// equivalent of emptyterm1 for a writable database +static bool test_emptyterm2() +{ + Xapian::WritableDatabase db(get_writable_database("apitest_manydocs")); + TEST_EQUAL(db.get_doccount(), 512); + test_emptyterm2_helper(db); + db.delete_document(1); + TEST_EQUAL(db.get_doccount(), 511); + test_emptyterm2_helper(db); + db.delete_document(50); + TEST_EQUAL(db.get_doccount(), 510); + test_emptyterm2_helper(db); + db.delete_document(512); + TEST_EQUAL(db.get_doccount(), 509); + test_emptyterm2_helper(db); + + db = get_writable_database("apitest_onedoc"); + TEST_EQUAL(db.get_doccount(), 1); + test_emptyterm2_helper(db); + db.delete_document(1); + TEST_EQUAL(db.get_doccount(), 0); + test_emptyterm2_helper(db); + + db = get_writable_database(""); + TEST_EQUAL(db.get_doccount(), 0); + test_emptyterm2_helper(db); + + return true; +} + // Check that PHRASE/NEAR becomes AND if there's no positional info in the // database. static bool test_phraseorneartoand1() @@ -1035,7 +1120,14 @@ {"replacedoc3", test_replacedoc3}, {"replacedoc4", test_replacedoc4}, {"uniqueterm1", test_uniqueterm1}, + {"emptyterm2", test_emptyterm2}, {"phraseorneartoand1", test_phraseorneartoand1}, {"longpositionlist1", test_longpositionlist1}, {0, 0} }; + +/// The tests which use a writable, but local, backend +test_desc writablelocaldb_tests[] = { + {"allpostlist2", test_allpostlist2}, + {0, 0} +}; Index: tests/api_wrdb.h =================================================================== --- tests/api_wrdb.h (revision 7333) +++ tests/api_wrdb.h (working copy) @@ -3,6 +3,7 @@ * ----START-LICENCE---- * Copyright 1999,2000,2001 BrightStation PLC * Copyright 2004 Olly Betts + * Copyright 2006 Richard Boulton * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as @@ -27,5 +28,6 @@ #include "testsuite.h" extern test_desc writabledb_tests[]; +extern test_desc writablelocaldb_tests[]; #endif /* XAPIAN_HGUARD_API_WRDB_H */ Index: include/xapian/database.h =================================================================== --- include/xapian/database.h (revision 7333) +++ include/xapian/database.h (working copy) @@ -4,6 +4,7 @@ /* Copyright 1999,2000,2001 BrightStation PLC * Copyright 2002 Ananova Ltd * Copyright 2002,2003,2004,2005,2006 Olly Betts + * Copyright 2006 Richard Boulton * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as @@ -108,6 +109,11 @@ /** An iterator pointing to the start of the postlist * for a given term. + * + * If the term name is the empty string, the iterator returned + * will list all the documents in the database. Such an iterator + * will always return a WDF value of 1, since there is no obvious + * meaning for this quantity in this case. */ PostingIterator postlist_begin(const std::string &tname) const; Index: ChangeLog =================================================================== --- ChangeLog (revision 7333) +++ ChangeLog (working copy) @@ -1,3 +1,33 @@ +Wed Oct 11 19:05:38 BST 2006 Richard Boulton + + * common/database.h,api/omdatabase.cc, + backends/inmemory/inmemory_database.cc, + backends/inmemory/inmemory_database.h, + backends/quartz/Makefile.am,backends/quartz/quartz_database.cc, + backends/quartz/quartz_alldocspostlist.h, + backends/quartz/quartz_alldocspostlist.cc, + backends/flint/Makefile.am,backends/flint/flint_database.cc, + backends/flint/flint_alldocspostlist.cc, + backends/flint/flint_alldocspostlist.h: + Implement posting lists which return a list of all documents in + the database. Such a posting list is obtained by calling + Xapian::Database::postlist_begin() with an empty term (ie, ""). + Also, all Xapian::Database methods which take a termname now + accept an empty term, and return appropriate values (ie, + get_termfreq("") and get_collection_freq("") return the number of + documents in the database, and term_exists("") returns true + unless the database is empty). + * docs/quartzdesign.html: Document the inefficiency of all-document + postlists for Quartz. + * tests/apitest.cc,tests/apitest.h,tests/api_db.cc, + tests/api_wrdb.cc,tests/api_wrdb.h: Add tests for all-document + postlists, and for passing an empty term to all the applicable + datbase methods. This defines the new tests allpostlist[12], and + emptyterm[12] Includes defining a new test category for databases + which are local and writable, and adding a function to get an + empty database (for testing the empty term methods with such a + database). + Tue Oct 10 17:24:00 BST 2006 Olly Betts * NEWS: Bump release date. Index: common/database.h =================================================================== --- common/database.h (revision 7333) +++ common/database.h (working copy) @@ -3,6 +3,7 @@ * Copyright 1999,2000,2001 BrightStation PLC * Copyright 2002 Ananova Ltd * Copyright 2002,2003,2004,2005,2006 Olly Betts + * Copyright 2006 Richard Boulton * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as @@ -188,7 +189,7 @@ * use. */ LeafPostList * open_post_list(const string & tname) const { - if (!term_exists(tname)) { + if (!tname.empty() && !term_exists(tname)) { DEBUGLINE(MATCH, tname + " is not in database."); // Term doesn't exist in this database. However, we create // a (empty) postlist for it to help make distributed searching Index: api/omdatabase.cc =================================================================== --- api/omdatabase.cc (revision 7333) +++ api/omdatabase.cc (working copy) @@ -3,6 +3,7 @@ * Copyright 1999,2000,2001 BrightStation PLC * Copyright 2001,2002 Ananova Ltd * Copyright 2002,2003,2004,2005,2006 Olly Betts + * Copyright 2006 Richard Boulton * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as @@ -101,8 +102,6 @@ Database::postlist_begin(const string &tname) const { DEBUGAPICALL(PostingIterator, "Database::postlist_begin", tname); - if (tname.empty()) - throw InvalidArgumentError("Zero length terms are invalid"); // Don't bother checking that the term exists first. If it does, we // just end up doing more work, and if it doesn't, we save very little @@ -248,8 +247,9 @@ Database::get_termfreq(const string & tname) const { DEBUGAPICALL(Xapian::doccount, "Database::get_termfreq", tname); - if (tname.empty()) - throw InvalidArgumentError("Zero length terms are invalid"); + if (tname.empty()) { + return get_doccount(); + } Xapian::doccount tf = 0; vector >::const_iterator i; for (i = internal.begin(); i != internal.end(); i++) { @@ -262,8 +262,9 @@ Database::get_collection_freq(const string & tname) const { DEBUGAPICALL(Xapian::termcount, "Database::get_collection_freq", tname); - if (tname.empty()) - throw InvalidArgumentError("Zero length terms are invalid"); + if (tname.empty()) { + return get_doccount(); + } Xapian::termcount cf = 0; vector >::const_iterator i; @@ -303,8 +304,9 @@ bool Database::term_exists(const string & tname) const { - if (tname.empty()) - throw InvalidArgumentError("Zero length terms are invalid"); + if (tname.empty()) { + return get_doccount() != 0; + } vector >::const_iterator i; for (i = internal.begin(); i != internal.end(); ++i) { if ((*i)->term_exists(tname)) return true; Index: backends/inmemory/inmemory_database.cc =================================================================== --- backends/inmemory/inmemory_database.cc (revision 7333) +++ backends/inmemory/inmemory_database.cc (working copy) @@ -3,6 +3,7 @@ * Copyright 1999,2000,2001 BrightStation PLC * Copyright 2002 Ananova Ltd * Copyright 2002,2003,2004,2005,2006 Olly Betts + * Copyright 2006 Richard Boulton * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as @@ -260,6 +261,89 @@ return Xapian::PositionIterator(db->open_position_list(did, (*pos).tname)); } +///////////////////////////// +// InMemoryAllDocsPostList // +///////////////////////////// + +InMemoryAllDocsPostList::InMemoryAllDocsPostList(Xapian::Internal::RefCntPtr db_) + : did(0), db(db_) +{ +} + +Xapian::doccount +InMemoryAllDocsPostList::get_termfreq() const +{ + return db->totdocs; +} + +Xapian::docid +InMemoryAllDocsPostList::get_docid() const +{ + Assert(did > 0); + Assert(did <= db->termlists.size()); + Assert(db->termlists[did - 1].is_valid); + return did; +} + +Xapian::doclength +InMemoryAllDocsPostList::get_doclength() const +{ + return db->get_doclength(did); +} + +Xapian::termcount +InMemoryAllDocsPostList::get_wdf() const +{ + return 1; +} + +PositionList * +InMemoryAllDocsPostList::read_position_list() +{ + throw Xapian::UnimplementedError("Can't open position list for all docs iterator"); +} + +PositionList * +InMemoryAllDocsPostList::open_position_list() const +{ + throw Xapian::UnimplementedError("Can't open position list for all docs iterator"); +} + +PostList * +InMemoryAllDocsPostList::next(Xapian::weight /*w_min*/) +{ + Assert(!at_end()); + do { + ++did; + } while (did <= db->termlists.size() && !db->termlists[did - 1].is_valid); + return NULL; +} + +PostList * +InMemoryAllDocsPostList::skip_to(Xapian::docid did_, Xapian::weight /*w_min*/) +{ + Assert(!at_end()); + if (did <= did_) { + did = did_; + while (did <= db->termlists.size() && !db->termlists[did - 1].is_valid) { + ++did; + } + } + return NULL; +} + +bool +InMemoryAllDocsPostList::at_end() const +{ + return (did > db->termlists.size()); +} + +string +InMemoryAllDocsPostList::get_description() const +{ + return "InMemoryAllDocsPostList" + om_tostring(did); +} + /////////////////////////// // Actual database class // /////////////////////////// @@ -279,7 +363,11 @@ LeafPostList * InMemoryDatabase::do_open_post_list(const string & tname) const { - Assert(tname.size() != 0); + if (tname.empty()) { + if (termlists.empty()) + return new EmptyPostList(); + return new InMemoryAllDocsPostList(Xapian::Internal::RefCntPtr(this)); + } map::const_iterator i = postlists.find(tname); if (i == postlists.end() || i->second.term_freq == 0) return new EmptyPostList(); Index: backends/inmemory/inmemory_database.h =================================================================== --- backends/inmemory/inmemory_database.h (revision 7333) +++ backends/inmemory/inmemory_database.h (working copy) @@ -3,6 +3,7 @@ * Copyright 1999,2000,2001 BrightStation PLC * Copyright 2002 Ananova Ltd * Copyright 2002,2003,2004,2005,2006 Olly Betts + * Copyright 2006 Richard Boulton * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as @@ -159,6 +160,35 @@ string get_description() const; }; +/** A PostList over all docs in an inmemory database. + */ +class InMemoryAllDocsPostList : public LeafPostList { + friend class InMemoryDatabase; + private: + Xapian::docid did; + + Xapian::Internal::RefCntPtr db; + + InMemoryAllDocsPostList(Xapian::Internal::RefCntPtr db); + public: + Xapian::doccount get_termfreq() const; + + Xapian::docid get_docid() const; // Gets current docid + Xapian::doclength get_doclength() const; // Length of current document + Xapian::termcount get_wdf() const; // Within Document Frequency + PositionList * read_position_list(); + PositionList * open_position_list() const; + + PostList *next(Xapian::weight w_min); // Moves to next docid + + PostList *skip_to(Xapian::docid did, Xapian::weight w_min); // Moves to next docid >= specified docid + + // True if we're off the end of the list + bool at_end() const; + + string get_description() const; +}; + // Term List class InMemoryTermList : public LeafTermList { friend class InMemoryDatabase; @@ -193,6 +223,7 @@ * This is a prototype database, mainly used for debugging and testing. */ class InMemoryDatabase : public Xapian::Database::Internal { + friend class InMemoryAllDocsPostList; private: map postlists; vector termlists; Index: backends/quartz/quartz_alldocspostlist.h =================================================================== --- backends/quartz/quartz_alldocspostlist.h (revision 0) +++ backends/quartz/quartz_alldocspostlist.h (revision 0) @@ -0,0 +1,215 @@ +/* quartz_alldocspostlist.h: All document postlists in quartz databases + * + * ----START-LICENCE---- + * Copyright 1999,2000,2001 BrightStation PLC + * Copyright 2002 Ananova Ltd + * Copyright 2002,2003,2004,2005 Olly Betts + * Copyright 2006 Richard Boulton + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA + * -----END-LICENCE----- + */ + +#ifndef OM_HGUARD_QUARTZ_ALLDOCSPOSTLIST_H +#define OM_HGUARD_QUARTZ_ALLDOCSPOSTLIST_H + +#include +#include + +#include "leafpostlist.h" +#include +#include +#include "database.h" +#include "omassert.h" +#include "quartz_types.h" +#include "btree.h" + +using namespace std; + +class Bcursor; + +class QuartzDocIdList; + +class QuartzDocIdListIterator { + private: + const map * ranges; + map::const_iterator currrange; + Xapian::docid currdocid; + + friend class QuartzDocIdList; + + QuartzDocIdListIterator(const map * ranges_); + QuartzDocIdListIterator(const map * ranges_, int); + + public: + Xapian::docid operator*() { + return currdocid; + } + + friend bool operator==(const QuartzDocIdListIterator &a, + const QuartzDocIdListIterator &b); + + QuartzDocIdListIterator(); + ~QuartzDocIdListIterator() {} + QuartzDocIdListIterator(const QuartzDocIdListIterator & other); + void operator=(const QuartzDocIdListIterator & other); + + QuartzDocIdListIterator & operator++(); + + Xapian::DocIDWrapper operator++(int) { + Xapian::docid tmp = **this; + operator++(); + return Xapian::DocIDWrapper(tmp); + } + + Xapian::docid operator *() const { return currdocid; } + + /// Allow use as an STL iterator + //@{ + typedef std::input_iterator_tag iterator_category; + typedef Xapian::docid value_type; + typedef Xapian::doccount_diff difference_type; + typedef Xapian::docid * pointer; + typedef Xapian::docid & reference; + //@} +}; + +inline bool operator==(const QuartzDocIdListIterator &a, + const QuartzDocIdListIterator &b) +{ + if (a.ranges != b.ranges) + return false; + return a.currdocid == b.currdocid; +} + +inline bool operator!=(const QuartzDocIdListIterator &a, + const QuartzDocIdListIterator &b) +{ + return !(a==b); +} + +class QuartzDocIdList { + private: + /** Map from start of a range to end of a range. + */ + map ranges; + + public: + QuartzDocIdList() {} + void addDocId(Xapian::docid did); + + QuartzDocIdListIterator begin() const { + return QuartzDocIdListIterator(&ranges); + } + + QuartzDocIdListIterator end() const { + return QuartzDocIdListIterator(&ranges, 1); + } +}; + +/** A postlist in a quartz database. + */ +class QuartzAllDocsPostList : public LeafPostList { + private: + /// Pointer to database. + Xapian::Internal::RefCntPtr this_db; + + /// List of docids. + QuartzDocIdList docids; + + /// Iterator through docids. + QuartzDocIdListIterator dociditer; + + /// Number of documents in the database. + Xapian::doccount doccount; + + /// Whether we've started yet. + bool have_started; + + /// Copying is not allowed. + QuartzAllDocsPostList(const QuartzAllDocsPostList &); + + /// Assignment is not allowed. + void operator=(const QuartzAllDocsPostList &); + + public: + /// Default constructor. + QuartzAllDocsPostList(Xapian::Internal::RefCntPtr this_db_, + const Btree * table, + Xapian::doccount doccount_); + + /// Destructor. + ~QuartzAllDocsPostList(); + + /** Returns length of the all documents postlist. + * + * This is also the number of documents in the database. + */ + Xapian::doccount get_termfreq() const { return doccount; } + + /** Returns the number of occurrences of the term in the database. + * + * We pretend that each document has one "empty" term, so this is + * also the number of documents in the database. + */ + Xapian::termcount get_collection_freq() const { return doccount; } + + /// Returns the current docid. + Xapian::docid get_docid() const { + Assert(have_started); + return *dociditer; + } + + /// Returns the length of current document. + Xapian::doclength get_doclength() const { + Assert(have_started); + return this_db->get_doclength(*dociditer); + } + + /** Returns the Within Document Frequency of the term in the current + * document. + */ + Xapian::termcount get_wdf() const { + Assert(have_started); + return static_cast(1); + } + + /** Get the list of positions of the term in the current document. + */ + PositionList *read_position_list() { + throw Xapian::InvalidOperationError("Can't read position list from all docs postlist."); + } + + /** Get the list of positions of the term in the current document. + */ + PositionList * open_position_list() const { + throw Xapian::InvalidOperationError("Can't read position list from all docs postlist."); + } + + /// Move to the next document. + PostList * next(Xapian::weight w_min); + + /// Skip to next document with docid >= docid. + PostList * skip_to(Xapian::docid desired_did, Xapian::weight w_min); + + /// Return true if and only if we're off the end of the list. + bool at_end() const { return (have_started && dociditer == docids.end()); } + + /// Get a description of the postlist. + std::string get_description() const; +}; + +#endif /* OM_HGUARD_QUARTZ_ALLDOCSPOSTLIST_H */ Index: backends/quartz/quartz_database.cc =================================================================== --- backends/quartz/quartz_database.cc (revision 7333) +++ backends/quartz/quartz_database.cc (working copy) @@ -4,6 +4,7 @@ * Copyright 2001 Hein Ragas * Copyright 2002 Ananova Ltd * Copyright 2002,2003,2004,2005,2006 Olly Betts + * Copyright 2006 Richard Boulton * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as @@ -33,6 +34,7 @@ #include #include "quartz_postlist.h" +#include "quartz_alldocspostlist.h" #include "quartz_termlist.h" #include "quartz_positionlist.h" #include "quartz_utils.h" @@ -598,10 +600,15 @@ QuartzDatabase::do_open_post_list(const string& tname) const { DEBUGCALL(DB, LeafPostList *, "QuartzDatabase::do_open_post_list", tname); - Assert(!tname.empty()); + Xapian::Internal::RefCntPtr ptrtothis(this); - Xapian::Internal::RefCntPtr ptrtothis(this); - return(new QuartzPostList(ptrtothis, + if (tname.empty()) { + RETURN(new QuartzAllDocsPostList(ptrtothis, + &termlist_table, + get_doccount())); + } + + RETURN(new QuartzPostList(ptrtothis, &postlist_table, &positionlist_table, tname)); @@ -1111,8 +1118,14 @@ QuartzWritableDatabase::do_open_post_list(const string& tname) const { DEBUGCALL(DB, LeafPostList *, "QuartzWritableDatabase::do_open_post_list", tname); - Assert(!tname.empty()); + Xapian::Internal::RefCntPtr ptrtothis(this); + if (tname.empty()) { + RETURN(new QuartzAllDocsPostList(ptrtothis, + &database_ro.termlist_table, + get_doccount())); + } + // Need to flush iff we've got buffered changes to this term's postlist. map > >::const_iterator j; j = mod_plists.find(tname); @@ -1122,8 +1135,7 @@ do_flush_const(); } - Xapian::Internal::RefCntPtr ptrtothis(this); - return(new QuartzPostList(ptrtothis, + RETURN(new QuartzPostList(ptrtothis, &database_ro.postlist_table, &database_ro.positionlist_table, tname)); Index: backends/quartz/Makefile.am =================================================================== --- backends/quartz/Makefile.am (revision 7333) +++ backends/quartz/Makefile.am (working copy) @@ -12,6 +12,7 @@ quartz_utils.h \ quartz_log.h \ quartz_document.h \ + quartz_alldocspostlist.h \ quartz_alltermslist.h \ quartz_metafile.h \ btree.h \ @@ -27,6 +28,7 @@ quartz_values.cc \ quartz_log.cc \ quartz_document.cc \ + quartz_alldocspostlist.cc \ quartz_alltermslist.cc \ quartz_metafile.cc \ btree.cc \ Index: backends/quartz/quartz_alldocspostlist.cc =================================================================== --- backends/quartz/quartz_alldocspostlist.cc (revision 0) +++ backends/quartz/quartz_alldocspostlist.cc (revision 0) @@ -0,0 +1,258 @@ +/* quartz_alldocspostlist.cc: All-document postlists in quartz databases + * + * ----START-LICENCE---- + * Copyright 1999,2000,2001 BrightStation PLC + * Copyright 2002,2003,2004,2005 Olly Betts + * Copyright 2006 Richard Boulton + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA + * -----END-LICENCE----- + */ + +#include +#include "omdebug.h" +#include "quartz_alldocspostlist.h" +#include "quartz_utils.h" +#include "bcursor.h" +#include "database.h" +#include + +QuartzDocIdListIterator::QuartzDocIdListIterator() + : ranges(NULL), + currrange(), + currdocid(0) +{ + DEBUGCALL(DB, void, + "QuartzDocIdListIterator::QuartzDocIdListIterator", ""); +} + +QuartzDocIdListIterator::QuartzDocIdListIterator(const map * ranges_, int) + : ranges(ranges_), + currrange(ranges_->end()), + currdocid(0) +{ +} + +QuartzDocIdListIterator::QuartzDocIdListIterator(const map * ranges_) + : ranges(ranges_), + currrange(ranges_->begin()), + currdocid(0) +{ + DEBUGCALL(DB, void, + "QuartzDocIdListIterator::QuartzDocIdListIterator", "ranges"); + if (currrange != ranges_->end()) { + currdocid = currrange->first; + } + + map::const_iterator i; + for (i = ranges->begin(); i != ranges->end(); i++) { + DEBUGLINE(DB, "Docid range begin=" << i->first << ", end=" << i->second); + } +} + +QuartzDocIdListIterator::QuartzDocIdListIterator(const QuartzDocIdListIterator & other) + : ranges(other.ranges), + currrange(other.currrange), + currdocid(other.currdocid) +{ + DEBUGCALL(DB, void, + "QuartzDocIdListIterator::~QuartzDocIdListIterator", "other"); +} + +void +QuartzDocIdListIterator::operator=(const QuartzDocIdListIterator & other) +{ + DEBUGCALL(DB, void, + "QuartzDocIdListIterator::operator=", "other"); + ranges = other.ranges; + currrange = other.currrange; + currdocid = other.currdocid; +} + +QuartzDocIdListIterator & +QuartzDocIdListIterator::operator++() +{ + DEBUGCALL(DB, void, + "QuartzDocIdListIterator::operator++", ""); + DEBUGLINE(DB, string("Moved from ") << + (currrange == ranges->end() ? string("end.") : string("docid = ") + + om_tostring(currdocid))); + + if (currrange != ranges->end()) { + Assert(currrange->first <= currdocid); + if (currdocid < currrange->second) { + currdocid++; + } else { + currrange++; + if (currrange == ranges->end()) { + currdocid = 0; + } else { + Assert(currrange->first > currdocid); + currdocid = currrange->first; + } + } + } + + DEBUGLINE(DB, string("Moved to ") << + (currrange == ranges->end() ? string("end.") : string("docid = ") + + om_tostring(currdocid))); + + return *this; +} + + +void +QuartzDocIdList::addDocId(Xapian::docid did) { + DEBUGCALL(DB, void, "QuartzDocIdList::addDocId", did); + + if(ranges.size() == 0) { + ranges.insert(pair(did, did)); + return; + } + + if (did < ranges.begin()->first) { + Xapian::docid newend; + if (did == ranges.begin()->first - 1) { + newend = ranges.begin()->second; + ranges.erase(ranges.begin()); + } else { + newend = did; + } + ranges[did] = newend; + return; + } + + map::iterator i; + i = ranges.lower_bound(did); + if (i == ranges.end()) { + i--; + Assert(did > i->first); + } else if (did < i->first) { + i--; + Assert(did > i->first); + } + Assert(did >= i->first); + + if (did <= i->second) { + // Do nothing - already in range + return; + } + + if (did == i->second + 1) { + // Extend range + i->second = did; + map::iterator j; + j = i; + j++; + if (j != ranges.end()) { + Assert(j->first > i->second); + if (j->first == i->second + 1) { + // Merge ranges + i->second = j->second; + ranges.erase(j); + } + } + } else { + ranges[did] = did; + } +} + + +QuartzAllDocsPostList::QuartzAllDocsPostList(Xapian::Internal::RefCntPtr this_db_, + const Btree * table, + Xapian::doccount doccount_) + : this_db(this_db_), + docids(), + dociditer(), + doccount(doccount_), + have_started(false) +{ + DEBUGCALL(DB, void, "QuartzAllDocsPostList::QuartzAllDocsPostList", + this_db_.get() << ", " << table << ", " << doccount_); + + // Move to initial NULL entry. + Bcursor * cursor = table->cursor_get(); + cursor->find_entry(""); + if (!cursor->after_end()) + cursor->next(); + while (!cursor->after_end()) { + string key = cursor->current_key; + const char * keystr = key.c_str(); + Xapian::docid did; + if (!unpack_uint_last(&keystr, keystr + key.length(), &did)) { + throw Xapian::RangeError("Document number in value table is too large"); + } + docids.addDocId(did); + cursor->next(); + } +} + +QuartzAllDocsPostList::~QuartzAllDocsPostList() +{ + DEBUGCALL(DB, void, "QuartzAllDocsPostList::~QuartzAllDocsPostList", ""); +} + +PostList * +QuartzAllDocsPostList::next(Xapian::weight w_min) +{ + DEBUGCALL(DB, PostList *, "QuartzAllDocsPostList::next", w_min); + (void)w_min; + + if (have_started) { + ++dociditer; + } else { + dociditer = docids.begin(); + have_started = true; + } + + DEBUGLINE(DB, string("Moved to ") << + (dociditer == docids.end() ? string("end.") : string("docid = ") + + om_tostring(*dociditer))); + + RETURN(NULL); +} + +PostList * +QuartzAllDocsPostList::skip_to(Xapian::docid desired_did, Xapian::weight w_min) +{ + DEBUGCALL(DB, PostList *, + "QuartzAllDocsPostList::skip_to", desired_did << ", " << w_min); + (void)w_min; // no warning + + // Don't skip back, and don't need to do anything if already there. + if (!have_started) { + dociditer = docids.begin(); + } + if (dociditer == docids.end()) RETURN(NULL); + if (desired_did <= *dociditer) RETURN(NULL); + + while (dociditer != docids.end() && *dociditer < desired_did) + { + ++dociditer; + } + + DEBUGLINE(DB, string("Skipped to ") << + (dociditer == docids.end() ? string("end.") : string("docid = ") + + om_tostring(*dociditer))); + + RETURN(NULL); +} + +string +QuartzAllDocsPostList::get_description() const +{ + return ":" + om_tostring(doccount); +} Index: backends/flint/flint_database.cc =================================================================== --- backends/flint/flint_database.cc (revision 7333) +++ backends/flint/flint_database.cc (working copy) @@ -4,6 +4,7 @@ * Copyright 2001 Hein Ragas * Copyright 2002 Ananova Ltd * Copyright 2002,2003,2004,2005,2006 Olly Betts + * Copyright 2006 Richard Boulton * * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License as @@ -34,6 +35,7 @@ #include "flint_modifiedpostlist.h" #include "flint_postlist.h" +#include "flint_alldocspostlist.h" #include "flint_termlist.h" #include "flint_positionlist.h" #include "flint_utils.h" @@ -455,10 +457,15 @@ FlintDatabase::do_open_post_list(const string& tname) const { DEBUGCALL(DB, LeafPostList *, "FlintDatabase::do_open_post_list", tname); - Assert(!tname.empty()); + Xapian::Internal::RefCntPtr ptrtothis(this); - Xapian::Internal::RefCntPtr ptrtothis(this); - return(new FlintPostList(ptrtothis, + if (tname.empty()) { + RETURN(new FlintAllDocsPostList(ptrtothis, + &termlist_table, + get_doccount())); + } + + RETURN(new FlintPostList(ptrtothis, &postlist_table, &positionlist_table, tname)); @@ -967,10 +974,14 @@ FlintWritableDatabase::do_open_post_list(const string& tname) const { DEBUGCALL(DB, LeafPostList *, "FlintWritableDatabase::do_open_post_list", tname); - Assert(!tname.empty()); - Xapian::Internal::RefCntPtr ptrtothis(this); + if (tname.empty()) { + RETURN(new FlintAllDocsPostList(ptrtothis, + &database_ro.termlist_table, + get_doccount())); + } + map > >::const_iterator j; j = mod_plists.find(tname); if (j != mod_plists.end()) { Index: backends/flint/flint_alldocspostlist.cc =================================================================== --- backends/flint/flint_alldocspostlist.cc (revision 0) +++ backends/flint/flint_alldocspostlist.cc (revision 0) @@ -0,0 +1,123 @@ +/* flint_alldocspostlist.cc: All-document postlists in flint databases + * + * ----START-LICENCE---- + * Copyright 1999,2000,2001 BrightStation PLC + * Copyright 2002,2003,2004,2005 Olly Betts + * Copyright 2006 Richard Boulton + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA + * -----END-LICENCE----- + */ + +#include +#include "omdebug.h" +#include "flint_alldocspostlist.h" +#include "flint_utils.h" +#include "flint_values.h" +#include "flint_cursor.h" +#include "database.h" + +/** The format of a postlist is: + */ +FlintAllDocsPostList::FlintAllDocsPostList(Xapian::Internal::RefCntPtr this_db_, + const FlintTable * table_, + Xapian::doccount doccount_) + : this_db(this_db_), + table(table_), + cursor(table->cursor_get()), + did(0), + is_at_end(false), + doccount(doccount_) +{ + DEBUGCALL(DB, void, "FlintAllDocsPostList::FlintAllDocsPostList", + this_db_.get() << ", " << table_ << ", " << doccount_); + + // Move to initial NULL entry. + cursor->find_entry(""); +} + +FlintAllDocsPostList::~FlintAllDocsPostList() +{ + DEBUGCALL(DB, void, "FlintAllDocsPostList::~FlintAllDocsPostList", ""); +} + +PostList * +FlintAllDocsPostList::next(Xapian::weight w_min) +{ + DEBUGCALL(DB, PostList *, "FlintAllDocsPostList::next", w_min); + (void)w_min; // no warning + + cursor->next(); + if (cursor->after_end()) { + is_at_end = true; + } else { + string key = cursor->current_key; + const char * keystr = key.c_str(); + if (!unpack_uint_preserving_sort(&keystr, keystr + key.length(), &did)) { + if (*keystr == 0) + throw Xapian::DatabaseCorruptError("Unexpected end of data when reading from termlist table"); + else + throw Xapian::RangeError("Document number in value table is too large"); + } + } + + DEBUGLINE(DB, string("Moved to ") << + (is_at_end ? string("end.") : string("docid = ") + + om_tostring(did))); + + RETURN(NULL); +} + +PostList * +FlintAllDocsPostList::skip_to(Xapian::docid desired_did, Xapian::weight w_min) +{ + DEBUGCALL(DB, PostList *, + "FlintAllDocsPostList::skip_to", desired_did << ", " << w_min); + (void)w_min; // no warning + + // Don't skip back, and don't need to do anything if already there. + if (desired_did <= did) RETURN(NULL); + if (is_at_end) RETURN(NULL); + + string desired_key = pack_uint_preserving_sort(desired_did); + bool exact_match = cursor->find_entry(desired_key); + if (!exact_match) + cursor->next(); + if (cursor->after_end()) { + is_at_end = true; + } else { + string key = cursor->current_key; + const char * keystr = key.c_str(); + if (!unpack_uint_preserving_sort(&keystr, keystr + key.length(), &did)) { + if (*keystr == 0) + throw Xapian::DatabaseCorruptError("Unexpected end of data when reading from termlist table"); + else + throw Xapian::RangeError("Document number in value table is too large"); + } + } + + DEBUGLINE(DB, string("Skipped to ") << + (is_at_end ? string("end.") : string("docid = ") + + om_tostring(did))); + + RETURN(NULL); +} + +string +FlintAllDocsPostList::get_description() const +{ + return ":" + om_tostring(doccount); +} Index: backends/flint/flint_alldocspostlist.h =================================================================== --- backends/flint/flint_alldocspostlist.h (revision 0) +++ backends/flint/flint_alldocspostlist.h (revision 0) @@ -0,0 +1,134 @@ +/* flint_alldocspostlist.h: All document postlists in flint databases + * + * ----START-LICENCE---- + * Copyright 1999,2000,2001 BrightStation PLC + * Copyright 2002 Ananova Ltd + * Copyright 2002,2003,2004,2005 Olly Betts + * Copyright 2006 Richard Boulton + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License as + * published by the Free Software Foundation; either version 2 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 + * USA + * -----END-LICENCE----- + */ + +#ifndef OM_HGUARD_FLINT_ALLDOCSPOSTLIST_H +#define OM_HGUARD_FLINT_ALLDOCSPOSTLIST_H + +#include +#include + +#include "leafpostlist.h" +#include "database.h" +#include "omassert.h" +#include "flint_types.h" + +using namespace std; + +class FlintCursor; +class FlintTable; + +/** A postlist in a flint database. + */ +class FlintAllDocsPostList : public LeafPostList { + private: + /** The database we are searching. This pointer is held so that the + * database doesn't get deleted before us. + */ + Xapian::Internal::RefCntPtr this_db; + + /// The table containing the values. + const FlintTable * table; + + /// Cursor pointing to current values. + AutoPtr cursor; + + /// Document id we're currently at. + Xapian::docid did; + + /// Whether we've run off the end of the list yet. + bool is_at_end; + + /// Number of documents in the database. + Xapian::doccount doccount; + + /// Copying is not allowed. + FlintAllDocsPostList(const FlintAllDocsPostList &); + + /// Assignment is not allowed. + void operator=(const FlintAllDocsPostList &); + + + public: + /// Default constructor. + FlintAllDocsPostList(Xapian::Internal::RefCntPtr this_db_, + const FlintTable * table_, + Xapian::doccount doccount_); + + /// Destructor. + ~FlintAllDocsPostList(); + + /** Returns length of the all documents postlist. + * + * This is also the number of documents in the database. + */ + Xapian::doccount get_termfreq() const { return doccount; } + + /** Returns the number of occurrences of the term in the database. + * + * We pretend that each document has one "empty" term, so this is + * also the number of documents in the database. + */ + Xapian::termcount get_collection_freq() const { return doccount; } + + /// Returns the current docid. + Xapian::docid get_docid() const { Assert(did != 0); return did; } + + /// Returns the length of current document. + Xapian::doclength get_doclength() const { + Assert(did != 0); + return this_db->get_doclength(did); + } + + /** Returns the Within Document Frequency of the term in the current + * document. + */ + Xapian::termcount get_wdf() const { Assert(did != 0); return static_cast(1); } + + /** Get the list of positions of the term in the current document. + */ + PositionList *read_position_list() { + throw Xapian::InvalidOperationError("Can't read position list from all docs postlist."); + } + + /** Get the list of positions of the term in the current document. + */ + PositionList * open_position_list() const { + throw Xapian::InvalidOperationError("Can't read position list from all docs postlist."); + } + + /// Move to the next document. + PostList * next(Xapian::weight w_min); + + /// Skip to next document with docid >= docid. + PostList * skip_to(Xapian::docid desired_did, Xapian::weight w_min); + + /// Return true if and only if we're off the end of the list. + bool at_end() const { return is_at_end; } + + /// Get a description of the postlist. + std::string get_description() const; +}; + +#endif /* OM_HGUARD_FLINT_ALLDOCSPOSTLIST_H */ Index: backends/flint/Makefile.am =================================================================== --- backends/flint/Makefile.am (revision 7333) +++ backends/flint/Makefile.am (working copy) @@ -12,6 +12,7 @@ flint_values.h \ flint_utils.h \ flint_document.h \ + flint_alldocspostlist.h \ flint_alltermslist.h \ flint_table.h \ flint_cursor.h \ @@ -29,6 +30,7 @@ flint_record.cc \ flint_values.cc \ flint_document.cc \ + flint_alldocspostlist.cc \ flint_alltermslist.cc \ flint_table.cc \ flint_cursor.cc \