Ticket #278: max_changesets.diff

xapian-core/backends/brass/brass_database.cc

@@ -7 +7 @@
  * Copyright 2006,2008 Lemur Consulting Ltd
  * Copyright 2009 Richard Boulton
  * Copyright 2009 Kan-Ru Chen
+ * Copyright 2011 Dan Colish
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -470 +471 @@
 
         throw;
     }
+    
+    // Only remove the oldest_changeset if we successfully write a new changeset and
+    // we have a revision number greater than max_changesets
+    if (changes_fd >= 0 && max_changesets < new_revision) {
+        // use the oldest changeset we know about to being deleting
+        // if nothing went wrong only one file should be deleted, otherwise
+        // attempts will be made to clean up more
+        brass_revision_number_t oldest_changeset = get_oldest_changeset();
+        unsigned int stop_changeset = new_revision - max_changesets;
+        while (oldest_changeset < stop_changeset) {
+            if (io_unlink(db_dir + "/changes" + str(oldest_changeset))) {
+                LOGLINE(DB, "Removed changeset " << oldest_changeset);
+            } else {
+                LOGLINE(DB, "Skipping changeset " << oldest_changeset << 
+                        ", likely removed before");
+            }
+            set_oldest_changeset(oldest_changeset++);
+        }
+    }
 }
 
 void
@@ -1008 +1028 @@
     RETURN(version_file.get_uuid_string());
 }
 
+brass_revision_number_t
+BrassDatabase::get_oldest_changeset() const
+{
+    LOGCALL(DB, brass_revision_number_t, "BrassDatabase::get_oldest_changeset", NO_ARGS);
+    return stats.get_oldest_changeset();
+}
+
+void
+BrassDatabase::set_oldest_changeset(brass_revision_number_t changeset)
+{
+    LOGCALL(DB, void, "BrassDatabase::set_oldest_changeset", changeset);
+    stats.set_oldest_changeset(changeset);
+}
+
 ///////////////////////////////////////////////////////////////////////////
 
 BrassWritableDatabase::BrassWritableDatabase(const string &dir, int action,

xapian-core/backends/brass/brass_database.h

@@ -4 +4 @@
  * Copyright 2002 Ananova Ltd
  * Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2010 Olly Betts
  * Copyright 2008 Lemur Consulting Ltd
+ * Copyright 2011 Dan Colish
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -216 +217 @@
         void get_changeset_revisions(const string & path,
                                      brass_revision_number_t * startrev,
                                      brass_revision_number_t * endrev) const;
+
+        /** Get the oldest changeset on disk
+         */
+        brass_revision_number_t get_oldest_changeset() const;
+        
+        /** Set the new oldest changeset on disk
+         */
+        void set_oldest_changeset(brass_revision_number_t oldest_changeset);
+
     public:
         /** Create and open a brass database.
          *

xapian-core/backends/brass/brass_dbstats.cc

@@ -2 +2 @@
  * @brief Brass class for database statistics.
  */
 /* Copyright (C) 2009 Olly Betts
+ * Copyright (C) 2011 Dan Colish
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -35 +36 @@
     string data;
     if (!postlist_table.get_exact_entry(DATABASE_STATS_KEY, data)) {
         // If there's no entry yet, then all the values are zero.
-        total_doclen = 0;
-        last_docid = 0;
-        doclen_lbound = 0;
-        doclen_ubound = 0;
-        wdf_ubound = 0;
+        zero();
         return;
     }
 
@@ -50 +47 @@
         unpack_uint(&p, end, &doclen_lbound) &&
         unpack_uint(&p, end, &wdf_ubound) &&
         unpack_uint(&p, end, &doclen_ubound) &&
+        unpack_uint(&p, end, &oldest_changeset) &&
         unpack_uint_last(&p, end, &total_doclen)) {
         // doclen_ubound should always be >= wdf_ubound, so we store the
         // difference as it may encode smaller.  wdf_ubound is likely to
@@ -75 +73 @@
     // difference as it may encode smaller.  wdf_ubound is likely to
     // be larger than doclen_lbound.
     pack_uint(data, doclen_ubound - wdf_ubound);
+    pack_uint(data, oldest_changeset);
     // Micro-optimisation: total_doclen is likely to be the largest value, so
     // store it last as pack_uint_last() uses a slightly more compact encoding
     // - this could save us a few bytes!

xapian-core/backends/brass/brass_dbstats.h

@@ -2 +2 @@
  * @brief Brass class for database statistics.
  */
 /* Copyright (C) 2009 Olly Betts
+ * Copyright (C) 2011 Dan Colish
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -51 +52 @@
     /// An upper bound on the greatest wdf in this database.
     Xapian::termcount wdf_ubound;
 
+    /// Oldest changeset removed when max_changesets is set
+    brass_revision_number_t oldest_changeset;
+
   public:
     BrassDatabaseStats()
         : total_doclen(0), last_docid(0), doclen_lbound(0), doclen_ubound(0),
-          wdf_ubound(0) { }
+          wdf_ubound(0), oldest_changeset(0) { }
 
     totlen_t get_total_doclen() const { return total_doclen; }
 
@@ -70 +74 @@
 
     Xapian::termcount get_wdf_upper_bound() const { return wdf_ubound; }
 
+    brass_revision_number_t get_oldest_changeset() const { return oldest_changeset; }
+
     void zero() {
         total_doclen = 0;
         last_docid = 0;
         doclen_lbound = 0;
         doclen_ubound = 0;
         wdf_ubound = 0;
+        oldest_changeset = 0;
     }
 
     void read(BrassPostListTable & postlist_table);
 
     void set_last_docid(Xapian::docid did) { last_docid = did; }
 
+    void set_oldest_changeset(brass_revision_number_t changeset) { oldest_changeset = changeset; }
+
     void add_document(Xapian::termcount doclen) {
         if (total_doclen == 0 || (doclen && doclen < doclen_lbound))
             doclen_lbound = doclen;

xapian-core/backends/brass/brass_version.cc

@@ -2 +2 @@
  * @brief BrassVersion class
  */
 /* Copyright (C) 2006,2007,2008,2009,2010 Olly Betts
+ * Copyright (C) 2011 Dan Colish
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -43 +44 @@
 using namespace std;
 
 // YYYYMMDDX where X allows multiple format revisions in a day
-#define BRASS_VERSION 200912150
+#define BRASS_VERSION 201103110
+// 201103110 1.2.5? Bump for new max changesets dbstats
 // 200912150 1.1.4 Brass debuts.
 
 #define MAGIC_STRING "IAmBrass"

xapian-core/backends/chert/chert_database.cc

@@ -7 +7 @@
  * Copyright 2006,2008 Lemur Consulting Ltd
  * Copyright 2009,2010 Richard Boulton
  * Copyright 2009 Kan-Ru Chen
+ * Copyright 2011 Dan Colish
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -465 +466 @@
 
         throw;
     }
+    
+    if (changes_fd >= 0 && max_changesets < new_revision) {
+        // While change sets less than N - max_changesets exist, delete them
+        // 1 must be subtracted so we don't delete the changset we just wrote
+        // when max_changesets = 1
+        unsigned rev = new_revision - max_changesets - 1;
+        while (io_unlink(db_dir + "/changes" + str(rev--))) { }
+    }
 }
 
 void

xapian-core/backends/flint/flint_database.cc

@@ -7 +7 @@
  * Copyright 2006,2008 Lemur Consulting Ltd
  * Copyright 2009,2010 Richard Boulton
  * Copyright 2009 Kan-Ru Chen
+ * Copyright 2011 Dan Colish
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -497 +498 @@
 
         throw;
     }
+
+    if (changes_fd >= 0 && max_changesets < new_revision) {
+        // while change sets less than N - max_changesets exist, delete them
+        // 1 must be subtracted so we don't delete the changset we just wrote
+        // when max_changesets = 1
+        unsigned rev = new_revision - max_changesets - 1;
+        while (io_unlink(db_dir + "/changes" + str(rev--)))  { }
+    }
 }
 
 void

xapian-core/tests/api_replicate.cc

@@ -3 +3 @@
  * Copyright 2008 Lemur Consulting Ltd
  * Copyright 2009,2010 Olly Betts
  * Copyright 2010 Richard Boulton
+ * Copyright 2011 Dan Colish
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -512 +513 @@
     return true;
 }
 
-// Basic test of replication functionality.
+// Tests for max_changesets
 DEFINE_TESTCASE(replicate4, replicas) {
     string tempdir = ".replicatmp";
     mktmpdir(tempdir);
     string masterpath = get_named_writable_database_path("master");
 
-    set_max_changesets(10);
+    set_max_changesets(1);
 
     Xapian::WritableDatabase orig(get_named_writable_database("master"));
     Xapian::DatabaseMaster master(masterpath);
@@ -555 +556 @@
     }
     check_equal_dbs(masterpath, replicapath);
 
+    // Add a document with no positions to the original database.
+    Xapian::Document doc2;
+    doc2.set_data(string("doc2"));
+    doc2.add_term("nopos");
+    orig.add_document(doc2);
+    orig.commit();
+
+    // Replicate, and check that we have the positional information.
+    count = replicate(master, replica, tempdir, 1, 0, 1);
+    TEST_EQUAL(count, 2);
+    {
+        Xapian::Database dbcopy(replicapath);
+        TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
+    }
+    check_equal_dbs(masterpath, replicapath);
+
+    int changes_fd = -1;
+    string path = masterpath + "/changes1";
+#ifdef __WIN32__
+    changes_fd = msvc_posix_open(path.c_str(), O_RDONLY);
+#else
+    changes_fd = open(path.c_str(), O_RDONLY);
+#endif
+    fdcloser closer(changes_fd);
+
+    TEST_EQUAL(changes_fd, -1);
+
     // Need to close the replica before we remove the temporary directory on
     // Windows.
     replica.close();