Ticket #278: max_changesets_post_review.diff

xapian-core/backends/brass/brass_database.cc

@@ -7 +7 @@
  * Copyright 2006,2008 Lemur Consulting Ltd
  * Copyright 2009 Richard Boulton
  * Copyright 2009 Kan-Ru Chen
+ * Copyright 2011 Dan Colish
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -120 +121 @@
         return;
     }
 
-    const char *p = getenv("XAPIAN_MAX_CHANGESETS");
-    if (p)
-        max_changesets = atoi(p);
-
     if (action != Xapian::DB_OPEN && !database_exists()) {
         // FIXME: if we allow Xapian::DB_OVERWRITE, check it here
 
@@ -400 +397 @@
 
     int changes_fd = -1;
     string changes_name;
+    
+    // always check max_changesets for modification since last revision
+    const char *p = getenv("XAPIAN_MAX_CHANGESETS");
+    if (p) {
+        max_changesets = atoi(p);
+    }
 
     if (max_changesets > 0) {
         brass_revision_number_t old_revision = get_revision_number();
@@ -470 +473 @@
 
         throw;
     }
+    
+    // Only remove the oldest_changeset if we successfully write a new changeset and
+    // we have a revision number greater than max_changesets
+    if (changes_fd >= 0 && max_changesets < new_revision) {
+        // use the oldest changeset we know about to begin deleting to the stop_changeset
+        // if nothing went wrong only one file should be deleted, otherwise
+        // attempts will be made to clean up more
+        brass_revision_number_t oldest_changeset = stats.get_oldest_changeset();
+        brass_revision_number_t stop_changeset = new_revision - max_changesets;
+        while (oldest_changeset < stop_changeset) {
+            if (io_unlink(db_dir + "/changes" + str(oldest_changeset))) {
+                LOGLINE(DB, "Removed changeset " << oldest_changeset);
+            } else {
+                LOGLINE(DB, "Skipping changeset " << oldest_changeset << 
+                        ", likely removed before");
+            }
+            stats.set_oldest_changeset(oldest_changeset++);
+        }
+    }
 }
 
 void

xapian-core/backends/brass/brass_database.h

@@ -4 +4 @@
  * Copyright 2002 Ananova Ltd
  * Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2010 Olly Betts
  * Copyright 2008 Lemur Consulting Ltd
+ * Copyright 2011 Dan Colish
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -216 +217 @@
         void get_changeset_revisions(const string & path,
                                      brass_revision_number_t * startrev,
                                      brass_revision_number_t * endrev) const;
+
     public:
         /** Create and open a brass database.
          *

xapian-core/backends/brass/brass_dbstats.cc

@@ -2 +2 @@
  * @brief Brass class for database statistics.
  */
 /* Copyright (C) 2009 Olly Betts
+ * Copyright (C) 2011 Dan Colish
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -35 +36 @@
     string data;
     if (!postlist_table.get_exact_entry(DATABASE_STATS_KEY, data)) {
         // If there's no entry yet, then all the values are zero.
-        total_doclen = 0;
-        last_docid = 0;
-        doclen_lbound = 0;
-        doclen_ubound = 0;
-        wdf_ubound = 0;
+        zero();
         return;
     }
 
@@ -50 +47 @@
         unpack_uint(&p, end, &doclen_lbound) &&
         unpack_uint(&p, end, &wdf_ubound) &&
         unpack_uint(&p, end, &doclen_ubound) &&
+        unpack_uint(&p, end, &oldest_changeset) &&
         unpack_uint_last(&p, end, &total_doclen)) {
         // doclen_ubound should always be >= wdf_ubound, so we store the
         // difference as it may encode smaller.  wdf_ubound is likely to
@@ -75 +73 @@
     // difference as it may encode smaller.  wdf_ubound is likely to
     // be larger than doclen_lbound.
     pack_uint(data, doclen_ubound - wdf_ubound);
+    pack_uint(data, oldest_changeset);
     // Micro-optimisation: total_doclen is likely to be the largest value, so
     // store it last as pack_uint_last() uses a slightly more compact encoding
     // - this could save us a few bytes!

xapian-core/backends/brass/brass_dbstats.h

@@ -2 +2 @@
  * @brief Brass class for database statistics.
  */
 /* Copyright (C) 2009 Olly Betts
+ * Copyright (C) 2011 Dan Colish
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -51 +52 @@
     /// An upper bound on the greatest wdf in this database.
     Xapian::termcount wdf_ubound;
 
+    /// Oldest changeset removed when max_changesets is set
+    brass_revision_number_t oldest_changeset;
+
   public:
     BrassDatabaseStats()
         : total_doclen(0), last_docid(0), doclen_lbound(0), doclen_ubound(0),
-          wdf_ubound(0) { }
+          wdf_ubound(0), oldest_changeset(0) { }
 
     totlen_t get_total_doclen() const { return total_doclen; }
 
@@ -70 +74 @@
 
     Xapian::termcount get_wdf_upper_bound() const { return wdf_ubound; }
 
+    brass_revision_number_t get_oldest_changeset() const { return oldest_changeset; }
+
     void zero() {
         total_doclen = 0;
         last_docid = 0;
         doclen_lbound = 0;
         doclen_ubound = 0;
         wdf_ubound = 0;
+        oldest_changeset = 0;
     }
 
     void read(BrassPostListTable & postlist_table);
 
     void set_last_docid(Xapian::docid did) { last_docid = did; }
 
+    void set_oldest_changeset(brass_revision_number_t changeset) { oldest_changeset = changeset; }
+
     void add_document(Xapian::termcount doclen) {
         if (total_doclen == 0 || (doclen && doclen < doclen_lbound))
             doclen_lbound = doclen;

xapian-core/backends/brass/brass_version.cc

@@ -2 +2 @@
  * @brief BrassVersion class
  */
 /* Copyright (C) 2006,2007,2008,2009,2010 Olly Betts
+ * Copyright (C) 2011 Dan Colish
  *
  * This program is free software; you can redistribute it and/or modify
  * it under the terms of the GNU General Public License as published by
@@ -43 +44 @@
 using namespace std;
 
 // YYYYMMDDX where X allows multiple format revisions in a day
-#define BRASS_VERSION 200912150
+#define BRASS_VERSION 201103110
+// 201103110 1.2.5? Bump for new max changesets dbstats
 // 200912150 1.1.4 Brass debuts.
 
 #define MAGIC_STRING "IAmBrass"

xapian-core/backends/chert/chert_database.cc

@@ -7 +7 @@
  * Copyright 2006,2008 Lemur Consulting Ltd
  * Copyright 2009,2010 Richard Boulton
  * Copyright 2009 Kan-Ru Chen
+ * Copyright 2011 Dan Colish
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -123 +124 @@
         return;
     }
 
-    const char *p = getenv("XAPIAN_MAX_CHANGESETS");
-    if (p)
-        max_changesets = atoi(p);
-
     if (action != Xapian::DB_OPEN && !database_exists()) {
         // FIXME: if we allow Xapian::DB_OVERWRITE, check it here
 
@@ -404 +401 @@
     int changes_fd = -1;
     string changes_name;
 
+    const char *p = getenv("XAPIAN_MAX_CHANGESETS");
+    if (p)
+        max_changesets = atoi(p);
+
     if (max_changesets > 0) {
         chert_revision_number_t old_revision = get_revision_number();
         if (old_revision) {
@@ -465 +466 @@
 
         throw;
     }
+    
+    if (changes_fd >= 0 && max_changesets < new_revision) {
+        // While change sets less than N - max_changesets exist, delete them
+        // 1 must be subtracted so we don't delete the changeset we just wrote
+        // when max_changesets = 1
+        unsigned rev = new_revision - max_changesets - 1;
+        while (io_unlink(db_dir + "/changes" + str(rev--))) { }
+    }
 }
 
 void

xapian-core/backends/flint/flint_database.cc

@@ -7 +7 @@
  * Copyright 2006,2008 Lemur Consulting Ltd
  * Copyright 2009,2010 Richard Boulton
  * Copyright 2009 Kan-Ru Chen
+ * Copyright 2011 Dan Colish
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -121 +122 @@
         return;
     }
 
-    const char *p = getenv("XAPIAN_MAX_CHANGESETS");
-    if (p)
-        max_changesets = atoi(p);
-
     if (action != Xapian::DB_OPEN && !database_exists()) {
         // FIXME: if we allow Xapian::DB_OVERWRITE, check it here
 
@@ -434 +431 @@
     int changes_fd = -1;
     string changes_name;
 
+    const char *p = getenv("XAPIAN_MAX_CHANGESETS");
+    if (p)
+        max_changesets = atoi(p);
+
     if (max_changesets > 0) {
         flint_revision_number_t old_revision = get_revision_number();
         if (old_revision) {
@@ -497 +498 @@
 
         throw;
     }
+
+    if (changes_fd >= 0 && max_changesets < new_revision) {
+        // while change sets less than N - max_changesets exist, delete them
+        // 1 must be subtracted so we don't delete the changset we just wrote
+        // when max_changesets = 1
+        unsigned rev = new_revision - max_changesets - 1;
+        while (io_unlink(db_dir + "/changes" + str(rev--)))  { }
+    }
 }
 
 void

xapian-core/tests/api_replicate.cc

@@ -3 +3 @@
  * Copyright 2008 Lemur Consulting Ltd
  * Copyright 2009,2010 Olly Betts
  * Copyright 2010 Richard Boulton
+ * Copyright 2011 Dan Colish
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -512 +513 @@
     return true;
 }
 
-// Basic test of replication functionality.
+// Tests for max_changesets
 DEFINE_TESTCASE(replicate4, replicas) {
     string tempdir = ".replicatmp";
     mktmpdir(tempdir);
     string masterpath = get_named_writable_database_path("master");
 
-    set_max_changesets(10);
+    set_max_changesets(1);
+
+    Xapian::WritableDatabase orig(get_named_writable_database("master"));
+    Xapian::DatabaseMaster master(masterpath);
+    string replicapath = tempdir + "/replica";
+    Xapian::DatabaseReplica replica(replicapath);
+
+    // Add a document with no positions to the original database.
+    Xapian::Document doc1;
+    doc1.set_data(string("doc1"));
+    doc1.add_term("nopos");
+    orig.add_document(doc1);
+    orig.commit();
+
+    // Apply the replication - we don't have changesets stored, so this should
+    // just do a database copy, and return a count of 1.
+    int count = replicate(master, replica, tempdir, 0, 1, 1);
+    TEST_EQUAL(count, 1);
+    {
+        Xapian::Database dbcopy(replicapath);
+        TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
+    }
+
+    // Add a document with positional information to the original database.
+    doc1.add_posting("pos", 1);
+    orig.add_document(doc1);
+    orig.commit();
+
+    // Replicate, and check that we have the positional information.
+    count = replicate(master, replica, tempdir, 1, 0, 1);
+    TEST_EQUAL(count, 2);
+    {
+        Xapian::Database dbcopy(replicapath);
+        TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
+    }
+    check_equal_dbs(masterpath, replicapath);
+
+    // Add a document with no positions to the original database.
+    Xapian::Document doc2;
+    doc2.set_data(string("doc2"));
+    doc2.add_term("nopos");
+    orig.add_document(doc2);
+    orig.commit();
+
+    // Replicate, and check that we have the positional information.
+    count = replicate(master, replica, tempdir, 1, 0, 1);
+    TEST_EQUAL(count, 2);
+    {
+        Xapian::Database dbcopy(replicapath);
+        TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
+    }
+    check_equal_dbs(masterpath, replicapath);
+    TEST(!file_exists(masterpath + "/changes1"));
+
+    // Need to close the replica before we remove the temporary directory on
+    // Windows.
+    replica.close();
+    rmtmpdir(tempdir);
+    return true;
+}
+
+
+// Tests for max_changesets
+DEFINE_TESTCASE(replicate5, replicas) {
+    SKIP_TEST_FOR_BACKEND("chert");
+    SKIP_TEST_FOR_BACKEND("flint");
+    string tempdir = ".replicatmp";
+    mktmpdir(tempdir);
+    string masterpath = get_named_writable_database_path("master");
+
+    set_max_changesets(2);
 
     Xapian::WritableDatabase orig(get_named_writable_database("master"));
     Xapian::DatabaseMaster master(masterpath);
@@ -555 +626 @@
     }
     check_equal_dbs(masterpath, replicapath);
 
+    // Add a document with no positions to the original database.
+    Xapian::Document doc2;
+    doc2.set_data(string("doc2"));
+    doc2.add_term("nopos");
+    orig.add_document(doc2);
+    orig.commit();
+
+    // Replicate, and check that we have the positional information.
+    count = replicate(master, replica, tempdir, 1, 0, 1);
+    TEST_EQUAL(count, 2);
+    {
+        Xapian::Database dbcopy(replicapath);
+        TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
+    }
+    check_equal_dbs(masterpath, replicapath);
+
+    // Add a document with no positions to the original database.
+    Xapian::Document doc3;
+    doc3.set_data(string("doc3"));
+    doc3.add_term("nonopos");
+    orig.add_document(doc3);
+    orig.commit();
+
+    // Replicate, and check that we have the positional information.
+    count = replicate(master, replica, tempdir, 1, 0, 1);
+    TEST_EQUAL(count, 2);
+    {
+        Xapian::Database dbcopy(replicapath);
+        TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
+    }
+    check_equal_dbs(masterpath, replicapath);
+    
+    // Ensure that only these changesets exists
+    TEST(!file_exists(masterpath + "/changes1"));
+    TEST(file_exists(masterpath + "/changes2"));
+    TEST(file_exists(masterpath + "/changes3"));
+
+    set_max_changesets(3);
+    masterpath = get_named_writable_database_path("master");
+
+    // Add a document with no positions to the original database.
+    Xapian::Document doc4;
+    doc4.set_data(string("doc4"));
+    doc4.add_term("nononopos");
+    orig.add_document(doc4);
+    orig.commit();
+
+    // Replicate, and check that we have the positional information.
+    count = replicate(master, replica, tempdir, 1, 0, 1);
+    TEST_EQUAL(count, 2);
+    {
+        Xapian::Database dbcopy(replicapath);
+        TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
+    }
+    check_equal_dbs(masterpath, replicapath);
+
+    // Add a document with no positions to the original database.
+    Xapian::Document doc5;
+    doc5.set_data(string("doc5"));
+    doc5.add_term("nonononopos");
+    orig.add_document(doc5);
+    orig.commit();
+
+    // Replicate, and check that we have the positional information.
+    count = replicate(master, replica, tempdir, 1, 0, 1);
+    TEST_EQUAL(count, 2);
+    {
+        Xapian::Database dbcopy(replicapath);
+        TEST_EQUAL(orig.get_uuid(), dbcopy.get_uuid());
+    }
+    check_equal_dbs(masterpath, replicapath);
+    
+    TEST(!file_exists(masterpath + "/changes2"));
+    TEST(file_exists(masterpath + "/changes3"));
+    TEST(file_exists(masterpath + "/changes4"));
+    TEST(file_exists(masterpath + "/changes5"));
+
     // Need to close the replica before we remove the temporary directory on
     // Windows.
     replica.close();