Ticket #385: diffwithtest.patch

xapian-core/common/pack.h

@@ -145 +145 @@
 {
     // Check U is an unsigned type.
     STATIC_ASSERT_UNSIGNED_TYPE(U);
-    static_assert(sizeof(U) <= SORTABLE_UINT_MAX_BYTES,
-                  "Template type U too wide for database format");
+    //static_assert(sizeof(U) <= SORTABLE_UINT_MAX_BYTES,
+      //"Template type U too wide for database format");
 
     char tmp[sizeof(U) + 1];
     char * p = tmp + sizeof(tmp);

xapian-core/configure.ac

@@ -456 +456 @@
   test -z "$PERL" && AC_MSG_ERROR([perl is required in maintainer mode])
 fi
 
+
+AC_ARG_ENABLE([64bit_docid],
+               [AS_HELP_STRING([--enable-64bit-docid], [enable 64bit docids[default=no]])],
+  [case ${enableval} in
+    yes|no) ;;
+  *) AC_MSG_ERROR([bad value ${enableval} for --enable-64bit-docid]) ;;
+  esac],
+  [enable_64bit_docid=no])
+
+AC_ARG_ENABLE([64bit_termcount],
+               [AS_HELP_STRING([--enable-64bit-termcount], [enable 64bit termcounts[default=no]])],
+  [case ${enableval} in
+    yes|no) ;;
+  *) AC_MSG_ERROR([bad value ${enableval} for --enable-64bit-termcount]) ;;
+  esac],
+  [enable_64bit_termcount=no])
+
 AC_ARG_ENABLE([documentation],
   [AS_HELP_STRING([--enable-documentation], [enable make rules to rebuild documentation [default=maintainer-mode]])],
   [case ${enableval} in
@@ -1258 +1275 @@
     cxxcpp_flags="$cxxcpp_flags -DXAPIAN_HAS_${backend}_BACKEND"
   fi
 done
+
+if test yes = "$enable_64bit_docid" ; then
+  cxxcpp_flags="$cxxcpp_flags -DXAPIAN_HAS_64BIT_DOCID"
+fi
+
+if test yes = "$enable_64bit_termcount" ; then
+  cxxcpp_flags="$cxxcpp_flags -DXAPIAN_HAS_64BIT_TERMCOUNT"
+fi
+
 dnl For GCC (and compilers which pretend to be GCC) and MSVC, we create some
 dnl ABI checks in the generated version.h, and $CXXFLAGS may contain options
 dnl which affect the ABI (e.g. -fabi-version for GCC) so we want to use these

xapian-core/include/xapian/types.h

@@ -25 +25 @@
 # error "Never use <xapian/types.h> directly; include <xapian.h> instead."
 #endif
 
+#include <xapian/version.h>
 #include <xapian/deprecated.h>
 
 namespace Xapian {
@@ -34 +35 @@
  *  This is used to hold values such as the number of documents in a database
  *  and the frequency of a term in the database.
  */
+#ifdef XAPIAN_HAS_64BIT_DOCID
+typedef unsigned long long doccount;
+#else
 typedef unsigned doccount;
+#endif
 
 /** A signed difference between two counts of documents.
  *
  *  This is used by the Xapian classes which are STL containers of documents
  *  for "difference_type".
  */
-typedef int doccount_diff; /* FIXME: can overflow with more than 2^31 docs. */
+#ifdef XAPIAN_HAS_64BIT_DOCID
+typedef long long doccount_diff;
+#else
+typedef int doccount_diff;
+#endif
 
 /** A unique identifier for a document.
  *
  *  Docid 0 is invalid, providing an "out of range" value which can be
  *  used to mean "not a valid document".
  */
+#ifdef XAPIAN_HAS_64BIT_DOCID
+typedef unsigned long long docid;
+#else
 typedef unsigned docid;
+#endif
 
 /** A normalised document length.
  *
@@ -68 +81 @@
  *
  *  This is used to hold values such as the Within Document Frequency (wdf).
  */
+#ifdef XAPIAN_HAS_64BIT_TERMCOUNT
+typedef unsigned long long termcount;
+#else
 typedef unsigned termcount;
+#endif
 
 /** A signed difference between two counts of terms.
  *
  *  This is used by the Xapian classes which are STL containers of terms
  *  for "difference_type".
  */
-typedef int termcount_diff; /* FIXME: can overflow with more than 2^31 terms. */
+#ifdef XAPIAN_HAS_64BIT_TERMCOUNT
+typedef long long termcount_diff;
+#else
+typedef int termcount_diff;
+#endif
 
 /** A term position within a document or query.
  */

xapian-core/include/xapian/version_h.cc

@@ -157 +157 @@
 " */",
 "#define XAPIAN_REVISION ",REVISION,
 "",
+"/// XAPIAN_HAS_64BIT_DOCID Defined if the 64bit docid is enabled.",
+#ifdef XAPIAN_HAS_64BIT_DOCID
+"#define XAPIAN_HAS_64BIT_DOCID 1",
+#else
+"/* #undef XAPIAN_HAS_64BIT_DOCID */",
+#endif
+"",
+"/// XAPIAN_HAS_64BIT_TERMCOUNT Defined if the 64bit termcount is enabled.",
+#ifdef XAPIAN_HAS_64BIT_TERMCOUNT
+"#define XAPIAN_HAS_64BIT_TERMCOUNT 1",
+#else
+"/* #undef XAPIAN_HAS_64BIT_TERMCOUNT */",
+#endif
+"",
 "/// XAPIAN_HAS_CHERT_BACKEND Defined if the chert backend is enabled.",
 #ifdef XAPIAN_HAS_CHERT_BACKEND
 "#define XAPIAN_HAS_CHERT_BACKEND 1",

xapian-core/net/length.cc

@@ -103 +103 @@
 }
 
 void
+decode_length(const char ** p, const char *end, unsigned long long & out)
+{
+    decode_length_(p, end, out);
+}
+
+void
 decode_length_and_check(const char ** p, const char *end, uint32_t & out)
 {
     decode_length_and_check_(p, end, out);
@@ -113 +119 @@
 {
     decode_length_and_check_(p, end, out);
 }
+
+void
+decode_length_and_check(const char ** p, const char *end, unsigned long long & out)
+{
+    decode_length_and_check_(p, end, out);
+}

xapian-core/net/length.h

@@ -66 +66 @@
 
 void decode_length(const char ** p, const char *end, uint64_t & out);
 
+void decode_length(const char ** p, const char *end, unsigned long long & out);
+
 /** Decode a length encoded by encode_length.
  *
  *  Also checks the result against the amount of data remaining after the
@@ -80 +82 @@
 
 void decode_length_and_check(const char ** p, const char *end, uint64_t & out);
 
+void decode_length_and_check(const char ** p, const char *end, unsigned long long & out);
+
 #endif //XAPIAN_INCLUDED_LENGTH_H

xapian-core/tests/api_backend.cc

@@ -93 +93 @@
     return true;
 }
 
+#ifdef XAPIAN_HAS_64BIT_DOCID
+// Assert that exceeding 32bit in combined database doesn't cause a problem
+// when using 64bit docids.
+DEFINE_TESTCASE(exceed32bitcombineddb1, writable) {
+    // The InMemory backend uses a vector for the documents, so trying to add
+    // document "-1" will fail because we can't allocate enough memory!
+    SKIP_TEST_FOR_BACKEND("inmemory");
+
+    Xapian::WritableDatabase db1 = get_writable_database();
+    Xapian::WritableDatabase db2 = get_writable_database();
+    Xapian::Document doc;
+    doc.set_data("prose");
+    doc.add_term("word");
+
+    Xapian::docid max_id = 0xffffffff;
+
+    db1.replace_document(max_id, doc);
+    db2.replace_document(max_id, doc);
+
+    Xapian::Database db = get_database("");
+    db.add_database(db1);
+    db.add_database(db2);
+
+    Xapian::Enquire enquire(db);
+    enquire.set_query(Xapian::Query::MatchAll);
+    Xapian::MSet mymset = enquire.get_mset(0, 10);
+
+    TEST_EQUAL(2, mymset.size());
+
+    for (Xapian::MSetIterator i = mymset.begin(); i != mymset.end(); ++i) {
+      TEST_EQUAL("prose", i.get_document().get_data());
+    }
+
+    return true;
+}
+#endif
+
 DEFINE_TESTCASE(dbstats1, backend) {
     Xapian::Database db = get_database("etext");
 

xapian-core/tests/api_wrdb.cc

@@ -1614 +1614 @@
     doc.set_data("prose");
     doc.add_term("word");
 
-    db.replace_document(Xapian::docid(-1), doc);
+    // FIXME: This probably should use the _MAX_DOCID values
+    Xapian::docid max_id = 0xffffffff;
+
+    db.replace_document(max_id, doc);
 
     TEST_EXCEPTION(Xapian::DatabaseError, db.add_document(doc));