Ticket #342: last_mod.patch

.cc

@@ -72 +72 @@
 
 static bool skip_duplicates = false;
 static bool follow_symlinks = false;
+static bool ignore_time = false;
 static string dbpath;
 static string root;
 static string indexroot;
@@ -195 +196 @@
     if (urlterm.length() > MAX_SAFE_TERM_LENGTH)
         urlterm = hash_long_term(urlterm, MAX_SAFE_TERM_LENGTH);
 
-    if (skip_duplicates && db.term_exists(urlterm)) {
-        cout << "duplicate. Ignored." << endl;
-        return;
-    }
+    {
+        // First find the docid with the urlterm.
+        Xapian::docid docid = 0;
+        Xapian::PostingIterator p = db.postlist_begin(urlterm);
+        if (p != db.postlist_end(urlterm)) {
+            docid = *p;
+            if (skip_duplicates) {
+                cout << "duplicate. Ignored." << endl;
+                return;
+            }
+        }
+        if (docid && !ignore_time) {
+            // Check the timestamp.
+            Xapian::Document doc = db.get_document(docid);
+            string value = doc.get_value(VALUE_LASTMOD);
+            time_t old_last_mod = binary_string_to_int(value);
+            if (old_last_mod >= last_mod) {
+                cout << "not newer. Ignored." << endl;
+            if (docid < updated.size()) {
+                updated[docid] = true;
+                return;
+            }}
+        }
+     }
 
     string md5;
     if (mimetype == "text/html") {
@@ -677 +698 @@
         { "depth-limit",required_argument,      NULL, 'l' },
         { "follow",     no_argument,            NULL, 'f' },
         { "stemmer",    required_argument,      NULL, 's' },
+        { "ignore-time",no_argument,            NULL, 'i' },
         { 0, 0, NULL, 0 }
     };
 
@@ -752 +774 @@
     mime_map["djv"] = "image/vnd.djvu";
     mime_map["djvu"] = "image/vnd.djvu";
 
-    while ((getopt_ret = gnu_getopt_long(argc, argv, "hvd:D:U:M:lpf", longopts, NULL)) != -1) {
+    while ((getopt_ret = gnu_getopt_long(argc, argv, "hvd:D:U:M:lpfi", longopts, NULL)) != -1) {
         switch (getopt_ret) {
         case 'h': {
             cout << PROG_NAME" - "PROG_DESC"\n\n"
@@ -766 +788 @@
 "  -M, --mime-type          additional MIME mapping ext:type\n"
 "  -l, --depth-limit=LIMIT  set recursion limit (0 = unlimited)\n"
 "  -f, --follow             follow symbolic links\n"
+"  -i, --ignore-time        ignore timestamp comparison\n"
 "      --overwrite          create the database anew (the default is to update\n"
 "                           if the database already exists)" << endl;
             print_stemmer_help("     ");
@@ -795 +818 @@
         case 'p': // don't delete unupdated documents
             preserve_unupdated = true;
             break;
+        case 'i': // --ignore-time: on updates parse the file again
+            ignore_time = true;
+            break;
         case 'l': { // Set recursion limit
             int arg = atoi(optarg);
             if (arg < 0) arg = 0;