Ticket #290: office2007.patch

.cc

@@ -72 +72 @@
 
 static bool skip_duplicates = false;
 static bool follow_symlinks = false;
+static bool ignore_time = false;
 static string dbpath;
 static string root;
 static string indexroot;
@@ -194 +195 @@
 
     if (urlterm.length() > MAX_SAFE_TERM_LENGTH)
         urlterm = hash_long_term(urlterm, MAX_SAFE_TERM_LENGTH);
-
-    if (skip_duplicates && db.term_exists(urlterm)) {
-        cout << "duplicate. Ignored." << endl;
-        return;
-    }
-
+      {
+        // First find the docid with the urlterm.
+        Xapian::docid docid = 0;
+        Xapian::PostingIterator p = db.postlist_begin(urlterm);
+        if (p != db.postlist_end(urlterm)) {
+            docid = *p;
+            if (skip_duplicates) {
+                cout << "duplicate. Ignored." << endl;
+                return;
+            }
+        }
+        if (docid && !ignore_time) {
+            // Check the timestamp.
+            Xapian::Document doc = db.get_document(docid);
+            string value = doc.get_value(VALUE_LASTMOD);
+            time_t old_last_mod = binary_string_to_int(value);
+            if (old_last_mod >= last_mod) {
+                cout << "not newer. Ignored." << endl;
+            if (docid < updated.size()) {
+                updated[docid] = true;
+                return;
+            }}
+        }
+     }
+    
     string md5;
     if (mimetype == "text/html") {
         string text;
@@ -351 +371 @@
             cout << "\"" << cmd << "\" failed - skipping\n";
             return;
         }
+    } else if (startswith(mimetype, "application/vnd.openxmlformats-officedocument.wordprocessingml.")) {
+    string safefile = shell_protect(file);
+    string cmd = "unzip -p " + safefile + " word/document.xml";
+    try {
+        XmlParser xmlparser;
+        xmlparser.parse_html(stdout_to_string(cmd));
+        dump = xmlparser.dump;
+    } catch (ReadError) {
+        cout << "\"" << cmd << "\" failed - skipping\n";
+        return;
+    }
+    } else if (startswith(mimetype, "application/vnd.openxmlformats-officedocument.spreadsheetml.")) {
+    string safefile = shell_protect(file);
+    string cmd = "unzip -p " + safefile + " xl/sharedStrings.xml";
+    try {
+        XmlParser xmlparser;
+        xmlparser.parse_html(stdout_to_string(cmd));
+        dump = xmlparser.dump;
+    } catch (ReadError) {
+        cout << "\"" << cmd << "\" failed - skipping\n";
+        return;
+    }
+    } else if (startswith(mimetype, "application/vnd.openxmlformats-officedocument.presentationml.")) {
+    string safefile = shell_protect(file);
+    string cmd = "unzip -p " + safefile + " ppt/slides/slide*.xml";
+    try {
+        XmlParser xmlparser;
+        xmlparser.parse_html(stdout_to_string(cmd));
+        dump = xmlparser.dump;
+    } catch (ReadError) {
+        cout << "\"" << cmd << "\" failed - skipping\n";
+        return;
+    }
     } else if (mimetype == "application/vnd.wordperfect") {
         // Looking at the source of wpd2html and wpd2text I think both output
         // utf-8, but it's hard to be sure without sample Unicode .wpd files
@@ -362 +415 @@
             cout << "\"" << cmd << "\" failed - skipping\n";
             return;
         }
+    } else if (mimetype == "application/vnd.ms-outlook") {
+    string cmd = "msg2txt.pl " + shell_protect(file) + " | strings";
+    try {
+        dump = stdout_to_string(cmd);
+    } catch (ReadError) {
+        cout << "\"" << cmd << "\" failed - skipping\n";
+        return;
+    }
     } else if (mimetype == "application/vnd.ms-works") {
         // wps2text produces UTF-8 output from the sample files I've tested.
         string cmd = "wps2text " + shell_protect(file);
@@ -677 +738 @@
         { "depth-limit",required_argument,      NULL, 'l' },
         { "follow",     no_argument,            NULL, 'f' },
         { "stemmer",    required_argument,      NULL, 's' },
+        { "ignore-time",no_argument,            NULL, 'i' },
         { 0, 0, NULL, 0 }
     };
 
@@ -730 +792 @@
     // Some other word processor formats:
     mime_map["doc"] = "application/msword";
     mime_map["dot"] = "application/msword"; // Word template
+    mime_map["docx"] = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"; //Word 2007
+    mime_map["dotx"] = "application/vnd.openxmlformats-officedocument.wordprocessingml.document"; //Word 2007 template
     mime_map["wpd"] = "application/vnd.wordperfect";
     mime_map["wps"] = "application/vnd.ms-works";
     mime_map["wpt"] = "application/vnd.ms-works"; // Works template
@@ -740 +804 @@
     mime_map["xls"] = "application/vnd.ms-excel";
     mime_map["xlb"] = "application/vnd.ms-excel";
     mime_map["xlt"] = "application/vnd.ms-excel"; // Excel template
+    mime_map["xlsx"] = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; //Excel 2007
+    mime_map["xltx"] = "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet"; //Excel 2007 template
     mime_map["ppt"] = "application/vnd.ms-powerpoint";
     mime_map["pps"] = "application/vnd.ms-powerpoint"; // Powerpoint slideshow
+    mime_map["pptx"] = "application/vnd.openxmlformats-officedocument.presentationml.presentation"; //PowerPoint 2007
+    mime_map["ppsx"] = "application/vnd.openxmlformats-officedocument.presentationml.presentation"; //PowerPoint 2007 slideshow
+    mime_map["potx"] = "application/vnd.openxmlformats-officedocument.presentationml.presentation"; //PowerPoint 2007 template
+    mime_map["msg"] = "application/vnd.ms-outlook"; //Outlook .msg
     // Perl:
     mime_map["pl"] = "text/x-perl";
     mime_map["pm"] = "text/x-perl";
@@ -752 +822 @@
     mime_map["djv"] = "image/vnd.djvu";
     mime_map["djvu"] = "image/vnd.djvu";
 
-    while ((getopt_ret = gnu_getopt_long(argc, argv, "hvd:D:U:M:lpf", longopts, NULL)) != -1) {
+    while ((getopt_ret = gnu_getopt_long(argc, argv, "hvd:D:U:M:lpfi", longopts, NULL)) != -1) {
         switch (getopt_ret) {
         case 'h': {
             cout << PROG_NAME" - "PROG_DESC"\n\n"
@@ -766 +836 @@
 "  -M, --mime-type          additional MIME mapping ext:type\n"
 "  -l, --depth-limit=LIMIT  set recursion limit (0 = unlimited)\n"
 "  -f, --follow             follow symbolic links\n"
+"  -i, --ignore-time        ignore timestamp comparison\n"
 "      --overwrite          create the database anew (the default is to update\n"
 "                           if the database already exists)" << endl;
             print_stemmer_help("     ");
@@ -795 +866 @@
         case 'p': // don't delete unupdated documents
             preserve_unupdated = true;
             break;
+        case 'i': // --ignore-time: on updates parse the file again
+            ignore_time = true;
+            break;
         case 'l': { // Set recursion limit
             int arg = atoi(optarg);
             if (arg < 0) arg = 0;