Ticket #282: omindex-assorted-enhancements.patch

omindex.cc

@@ -4 +4 @@
  * Copyright 2001,2005 James Aylett
  * Copyright 2001,2002 Ananova Ltd
  * Copyright 2002,2003,2004,2005,2006,2007,2008 Olly Betts
+ * Copyright 2006 AVL List GesmbH
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -42 +43 @@
 #include <xapian.h>
 
 #include "commonhelp.h"
+#include "configfile.h"
 #include "diritor.h"
 #include "hashterm.h"
 #include "loadfile.h"
@@ -72 +74 @@
 
 static bool skip_duplicates = false;
 static bool follow_symlinks = false;
+static bool ignore_time = false;
+static bool nocleanup = false;
 static string dbpath;
 static string root;
 static string indexroot;
 static string baseurl;
+static string error_log;
 static Xapian::WritableDatabase db;
 static Xapian::Stem stemmer("english");
 static Xapian::TermGenerator indexer;
@@ -111 +116 @@
     return safefile;
 }
 
+inline string
+run_cmd(const string & cmd)
+{
+    return stdout_to_string(cmd + error_log);
+}
+
 static bool ensure_tmpdir() {
     if (!tmpdir.empty()) return true;
 
@@ -140 +151 @@
 get_pdf_metainfo(const string & safefile, string &title, string &keywords)
 {
     try {
-        string pdfinfo = stdout_to_string("pdfinfo -enc UTF-8 " + safefile);
+        string pdfinfo = run_cmd("pdfinfo -enc UTF-8 " + safefile);
 
         string::size_type idx;
 
@@ -195 +206 @@
     if (urlterm.length() > MAX_SAFE_TERM_LENGTH)
         urlterm = hash_long_term(urlterm, MAX_SAFE_TERM_LENGTH);
 
-    if (skip_duplicates && db.term_exists(urlterm)) {
-        cout << "duplicate. Ignored." << endl;
-        return;
+    {
+        // First find the docid with the urlterm.
+        Xapian::docid docid = 0;
+        Xapian::PostingIterator p = db.postlist_begin(urlterm);
+        if (p != db.postlist_end(urlterm)) {
+            docid = *p;
+            if (skip_duplicates) {
+                cout << "duplicate. Ignored." << endl;
+                return;
+            }
+        }
+        if (docid && !ignore_time) {
+            // Check the timestamp.
+            Xapian::Document doc = db.get_document(docid);
+            string value = doc.get_value(VALUE_LASTMOD);
+            time_t old_last_mod = binary_string_to_int(value);
+            if (old_last_mod >= last_mod) {
+                cout << "not newer. Ignored." << endl;
+                return;
+            }
+        }
     }
 
     string md5;
@@ -253 +282 @@
         string safefile = shell_protect(file);
         string cmd = "pdftotext -enc UTF-8 " + safefile + " -";
         try {
-            dump = stdout_to_string(cmd);
+            dump = run_cmd(cmd);
         } catch (ReadError) {
             cout << "\"" << cmd << "\" failed - skipping\n";
             return;
@@ -276 +305 @@
         string safetmp = shell_protect(tmpfile);
         string cmd = "ps2pdf " + shell_protect(file) + " " + safetmp;
         try {
-            (void)stdout_to_string(cmd);
+            (void)run_cmd(cmd);
             cmd = "pdftotext -enc UTF-8 " + safetmp + " -";
-            dump = stdout_to_string(cmd);
+            dump = run_cmd(cmd);
         } catch (ReadError) {
             cout << "\"" << cmd << "\" failed - skipping" << endl;
             unlink(tmpfile.c_str());
@@ -302 +331 @@
         string cmd = "unzip -p " + safefile + " content.xml";
         try {
             XmlParser xmlparser;
-            xmlparser.parse_html(stdout_to_string(cmd));
+            xmlparser.parse_html(run_cmd(cmd));
             dump = xmlparser.dump;
         } catch (ReadError) {
             cout << "\"" << cmd << "\" failed - skipping\n";
@@ -312 +341 @@
         cmd = "unzip -p " + safefile + " meta.xml";
         try {
             MetaXmlParser metaxmlparser;
-            metaxmlparser.parse_html(stdout_to_string(cmd));
+            metaxmlparser.parse_html(run_cmd(cmd));
             title = metaxmlparser.title;
             keywords = metaxmlparser.keywords;
             sample = metaxmlparser.sample;
@@ -322 +351 @@
     } else if (mimetype == "application/msword") {
         string cmd = "antiword -mUTF-8.txt " + shell_protect(file);
         try {
-            dump = stdout_to_string(cmd);
+            dump = run_cmd(cmd);
         } catch (ReadError) {
             cout << "\"" << cmd << "\" failed - skipping\n";
             return;
@@ -330 +359 @@
     } else if (mimetype == "application/vnd.ms-excel") {
         string cmd = "xls2csv -q0 -dutf-8 " + shell_protect(file);
         try {
-            dump = stdout_to_string(cmd);
+            dump = run_cmd(cmd);
         } catch (ReadError) {
             cout << "\"" << cmd << "\" failed - skipping\n";
             return;
@@ -338 +367 @@
     } else if (mimetype == "application/vnd.ms-powerpoint") {
         string cmd = "catppt -dutf-8 " + shell_protect(file);
         try {
-            dump = stdout_to_string(cmd);
+            dump = run_cmd(cmd);
         } catch (ReadError) {
             cout << "\"" << cmd << "\" failed - skipping\n";
             return;
@@ -349 +378 @@
         // as they don't seem to be at all well documented.
         string cmd = "wpd2text " + shell_protect(file);
         try {
-            dump = stdout_to_string(cmd);
+            dump = run_cmd(cmd);
         } catch (ReadError) {
             cout << "\"" << cmd << "\" failed - skipping\n";
             return;
@@ -358 +387 @@
         // wps2text produces UTF-8 output from the sample files I've tested.
         string cmd = "wps2text " + shell_protect(file);
         try {
-            dump = stdout_to_string(cmd);
+            dump = run_cmd(cmd);
         } catch (ReadError) {
             cout << "\"" << cmd << "\" failed - skipping\n";
             return;
@@ -380 +409 @@
         string cmd = "gzip -dc " + shell_protect(file);
         try {
             XmlParser xmlparser;
-            xmlparser.parse_html(stdout_to_string(cmd));
+            xmlparser.parse_html(run_cmd(cmd));
             dump = xmlparser.dump;
         } catch (ReadError) {
             cout << "\"" << cmd << "\" failed - skipping\n";
@@ -389 +418 @@
     } else if (mimetype == "text/rtf") {
         // The --text option unhelpfully converts all non-ASCII characters to
         // "?" so we use --html instead, which produces HTML entities.
-        string cmd = "unrtf --nopict --html 2>/dev/null " + shell_protect(file);
+        string cmd = "unrtf --nopict --html " + shell_protect(file);
         MyHtmlParser p;
         try {
-            p.parse_html(stdout_to_string(cmd));
+            p.parse_html(run_cmd(cmd));
         } catch (ReadError) {
             cout << "\"" << cmd << "\" failed - skipping\n";
             return;
@@ -413 +442 @@
         // from inspecting the source it looks like it's probably iso-8859-1.
         string cmd = "pod2text " + shell_protect(file);
         try {
-            dump = stdout_to_string(cmd);
+            dump = run_cmd(cmd);
             convert_to_utf8(dump, "ISO-8859-1");
         } catch (ReadError) {
             cout << "\"" << cmd << "\" failed - skipping\n";
@@ -427 +456 @@
         // decompositions".
         string cmd = "catdvi -e2 -s " + shell_protect(file);
         try {
-            dump = stdout_to_string(cmd);
+            dump = run_cmd(cmd);
             convert_to_utf8(dump, "ISO-8859-1");
         } catch (ReadError) {
             cout << "\"" << cmd << "\" failed - skipping\n";
@@ -440 +469 @@
         // (as it is in CP1250).
         string cmd = "djvutxt " + shell_protect(file);
         try {
-            dump = stdout_to_string(cmd);
+            dump = run_cmd(cmd);
         } catch (ReadError) {
             cout << "\"" << cmd << "\" failed - skipping\n";
             return;
         }
+#if 0 // FIXME: this won't work as omindex will have the database locked...
+    } else if (mimetype == "message/rfc822") {
+        // => mbox2script
+        //for stemmer lang, parse stemmer.get_description => Xapian::Stem(bla)
+        string cmd = "(mbox2omega " + shell_protect(file) + "|"
+            "scriptindex " + shell_protect(dbpath) + " /usr/share/omega/mbox2script.script)";
+        try {
+            dump = run_cmd(cmd);
+        } catch (ReadError) {
+            cout << "\"" << cmd << "\" failed - skipping\n";
+            return;
+        }
+#endif
+    } else if (mimetype == "application/vnd.ms-outlook") { // msg
+        string cmd = "outlook2text " + shell_protect(file);
+        try {
+            dump = run_cmd(cmd);
+        } catch (ReadError) {
+            cout << "\"" << cmd << "\" failed - skipping\n";
+            return;
+        }
     } else {
         // Don't know how to index this type.
         cout << "unknown MIME type - skipping\n";
@@ -581 +631 @@
                     }
                     continue;
                 case DirectoryIterator::REGULAR_FILE: {
+                    if (strcmp(d.leafname(), "mbox") == 0) {
+                        // Special filename.
+                        off_t size = d.get_size();
+                        time_t mtime = d.get_mtime();
+                        index_file(indexroot + url, "message/rfc822", mtime,
+                                   size);
+                        continue;
+                    }
+
                     string ext;
                     string::size_type dot = url.find_last_of('.');
                     if (dot != string::npos) ext = url.substr(dot + 1);
@@ -613 +672 @@
 
                         // It's in our MIME map so we know how to index it.
                         const string & mimetype = mt->second;
+
+                        // NOTE: unpacking does not work on MSWin32 this way!
+                        // We'd really have to pull in utils.cc:rmdir from
+                        // xapian-core.
+#ifndef _MSC_VER
+                        if (ext == "rar") {
+                            // TODO: Check timestamp
+                            string x = root+indexroot+"/.rar";
+                            cout << "[UNRAR into " << shell_protect(x+url) << "]" << endl;
+                            run_cmd("mkdir -p "+shell_protect(x+url));
+                            string cmd = "unrar x -o+ " +shell_protect(file) + " " + shell_protect(x+url+"/");
+                            run_cmd(cmd);
+                            index_directory(5, "/.rar"+url, mime_map);
+                            if (!nocleanup) {
+                                cout << "[CLEANUP " << "rm -rf " << shell_protect(x) << "]" << endl;
+                                run_cmd("rm -rf "+shell_protect(x));
+                            }
+                        } else if (ext == "zip") {
+                            // TODO: Check timestamp
+                            string x = root+indexroot+"/.zip";
+                            cout << "[UNZIP into " << shell_protect(x+url) << "]" << endl;
+                            run_cmd("mkdir -p "+shell_protect(x+url));
+                            string cmd = "unzip -o " +shell_protect(file) + " -d " +shell_protect(x+url+"/");
+                            run_cmd(cmd);
+                            index_directory(5, "/.zip"+url, mime_map);
+                            if (!nocleanup) {
+                                cout << "[CLEANUP " << "rm -rf " << shell_protect(x) << "]" << endl;
+                                run_cmd("rm -rf "+shell_protect(x));
+                            }
+                        } else if (ext == "pst") {
+                            // TODO: Check timestamp
+                            string x = root+indexroot+"/.pst";
+                            cout << "[READPST into " << shell_protect(x+url) << "]" << endl;
+                            run_cmd("mkdir -p "+shell_protect(x+url));
+                            // unpack attachments also, together with mbox files
+                            string cmd = "readpst -r -cv -w -o "+shell_protect(x+url)+" "+shell_protect(file);
+                            run_cmd(cmd);
+                            cout << "[UNPACK mbox attachments in " << shell_protect(x+url) << "]" << endl;
+                            cmd = "/usr/bin/find "+shell_protect(x+url)+" -name mbox -execdir uudeview -a -o -i '{}' ';'";
+                            run_cmd(cmd);
+                            index_directory(5, "/.pst"+url, mime_map); // mbox handling
+                            if (!nocleanup) {
+                                cout << "[CLEANUP " << "rm -rf " << shell_protect(x) << "]" << endl;
+                                run_cmd("rm -rf "+shell_protect(x));
+                            }
+                        } else
+#endif
                         try {
                             time_t mtime = d.get_mtime();
                             index_file(indexroot + url, mimetype, mtime, size);
@@ -653 +759 @@
     // If preserve_unupdated is false, delete any documents we don't
     // replace (if in replace duplicates mode)
     bool preserve_unupdated = false;
+    // If ignore_time is true, the existing timestamps are not checked on
+    // updates and every file will be parsed.
     size_t depth_limit = 0;
 
     static const struct option longopts[] = {
@@ -667 +775 @@
         { "depth-limit",required_argument,      NULL, 'l' },
         { "follow",     no_argument,            NULL, 'f' },
         { "stemmer",    required_argument,      NULL, 's' },
+        { "ignore-time",no_argument,            NULL, 'i' },
+        { "nocleanup",  no_argument,            NULL, 'c' },
         { 0, 0, NULL, 0 }
     };
 
@@ -742 +852 @@
     mime_map["djv"] = "image/vnd.djvu";
     mime_map["djvu"] = "image/vnd.djvu";
 
-    while ((getopt_ret = gnu_getopt_long(argc, argv, "hvd:D:U:M:lpf", longopts, NULL)) != -1) {
+    mime_map["msg"] = "application/vnd.ms-outlook";     // outlook2text - single message
+    mime_map["pst"] = "application/vnd.ms-outlook-pst"; // readpst | mimeexpand (libpst, Mime-tools) Outlook messager folder
+    mime_map["mbox"] = "message/rfc822";                // => mbox2script
+
+#ifndef _MSC_VER
+    mime_map["zip"] = "application/x-zip"; // recursive scanning
+    mime_map["rar"] = "application/x-rar"; // recursive scanning
+#endif
+
+    read_config_file();
+
+    while ((getopt_ret = gnu_getopt_long(argc, argv, "hvd:D:U:M:lpfi", longopts, NULL)) != -1) {
         switch (getopt_ret) {
         case 'h': {
             cout << PROG_NAME" - "PROG_DESC"\n\n"
@@ -756 +877 @@
 "  -M, --mime-type          additional MIME mapping ext:type\n"
 "  -l, --depth-limit=LIMIT  set recursion limit (0 = unlimited)\n"
 "  -f, --follow             follow symbolic links\n"
+"  -i, --ignore-time        ignore timestamp comparison\n"
+"      --nocleanup          don't delete temporary created from zip/rar/pst\n"
 "      --overwrite          create the database anew (the default is to update\n"
 "                           if the database already exists)" << endl;
             print_stemmer_help("     ");
@@ -769 +892 @@
                  << "Copyright (c) 2001,2005 James Aylett\n"
                  << "Copyright (c) 2001,2002 Ananova Ltd\n"
                  << "Copyright (c) 2002,2003,2004,2005,2006 Olly Betts\n\n"
+                 << "Copyright (c) 2006 AVL List GesmbH\n\n"
                  << "This is free software, and may be redistributed under\n"
                  << "the terms of the GNU Public License." << endl;
             return 0;
@@ -785 +909 @@
         case 'p': // don't delete unupdated documents
             preserve_unupdated = true;
             break;
+        case 'i': // --ignore-time: on updates parse the file again
+            ignore_time = true;
+            break;
+        case 'c':
+            nocleanup = true;
+            break;
         case 'l': { // Set recursion limit
             int arg = atoi(optarg);
             if (arg < 0) arg = 0;
@@ -844 +974 @@
     if (baseurl.empty()) {
         cerr << PROG_NAME": --url not specified, assuming `/'.\n";
     }
+    error_log = " 2>>"+log_dir+"omindex-error.log";
     // baseurl mustn't end '/' or you end up with the wrong URL
     // (//thing is different to /thing). We could probably make this
     // safe a different way, by ensuring that we don't put a leading '/'

Makefile.am

@@ -103 +103 @@
 omindex_SOURCES = omindex.cc myhtmlparse.cc htmlparse.cc\
  common/getopt.cc commonhelp.cc utils.cc hashterm.cc loadfile.cc md5.cc\
  md5wrap.cc xmlparse.cc metaxmlparse.cc utf8convert.cc sample.cc diritor.cc\
- runfilter.cc freemem.cc common/msvc_dirent.cc
+ runfilter.cc freemem.cc common/msvc_dirent.cc configfile.cc
 if NEED_MKDTEMP
 omindex_SOURCES += portability/mkdtemp.cc
 endif