Ticket #114: libextractor.patch

omega/omindex.cc

@@ -26 +26 @@
 #include <algorithm>
 #include <fstream>
 #include <iostream>
+#include <sstream>
 #include <string>
 #include <map>
 #include <vector>
@@ -41 +42 @@
 
 #include <xapian.h>
 
+#include <magic.h>
+#include <extractor.h>
+
 #include "commonhelp.h"
 #include "hashterm.h"
 #include "indextext.h"
@@ -208 +212 @@
 }
 
 static void
-index_file(const string &url, const string &mimetype, time_t last_mod, off_t size)
+index_file(const string &url, EXTRACTOR_ExtractorList* extractors, time_t last_mod, off_t size, magic_t cookie)
 {
     string file = root + url;
     string title, sample, keywords, dump;
 
-    cout << "Indexing \"" << url << "\" as " << mimetype << " ... " << flush;
+    cout << "Indexing \"" << url << " ... " << flush;
 
     string urlterm("U");
     urlterm += baseurl;
@@ -228 +232 @@
     }
 
     string md5;
+#if 0
     if (mimetype == "text/html") {
         string text;
         try {
@@ -429 +434 @@
         cout << "unknown MIME type - skipping\n";
         return;
     }
+#endif
 
-    // Compute the MD5 of the file if we haven't already.
-    if (md5.empty() && md5_file(file, md5) == 0) {
-        cout << "failed to read file to calculate MD5 checksum - skipping\n";
-        return;
+    // Find mime type of file using libmagic
+    const char* mime = magic_file(cookie, file.c_str());
+    if (mime == NULL) {
+        std::cerr << magic_error(cookie) << std::endl;
+        throw;
     }
+    string mimetype(mime);
+    cout << "type " << mimetype << "...";
+    // If file is plain text, just index it as-is
+    // This ignores the charset determined by libmagic... i.e. us-ascii
+    if (strncmp(mime, "text/plain", strlen("text/plain")) == 0) {
+        try {
+            // Currently we assume that text files are UTF-8.
+            // FIXME: What charset is the file?  Look for BOM?  Look at contents?
+            dump = file_to_string(file);
+        } catch (ReadError) {
+            cout << "can't read \"" << file << "\" - skipping\n";
+            return;
+        }
+    }
+    else {
+        stringstream ss;
+        EXTRACTOR_KeywordList* exkeywords = EXTRACTOR_getKeywords(extractors, file.c_str());
+        //exkeywords = EXTRACTOR_removeDuplicateKeywords(exkeywords, EXTRACTOR_DUPLICATES_TYPELESS);
 
+        if (exkeywords == NULL) {
+            cout << "ignored by libextractor.\n";
+            return;
+        }
+
+        while (exkeywords != NULL) {
+            if (exkeywords->keywordType == EXTRACTOR_TITLE)
+                title = exkeywords->keyword;
+            else if (exkeywords->keywordType == EXTRACTOR_KEYWORDS)
+                keywords = exkeywords->keyword;
+            else if (exkeywords->keywordType == EXTRACTOR_DESCRIPTION)
+                sample = exkeywords->keyword;
+
+            ss << exkeywords->keyword << " ";
+            exkeywords = exkeywords->next;
+        }
+        dump = ss.str();
+        EXTRACTOR_freeKeywords(exkeywords);
+
+        // Compute the MD5 of the file if we haven't already.
+        if (md5.empty() && md5_file(file, md5) == 0) {
+            cout << "failed to read file to calculate MD5 checksum - skipping\n";
+            return;
+        }
+    }
+
     // Produce a sample
     if (sample.empty()) {
         sample = truncate_to_word(dump, SAMPLE_SIZE);
@@ -535 +586 @@
 {
     struct dirent *ent;
     string path = root + indexroot + dir;
+    EXTRACTOR_ExtractorList* extractors = EXTRACTOR_loadDefaultLibraries();
 
+#if 1
+    magic_t cookie = magic_open(MAGIC_MIME);
+    int result = magic_load(cookie, NULL);
+    if (result == -1) {
+        cout << "Can't open MIME database: " << magic_error(cookie) << endl;
+        throw;
+    }
+#endif
+
     cout << "[Entering directory " << dir << "]" << endl;
 
     DIR *d = opendir(path.c_str());
@@ -583 +644 @@
                 cout << "Skipping empty file: \"" << file << "\"" << endl;
                 continue;
             }
-            string ext;
-            string::size_type dot = url.find_last_of('.');
-            if (dot != string::npos) ext = url.substr(dot + 1);
 
-            map<string,string>::iterator mt = mime_map.find(ext);
-            if (mt != mime_map.end()) {
+            //if (mimetype != "data") {
                 // It's in our MIME map so we know how to index it.
-                const string & mimetype = mt->second;
                 try {
-                    index_file(indexroot + url, mimetype, statbuf.st_mtime,
-                               statbuf.st_size);
+                    index_file(indexroot + url, extractors, statbuf.st_mtime,
+                               statbuf.st_size, cookie);
                 } catch (NoSuchFilter) {
                     // FIXME: we ought to ignore by mime-type not extension.
-                    cout << "Filter for \"" << mimetype << "\" not installed - ignoring extension \"" << ext << "\"" << endl;
-                    mime_map.erase(mt);
+//                  cout << "Filter for \"" << mimetype << "\" not installed - ignoring file \"" << file << "\"" << endl;
                 }
-            }
+            //}
             continue;
         }
         cout << "Not a regular file \"" << file << "\" - skipping\n";
     }
     closedir(d);
+    EXTRACTOR_removeAll(extractors);
+    magic_close(cookie);
 }
 
 int