Ticket #595: atomparse.2.diff

xapian-applications/omega/Makefile.am

@@ -97 +97 @@
  md5.h md5wrap.h xmlparse.h metaxmlparse.h values.h utf8convert.h\
  namedentities.h pkglibbindir.h datematchdecider.h sample.h strcasecmp.h\
  utf8truncate.h diritor.h runfilter.h freemem.h xpsxmlparse.h transform.h\
- weight.h svgparse.h tmpdir.h urldecode.h urlencode.h unixperm.h
+ weight.h svgparse.h tmpdir.h urldecode.h urlencode.h unixperm.h atomparse.h
 
 # headers maintained in xapian-core
 noinst_HEADERS +=\
@@ -138 +138 @@
  common/getopt.cc commonhelp.cc utils.cc hashterm.cc loadfile.cc md5.cc\
  md5wrap.cc xmlparse.cc metaxmlparse.cc utf8convert.cc sample.cc diritor.cc\
  runfilter.cc freemem.cc common/msvc_dirent.cc xpsxmlparse.cc common/str.cc\
- pkglibbindir.cc svgparse.cc tmpdir.cc urlencode.cc
+ pkglibbindir.cc svgparse.cc tmpdir.cc urlencode.cc atomparse.cc
 if NEED_MKDTEMP
 omindex_SOURCES += portability/mkdtemp.cc
 endif

new file xapian-applications/omega/atomparse.cc

@@ +1 @@
+/** @file atomparse.cc
+ * @brief Extract text from an RSS atom file.
+ */
+/* Copyright (C) 2010,2011 Olly Betts
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ */
+
+#include <config.h>
+#include <iostream>
+
+#include "atomparse.h"
+#include "stringutils.h"
+#include "myhtmlparse.h"
+
+using namespace std;
+
+void
+AtomParser::process_text(const string &text)
+{
+        if (is_escaped)
+                return;
+
+    string * target = NULL;
+        string text_copy = text;
+
+    switch (state) {
+        case TEXT:
+            target = &dump;
+            break;
+        case TITLE:
+            target = &title;
+            break;
+        case KEYWORDS:
+            target = &keywords;
+            break;
+        case AUTHOR:
+            target = &author;
+            break;
+        case OTHER:
+            // Ignore context in other places.
+            return;
+    }
+
+        if (type == "html") {
+                MyHtmlParser p;
+                p.parse_html(text_copy, "iso-8859-1", false);
+                text_copy = p.dump;
+        }
+
+    if (!target->empty())
+                *target += ' ';
+
+        *target += text_copy;
+}
+
+bool
+AtomParser::opening_tag(const string &tag)
+{
+        if (state == OTHER) {
+                if (tag == "title")
+                        state = in_entry ? KEYWORDS : TITLE;
+                else if (tag == "summary" || tag == "subtitle" || tag == "content")
+                        state = TEXT;
+                else if (tag == "author")
+                        state = AUTHOR;
+                else if (tag == "entry")
+                        in_entry = true;
+                else if (tag == "category") {
+                        //handle category term sepparatley
+                        string new_keyword;
+                        get_parameter("term", new_keyword);
+                        keywords = keywords + ' ' + new_keyword;
+                }
+        } else if (state == AUTHOR) {
+                if (tag == "uri")
+                        is_escaped = true;
+        }
+
+        get_parameter("type", type);
+    return true;
+}
+
+bool
+AtomParser::closing_tag(const string &tag)
+{
+        if (tag == "entry")
+                in_entry = false;
+        if (tag == "uri")
+                is_escaped = false;
+
+        else if (tag == "title" || tag == "summary" || tag == "subtitle" ||
+                 tag == "author" || tag == "content")
+                state = OTHER;
+    return true;
+}

new file xapian-applications/omega/atomparse.h

@@ +1 @@
+/** @file atomparse.h
+ * @brief Extract text from an RSS atom file.
+ */
+/* Copyright (C) 2010,2011 Olly Betts
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation; either version 2 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301 USA
+ */
+
+#ifndef OMEGA_INCLUDED_ATOMPARSE_H
+#define OMEGA_INCLUDED_ATOMPARSE_H
+
+#include "htmlparse.h"
+
+class AtomParser : public HtmlParser {
+    enum { OTHER, TITLE, AUTHOR, KEYWORDS, TEXT } state;
+        bool in_entry, is_escaped;
+        string type;
+  public:
+    AtomParser() : state(OTHER), in_entry(false), is_escaped(false) { }
+    void process_text(const string &text);
+    bool opening_tag(const string &tag);
+    bool closing_tag(const string &tag);
+    string title, keywords, dump, author;
+};
+
+#endif // OMEGA_INCLUDED_ATOMPARSE_H

xapian-applications/omega/omega.conf

@@ -1 +1 @@
 # Directory containing Xapian databases:
-database_dir /var/lib/omega/data
+database_dir /home/mihai/sandbox/xapiandb/
 
 # Directory containing OmegaScript templates:
-template_dir /var/lib/omega/templates
+template_dir /home/mihai/repos/xapian/xapian-applications/omega/templates
 
 # Directory to write Omega logs to:
 log_dir /var/log/omega

xapian-applications/omega/omindex.cc

@@ -62 +62 @@
 #include "values.h"
 #include "xmlparse.h"
 #include "xpsxmlparse.h"
+#include "atomparse.h"
 
 #include "gnu_getopt.h"
 
@@ -765 +766 @@
             if (idx != string::npos) {
                 dump.assign(desc, idx + 1, string::npos);
             }
+        } else if (mimetype == "application/atom+xml") {
+                AtomParser atomparser;
+                atomparser.parse_html(d.file_to_string());
+                dump = atomparser.dump;
+                title = atomparser.title;
+                keywords = atomparser.keywords;
+                author = atomparser.author;
         } else {
             // Don't know how to index this type.
             skip_unknown_mimetype(file, mimetype);
@@ -1153 +1161 @@
     // RPM packages:
     mime_map["rpm"] = "application/x-redhat-package-manager";
 
+        // Atom feeds:
+        mime_map["atom"] = "application/atom+xml";
+
     // Extensions to quietly ignore:
     mime_map["a"] = "ignore";
     mime_map["bin"] = "ignore";