Ticket #292: omega-rssh-292.patch

xapian-omega-1.0.7

@@ -22 +22 @@
 #include <config.h>
 
 #include "myhtmlparse.h"
+#include "utf8convert.h"
+
 
 #include <ctype.h>
 #include <string.h>
 
+#include <iostream>
+
 inline void
 lowercase_string(string &str)
 {
@@ -37 +41 @@
 void
 MyHtmlParser::parse_html(const string &text)
 {
-    // Default HTML character set is latin 1, though not specifying one is
+    // Default HTML character set is utf-8, though not specifying one is
     // deprecated these days.
-    charset = "ISO-8859-1";
+    //   (first 256 position of UTF-8 are the same, as in "ISO-8859-1")
+    charset = "UTF-8";
     HtmlParser::parse_html(text);
 }
 
@@ -172 +177 @@
                                     ++end;
                                 }
                             }
+                            string prev_charset = charset;
                             charset = value.substr(start, end - start);
+                            if (prev_charset!=charset) {
+                              // title and values can be filled 
+                              //   before setting charset
+                              if (
+                                prev_charset.size()==0
+                                ||
+                                strcasecmp(prev_charset.c_str(),"utf-8")==0
+                                ||
+                                strcasecmp(prev_charset.c_str(),"utf8")==0
+                              ) {
+                                if (title.size()!=0) {
+                                   convert_to_utf8(title,charset);
+                                }
+                                if (sample.size()!=0) {
+                                   convert_to_utf8(sample,charset);
+                                }
+                              } else {
+                                if (title.size()!=0) {
+                                  string title1 = title;
+                                  convert_to_utf8(title1,charset);
+                                  if (title1!=title) {
+                                     title=string("[incorrect conversion]");
+                                  }
+                                }
+                                if (sample.size()!=0) {
+                                  string sample1 = sample;
+                                  convert_to_utf8(sample1,charset);
+                                  if (sample1!=sample) {
+                                     sample=string("[incorrect conversion]");
+                                  }
+                                }
+                              }
+                            }
                         }
                     }
                 }