Ticket #282: xapian-omega-trunk-r16879-from-ticket-285-and-cleaned-up-updated-2012-11-13.patch

xapian-applications/omega/ChangeLog

@@ +1 @@
+2006-08-22 09:30:12 Reini Urban <reinhard.urban@avl.com>
+
+        omega-0.9.6c:
+        * omindex.cc: Fix wrong timestamp comparison in cache logic
+        * outlook2text.in: New script
+
+2006-08-17 18:06:26 Reini Urban <reinhard.urban@avl.com>
+
+        omega-0.9.6a:
+        * omindex.cc: Added cached virtual directories zip,msg,pst,...).
+        Consistently log stderr to /var/log/omega/omindex-error.log.
+
 Tue Sep 25 23:57:12 GMT 2012  Olly Betts <olly@survex.com>
 
         * Makefile.am,omindex.cc: Replace shell_protect() with

xapian-applications/omega/Makefile.am

@@ -78 +78 @@
 pkglibbin_PROGRAMS = omega
 dist_pkglibbin_SCRIPTS = outlookmsg2html
 bin_PROGRAMS = omindex scriptindex
+dist_libexec_SCRIPTS = outlook2text mimeexplode msgconvert.pl
 dist_bin_SCRIPTS = dbi2omega htdig2omega mbox2omega
 
 check_PROGRAMS = atomparsetest htmlparsetest md5test urlenctest utf8converttest
@@ -173 +174 @@
 MAINTAINERCLEANFILES = $(dist_man_MANS)
 endif
 
+CLEANFILES = outlook2text
+
+outlook2text: $(srcdir)/outlook2text.in Makefile
+        sed "s,@MSGCONVERT@,$(pkglibbindir)/msgconvert.pl,;s,@MIMEEXPLODE@,$(pkglibbindir)/mimeexplode," $(srcdir)/outlook2text.in > $@
+
 if DOCUMENTATION_RULES
 omindex.1: omindex$(EXEEXT) makemanpage
         ./makemanpage ./omindex $(srcdir)/omindex.cc omindex.1

new file xapian-applications/omega/mimeexplode

@@ +1 @@
+#!/usr/bin/perl -w
+
+=head1 NAME
+
+mimeexplode - explode one or more MIME messages
+
+=head1 SYNOPSIS
+
+    mimeexplode [-d <dir>] <mime-msg-file> <mime-msg-file> ...
+
+    someprocess | mimeexplode -
+
+=head1 DESCRIPTION
+
+Takes one or more files from the command line that contain MIME
+messages, and explodes their contents out into subdirectories
+of the current working directory.  The subdirectories are
+just called C<msg0>, C<msg1>, C<msg2>, etc.  Existing directories are
+skipped over.
+
+The message information is output to the stdout, like this:
+
+    Message: msg3 (inputfile1.msg)
+        Part: msg3/filename-1.dat (text/plain)
+        Part: msg3/filename-2.dat (text/plain)
+    Message: msg5 (input-file2.msg)
+        Part: msg5/dir.gif (image/gif)
+        Part: msg5/face.jpg (image/jpeg)
+    Message: msg6 (infile3)
+        Part: msg6/filename-1.dat (text/plain)
+
+This was written as an example of the MIME:: modules in the
+MIME-parser package I wrote.  It may prove useful as a quick-and-dirty
+way of splitting a MIME message if you need to decode something, and
+you don't have a MIME mail reader on hand.
+
+=head1 COMMAND LINE OPTIONS
+
+-d outdir
+
+=head1 AUTHOR
+
+Eryq C<eryq@zeegee.com>, in a big hurry...
+Reini Urban C<rurban@x-ray.at>: -d option to always explode into the same dir
+
+=cut
+
+#BEGIN { unshift @INC, ".." }    # to test MIME:: stuff before installing it!
+
+require 5.001;
+
+use strict;
+use vars;
+
+use MIME::Parser;
+use Getopt::Std;
+my %opts;
+my $outbase = '';
+my $postfix = '';
+
+#------------------------------------------------------------
+# make_msg - make and return the name of a msgXXX directory
+#------------------------------------------------------------
+
+#ignored
+#sub make_msg {
+#    while (-d "msg$Msgno") {
+#       ++$Msgno;
+#       die "self-imposed limit reached" if $Msgno == 256;
+#    }
+#   mkdir "msg$Msgno",0755 or die "couldn't make msg$Msgno: $!";
+#    "msg$Msgno";
+#}
+
+#------------------------------------------------------------
+# dump_entity - dump an entity's file info
+#------------------------------------------------------------
+sub dump_entity {
+    my $ent = shift;
+    my @parts = $ent->parts;
+
+    if (@parts) {        # multipart...
+        map { dump_entity($_) } @parts;
+    }
+    else {               # single part...
+        print "    Part: ", $ent->bodyhandle->path,
+              " (", scalar($ent->head->mime_type), ")\n";
+    }
+}
+
+#------------------------------------------------------------
+# main
+#------------------------------------------------------------
+sub main {
+    my $file;
+    my $entity;
+
+    # make sure the same message gets exploded into the same dir
+    getopts('d:', \%opts);
+    $outbase = $opts{d} ? $opts{d} : "msg0";
+    my $outdir = $outbase;
+
+    # Go through messages:
+    @ARGV or unshift @ARGV, "-";
+    while (defined($file = shift @ARGV)) {
+
+      # Sanity:
+      (-d $outdir) or mkdir "$outdir",0755;
+      (-w "$outdir") or die "cwd $outdir not writable!";
+      #my $msgdir = make_msg();
+      #print "Message: $msgdir ($file)\n";
+
+      # Create a new parser object:
+      my $parser = new MIME::Parser;
+      ### $parser->parse_nested_messages('REPLACE');
+
+      # Optional: set up parameters that will affect how it extracts
+      #   documents from the input stream:
+      $parser->output_dir($outdir);
+
+      # Parse an input stream:
+      open FILE, $file or die "couldn't open $file";
+      $entity = $parser->read(\*FILE) or
+        print STDERR "Couldn't parse MIME in $file; continuing...\n";
+      close FILE;
+
+      # Congratulations: you now have a (possibly multipart) MIME entity!
+      dump_entity($entity) if $entity;
+      ### $entity->dump_skeleton if $entity;
+
+      $postfix++;
+      $outdir = $outbase.$postfix;
+    }
+    1;
+}
+
+exit (&main ? 0 : -1);
+#------------------------------------------------------------
+1;
+

new file xapian-applications/omega/msgconvert.pl

@@ +1 @@
+#!/usr/bin/perl -w
+#
+# msgconvert.pl:
+#
+# Convert .MSG files (made by Outlook (Express)) to multipart MIME messages.
+#
+# Copyright 2002, 2004, 2006 Matijs van Zuijlen
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+# Public License for more details.
+#
+# CHANGES:
+# 20020715  Recognize new items 'Cc', mime type of attachment, long
+#           filename of attachment, and full headers. Attachments turn out
+#           to be numbered, so a regexp is now used to recognize label of
+#           items that are attachments.
+# 20020831  long file name will definitely be used if present. Full headers
+#           and mime type information are used when present. Created
+#           generic system for specifying known items to be skipped.
+#           Unexpected contents is never reason to bail out anymore. Added
+#           support for usage message and option processing (--verbose).
+# 20040104  Handle address data slightly better, make From line less fake,
+#           make $verbose and $skippable_entries global vars, handle HTML
+#           variant of body text if present (though not optimally).
+# 20040214  Fix typos and incorrect comments.
+# 20040307  - Complete rewrite: All functional parts are now in the package
+#             MSGParser;
+#           - Creation of MIME::Entity object is delayed until the output
+#             routines, which means all data is known; This means I can
+#             create a multipart/alternative body.
+#           - Item names are parsed (thanks to bfrederi@alumni.sfu.ca for
+#             the information).
+# 20040514  Check if $self->{HEAD} actually exists before trying to add its
+#           contents to the output Mime object's header data.
+#           (Bug reported by Thomas Ng).
+#           Don't produce multipart messages if not needed.
+#           (Bug reported by Justin B. Scout).
+# 20040529  Correctly format OLEDATE.
+# 20040530  - Extract date from property 0047 (thanks, Marc Goodman).
+#           - Use address data to make To: and Cc: lines complete
+#           - Use the in-reply-to property
+#           - More unknown properties named.
+#           - Found another property containing an SMTP address.
+#           - Put non-SMTP type addresses back in output.
+# 20040825  Replace 'our' to declare globals with 'use vars'. This means
+#           the globals our now properly scoped inside the package and not
+#           the file.
+#           This also fixes the bug that this program did not work on perl
+#           versions below 5.6. (Bug reported by Tim Gustafson)
+# 20060218  More sensible encoding warnings.
+# 20060219  Move OLE parsing to main program.
+#           Parse nested MSG files (Bug reported by Christof Lukas).
+# 20060225  Simplify code.
+#
+
+#
+# Import modules.
+#
+package MSGParser;
+use strict;
+use OLE::Storage_Lite;
+use MIME::Entity;
+use MIME::Parser;
+use Date::Format;
+use POSIX qw(mktime);
+use constant DIR_TYPE => 1;
+use constant FILE_TYPE => 2;
+
+use vars qw($skipproperties $skipheaders);
+#
+# Descriptions partially based on mapitags.h
+#
+$skipproperties = {
+  # Envelope properties
+  '000B' => "Conversation key?",
+  '001A' => "Type of message",
+  '003B' => "Sender address variant",
+  '003D' => "Contains 'Re: '",
+  '003F' => "'recieved by' id",
+  '0040' => "'recieved by' name",
+  '0041' => "Sender variant address id",
+  '0042' => "Sender variant name",
+  '0043' => "'recieved representing' id",
+  '0044' => "'recieved representing' name",
+  '0046' => "Read receipt address id",
+  '0051' => "'recieved by' search key",
+  '0052' => "'recieved representing' search key",
+  '0053' => "Read receipt search key",
+  '0064' => "Sender variant address type",
+  '0065' => "Sender variant address",
+  '0070' => "Conversation topic",
+  '0071' => "Conversation index",
+  '0075' => "'recieved by' address type",
+  '0076' => "'recieved by' email address",
+  '0077' => "'recieved representing' address type",
+  '0078' => "'recieved representing' email address",
+  '007F' => "something like a message id",
+  # Recipient properties
+  '0C19' => "Reply address variant",
+  '0C1D' => "Reply address variant",
+  '0C1E' => "Reply address type",
+  # Non-transmittable properties
+  '0E02' => "?Should BCC be displayed",
+  '0E0A' => "sent mail id",
+  '0E1D' => "Subject w/o Re",
+  '0E27' => "64 bytes: Unknown",
+  '0FF6' => "Index",
+  '0FF9' => "Index",
+  '0FFF' => "Address variant",
+  # Content properties
+  '1008' => "Summary or something",
+  '1009' => "RTF Compressed",
+  # 'Common property'
+  '3001' => "Display name",
+  '3002' => "Address Type",
+  '300B' => "'Search key'",
+  # Attachment properties
+  '3702' => "Attachment encoding",
+  '3703' => "Attachment extension",
+  '3709' => "'Attachment rendering'", # Maybe an icon or something?
+  '3713' => "Icon URL?",
+  # 'Mail user'
+  '3A20' => "Address variant",
+  # 3900 -- 39FF: 'Address book'
+  '39FF' => "7 bit display name",
+  # 'Display table properties'
+  '3FF8' => "Routing data?",
+  '3FF9' => "Routing data?",
+  '3FFA' => "Routing data?",
+  '3FFB' => "Routing data?",
+  # 'Transport-defined envelope property'
+  '4029' => "Sender variant address type",
+  '402A' => "Sender variant address",
+  '402B' => "Sender variant name",
+  '5FF6' => "Recipient name",
+  '5FF7' => "Recipient address variant",
+  # 'Provider-defined internal non-transmittable property'
+  '6740' => "Unknown, binary data",
+  # User defined id's
+  '8000' => "Content Class",
+  '8002' => "Unknown, binary data",
+};
+
+$skipheaders = {
+  "MIME-Version" => 1,
+  "Content-Type" => 1,
+  "Content-Transfer-Encoding" => 1,
+  "X-Mailer" => 1,
+  "X-Msgconvert" => 1,
+  "X-MS-Tnef-Correlator" => 1,
+  "X-MS-Has-Attach" => 1,
+};
+
+use constant ENCODING_UNICODE => '001F';
+use constant KNOWN_ENCODINGS => {
+    '000D' => 'Directory',
+    '001F' => 'Unicode',
+    '001E' => 'Ascii?',
+    '0102' => 'Binary',
+};
+
+use constant MAP_ATTACHMENT_FILE => {
+  '3701' => ["DATA",        0], # Data
+  '3704' => ["SHORTNAME",   1], # Short file name
+  '3707' => ["LONGNAME",    1], # Long file name
+  '370E' => ["MIMETYPE",    1], # mime type
+  '3716' => ["DISPOSITION", 1], # disposition
+};
+
+use constant MAP_SUBITEM_FILE => {
+  '1000' => ["BODY_PLAIN",      0], # Body
+  '1013' => ["BODY_HTML",       0], # HTML Version of body
+  '0037' => ["SUBJECT",         1], # Subject
+  '0047' => ["SUBMISSION_ID",   1], # Seems to contain the date
+  '007D' => ["HEAD",            1], # Full headers
+  '0C1A' => ["FROM",            1], # Reply-To: Name
+  '0C1E' => ["FROM_ADDR_TYPE",  1], # From: Address type
+  '0C1F' => ["FROM_ADDR",       1], # Reply-To: Address
+  '0E04' => ["TO",              1], # To: Names
+  '0E03' => ["CC",              1], # Cc: Names
+  '1035' => ["MESSAGEID",       1], # Message-Id
+  '1042' => ["INREPLYTO",       1], # In reply to Message-Id
+};
+
+use constant MAP_ADDRESSITEM_FILE => {
+  '3001' => ["NAME",            1], # Real name
+  '3002' => ["TYPE",            1], # Address type
+  '403D' => ["TYPE",            1], # Address type
+  '3003' => ["ADDRESS",         1], # Address
+  '403E' => ["ADDRESS",         1], # Address
+  '39FE' => ["SMTPADDRESS",     1], # SMTP Address variant
+};
+
+#
+# Main body of module
+#
+
+sub new {
+  my $that = shift;
+  my $class = ref $that || $that;
+
+  my $self = {
+    ATTACHMENTS => [],
+    ADDRESSES => [],
+    VERBOSE => 0,
+    HAS_UNICODE => 0,
+    FROM_ADDR_TYPE => "",
+  };
+  bless $self, $class;
+}
+
+#
+# Main sub: parse the PPS tree, and return 
+#
+sub parse {
+  my $self = shift;
+  my $PPS = shift or die "Internal error: No PPS tree";
+  $self->_RootDir($PPS);
+}
+
+sub mime_object {
+  my $self = shift;
+
+  my $bodymime;
+  my $mime;
+
+  if ($self->_IsMultiPart) {
+    # Construct a multipart message object
+
+    $mime = MIME::Entity->build(Type => "multipart/mixed");
+
+    # Set the entity that we'll save the body parts to. If there's more than
+    # one part, it's a new entity, otherwise, it's the main $mime object.
+    if ($self->{BODY_HTML} and $self->{BODY_PLAIN}) {
+      $bodymime = MIME::Entity->build(
+        Type => "multipart/alternative",
+        Encoding => "8bit",
+      );
+      $mime->add_part($bodymime);
+    } else {
+      $bodymime = $mime;
+    }
+    if ($self->{BODY_PLAIN}) {
+      $self->_SaveAttachment($bodymime, {
+        MIMETYPE => 'text/plain; charset=ISO-8859-1',
+        ENCODING => '8bit',
+        DATA => $self->{BODY_PLAIN},
+        DISPOSITION => 'inline',
+      });
+    }
+    if ($self->{BODY_HTML}) {
+      $self->_SaveAttachment($bodymime, {
+        MIMETYPE => 'text/html',
+        ENCODING => '8bit',
+        DATA => $self->{BODY_HTML},
+        DISPOSITION => 'inline',
+      });
+    }
+    foreach my $att (@{$self->{ATTACHMENTS}}) {
+      $self->_SaveAttachment($mime, $att);
+    }
+  } elsif ($self->{BODY_PLAIN}) {
+    # Construct a single part message object with a plain text body
+    $mime = MIME::Entity->build(
+      Type => "text/plain",
+      Data => $self->{BODY_PLAIN}
+    );
+  } elsif ($self->{BODY_HTML}) {
+    # Construct a single part message object with an HTML body
+    $mime = MIME::Entity->build(
+      Type => "text/html",
+      Data => $self->{BODY_HTML}
+    );
+  }
+
+  $self->_CopyHeaderData($mime);
+
+  $self->_SetHeaderFields($mime);
+
+  return $mime;
+}
+
+# Actually output the message in mbox format
+sub print {
+  my $self = shift;
+
+  my $mime = $self->mime_object;
+
+  # Construct From line from whatever we know.
+  my $string = "";
+  $string = (
+    $self->{FROM_ADDR_TYPE} eq "SMTP" ?
+    $self->{FROM_ADDR} :
+    'someone@somewhere'
+  );
+  $string =~ s/\n//g;
+
+  # The date used here is not really important.
+  print "From ", $string, " ", scalar localtime, "\n";
+  $mime->print(\*STDOUT);
+  print "\n";
+}
+
+sub set_verbosity {
+  my ($self, $verbosity) = @_;
+  defined $verbosity or die "Internal error: no verbosity level";
+  $self->{VERBOSE} = $verbosity;
+}
+
+#
+# Below are functions that walk the PPS tree. The *Dir functions handle
+# processing the directory nodes of the tree (mainly, iterating over the
+# children), whereas the *Item functions handle processing the items in the
+# directory (if such an item is itself a directory, it will in turn be
+# processed by the relevant *Dir function).
+#
+
+#
+# RootItem: Check Root Entry, parse sub-entries.
+# The OLE file consists of a single entry called Root Entry, which has
+# several children. These children are parsed in the sub SubItem.
+# 
+sub _RootDir {
+  my ($self, $PPS) = @_;
+
+  foreach my $child (@{$PPS->{Child}}) {
+    $self->_SubItem($child);
+  }
+}
+
+sub _SubItem {
+  my ($self, $PPS) = @_;
+  
+  if ($PPS->{Type} == DIR_TYPE) {
+    $self->_SubItemDir($PPS);
+  } elsif ($PPS->{Type} == FILE_TYPE) {
+    $self->_SubItemFile($PPS);
+  } else {
+    warn "Unknown entry type: $PPS->{Type}";
+  }
+}
+
+sub _SubItemDir {
+  my ($self, $PPS) = @_;
+
+  $self->_GetOLEDate($PPS);
+
+  my $name = $self->_GetName($PPS);
+
+  if ($name =~ /__recip_version1 0_ /) { # Address of one recipient
+    $self->_AddressDir($PPS);
+  } elsif ($name =~ '__attach_version1 0_ ') { # Attachment
+    $self->_AttachmentDir($PPS);
+  } else {
+    $self->_UnknownDir($self->_GetName($PPS));
+  }
+}
+
+sub _SubItemFile {
+  my ($self, $PPS) = @_;
+
+  my $name = $self->_GetName($PPS);
+  my ($property, $encoding) = $self->_ParseItemName($name);
+
+  $self->_MapProperty($self, $PPS->{Data}, $property,
+    MAP_SUBITEM_FILE) or $self->_UnknownFile($name);
+}
+
+sub _AddressDir {
+  my ($self, $PPS) = @_;
+
+  my $address = {
+    NAME        => undef,
+    ADDRESS     => undef,
+    TYPE        => "",
+  };
+  foreach my $child (@{$PPS->{Child}}) {
+    $self->_AddressItem($child, $address);
+  }
+  push @{$self->{ADDRESSES}}, $address;
+}
+
+sub _AddressItem {
+  my ($self, $PPS, $addr_info) = @_;
+
+  my $name = $self->_GetName($PPS);
+
+  # DIR Entries: There should be none.
+  if ($PPS->{Type} == DIR_TYPE) {
+    $self->_UnknownDir($name);
+  } elsif ($PPS->{Type} == FILE_TYPE) {
+    my ($property, $encoding) = $self->_ParseItemName($name);
+    $self->_MapProperty($addr_info, $PPS->{Data}, $property,
+      MAP_ADDRESSITEM_FILE) or $self->_UnknownFile($name);
+  } else {
+    warn "Unknown entry type: $PPS->{Type}";
+  }
+}
+
+sub _AttachmentDir {
+  my ($self, $PPS) = @_;
+
+  my $attachment = {
+    SHORTNAME   => undef,
+    LONGNAME    => undef,
+    MIMETYPE    => 'application/octet-stream',
+    ENCODING    => 'base64',
+    DISPOSITION => 'attachment',
+    DATA        => undef
+  };
+  foreach my $child (@{$PPS->{Child}}) {
+    $self->_AttachmentItem($child, $attachment);
+  }
+  push @{$self->{ATTACHMENTS}}, $attachment;
+}
+
+sub _AttachmentItem {
+  my ($self, $PPS, $att_info) = @_;
+
+  my $name = $self->_GetName($PPS);
+
+  my ($property, $encoding) = $self->_ParseItemName($name);
+
+  if ($PPS->{Type} == DIR_TYPE) {
+
+    if ($property eq '3701') {  # Nested MSG file
+      my $msgp = new MSGParser();
+      $msgp->parse($PPS);
+      my $data = $msgp->mime_object->as_string;
+      $att_info->{DATA} = $data;
+      $att_info->{MIMETYPE} = 'message/rfc822';
+      $att_info->{ENCODING} = '8bit';
+    } else {
+      $self->_UnknownDir($name);
+    }
+
+  } elsif ($PPS->{Type} == FILE_TYPE) {
+    $self->_MapProperty($att_info, $PPS->{Data}, $property,
+      MAP_ATTACHMENT_FILE) or $self->_UnknownFile($name);
+  } else {
+    warn "Unknown entry type: $PPS->{Type}";
+  }
+}
+
+sub _MapProperty {
+  my ($self, $hash, $data, $property, $map) = @_;
+
+  defined $property or return 0;
+  my $arr = $map->{$property} or return 0;
+
+  $arr->[1] and $data =~ s/\000//g;
+  $hash->{$arr->[0]} = $data;
+
+  return 1;
+}
+
+sub _UnknownDir {
+  my ($self, $name) = @_;
+
+  if ($name eq '__nameid_version1 0') {
+    $self->{VERBOSE}
+      and warn "Skipping DIR entry $name (Introductory stuff)\n";
+    return;
+  }
+  warn "Unknown DIR entry $name\n";
+}
+
+sub _UnknownFile {
+  my ($self, $name) = @_;
+
+  if ($name eq '__properties_version1 0') {
+    $self->{VERBOSE}
+      and warn "Skipping FILE entry $name (Properties)\n";
+    return;
+  }
+
+  my ($property, $encoding) = $self->_ParseItemName($name);
+  unless (defined $property) {
+    warn "Unknown FILE entry $name\n";
+    return;
+  }
+  if ($skipproperties->{$property}) {
+    $self->{VERBOSE}
+      and warn "Skipping property $property ($skipproperties->{$property})\n";
+    return;
+  } elsif ($property =~ /^80/) {
+    $self->{VERBOSE}
+      and warn "Skipping property $property (user-defined property)\n";
+    return;
+  } else {
+    warn "Unknown property $property\n";
+    return;
+  }
+}
+
+#
+# Helper functions
+#
+
+sub _GetName {
+  my ($self, $PPS) = @_;
+  return $self->_NormalizeWhiteSpace(OLE::Storage_Lite::Ucs2Asc($PPS->{Name}));
+}
+
+sub _NormalizeWhiteSpace {
+  my ($self, $name) = @_;
+  $name =~ s/\W/ /g;
+  return $name;
+}
+
+sub _GetOLEDate {
+  my ($self, $PPS) = @_;
+  unless (defined ($self->{OLEDATE})) {
+    # Make Date
+    my $datearr;
+    $datearr = $PPS->{Time2nd};
+    $datearr = $PPS->{Time1st} unless($datearr);
+    $self->{OLEDATE} = $self->_FormatDate($datearr) if $datearr;
+  }
+}
+
+sub _FormatDate {
+  my ($self, $datearr) = @_;
+
+  # TODO: This is a little convoluted. Directly using strftime didn't seem
+  # to work.
+  my $datetime = mktime(@$datearr);
+  return time2str("%a, %d %h %Y %X %z", $datetime);
+}
+
+# If we didn't get the date from the original header data, we may be able
+# to get it from the SUBMISSION_ID:
+# It seems to have the format of a semicolon-separated list of key=value
+# pairs. The key l has a value with the format:
+# <SERVER>-<DATETIME>Z-<NUMBER>, where DATETIME is the date and time in
+# the format YYMMDDHHMMSS.
+sub _SubmissionIdDate {
+  my $self = shift;
+
+  my $submission_id = $self->{SUBMISSION_ID} or return undef;
+  $submission_id =~ m/l=.*-(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)Z-.*/
+    or return undef;
+  my $year = $1;
+  $year += 100 if $year < 20;
+  return $self->_FormatDate([$6,$5,$4,$3,$2-1,$year]);
+}
+
+sub _ParseItemName {
+  my ($self, $name) = @_;
+
+  if ($name =~ /^__substg1 0_(....)(....)$/) {
+    my ($property, $encoding) = ($1, $2);
+    if ($encoding eq ENCODING_UNICODE and not ($self->{HAS_UNICODE})) {
+      warn "This MSG file contains Unicode fields." 
+        . " This is currently unsupported.\n";
+      $self->{HAS_UNICODE} = 1;
+    } elsif (not (KNOWN_ENCODINGS()->{$encoding})) {
+      warn "Unknown encoding $encoding. Results may be strange or wrong.\n";
+    }
+    return ($property, $encoding);
+  } else {
+    return (undef, undef);
+  }
+}
+
+sub _SaveAttachment {
+  my ($self, $mime, $att) = @_;
+
+  my $ent = $mime->attach(
+    Type => $att->{MIMETYPE},
+    Encoding => $att->{ENCODING},
+    Data => [],
+    Filename => ($att->{LONGNAME} ? $att->{LONGNAME} : $att->{SHORTNAME}),
+    Disposition => $att->{DISPOSITION}
+  );
+
+  my $handle;
+  if ($handle = $ent->open("w")) {
+    $handle->print($att->{DATA});
+    $handle->close;
+  } else {
+    warn "Could not write data!";
+  }
+}
+
+sub _SetAddressPart {
+  my ($self, $adrname, $partname, $data) = @_;
+
+  my $address = $self->{ADDRESSES}->{$adrname};
+  $data =~ s/\000//g;
+  #warn "Processing address data part $partname : $data\n";
+  if (defined ($address->{$partname})) {
+    if ($address->{$partname} eq $data) {
+      warn "Skipping duplicate but identical address information for"
+      . " $partname\n" if $self->{VERBOSE};
+    } else {
+      warn "Address information $partname inconsistent:\n";
+      warn "    Original data: $address->{$partname}\n";
+      warn "    New data: $data\n";
+    }
+  } else {
+    $address->{$partname} = $data;
+  }
+}
+
+# Set header fields
+sub _AddHeaderField {
+  my ($self, $mime, $fieldname, $value) = @_;
+
+  my $oldvalue = $mime->head->get($fieldname);
+  return if $oldvalue;
+  $mime->head->add($fieldname, $value) if $value;
+}
+
+sub _Address {
+  my ($self, $tag) = @_;
+  my $name = $self->{$tag} || "";
+  my $address = $self->{$tag . "_ADDR"} || "";
+  return "$name <$address>";
+}
+
+# Find SMTP addresses for the given list of names
+sub _ExpandAddressList {
+  my ($self, $names) = @_;
+
+  my $addresspool = $self->{ADDRESSES};
+  my @namelist = split /; */, $names;
+  my @result;
+  name: foreach my $name (@namelist) {
+    foreach my $address (@$addresspool) {
+      if ($name eq $address->{NAME}) {
+        my $addresstext = $address->{NAME} . " <";
+        if (defined ($address->{SMTPADDRESS})) {
+          $addresstext .= $address->{SMTPADDRESS};
+        } elsif ($address->{TYPE} eq "SMTP") {
+          $addresstext .= $address->{ADDRESS};
+        }
+        $addresstext .= ">";
+        push @result, $addresstext;
+        next name;
+      }
+    }
+    push @result, $name;
+  }
+  return join ", ", @result;
+}
+
+sub _ParseHead {
+  my ($self, $data) = @_;
+  defined $data or return undef;
+  # Parse full header date if we got that.
+  my $parser = new MIME::Parser();
+  $parser->output_to_core(1);
+  $parser->decode_headers(1);
+  $data =~ s/^Microsoft Mail.*$/X-MSGConvert: yes/m;
+  my $entity = $parser->parse_data($data)
+    or warn "Couldn't parse full headers!"; 
+  my $head = $entity->head;
+  $head->unfold;
+  return $head;
+}
+
+# Find out if we need to construct a multipart message
+sub _IsMultiPart {
+  my $self = shift;
+
+  return (
+    ($self->{BODY_HTML} and $self->{BODY_PLAIN})
+      or @{$self->{ATTACHMENTS}}>0
+  );
+}
+
+# Copy original header data.
+# Note: This should contain the Date: header.
+sub _CopyHeaderData {
+  my ($self, $mime) = @_;
+
+  my $head = $self->_ParseHead($self->{HEAD}) or return;
+
+  foreach my $tag (grep {!$skipheaders->{$_}} $head->tags) {
+    foreach my $value ($head->get_all($tag)) {
+      $mime->head->add($tag, $value);
+    }
+  }
+}
+
+# Set header fields
+sub _SetHeaderFields {
+  my ($self, $mime) = @_;
+
+  # If we didn't get the date from the original header data, we may be able
+  # to get it from the SUBMISSION_ID:
+  $self->_AddHeaderField($mime, 'Date', $self->_SubmissionIdDate());
+
+  # Third and last chance to set the Date: header; this uses the date the
+  # MSG file was saved.
+  $self->_AddHeaderField($mime, 'Date', $self->{OLEDATE});
+  $self->_AddHeaderField($mime, 'Subject', $self->{SUBJECT});
+  $self->_AddHeaderField($mime, 'From', $self->_Address("FROM"));
+  #$self->_AddHeaderField($mime, 'Reply-To', $self->_Address("REPLYTO"));
+  $self->_AddHeaderField($mime, 'To', $self->_ExpandAddressList($self->{TO}));
+  $self->_AddHeaderField($mime, 'Cc', $self->_ExpandAddressList($self->{CC}));
+  $self->_AddHeaderField($mime, 'Message-Id', $self->{MESSAGEID});
+  $self->_AddHeaderField($mime, 'In-Reply-To', $self->{INREPLYTO});
+}
+
+package main;
+use Getopt::Long;
+use Pod::Usage;
+
+# Setup command line processing.
+my $verbose = '';
+my $help = '';      # Print help message and exit.
+GetOptions('verbose' => \$verbose, 'help|?' => \$help) or pod2usage(2);
+pod2usage(1) if $help;
+
+# Get file name
+my $file = $ARGV[0];
+defined $file or pod2usage(2);
+warn "Will parse file: $file\n" if $verbose; 
+
+# Load and parse MSG file (is OLE)
+my $Msg = OLE::Storage_Lite->new($file);
+my $PPS = $Msg->getPpsTree(1);
+$PPS or die "$file must be an OLE file";
+
+# parse PPS tree
+my $parser = new MSGParser();
+$parser->set_verbosity(1) if $verbose;
+$parser->parse($PPS);
+$parser->print();
+
+#
+# Usage info follows.
+#
+__END__
+
+=head1 NAME
+
+msgconvert.pl - Convert Outlook .msg files to mbox format
+
+=head1 SYNOPSIS
+
+msgconvert.pl [options] <file.msg>
+
+  Options:
+    --verbose   be verbose
+    --help      help message
+
+=head1 OPTIONS
+
+=over 8
+
+=item B<--verbose>
+
+    Print information about skipped parts of the .msg file.
+
+=item B<--help>
+
+    Print a brief help message.
+
+=head1 DESCRIPTION
+
+This program will output the message contained in file.msg in mbox format
+on stdout. It will complain about unrecognized OLE parts on
+stderr.
+
+=head1 BUGS
+
+Not all data that's in the .MSG file is converted. There simply are some
+parts whose meaning escapes me. One of these must contain the date the
+message was sent, for example. Formatting of text messages will also be
+lost. YMMV.
+
+=cut

xapian-applications/omega/omindex.cc

@@ -5 +5 @@
  * Copyright 2001,2002 Ananova Ltd
  * Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2010,2011,2012 Olly Betts
  * Copyright 2009 Frank J Bruzzaniti
+ * Copyright 2006,2007,2008 AVL List GesmbH
  * Copyright 2012 Mihai Bivol
  *
  * This program is free software; you can redistribute it and/or
@@ -70 +71 @@
 
 #include "gnu_getopt.h"
 
+#ifndef LIBEXECDIR
+// must have ending slash
+//# define LIBEXECDIR "/usr/lib/omega/bin/"
+# define LIBEXECDIR ""
+#endif
+#ifndef PKGDATADIR
+// must have ending slash
+# define PKGDATADIR "/usr/share/omega/"
+#endif
+
 using namespace std;
 
 #define TITLE_SIZE 128
@@ -84 +95 @@
 static bool spelling = false;
 static off_t  max_size = 0;
 static bool verbose = false;
+string error_log; /* used in runfilter.cc */
+static string baseurl;
+static string dbpath;
+static string cache_dir;
 static enum {
     EMPTY_BODY_WARN, EMPTY_BODY_INDEX, EMPTY_BODY_SKIP
 } empty_body = EMPTY_BODY_WARN;
@@ -104 +119 @@
 // text are common, so we handle these with a std::map.
 static map<string, string> commands;
 
+static void
+index_directory(const string &path, const string &url_, size_t depth_limit,
+                map<string, string>& mime_map, size_t sample_size);
+
 inline static bool
 p_notalnum(unsigned int c)
 {
@@ -258 +277 @@
 
 void
 index_mimetype(const string & file, const string & url, const string & ext,
-               const string &mimetype, DirectoryIterator &d, size_t sample_size);
+               const string &mimetype, DirectoryIterator &d, size_t sample_size,
+               map<string, string>& mime_map, size_t depth_limit);
+
+static
+void mkdir_p(const string &path, mode_t mode) {
+    (void)mode; // FIXME
+#ifdef __WIN32__
+    string cmd = "mkdir ";
+    append_filename_argument(cmd, path);
+    system(cmd.c_str());
+#else
+    string cmd = "mkdir -p ";
+    append_filename_argument(cmd, path);
+    if (system(cmd.c_str()) < 0) { /* FIXME */ }
+#endif
+}
+
+/*
+ * unpack .msg/.pst/.rar/.zip into local cache dir and recurse there
+ */
+static void
+index_cached_directory(size_t depth_limit,
+                       const string &file,
+                       const string &url,
+                       const string &ext,
+                       const string &cmd,
+                       map<string, string>& mime_map,
+                       size_t sample_size)
+{
+    string oldroot = root;
+    root = cache_dir;
+    string cache = root+"/."+ext;
+    string cachedir = cache+url;
+    struct stat statfile, statcache;
+    bool extract_cache;
+#ifdef HAVE_LSTAT
+    lstat(file.c_str(), &statfile);
+    lstat(cachedir.c_str(), &statcache);
+#else
+    stat(file.c_str(), &statfile);
+    stat(cachedir.c_str(), &statcache);
+#endif
+    extract_cache = true;
+    // if cachedir exists and if file is older as cachedir and if cachedir existed 5 secs ago,
+    // then it was already extracted.
+    if (S_ISDIR(statcache.st_mode) &&
+        S_ISREG(statfile.st_mode) &&
+        (statfile.st_mtime < statcache.st_mtime) &&
+        (statcache.st_mtime < (time_t)(time(NULL)-500))) // not created by nested mkdir call
+    {
+        // but is it in the database also? prevent from deleting skipped files
+        if (verbose)
+            cout << "Unchanged cache \"" << cachedir << "\" - \"" << file << "\" - skip extraction "
+                 // << statfile.st_mtime << " < " << statcache.st_mtime
+                 << endl;
+        extract_cache = false;
+    }
+    if (S_ISDIR(statcache.st_mode) && S_ISREG(statfile.st_mode) ) {
+        // If last_mod > last_mod_max, we know for sure that the file is new
+        // or updated.
+        if (statfile.st_mtime <= last_mod_max) {
+            // check database timestamp for cached container, esp. for cleaned up caches.
+            // if already in db we need not to extract again
+            string urlterm("U");
+            urlterm += baseurl;
+            urlterm += "/."+ext+url;
+            if (urlterm.length() > MAX_SAFE_TERM_LENGTH)
+                urlterm = hash_long_term(urlterm, MAX_SAFE_TERM_LENGTH);
+
+            Xapian::PostingIterator p = db.postlist_begin(urlterm);
+            if (p != db.postlist_end(urlterm)) {
+                Xapian::docid docid = *p;
+                Xapian::Document doc = db.get_document(docid);
+                string value = doc.get_value(VALUE_LASTMOD);
+                time_t old_last_mod = binary_string_to_int(value);
+                if (statfile.st_mtime <= old_last_mod) {
+                    if (verbose)
+                        cout << "Cache "<< "."+ext+url << " not newer. Ignored." << endl;
+                    // The docid should be in updated - the only valid
+                    // exception is if the URL was long and hashed to the
+                    // same URL as an existing document indexed in the same
+                    // batch.
+                    if (usual(docid < updated.size() && !updated[docid])) {
+                        updated[docid] = true;
+                        --old_docs_not_seen;
+                    }
+                    root = oldroot;
+                    return;
+                }
+            }
+        }
+    }
+
+    if (extract_cache) {
+        if (verbose)
+            cout << "[EXTRACT into cache " << cachedir << "]" << endl;
+        if (verbose && S_ISDIR(statcache.st_mode) && S_ISREG(statfile.st_mode))
+            cout << " ...changed cache \"" << cachedir << "\" - \"" << file << "\" "
+                 << statfile.st_mtime << " < " << statcache.st_mtime << " time: " << time(NULL)
+                 << endl;
+        if (!S_ISDIR(statcache.st_mode))
+            mkdir_p(cachedir, 0755);
+        stdout_to_string(cmd);
+#ifndef __WIN32__
+        string chmod_cmd = "chmod -R u+rwx ";
+        append_filename_argument(chmod_cmd, cachedir);
+        stdout_to_string(chmod_cmd);
+#endif
+#ifdef HAVE_LSTAT
+        lstat(cachedir.c_str(), &statcache);
+#else
+        stat(cachedir.c_str(), &statcache);
+#endif
+    }
+
+    if (S_ISDIR(statcache.st_mode)) {
+        if (depth_limit == 1) {
+            cout << "Recursion limit reached for \""<< url << "\" - skipping " << endl;
+        } else {
+            // max loop 5, magic start: /.ext+file
+            index_directory(cachedir + "/", url, depth_limit + 5, mime_map, sample_size);
+            if (verbose)
+                cout << "[CLEANUP '" << cachedir << "']" << endl;
+            rm_rf(cachedir);
+        }
+    } else { // no -p would be fatal here
+        cout << "cachedir '" << cachedir << "' does not exist - skipped" << endl;
+    }
+    root = oldroot;
+}
 
 static void
 index_file(const string &file, const string &url, DirectoryIterator & d,
-           map<string, string>& mime_map, size_t sample_size)
+           map<string, string>& mime_map, size_t sample_size,
+           size_t depth_limit)
 {
     string ext;
     const char * dot_ptr = strrchr(d.leafname(), '.');
@@ -290 +439 @@
 
     string mimetype;
     if (mt == mime_map.end()) {
+        if (strcasecmp(d.leafname(), "mbox") == 0) {
+            // Special filename.
+            mimetype = "message/rfc822";
+            goto got_mimetype;
+        }
+
         mimetype = d.get_magic_mimetype();
         if (mimetype.empty()) {
             skip(file, "Unknown extension and unrecognised format",
@@ -302 +457 @@
         mimetype = mt->second;
     }
 
+got_mimetype:
+
     if (verbose)
         cout << "Indexing \"" << file.substr(root.size()) << "\" as "
              << mimetype << " ... ";
@@ -319 +476 @@
         return;
     }
 
-    index_mimetype(file, url, ext, mimetype, d, sample_size);
+    index_mimetype(file, url, ext, mimetype, d, sample_size, mime_map, depth_limit);
 }
 
 void
 index_mimetype(const string & file, const string & url, const string & ext,
-               const string &mimetype, DirectoryIterator &d, size_t sample_size)
+               const string &mimetype, DirectoryIterator &d, size_t sample_size,
+               map<string, string>& mime_map, size_t depth_limit)
 {
     string urlterm("U");
     urlterm += url;
@@ -373 +531 @@
             }
         }
     }
+    // add the db basename to cache_dir
+    {
+        cache_dir = get_tmpdir();
+        const char *p = strrchr(dbpath.c_str(), '/');
+        // on windows only
+        if (!p) p = strrchr(dbpath.c_str(), '\\');
+        if (p) { p++; } else { p = dbpath.c_str(); }
+        cache_dir += p;
+    }
 
     if (verbose) cout << flush;
 
@@ -437 +604 @@
             } else {
                 // FIXME: What charset is the file?  Look at contents?
             }
+#if 0 // FIXME: this won't work as omindex will have the database locked...
+    } else if (mimetype == "message/rfc822") { // // => mbox2script
+        //for stemmer lang, parse stemmer.get_description => Xapian::Stem(bla)
+        string cmd = LIBEXECDIR"mbox2omega";
+        append_filename_argument(cmd, file);
+        cmd += error_log+"| scriptindex";
+        append_filename_argument(cmd, dbpath);
+        cmd += " "PKGDATADIR"mbox2script.script";
+        try {
+            dump = stdout_to_string(cmd);
+        } catch (ReadError) {
+            cout << "\"" << cmd << "\" failed - skipping" << endl;
+            return;
+        }
+#endif
         } else if (mimetype == "application/pdf") {
             string cmd = "pdftotext -enc UTF-8";
             append_filename_argument(cmd, file);
@@ -702 +884 @@
 
             generate_sample_from_csv(dump, sample, sample_size);
         } else if (mimetype == "application/vnd.ms-outlook") {
-            string cmd = get_pkglibbindir() + "/outlookmsg2html";
+            string oldroot = root;
+            struct stat statcache;
+            char olddir[256];
+
+            if (depth_limit == 1) {
+                skip(file, "Recursion limit reached for compound file", SKIP_SHOW_FILENAME);
+                return;
+            }
+            string cmd = LIBEXECDIR"outlook2text";
             append_filename_argument(cmd, file);
-            MyHtmlParser p;
-            p.ignore_metarobots();
+            // unpack multiparts and attachments. so we have to chdir first
+            string fulldir = cache_dir+"/."+ext+url;
+            if (getcwd(olddir, 256) == NULL) { /* FIXME */ }
+#ifdef HAVE_LSTAT
+            lstat(fulldir.c_str(), &statcache);
+#else
+            stat(fulldir.c_str(), &statcache);
+#endif
+            if (!S_ISDIR(statcache.st_mode)) {
+                mkdir_p(fulldir, 0755);
+            }
             try {
-                dump = stdout_to_string(cmd);
-                // FIXME: what should the default charset be?
-                p.parse_html(dump, "iso-8859-1", false);
-            } catch (const string & newcharset) {
-                p.reset();
-                p.ignore_metarobots();
-                p.parse_html(dump, newcharset, true);
+                if (chdir(fulldir.c_str()) < 0) { /* FIXME */ }
+                size_t new_limit = depth_limit;
+                if (new_limit) --new_limit;
+                index_cached_directory(new_limit, file, url, ext, cmd, mime_map, sample_size);
+                if (chdir(olddir) < 0) { /* FIXME */ }
             } catch (ReadError) {
-                skip_cmd_failed(file, cmd);
-                return;
+                cout << "failed " << cmd << " << in index_cached_directory" << endl;
+                if (chdir(olddir) < 0) { /* FIXME */ }
+                root = oldroot;
+            } catch (...) {
+                cout << "Caught unknown exception in index_cached_directory, rethrowing" << endl;
+                if (chdir(olddir) < 0) { /* FIXME */ }
+                root = oldroot;
+                throw;
             }
-            dump = p.dump;
-            title = p.title;
-            keywords = p.keywords;
-            sample = p.sample;
-            author = p.author;
+            return;
         } else if (mimetype == "image/svg+xml") {
             SvgParser svgparser;
             svgparser.parse_html(d.file_to_string());
@@ -937 +1136 @@
                     cout << "added" << endl;
                 }
             }
+        } else if (mimetype == "application/x-zip") {
+            string oldroot = root;
+            if (depth_limit == 1) {
+                skip(file, "Recursion limit reached for compound file", SKIP_SHOW_FILENAME);
+                return;
+            }
+            // overwrite
+            string cmd = "unzip -u -P. -o";
+            append_filename_argument(cmd, file);
+            cmd += " -d";
+            append_filename_argument(cmd, cache_dir+"/."+ext+url+"/");
+            try {
+                size_t new_limit = depth_limit;
+                if (new_limit) --new_limit;
+                index_cached_directory(new_limit, file, url, ext, cmd, mime_map, sample_size);
+            } catch (ReadError) {
+                cout << "failed " << cmd << " << in index_cached_directory" << endl;
+                root = oldroot;
+            } catch (...) {
+                cout << "Caught unknown exception in index_cached_directory, rethrowing" << endl;
+                root = oldroot;
+                throw;
+            }
+            return;
+        } else if (mimetype == "application/x-rar") {
+            string oldroot = root;
+            if (depth_limit == 1) {
+                skip(file, "Recursion limit reached for compound file", SKIP_SHOW_FILENAME);
+                return;
+            }
+
+            // overwrite
+            string cmd = "unrar x -o+";
+            append_filename_argument(cmd, file);
+            append_filename_argument(cmd, cache_dir+"/."+ext+url+"/");
+            try {
+                size_t new_limit = depth_limit;
+                if (new_limit) --new_limit;
+                index_cached_directory(new_limit, file, url, ext, cmd, mime_map, sample_size);
+            } catch (ReadError) {
+                cout << "failed " << cmd << " << in index_cached_directory" << endl;
+                root = oldroot;
+            } catch (...) {
+                cout << "Caught unknown exception in index_cached_directory, rethrowing" << endl;
+                root = oldroot;
+                throw;
+            }
+            return;
+        } else if (mimetype == "application/vnd.ms-outlook-pst") {
+            string oldroot = root;
+            if (depth_limit == 1) {
+                skip(file, "Recursion limit reached for compound file", SKIP_SHOW_FILENAME);
+                return;
+            }
+            // unpack attachments also, together with mbox files
+            string cmd = "readpst -r -cv -w -o";
+            append_filename_argument(cmd, cache_dir+"/."+ext+url+"/");
+            append_filename_argument(cmd, file);
+            try {
+                size_t new_limit = depth_limit;
+                if (new_limit) --new_limit;
+                index_cached_directory(new_limit, file, url, ext, cmd, mime_map, sample_size);
+            } catch (ReadError) {
+                root = oldroot;
+                cout << "failed " << cmd << " << in index_cached_directory" << endl;
+            } catch (...) {
+                root = oldroot;
+                cout << "Caught unknown exception in index_cached_directory, rethrowing" << endl;
+                throw;
+            }
+            return;
         } else {
             // If this were a duplicate, we'd have skipped it above.
             db.add_document(newdocument);
@@ -984 +1254 @@
                         break;
                     }
                     case DirectoryIterator::REGULAR_FILE:
-                        index_file(file, url, d, mime_map, sample_size);
+                        index_file(file, url, d, mime_map, sample_size, depth_limit);
                         break;
                     default:
                         skip(file, "Not a regular file",
@@ -1037 +1307 @@
     bool overwrite = false;
     // If delete_removed_documents is true, delete any documents we don't see.
     bool delete_removed_documents = true;
-    string baseurl;
     size_t depth_limit = 0;
     size_t sample_size = SAMPLE_SIZE;
 
@@ -1165 +1434 @@
     mime_map["pps"] = "application/vnd.ms-powerpoint"; // Powerpoint slideshow
     mime_map["msg"] = "application/vnd.ms-outlook"; // Outlook .msg email
 
+    //  Outlook message folder:
+    mime_map["pst"] = "application/vnd.ms-outlook-pst"; // readpst | uudeview (libpst)
+
+    // Miscellaneous compound formats:
+    mime_map["mbox"] = "message/rfc822";                // => mbox2omega
+    mime_map["mbx"] = "message/rfc822";                // => mbox2omega
+#ifndef _MSC_VER
+    mime_map["zip"] = "application/x-zip"; // recursive scanning
+    mime_map["rar"] = "application/x-rar"; // recursive scanning
+#endif
+
     // Perl:
     mime_map["pl"] = "text/x-perl";
     mime_map["pm"] = "text/x-perl";
@@ -1238 +1518 @@
         argv[1] = const_cast<char *>("--version");
     }
 
-    string dbpath;
     int getopt_ret;
     while ((getopt_ret = gnu_getopt_long(argc, argv, "hvd:D:U:M:F:l:s:pfSVe:im:E:",
                                          longopts, NULL)) != -1) {
@@ -1413 +1692 @@
         baseurl += '/';
     }
 
+    string log_dir = "./"; // FIXME: need to set log_dir to something appropriate.
+    error_log = " 2>>"+log_dir+"omindex-error.log";
+
     if (optind >= argc || optind + 2 < argc) {
         cerr << PROG_NAME": you must specify a directory to index.\n"
 "Do this either as a single directory (corresponding to the base URL)\n"

new file xapian-applications/omega/outlook2text.in

@@ +1 @@
+#! /bin/sh
+# converts msg to mbox and extract attachments
+# either be in the cache dir, or accept it as 2nd arg
+if [ -n $2 ]; then 
+  @MSGCONVERT@ "$1" | @MIMEEXPLODE@ -d "$2"
+else
+  # already is in the cache dir
+  base=`basename "$1" .msg`
+  @MSGCONVERT@ "$1" | @MIMEEXPLODE@ -d "${base}"
+fi

xapian-applications/omega/runfilter.cc

@@ -55 +55 @@
 
 using namespace std;
 
+extern string error_log;
+
 string
 stdout_to_string(const string &cmd)
 {
     string out;
+    string tmp = cmd;
+    tmp += error_log;
 #if defined HAVE_FORK && defined HAVE_SOCKETPAIR && defined HAVE_SETRLIMIT
     // We want to be able to get the exit status of the child process.
     signal(SIGCHLD, SIG_DFL);
@@ -104 +108 @@
         }
 #endif
 
-        execl("/bin/sh", "/bin/sh", "-c", cmd.c_str(), (void*)NULL);
+        execl("/bin/sh", "/bin/sh", "-c", tmp.c_str(), (void*)NULL);
         _exit(-1);
     }
 
@@ -171 +175 @@
         throw ReadError();
     }
 #else
-    FILE * fh = popen(cmd.c_str(), "r");
+    FILE * fh = popen(tmp.c_str(), "r");
     if (fh == NULL) throw ReadError();
     while (!feof(fh)) {
         char buf[4096];

xapian-applications/omega/utils.cc

@@ -23 +23 @@
 
 #include "utils.h"
 
+#include "append_filename_arg.h"
+
 #include <cassert>
 #include <stdio.h> // for sprintf/snprintf
 #include <cstdlib>
+#include <cstring>
+#include "safesysstat.h"
 
 #include <string>
 
 using namespace std;
 
+#ifdef __WIN32__
+#include "safewindows.h"
+#endif
+
+/// Remove a directory and contents.
+void
+rm_rf(const string &filename)
+{
+    // Check filename exists and is actually a directory
+    struct stat sb;
+    if (stat(filename.c_str(), &sb) != 0 || !S_ISDIR(sb.st_mode)) return;
+
+#ifdef __WIN32__
+    static int win95 = -1;
+    if (win95 == -1) {
+        OSVERSIONINFO info;
+        memset(&info, 0, sizeof(OSVERSIONINFO));
+        info.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
+        if (GetVersionEx(&info)) {
+            win95 = (info.dwPlatformId == VER_PLATFORM_WIN32_WINDOWS);
+        }
+    }
+
+    string cmd = win95 ?
+        "deltree /y" : // for 95-like systems.
+        "rd /s /q"; // for NT-like systems.
+#else
+    string cmd = "rm -rf";
+#endif
+    append_filename_argument(cmd, filename);
+    if (system(cmd.c_str())) { /* FIXME */ }
+}
+
 // This ought to be enough for any of the conversions below.
 #define BUFSIZE 100
 

xapian-applications/omega/utils.h

@@ -34 +34 @@
 /** Remove any leading and/or trailing whitespace from @a s. */
 void trim(std::string & s);
 
+void rm_rf(const std::string &filename);
+
 #endif