Ticket #282: xapian-omega-trunk-r16058-from-ticket-285-and-cleaned-up-updated-2011-12-06.patch

xapian-applications/omega/ChangeLog

@@ +1 @@
+2006-08-22 09:30:12 Reini Urban <reinhard.urban@avl.com>
+
+        omega-0.9.6c:
+        * omindex.cc: Fix wrong timestamp comparison in cache logic
+        * outlook2text.in: New script
+
+2006-08-17 18:06:26 Reini Urban <reinhard.urban@avl.com>
+
+        omega-0.9.6a: 
+        * omindex.cc: Added cached virtual directories zip,msg,pst,...).
+        Consistently log stderr to /var/log/omega/omindex-error.log.
+
 Sat Oct 29 14:49:40 GMT 2011  Olly Betts <olly@survex.com>
 
         * docs/omegascript.rst: Add note to discourage use of percentage

xapian-applications/omega/Makefile.am

@@ -78 +78 @@
 pkglibbin_PROGRAMS = omega
 dist_pkglibbin_SCRIPTS = outlookmsg2html
 bin_PROGRAMS = omindex scriptindex
+dist_libexec_SCRIPTS = outlook2text mimeexplode msgconvert.pl
 dist_bin_SCRIPTS = dbi2omega htdig2omega mbox2omega
 
 check_PROGRAMS = htmlparsetest md5test utf8converttest
@@ -160 +161 @@
 MAINTAINERCLEANFILES = $(dist_man_MANS)
 endif
 
+CLEANFILES = outlook2text
+
+outlook2text: $(srcdir)/outlook2text.in Makefile
+        sed "s,@MSGCONVERT@,$(MSGCONVERT),;s,@MIMEEXPLODE@,$(pkglibbindir)/mimeexplode," $(srcdir)/outlook2text.in > $@
+
 if DOCUMENTATION_RULES
 omindex.1: omindex$(EXEEXT) makemanpage
         ./makemanpage ./omindex $(srcdir)/omindex.cc omindex.1

new file xapian-applications/omega/mimeexplode

@@ +1 @@
+#!/usr/bin/perl -w
+
+=head1 NAME
+
+mimeexplode - explode one or more MIME messages
+
+=head1 SYNOPSIS
+
+    mimeexplode [-d <dir>] <mime-msg-file> <mime-msg-file> ...
+
+    someprocess | mimeexplode -
+
+=head1 DESCRIPTION
+
+Takes one or more files from the command line that contain MIME
+messages, and explodes their contents out into subdirectories
+of the current working directory.  The subdirectories are
+just called C<msg0>, C<msg1>, C<msg2>, etc.  Existing directories are
+skipped over.
+
+The message information is output to the stdout, like this:
+
+    Message: msg3 (inputfile1.msg)
+        Part: msg3/filename-1.dat (text/plain)
+        Part: msg3/filename-2.dat (text/plain)
+    Message: msg5 (input-file2.msg)
+        Part: msg5/dir.gif (image/gif)
+        Part: msg5/face.jpg (image/jpeg)
+    Message: msg6 (infile3)
+        Part: msg6/filename-1.dat (text/plain)
+
+This was written as an example of the MIME:: modules in the
+MIME-parser package I wrote.  It may prove useful as a quick-and-dirty
+way of splitting a MIME message if you need to decode something, and
+you don't have a MIME mail reader on hand.
+
+=head1 COMMAND LINE OPTIONS
+
+-d outdir
+
+=head1 AUTHOR
+
+Eryq C<eryq@zeegee.com>, in a big hurry...
+Reini Urban C<rurban@x-ray.at>: -d option to always explode into the same dir
+
+=cut
+
+#BEGIN { unshift @INC, ".." }    # to test MIME:: stuff before installing it!
+
+require 5.001;
+
+use strict;
+use vars;
+
+use MIME::Parser;
+use Getopt::Std;
+my %opts;
+my $outbase = '';
+my $postfix = '';
+
+#------------------------------------------------------------
+# make_msg - make and return the name of a msgXXX directory
+#------------------------------------------------------------
+
+#ignored
+#sub make_msg {
+#    while (-d "msg$Msgno") {
+#       ++$Msgno;
+#       die "self-imposed limit reached" if $Msgno == 256;
+#    }
+#   mkdir "msg$Msgno",0755 or die "couldn't make msg$Msgno: $!";
+#    "msg$Msgno";
+#}
+
+#------------------------------------------------------------
+# dump_entity - dump an entity's file info
+#------------------------------------------------------------
+sub dump_entity {
+    my $ent = shift;
+    my @parts = $ent->parts;
+
+    if (@parts) {        # multipart...
+        map { dump_entity($_) } @parts;
+    }
+    else {               # single part...
+        print "    Part: ", $ent->bodyhandle->path,
+              " (", scalar($ent->head->mime_type), ")\n";
+    }
+}
+
+#------------------------------------------------------------
+# main
+#------------------------------------------------------------
+sub main {
+    my $file;
+    my $entity;
+
+    # make sure the same message gets exploded into the same dir
+    getopts('d:', \%opts);
+    $outbase = $opts{d} ? $opts{d} : "msg0";
+    my $outdir = $outbase;
+
+    # Go through messages:
+    @ARGV or unshift @ARGV, "-";
+    while (defined($file = shift @ARGV)) {
+
+      # Sanity:
+      (-d $outdir) or mkdir "$outdir",0755;
+      (-w "$outdir") or die "cwd $outdir not writable!";
+      #my $msgdir = make_msg();
+      #print "Message: $msgdir ($file)\n";
+
+      # Create a new parser object:
+      my $parser = new MIME::Parser;
+      ### $parser->parse_nested_messages('REPLACE');
+
+      # Optional: set up parameters that will affect how it extracts
+      #   documents from the input stream:
+      $parser->output_dir($outdir);
+
+      # Parse an input stream:
+      open FILE, $file or die "couldn't open $file";
+      $entity = $parser->read(\*FILE) or
+        print STDERR "Couldn't parse MIME in $file; continuing...\n";
+      close FILE;
+
+      # Congratulations: you now have a (possibly multipart) MIME entity!
+      dump_entity($entity) if $entity;
+      ### $entity->dump_skeleton if $entity;
+
+      $postfix++;
+      $outdir = $outbase.$postfix;
+    }
+    1;
+}
+
+exit (&main ? 0 : -1);
+#------------------------------------------------------------
+1;
+

new file xapian-applications/omega/msgconvert.pl

@@ +1 @@
+#!/usr/bin/perl -w
+#
+# msgconvert.pl:
+#
+# Convert .MSG files (made by Outlook (Express)) to multipart MIME messages.
+#
+# Copyright 2002, 2004, 2006 Matijs van Zuijlen
+#
+# This program is free software; you can redistribute it and/or modify it
+# under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+# Public License for more details.
+#
+# CHANGES:
+# 20020715  Recognize new items 'Cc', mime type of attachment, long
+#           filename of attachment, and full headers. Attachments turn out
+#           to be numbered, so a regexp is now used to recognize label of
+#           items that are attachments.
+# 20020831  long file name will definitely be used if present. Full headers
+#           and mime type information are used when present. Created
+#           generic system for specifying known items to be skipped.
+#           Unexpected contents is never reason to bail out anymore. Added
+#           support for usage message and option processing (--verbose).
+# 20040104  Handle address data slightly better, make From line less fake,
+#           make $verbose and $skippable_entries global vars, handle HTML
+#           variant of body text if present (though not optimally).
+# 20040214  Fix typos and incorrect comments.
+# 20040307  - Complete rewrite: All functional parts are now in the package
+#             MSGParser;
+#           - Creation of MIME::Entity object is delayed until the output
+#             routines, which means all data is known; This means I can
+#             create a multipart/alternative body.
+#           - Item names are parsed (thanks to bfrederi@alumni.sfu.ca for
+#             the information).
+# 20040514  Check if $self->{HEAD} actually exists before trying to add its
+#           contents to the output Mime object's header data.
+#           (Bug reported by Thomas Ng).
+#           Don't produce multipart messages if not needed.
+#           (Bug reported by Justin B. Scout).
+# 20040529  Correctly format OLEDATE.
+# 20040530  - Extract date from property 0047 (thanks, Marc Goodman).
+#           - Use address data to make To: and Cc: lines complete
+#           - Use the in-reply-to property
+#           - More unknown properties named.
+#           - Found another property containing an SMTP address.
+#           - Put non-SMTP type addresses back in output.
+# 20040825  Replace 'our' to declare globals with 'use vars'. This means
+#           the globals our now properly scoped inside the package and not
+#           the file.
+#           This also fixes the bug that this program did not work on perl
+#           versions below 5.6. (Bug reported by Tim Gustafson)
+# 20060218  More sensible encoding warnings.
+# 20060219  Move OLE parsing to main program.
+#           Parse nested MSG files (Bug reported by Christof Lukas).
+# 20060225  Simplify code.
+#
+
+#
+# Import modules.
+#
+package MSGParser;
+use strict;
+use OLE::Storage_Lite;
+use MIME::Entity;
+use MIME::Parser;
+use Date::Format;
+use POSIX qw(mktime);
+use constant DIR_TYPE => 1;
+use constant FILE_TYPE => 2;
+
+use vars qw($skipproperties $skipheaders);
+#
+# Descriptions partially based on mapitags.h
+#
+$skipproperties = {
+  # Envelope properties
+  '000B' => "Conversation key?",
+  '001A' => "Type of message",
+  '003B' => "Sender address variant",
+  '003D' => "Contains 'Re: '",
+  '003F' => "'recieved by' id",
+  '0040' => "'recieved by' name",
+  '0041' => "Sender variant address id",
+  '0042' => "Sender variant name",
+  '0043' => "'recieved representing' id",
+  '0044' => "'recieved representing' name",
+  '0046' => "Read receipt address id",
+  '0051' => "'recieved by' search key",
+  '0052' => "'recieved representing' search key",
+  '0053' => "Read receipt search key",
+  '0064' => "Sender variant address type",
+  '0065' => "Sender variant address",
+  '0070' => "Conversation topic",
+  '0071' => "Conversation index",
+  '0075' => "'recieved by' address type",
+  '0076' => "'recieved by' email address",
+  '0077' => "'recieved representing' address type",
+  '0078' => "'recieved representing' email address",
+  '007F' => "something like a message id",
+  # Recipient properties
+  '0C19' => "Reply address variant",
+  '0C1D' => "Reply address variant",
+  '0C1E' => "Reply address type",
+  # Non-transmittable properties
+  '0E02' => "?Should BCC be displayed",
+  '0E0A' => "sent mail id",
+  '0E1D' => "Subject w/o Re",
+  '0E27' => "64 bytes: Unknown",
+  '0FF6' => "Index",
+  '0FF9' => "Index",
+  '0FFF' => "Address variant",
+  # Content properties
+  '1008' => "Summary or something",
+  '1009' => "RTF Compressed",
+  # 'Common property'
+  '3001' => "Display name",
+  '3002' => "Address Type",
+  '300B' => "'Search key'",
+  # Attachment properties
+  '3702' => "Attachment encoding",
+  '3703' => "Attachment extension",
+  '3709' => "'Attachment rendering'", # Maybe an icon or something?
+  '3713' => "Icon URL?",
+  # 'Mail user'
+  '3A20' => "Address variant",
+  # 3900 -- 39FF: 'Address book'
+  '39FF' => "7 bit display name",
+  # 'Display table properties'
+  '3FF8' => "Routing data?",
+  '3FF9' => "Routing data?",
+  '3FFA' => "Routing data?",
+  '3FFB' => "Routing data?",
+  # 'Transport-defined envelope property'
+  '4029' => "Sender variant address type",
+  '402A' => "Sender variant address",
+  '402B' => "Sender variant name",
+  '5FF6' => "Recipient name",
+  '5FF7' => "Recipient address variant",
+  # 'Provider-defined internal non-transmittable property'
+  '6740' => "Unknown, binary data",
+  # User defined id's
+  '8000' => "Content Class",
+  '8002' => "Unknown, binary data",
+};
+
+$skipheaders = {
+  "MIME-Version" => 1,
+  "Content-Type" => 1,
+  "Content-Transfer-Encoding" => 1,
+  "X-Mailer" => 1,
+  "X-Msgconvert" => 1,
+  "X-MS-Tnef-Correlator" => 1,
+  "X-MS-Has-Attach" => 1,
+};
+
+use constant ENCODING_UNICODE => '001F';
+use constant KNOWN_ENCODINGS => {
+    '000D' => 'Directory',
+    '001F' => 'Unicode',
+    '001E' => 'Ascii?',
+    '0102' => 'Binary',
+};
+
+use constant MAP_ATTACHMENT_FILE => {
+  '3701' => ["DATA",        0], # Data
+  '3704' => ["SHORTNAME",   1], # Short file name
+  '3707' => ["LONGNAME",    1], # Long file name
+  '370E' => ["MIMETYPE",    1], # mime type
+  '3716' => ["DISPOSITION", 1], # disposition
+};
+
+use constant MAP_SUBITEM_FILE => {
+  '1000' => ["BODY_PLAIN",      0], # Body
+  '1013' => ["BODY_HTML",       0], # HTML Version of body
+  '0037' => ["SUBJECT",         1], # Subject
+  '0047' => ["SUBMISSION_ID",   1], # Seems to contain the date
+  '007D' => ["HEAD",            1], # Full headers
+  '0C1A' => ["FROM",            1], # Reply-To: Name
+  '0C1E' => ["FROM_ADDR_TYPE",  1], # From: Address type
+  '0C1F' => ["FROM_ADDR",       1], # Reply-To: Address
+  '0E04' => ["TO",              1], # To: Names
+  '0E03' => ["CC",              1], # Cc: Names
+  '1035' => ["MESSAGEID",       1], # Message-Id
+  '1042' => ["INREPLYTO",       1], # In reply to Message-Id
+};
+
+use constant MAP_ADDRESSITEM_FILE => {
+  '3001' => ["NAME",            1], # Real name
+  '3002' => ["TYPE",            1], # Address type
+  '403D' => ["TYPE",            1], # Address type
+  '3003' => ["ADDRESS",         1], # Address
+  '403E' => ["ADDRESS",         1], # Address
+  '39FE' => ["SMTPADDRESS",     1], # SMTP Address variant
+};
+
+#
+# Main body of module
+#
+
+sub new {
+  my $that = shift;
+  my $class = ref $that || $that;
+
+  my $self = {
+    ATTACHMENTS => [],
+    ADDRESSES => [],
+    VERBOSE => 0,
+    HAS_UNICODE => 0,
+    FROM_ADDR_TYPE => "",
+  };
+  bless $self, $class;
+}
+
+#
+# Main sub: parse the PPS tree, and return 
+#
+sub parse {
+  my $self = shift;
+  my $PPS = shift or die "Internal error: No PPS tree";
+  $self->_RootDir($PPS);
+}
+
+sub mime_object {
+  my $self = shift;
+
+  my $bodymime;
+  my $mime;
+
+  if ($self->_IsMultiPart) {
+    # Construct a multipart message object
+
+    $mime = MIME::Entity->build(Type => "multipart/mixed");
+
+    # Set the entity that we'll save the body parts to. If there's more than
+    # one part, it's a new entity, otherwise, it's the main $mime object.
+    if ($self->{BODY_HTML} and $self->{BODY_PLAIN}) {
+      $bodymime = MIME::Entity->build(
+        Type => "multipart/alternative",
+        Encoding => "8bit",
+      );
+      $mime->add_part($bodymime);
+    } else {
+      $bodymime = $mime;
+    }
+    if ($self->{BODY_PLAIN}) {
+      $self->_SaveAttachment($bodymime, {
+        MIMETYPE => 'text/plain; charset=ISO-8859-1',
+        ENCODING => '8bit',
+        DATA => $self->{BODY_PLAIN},
+        DISPOSITION => 'inline',
+      });
+    }
+    if ($self->{BODY_HTML}) {
+      $self->_SaveAttachment($bodymime, {
+        MIMETYPE => 'text/html',
+        ENCODING => '8bit',
+        DATA => $self->{BODY_HTML},
+        DISPOSITION => 'inline',
+      });
+    }
+    foreach my $att (@{$self->{ATTACHMENTS}}) {
+      $self->_SaveAttachment($mime, $att);
+    }
+  } elsif ($self->{BODY_PLAIN}) {
+    # Construct a single part message object with a plain text body
+    $mime = MIME::Entity->build(
+      Type => "text/plain",
+      Data => $self->{BODY_PLAIN}
+    );
+  } elsif ($self->{BODY_HTML}) {
+    # Construct a single part message object with an HTML body
+    $mime = MIME::Entity->build(
+      Type => "text/html",
+      Data => $self->{BODY_HTML}
+    );
+  }
+
+  $self->_CopyHeaderData($mime);
+
+  $self->_SetHeaderFields($mime);
+
+  return $mime;
+}
+
+# Actually output the message in mbox format
+sub print {
+  my $self = shift;
+
+  my $mime = $self->mime_object;
+
+  # Construct From line from whatever we know.
+  my $string = "";
+  $string = (
+    $self->{FROM_ADDR_TYPE} eq "SMTP" ?
+    $self->{FROM_ADDR} :
+    'someone@somewhere'
+  );
+  $string =~ s/\n//g;
+
+  # The date used here is not really important.
+  print "From ", $string, " ", scalar localtime, "\n";
+  $mime->print(\*STDOUT);
+  print "\n";
+}
+
+sub set_verbosity {
+  my ($self, $verbosity) = @_;
+  defined $verbosity or die "Internal error: no verbosity level";
+  $self->{VERBOSE} = $verbosity;
+}
+
+#
+# Below are functions that walk the PPS tree. The *Dir functions handle
+# processing the directory nodes of the tree (mainly, iterating over the
+# children), whereas the *Item functions handle processing the items in the
+# directory (if such an item is itself a directory, it will in turn be
+# processed by the relevant *Dir function).
+#
+
+#
+# RootItem: Check Root Entry, parse sub-entries.
+# The OLE file consists of a single entry called Root Entry, which has
+# several children. These children are parsed in the sub SubItem.
+# 
+sub _RootDir {
+  my ($self, $PPS) = @_;
+
+  foreach my $child (@{$PPS->{Child}}) {
+    $self->_SubItem($child);
+  }
+}
+
+sub _SubItem {
+  my ($self, $PPS) = @_;
+  
+  if ($PPS->{Type} == DIR_TYPE) {
+    $self->_SubItemDir($PPS);
+  } elsif ($PPS->{Type} == FILE_TYPE) {
+    $self->_SubItemFile($PPS);
+  } else {
+    warn "Unknown entry type: $PPS->{Type}";
+  }
+}
+
+sub _SubItemDir {
+  my ($self, $PPS) = @_;
+
+  $self->_GetOLEDate($PPS);
+
+  my $name = $self->_GetName($PPS);
+
+  if ($name =~ /__recip_version1 0_ /) { # Address of one recipient
+    $self->_AddressDir($PPS);
+  } elsif ($name =~ '__attach_version1 0_ ') { # Attachment
+    $self->_AttachmentDir($PPS);
+  } else {
+    $self->_UnknownDir($self->_GetName($PPS));
+  }
+}
+
+sub _SubItemFile {
+  my ($self, $PPS) = @_;
+
+  my $name = $self->_GetName($PPS);
+  my ($property, $encoding) = $self->_ParseItemName($name);
+
+  $self->_MapProperty($self, $PPS->{Data}, $property,
+    MAP_SUBITEM_FILE) or $self->_UnknownFile($name);
+}
+
+sub _AddressDir {
+  my ($self, $PPS) = @_;
+
+  my $address = {
+    NAME        => undef,
+    ADDRESS     => undef,
+    TYPE        => "",
+  };
+  foreach my $child (@{$PPS->{Child}}) {
+    $self->_AddressItem($child, $address);
+  }
+  push @{$self->{ADDRESSES}}, $address;
+}
+
+sub _AddressItem {
+  my ($self, $PPS, $addr_info) = @_;
+
+  my $name = $self->_GetName($PPS);
+
+  # DIR Entries: There should be none.
+  if ($PPS->{Type} == DIR_TYPE) {
+    $self->_UnknownDir($name);
+  } elsif ($PPS->{Type} == FILE_TYPE) {
+    my ($property, $encoding) = $self->_ParseItemName($name);
+    $self->_MapProperty($addr_info, $PPS->{Data}, $property,
+      MAP_ADDRESSITEM_FILE) or $self->_UnknownFile($name);
+  } else {
+    warn "Unknown entry type: $PPS->{Type}";
+  }
+}
+
+sub _AttachmentDir {
+  my ($self, $PPS) = @_;
+
+  my $attachment = {
+    SHORTNAME   => undef,
+    LONGNAME    => undef,
+    MIMETYPE    => 'application/octet-stream',
+    ENCODING    => 'base64',
+    DISPOSITION => 'attachment',
+    DATA        => undef
+  };
+  foreach my $child (@{$PPS->{Child}}) {
+    $self->_AttachmentItem($child, $attachment);
+  }
+  push @{$self->{ATTACHMENTS}}, $attachment;
+}
+
+sub _AttachmentItem {
+  my ($self, $PPS, $att_info) = @_;
+
+  my $name = $self->_GetName($PPS);
+
+  my ($property, $encoding) = $self->_ParseItemName($name);
+
+  if ($PPS->{Type} == DIR_TYPE) {
+
+    if ($property eq '3701') {  # Nested MSG file
+      my $msgp = new MSGParser();
+      $msgp->parse($PPS);
+      my $data = $msgp->mime_object->as_string;
+      $att_info->{DATA} = $data;
+      $att_info->{MIMETYPE} = 'message/rfc822';
+      $att_info->{ENCODING} = '8bit';
+    } else {
+      $self->_UnknownDir($name);
+    }
+
+  } elsif ($PPS->{Type} == FILE_TYPE) {
+    $self->_MapProperty($att_info, $PPS->{Data}, $property,
+      MAP_ATTACHMENT_FILE) or $self->_UnknownFile($name);
+  } else {
+    warn "Unknown entry type: $PPS->{Type}";
+  }
+}
+
+sub _MapProperty {
+  my ($self, $hash, $data, $property, $map) = @_;
+
+  defined $property or return 0;
+  my $arr = $map->{$property} or return 0;
+
+  $arr->[1] and $data =~ s/\000//g;
+  $hash->{$arr->[0]} = $data;
+
+  return 1;
+}
+
+sub _UnknownDir {
+  my ($self, $name) = @_;
+
+  if ($name eq '__nameid_version1 0') {
+    $self->{VERBOSE}
+      and warn "Skipping DIR entry $name (Introductory stuff)\n";
+    return;
+  }
+  warn "Unknown DIR entry $name\n";
+}
+
+sub _UnknownFile {
+  my ($self, $name) = @_;
+
+  if ($name eq '__properties_version1 0') {
+    $self->{VERBOSE}
+      and warn "Skipping FILE entry $name (Properties)\n";
+    return;
+  }
+
+  my ($property, $encoding) = $self->_ParseItemName($name);
+  unless (defined $property) {
+    warn "Unknown FILE entry $name\n";
+    return;
+  }
+  if ($skipproperties->{$property}) {
+    $self->{VERBOSE}
+      and warn "Skipping property $property ($skipproperties->{$property})\n";
+    return;
+  } elsif ($property =~ /^80/) {
+    $self->{VERBOSE}
+      and warn "Skipping property $property (user-defined property)\n";
+    return;
+  } else {
+    warn "Unknown property $property\n";
+    return;
+  }
+}
+
+#
+# Helper functions
+#
+
+sub _GetName {
+  my ($self, $PPS) = @_;
+  return $self->_NormalizeWhiteSpace(OLE::Storage_Lite::Ucs2Asc($PPS->{Name}));
+}
+
+sub _NormalizeWhiteSpace {
+  my ($self, $name) = @_;
+  $name =~ s/\W/ /g;
+  return $name;
+}
+
+sub _GetOLEDate {
+  my ($self, $PPS) = @_;
+  unless (defined ($self->{OLEDATE})) {
+    # Make Date
+    my $datearr;
+    $datearr = $PPS->{Time2nd};
+    $datearr = $PPS->{Time1st} unless($datearr);
+    $self->{OLEDATE} = $self->_FormatDate($datearr) if $datearr;
+  }
+}
+
+sub _FormatDate {
+  my ($self, $datearr) = @_;
+
+  # TODO: This is a little convoluted. Directly using strftime didn't seem
+  # to work.
+  my $datetime = mktime(@$datearr);
+  return time2str("%a, %d %h %Y %X %z", $datetime);
+}
+
+# If we didn't get the date from the original header data, we may be able
+# to get it from the SUBMISSION_ID:
+# It seems to have the format of a semicolon-separated list of key=value
+# pairs. The key l has a value with the format:
+# <SERVER>-<DATETIME>Z-<NUMBER>, where DATETIME is the date and time in
+# the format YYMMDDHHMMSS.
+sub _SubmissionIdDate {
+  my $self = shift;
+
+  my $submission_id = $self->{SUBMISSION_ID} or return undef;
+  $submission_id =~ m/l=.*-(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)(\d\d)Z-.*/
+    or return undef;
+  my $year = $1;
+  $year += 100 if $year < 20;
+  return $self->_FormatDate([$6,$5,$4,$3,$2-1,$year]);
+}
+
+sub _ParseItemName {
+  my ($self, $name) = @_;
+
+  if ($name =~ /^__substg1 0_(....)(....)$/) {
+    my ($property, $encoding) = ($1, $2);
+    if ($encoding eq ENCODING_UNICODE and not ($self->{HAS_UNICODE})) {
+      warn "This MSG file contains Unicode fields." 
+        . " This is currently unsupported.\n";
+      $self->{HAS_UNICODE} = 1;
+    } elsif (not (KNOWN_ENCODINGS()->{$encoding})) {
+      warn "Unknown encoding $encoding. Results may be strange or wrong.\n";
+    }
+    return ($property, $encoding);
+  } else {
+    return (undef, undef);
+  }
+}
+
+sub _SaveAttachment {
+  my ($self, $mime, $att) = @_;
+
+  my $ent = $mime->attach(
+    Type => $att->{MIMETYPE},
+    Encoding => $att->{ENCODING},
+    Data => [],
+    Filename => ($att->{LONGNAME} ? $att->{LONGNAME} : $att->{SHORTNAME}),
+    Disposition => $att->{DISPOSITION}
+  );
+
+  my $handle;
+  if ($handle = $ent->open("w")) {
+    $handle->print($att->{DATA});
+    $handle->close;
+  } else {
+    warn "Could not write data!";
+  }
+}
+
+sub _SetAddressPart {
+  my ($self, $adrname, $partname, $data) = @_;
+
+  my $address = $self->{ADDRESSES}->{$adrname};
+  $data =~ s/\000//g;
+  #warn "Processing address data part $partname : $data\n";
+  if (defined ($address->{$partname})) {
+    if ($address->{$partname} eq $data) {
+      warn "Skipping duplicate but identical address information for"
+      . " $partname\n" if $self->{VERBOSE};
+    } else {
+      warn "Address information $partname inconsistent:\n";
+      warn "    Original data: $address->{$partname}\n";
+      warn "    New data: $data\n";
+    }
+  } else {
+    $address->{$partname} = $data;
+  }
+}
+
+# Set header fields
+sub _AddHeaderField {
+  my ($self, $mime, $fieldname, $value) = @_;
+
+  my $oldvalue = $mime->head->get($fieldname);
+  return if $oldvalue;
+  $mime->head->add($fieldname, $value) if $value;
+}
+
+sub _Address {
+  my ($self, $tag) = @_;
+  my $name = $self->{$tag} || "";
+  my $address = $self->{$tag . "_ADDR"} || "";
+  return "$name <$address>";
+}
+
+# Find SMTP addresses for the given list of names
+sub _ExpandAddressList {
+  my ($self, $names) = @_;
+
+  my $addresspool = $self->{ADDRESSES};
+  my @namelist = split /; */, $names;
+  my @result;
+  name: foreach my $name (@namelist) {
+    foreach my $address (@$addresspool) {
+      if ($name eq $address->{NAME}) {
+        my $addresstext = $address->{NAME} . " <";
+        if (defined ($address->{SMTPADDRESS})) {
+          $addresstext .= $address->{SMTPADDRESS};
+        } elsif ($address->{TYPE} eq "SMTP") {
+          $addresstext .= $address->{ADDRESS};
+        }
+        $addresstext .= ">";
+        push @result, $addresstext;
+        next name;
+      }
+    }
+    push @result, $name;
+  }
+  return join ", ", @result;
+}
+
+sub _ParseHead {
+  my ($self, $data) = @_;
+  defined $data or return undef;
+  # Parse full header date if we got that.
+  my $parser = new MIME::Parser();
+  $parser->output_to_core(1);
+  $parser->decode_headers(1);
+  $data =~ s/^Microsoft Mail.*$/X-MSGConvert: yes/m;
+  my $entity = $parser->parse_data($data)
+    or warn "Couldn't parse full headers!"; 
+  my $head = $entity->head;
+  $head->unfold;
+  return $head;
+}
+
+# Find out if we need to construct a multipart message
+sub _IsMultiPart {
+  my $self = shift;
+
+  return (
+    ($self->{BODY_HTML} and $self->{BODY_PLAIN})
+      or @{$self->{ATTACHMENTS}}>0
+  );
+}
+
+# Copy original header data.
+# Note: This should contain the Date: header.
+sub _CopyHeaderData {
+  my ($self, $mime) = @_;
+
+  my $head = $self->_ParseHead($self->{HEAD}) or return;
+
+  foreach my $tag (grep {!$skipheaders->{$_}} $head->tags) {
+    foreach my $value ($head->get_all($tag)) {
+      $mime->head->add($tag, $value);
+    }
+  }
+}
+
+# Set header fields
+sub _SetHeaderFields {
+  my ($self, $mime) = @_;
+
+  # If we didn't get the date from the original header data, we may be able
+  # to get it from the SUBMISSION_ID:
+  $self->_AddHeaderField($mime, 'Date', $self->_SubmissionIdDate());
+
+  # Third and last chance to set the Date: header; this uses the date the
+  # MSG file was saved.
+  $self->_AddHeaderField($mime, 'Date', $self->{OLEDATE});
+  $self->_AddHeaderField($mime, 'Subject', $self->{SUBJECT});
+  $self->_AddHeaderField($mime, 'From', $self->_Address("FROM"));
+  #$self->_AddHeaderField($mime, 'Reply-To', $self->_Address("REPLYTO"));
+  $self->_AddHeaderField($mime, 'To', $self->_ExpandAddressList($self->{TO}));
+  $self->_AddHeaderField($mime, 'Cc', $self->_ExpandAddressList($self->{CC}));
+  $self->_AddHeaderField($mime, 'Message-Id', $self->{MESSAGEID});
+  $self->_AddHeaderField($mime, 'In-Reply-To', $self->{INREPLYTO});
+}
+
+package main;
+use Getopt::Long;
+use Pod::Usage;
+
+# Setup command line processing.
+my $verbose = '';
+my $help = '';      # Print help message and exit.
+GetOptions('verbose' => \$verbose, 'help|?' => \$help) or pod2usage(2);
+pod2usage(1) if $help;
+
+# Get file name
+my $file = $ARGV[0];
+defined $file or pod2usage(2);
+warn "Will parse file: $file\n" if $verbose; 
+
+# Load and parse MSG file (is OLE)
+my $Msg = OLE::Storage_Lite->new($file);
+my $PPS = $Msg->getPpsTree(1);
+$PPS or die "$file must be an OLE file";
+
+# parse PPS tree
+my $parser = new MSGParser();
+$parser->set_verbosity(1) if $verbose;
+$parser->parse($PPS);
+$parser->print();
+
+#
+# Usage info follows.
+#
+__END__
+
+=head1 NAME
+
+msgconvert.pl - Convert Outlook .msg files to mbox format
+
+=head1 SYNOPSIS
+
+msgconvert.pl [options] <file.msg>
+
+  Options:
+    --verbose   be verbose
+    --help      help message
+
+=head1 OPTIONS
+
+=over 8
+
+=item B<--verbose>
+
+    Print information about skipped parts of the .msg file.
+
+=item B<--help>
+
+    Print a brief help message.
+
+=head1 DESCRIPTION
+
+This program will output the message contained in file.msg in mbox format
+on stdout. It will complain about unrecognized OLE parts on
+stderr.
+
+=head1 BUGS
+
+Not all data that's in the .MSG file is converted. There simply are some
+parts whose meaning escapes me. One of these must contain the date the
+message was sent, for example. Formatting of text messages will also be
+lost. YMMV.
+
+=cut

xapian-applications/omega/omindex.cc

@@ -5 +5 @@
  * Copyright 2001,2002 Ananova Ltd
  * Copyright 2002,2003,2004,2005,2006,2007,2008,2009,2010,2011 Olly Betts
  * Copyright 2009 Frank J Bruzzaniti
+ * Copyright 2006,2007,2008 AVL List GesmbH
  *
  * This program is free software; you can redistribute it and/or
  * modify it under the terms of the GNU General Public License as
@@ -65 +66 @@
 
 #include "gnu_getopt.h"
 
+#ifndef LIBEXECDIR 
+// must have ending slash
+//# define LIBEXECDIR "/usr/lib/omega/bin/"
+# define LIBEXECDIR ""
+#endif
+#ifndef PKGDATADIR 
+// must have ending slash
+# define PKGDATADIR "/usr/share/omega/"
+#endif
+
 using namespace std;
 
 #define TITLE_SIZE 128
@@ -78 +89 @@
 static bool ignore_exclusions = false;
 static bool spelling = false;
 static bool verbose = false;
+string error_log; /* used in runfilter.cc */
+static string baseurl;
+static string dbpath;
+static string cache_dir;
 static enum {
     EMPTY_BODY_WARN, EMPTY_BODY_INDEX, EMPTY_BODY_SKIP
 } empty_body = EMPTY_BODY_WARN;
@@ -98 +113 @@
 // text are common, so we handle these with a std::map.
 static map<string, string> commands;
 
+static void
+index_directory(const string &path, const string &url_, size_t depth_limit,
+                map<string, string>& mime_map);
+
 inline static bool
 p_notalnum(unsigned int c)
 {
@@ -310 +329 @@
 index_mimetype(const string & file, const string & url, const string & ext,
                const string &mimetype, DirectoryIterator &d);
 
+static
+void mkdir_p(const string &path, mode_t mode) {
+    (void)mode; // FIXME
+#ifdef __WIN32__
+    system(("mkdir \"" + shell_protect(path) + "\"").c_str());
+#else
+    if (system(("mkdir -p " + shell_protect(path)).c_str()) < 0) { /* FIXME */ }
+#endif
+}
+
+/* 
+ * unpack .msg/.pst/.rar/.zip into local cache dir and recurse there 
+ */
+static void
+index_cached_directory(size_t depth_limit, 
+                       const string &file,
+                       const string &url,
+                       const string &ext, 
+                       const string &cmd,
+                       map<string, string>& mime_map)
+{
+    string oldroot = root;
+    root = cache_dir;
+    string cache = root+"/."+ext;
+    string cachedir = cache+url;
+    struct stat statfile, statcache;
+    bool extract_cache;
+#ifdef HAVE_LSTAT
+    lstat(file.c_str(), &statfile);
+    lstat(cachedir.c_str(), &statcache);
+#else
+    stat(file.c_str(), &statfile);
+    stat(cachedir.c_str(), &statcache);
+#endif
+    extract_cache = true;
+    // if cachedir exists and if file is older as cachedir and if cachedir existed 5 secs ago, 
+    // then it was already extracted.
+    if (S_ISDIR(statcache.st_mode) 
+        && S_ISREG(statfile.st_mode) 
+        && (statfile.st_mtime < statcache.st_mtime)
+        && (statcache.st_mtime < (time_t)(time(NULL)-500))) // not created by nested mkdir call
+    {
+        // but is it in the database also? prevent from deleting skipped files
+        if (verbose)
+            cout << "Unchanged cache \"" << cachedir << "\" - \"" << file << "\" - skip extraction " 
+                 // << statfile.st_mtime << " < " << statcache.st_mtime 
+                 << endl;
+        extract_cache = false;
+    }
+    if (S_ISDIR(statcache.st_mode) && S_ISREG(statfile.st_mode) ) {
+        // If last_mod > last_mod_max, we know for sure that the file is new
+        // or updated.
+        if (statfile.st_mtime <= last_mod_max) {
+            // check database timestamp for cached container, esp. for cleaned up caches.
+            // if already in db we need not to extract again
+            string urlterm("U");
+            urlterm += baseurl;
+            urlterm += "/."+ext+url;
+            if (urlterm.length() > MAX_SAFE_TERM_LENGTH)
+                urlterm = hash_long_term(urlterm, MAX_SAFE_TERM_LENGTH);
+        
+            Xapian::PostingIterator p = db.postlist_begin(urlterm);
+            if (p != db.postlist_end(urlterm)) {
+                Xapian::docid docid = *p;
+                Xapian::Document doc = db.get_document(docid);
+                string value = doc.get_value(VALUE_LASTMOD);
+                time_t old_last_mod = binary_string_to_int(value);
+                if (statfile.st_mtime <= old_last_mod) {
+                    if (verbose)
+                        cout << "Cache "<< "."+ext+url << " not newer. Ignored." << endl;
+                    // The docid should be in updated - the only valid
+                    // exception is if the URL was long and hashed to the
+                    // same URL as an existing document indexed in the same
+                    // batch.
+                    if (usual(docid < updated.size() && !updated[docid])) {
+                        updated[docid] = true;
+                        --old_docs_not_seen;
+                    }
+                    root = oldroot;
+                    return;
+                }
+            }
+        }
+    }
+
+    if (extract_cache) {
+        if (verbose)
+            cout << "[EXTRACT into cache " << cachedir << "]" << endl;
+        if (verbose && S_ISDIR(statcache.st_mode) && S_ISREG(statfile.st_mode))
+            cout << " ...changed cache \"" << cachedir << "\" - \"" << file << "\" " 
+                 << statfile.st_mtime << " < " << statcache.st_mtime << " time: " << time(NULL) 
+                 << endl;
+        if (!S_ISDIR(statcache.st_mode))
+            mkdir_p(cachedir, 0755);
+        stdout_to_string(cmd);
+#ifndef __WIN32__
+        stdout_to_string("chmod -R u+rwx " + shell_protect(cachedir));
+#endif
+#ifdef HAVE_LSTAT
+        lstat(cachedir.c_str(), &statcache);
+#else
+        stat(cachedir.c_str(), &statcache);
+#endif
+    }
+
+    if (S_ISDIR(statcache.st_mode)) {
+        if (depth_limit == 1) {
+            cout << "Recursion limit reached for \""<< url << "\" - skipping " << endl;
+        } else {
+            // max loop 5, magic start: /.ext+file
+            index_directory(file + "/."+ext+url, url, depth_limit + 5, mime_map);
+            if (verbose)
+                cout << "[CLEANUP " << "rm -rf " << shell_protect(cachedir) << "]" << endl;
+            rm_rf(cachedir);
+        }
+    }
+    else { // no -p would be fatal here
+        cout << "cachedir " << shell_protect(cachedir) << " does not exist - skipped" << endl;
+    }
+    root = oldroot;
+}
+
 static void
 index_file(const string &file, const string &url, DirectoryIterator & d,
-           map<string, string>& mime_map)
+           map<string, string>& mime_map, size_t depth_limit)
 {
     string ext;
     const char * dot_ptr = strrchr(d.leafname(), '.');
@@ -340 +481 @@
 
     string mimetype;
     if (mt == mime_map.end()) {
+        if (strcasecmp(d.leafname(), "mbox") == 0) {
+            // Special filename.
+            mimetype = "message/rfc822";
+            goto got_mimetype;
+        }
+
         mimetype = d.get_magic_mimetype();
         if (mimetype.empty()) {
             skip(file, "Unknown extension and unrecognised format",
@@ -352 +499 @@
         mimetype = mt->second;
     }
 
+got_mimetype:
+
     if (verbose)
         cout << "Indexing \"" << file.substr(root.size()) << "\" as "
              << mimetype << " ... ";
@@ -418 +567 @@
             }
         }
     }
+    // add the db basename to cache_dir
+    {
+        ensure_tmpdir(); // FIXME: be lazy!
+        cache_dir = tmpdir;
+        const char *p = strrchr(dbpath.c_str(), '/');
+        // on windows only
+        if (!p) p = strrchr(dbpath.c_str(), '\\');
+        if (p) { p++; } else { p = dbpath.c_str(); }
+        cache_dir += p;
+    }
 
     if (verbose) cout << flush;
 
@@ -482 +641 @@
             } else {
                 // FIXME: What charset is the file?  Look at contents?
             }
+#if 0 // FIXME: this won't work as omindex will have the database locked...
+    } else if (mimetype == "message/rfc822") { // // => mbox2script
+        //for stemmer lang, parse stemmer.get_description => Xapian::Stem(bla)
+        string cmd = LIBEXECDIR"mbox2omega " + shell_protect(file) + error_log+"| "
+            "scriptindex " + shell_protect(dbpath) + " "PKGDATADIR"mbox2script.script";
+        try {
+            dump = stdout_to_string(cmd);
+        } catch (ReadError) {
+            cout << "\"" << cmd << "\" failed - skipping" << endl;
+            return;
+        }
+#endif
         } else if (mimetype == "application/pdf") {
             string safefile = shell_protect(file);
             string cmd = "pdftotext -enc UTF-8 " + safefile + " -";
@@ -711 +882 @@
 
             generate_sample_from_csv(dump, sample);
         } else if (mimetype == "application/vnd.ms-outlook") {
-            string cmd = get_pkglibbindir() + "/outlookmsg2html " + shell_protect(file);
-            MyHtmlParser p;
-            p.ignore_metarobots();
+            string oldroot = root;
+            struct stat statcache;
+            char olddir[256];
+            
+            if (depth_limit == 1) {
+                skip(file, "Recursion limit reached for compound file", SKIP_SHOW_FILENAME);
+                return;
+            }
+            string cmd = LIBEXECDIR"outlook2text "+shell_protect(file);
+            // unpack multiparts and attachments. so we have to chdir first
+            string fulldir = cache_dir+"/.msg"+url;
+            if (getcwd(olddir, 256) == NULL) { /* FIXME */ }
+#ifdef HAVE_LSTAT
+            lstat(fulldir.c_str(), &statcache);
+#else
+            stat(fulldir.c_str(), &statcache);
+#endif
+            if (!S_ISDIR(statcache.st_mode)) {
+                mkdir_p(fulldir, 0755); 
+            }
             try {
-                dump = stdout_to_string(cmd);
-                // FIXME: what should the default charset be?
-                p.parse_html(dump, "iso-8859-1", false);
-            } catch (const string & newcharset) {
-                p.reset();
-                p.ignore_metarobots();
-                p.parse_html(dump, newcharset, true);
+                if (chdir(fulldir.c_str()) < 0) { /* FIXME */ }
+                size_t new_limit = depth_limit;
+                if (new_limit) --new_limit;
+                index_cached_directory(new_limit, file, url, ext, cmd, mime_map);
+                if (chdir(olddir) < 0) { /* FIXME */ }
             } catch (ReadError) {
-                skip_cmd_failed(file, cmd);
-                return;
+                cout << "failed " << cmd << " << in index_cached_directory" << endl;
+                if (chdir(olddir) < 0) { /* FIXME */ }
+                root = oldroot;
+            } catch (...) {
+                cout << "Caught unknown exception in index_cached_directory, rethrowing" << endl;
+                if (chdir(olddir) < 0) { /* FIXME */ }
+                root = oldroot;
+                throw;
             }
-            dump = p.dump;
-            title = p.title;
-            keywords = p.keywords;
-            sample = p.sample;
-            author = p.author;
+            return;
         } else if (mimetype == "image/svg+xml") {
             SvgParser svgparser;
             svgparser.parse_html(d.file_to_string());
@@ -759 +947 @@
             if (idx != string::npos) {
                 dump.assign(desc, idx + 1, string::npos);
             }
+        } else if (mimetype == "application/x-zip") {
+            string oldroot = root;
+            if (depth_limit == 1) {
+                skip(file, "Recursion limit reached for compound file", SKIP_SHOW_FILENAME);
+                return;
+            }
+            // overwrite
+            string cmd = "unzip -u -P. -o " +shell_protect(file) + " -d " +shell_protect(cache_dir+"/.zip"+url+"/");
+            try {
+                size_t new_limit = depth_limit;
+                if (new_limit) --new_limit;
+                index_cached_directory(new_limit, file, url, ext, cmd, mime_map);
+            } catch (ReadError) {
+                cout << "failed " << cmd << " << in index_cached_directory" << endl;
+                root = oldroot;
+            } catch (...) {
+                cout << "Caught unknown exception in index_cached_directory, rethrowing" << endl;
+                root = oldroot;
+                throw;
+            }
+            return;
+        } else if (mimetype == "application/x-rar") {
+            string oldroot = root;
+            if (depth_limit == 1) {
+                skip(file, "Recursion limit reached for compound file", SKIP_SHOW_FILENAME);
+                return;
+            }
+
+            // overwrite
+            string cmd = "unrar x -o+ " +shell_protect(file) + " " 
+                + shell_protect(cache_dir+"/.rar"+url+"/");
+            try {
+                size_t new_limit = depth_limit;
+                if (new_limit) --new_limit;
+                index_cached_directory(new_limit, file, url, ext, cmd, mime_map);
+            } catch (ReadError) {
+                cout << "failed " << cmd << " << in index_cached_directory" << endl;
+                root = oldroot;
+            } catch (...) {
+                cout << "Caught unknown exception in index_cached_directory, rethrowing" << endl;
+                root = oldroot;
+                throw;
+            }
+            return;
+        } else if (mimetype == "application/vnd.ms-outlook-pst") {
+            string oldroot = root;
+            if (depth_limit == 1) {
+                skip(file, "Recursion limit reached for compound file", SKIP_SHOW_FILENAME);
+                return;
+            }
+            // unpack attachments also, together with mbox files
+            string cmd = "readpst -r -cv -w -o "
+                + shell_protect(cache_dir+"/.pst"+url+"/")+" "+shell_protect(file);
+            try {
+                size_t new_limit = depth_limit;
+                if (new_limit) --new_limit;
+                index_cached_directory(new_limit, file, url, ext, cmd, mime_map);
+            } catch (ReadError) {
+                root = oldroot;
+                cout << "failed " << cmd << " << in index_cached_directory" << endl;
+            } catch (...) {
+                root = oldroot;
+                cout << "Caught unknown exception in index_cached_directory, rethrowing" << endl;
+                throw;
+            }
+            return;
         } else {
             // Don't know how to index this type.
             skip_unknown_mimetype(file, mimetype);
@@ -975 +1229 @@
                         break;
                     }
                     case DirectoryIterator::REGULAR_FILE:
-                        index_file(file, url, d, mime_map);
+                        index_file(file, url, d, mime_map, depth_limit);
                         break;
                     default:
                         skip(file, "Not a regular file",
@@ -999 +1253 @@
     bool overwrite = false;
     // If delete_removed_documents is true, delete any documents we don't see.
     bool delete_removed_documents = true;
-    string baseurl;
     size_t depth_limit = 0;
 
     static const struct option longopts[] = {
@@ -1124 +1377 @@
     mime_map["pps"] = "application/vnd.ms-powerpoint"; // Powerpoint slideshow
     mime_map["msg"] = "application/vnd.ms-outlook"; // Outlook .msg email
 
+    //  Outlook message folder:
+    mime_map["pst"] = "application/vnd.ms-outlook-pst"; // readpst | uudeview (libpst)
+
+    // Miscellaneous compound formats:
+    mime_map["mbox"] = "message/rfc822";                // => mbox2omega
+    mime_map["mbx"] = "message/rfc822";                // => mbox2omega
+#ifndef _MSC_VER
+    mime_map["zip"] = "application/x-zip"; // recursive scanning
+    mime_map["rar"] = "application/x-rar"; // recursive scanning
+#endif
+
     // Perl:
     mime_map["pl"] = "text/x-perl";
     mime_map["pm"] = "text/x-perl";
@@ -1180 +1444 @@
         argv[1] = const_cast<char *>("--version");
     }
 
-    string dbpath;
     int getopt_ret;
     while ((getopt_ret = gnu_getopt_long(argc, argv, "hvd:D:U:M:F:l:s:pfSVe:i",
                                          longopts, NULL)) != -1) {
@@ -1330 +1593 @@
         baseurl += '/';
     }
 
+    string log_dir = "./"; // FIXME: need to set log_dir to something appropriate.
+    error_log = " 2>>"+log_dir+"omindex-error.log";
+
     if (optind >= argc || optind + 2 < argc) {
         cerr << PROG_NAME": you must specify a directory to index.\n"
 "Do this either as a single directory (corresponding to the base URL)\n"

new file xapian-applications/omega/outlook2text.in

@@ +1 @@
+#! /bin/sh
+# converts msg to mbox and extract attachments
+# either be in the cache dir, or accept it as 2nd arg
+if [ -n $2 ]; then 
+  @MSGCONVERT@ "$1" | @MIMEEXPLODE@ -d "$2"
+else
+  # already is in the cache dir
+  base=`basename "$1" .msg`
+  @MSGCONVERT@ "$1" | @MIMEEXPLODE@ -d "${base}"
+fi

xapian-applications/omega/runfilter.cc

@@ -55 +55 @@
 
 using namespace std;
 
+extern string error_log;
+
 string
 stdout_to_string(const string &cmd)
 {
     string out;
+    string tmp = cmd;
+    tmp += error_log;
 #if defined HAVE_FORK && defined HAVE_SOCKETPAIR && defined HAVE_SETRLIMIT
     // We want to be able to get the exit status of the child process.
     signal(SIGCHLD, SIG_DFL);
@@ -101 +105 @@
         }
 #endif
 
-        execl("/bin/sh", "/bin/sh", "-c", cmd.c_str(), (void*)NULL);
+        execl("/bin/sh", "/bin/sh", "-c", tmp.c_str(), (void*)NULL);
         _exit(-1);
     }
 
@@ -168 +172 @@
         throw ReadError();
     }
 #else
-    FILE * fh = popen(cmd.c_str(), "r");
+    FILE * fh = popen(tmp.c_str(), "r");
     if (fh == NULL) throw ReadError();
     while (!feof(fh)) {
         char buf[4096];

xapian-applications/omega/utils.cc

@@ -26 +26 @@
 #include <cassert>
 #include <stdio.h> // for sprintf/snprintf
 #include <cstdlib>
+#include <cstring>
+#include "safesysstat.h"
 
 #include <string>
 
 using namespace std;
 
+#ifdef __WIN32__
+#include "safewindows.h"
+#endif
+
 // This ought to be enough for any of the conversions below.
 #define BUFSIZE 100
 
@@ -40 +46 @@
     int len = SNPRINTF(buf, BUFSIZE, (FMT), val);\
     if (len == -1 || len > BUFSIZE) return string(buf, BUFSIZE);\
     return string(buf, len);
+/// Allow system to work directly on C++ strings.
+inline int system(const string &command) { return system(command.c_str()); }
+
+// Duplicated from omindex.cc - FIXME
+static string
+shell_protect(const string & file)
+{
+    string safefile = file;
+#ifdef __WIN32__
+    bool need_to_quote = false;
+    for (string::iterator i = safefile.begin(); i != safefile.end(); ++i) {
+        unsigned char ch = *i;
+        if (!isalnum(ch) && ch < 128) {
+            if (ch == '/') {
+                // Convert Unix path separators to backslashes.  C library
+                // functions understand "/" in paths, but external commands
+                // generally don't, and also may interpret a leading '/' as
+                // introducing a command line option.
+                *i = '\\';
+            } else if (ch == ' ') {
+                need_to_quote = true;
+            } else if (ch < 32 || strchr("<>\"|*?", ch)) {
+                // Check for invalid characters in the filename.
+                string m("Invalid character '");
+                m += ch;
+                m += "' in filename \"";
+                m += file;
+                m += '"';
+                throw m;
+            }
+        }
+    }
+    if (safefile[0] == '-') {
+        // If the filename starts with a '-', protect it from being treated as
+        // an option by prepending ".\".
+        safefile.insert(0, ".\\");
+    }
+    if (need_to_quote) {
+        safefile.insert(0, "\"");
+        safefile += '"';
+    }
+#else
+    string::size_type p = 0;
+    if (!safefile.empty() && safefile[0] == '-') {
+        // If the filename starts with a '-', protect it from being treated as
+        // an option by prepending "./".
+        safefile.insert(0, "./");
+        p = 2;
+    }
+    while (p < safefile.size()) {
+        // Don't escape some safe characters which are common in filenames.
+        unsigned char ch = safefile[p];
+        if (!isalnum(ch) && strchr("/._-", ch) == NULL) {
+            safefile.insert(p, "\\");
+            ++p;
+        }
+        ++p;
+    }
+#endif
+    return safefile;
+}
+
+/// Remove a directory and contents.
+void
+rm_rf(const string &filename)
+{
+    // Check filename exists and is actually a directory
+    struct stat sb;
+    if (stat(filename.c_str(), &sb) != 0 || !S_ISDIR(sb.st_mode)) return;
+
+    string safefile = shell_protect(filename);
+#ifdef __WIN32__
+# if 1
+    static int win95 = -1;
+    if (win95 == -1) {
+        OSVERSIONINFO info;
+        memset(&info, 0, sizeof(OSVERSIONINFO));
+        info.dwOSVersionInfoSize = sizeof(OSVERSIONINFO);
+        if (GetVersionEx(&info)) {
+            win95 = (info.dwPlatformId == VER_PLATFORM_WIN32_WINDOWS);
+        }
+    }
+
+    if (win95) {
+        // for 95 like systems:
+        system("deltree /y \"" + safefile + "\"");
+    } else {
+        // for NT like systems:
+        system("rd /s /q \"" + safefile + "\"");
+    }
+# else
+    safefile.append("\0", 2);
+    SHFILEOPSTRUCT shfo;
+    memset((void*)&shfo, 0, sizeof(shfo));
+    shfo.hwnd = 0;
+    shfo.wFunc = FO_DELETE;
+    shfo.pFrom = safefile.data();
+    shfo.fFlags = FOF_NOCONFIRMATION|FOF_NOERRORUI|FOF_SILENT;
+    (void)SHFileOperation(&shfo);
+# endif
+#else
+    system("rm -rf " + safefile);
+#endif
+}
 #else
 #define CONVERT_TO_STRING(FMT) \
     char buf[BUFSIZE];\

xapian-applications/omega/utils.h

@@ -34 +34 @@
 /** Remove any leading and/or trailing whitespace from @a s. */
 void trim(std::string & s);
 
+void rm_rf(const std::string &filename);
+
 #endif