Ticket #325: 0001-Refactor-zlib-usage-in-FlintTable-to-avoid-reallocat.patch

backends/flint/flint_table.cc

@@ -1037 +1037 @@
         CompileTimeAssert(DONT_COMPRESS != Z_RLE);
 #endif
 
-        z_stream stream;
-
-        stream.zalloc = reinterpret_cast<alloc_func>(0);
-        stream.zfree = reinterpret_cast<free_func>(0);
-        stream.opaque = (voidpf)0;
-
-        // -15 means raw deflate with 32K LZ77 window (largest)
-        // memLevel 9 is the highest (8 is default)
-        int err;
-        err = deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 9,
-                           compress_strategy);
-        if (err != Z_OK) {
-            if (err == Z_MEM_ERROR) throw std::bad_alloc();
-            string msg = "deflateInit2 failed";
-            if (stream.msg) {
-                msg += " (";
-                msg += stream.msg;
-                msg += ')';
-            }
-            throw Xapian::DatabaseError(msg);
-        }
+        lazy_alloc_deflate_zstream();
+        deflateReset(deflate_zstream);
 
-        stream.next_in = (Bytef *)const_cast<char *>(tag.data());
-        stream.avail_in = (uInt)tag.size();
+        deflate_zstream->next_in = (Bytef *)const_cast<char *>(tag.data());
+        deflate_zstream->avail_in = (uInt)tag.size();
 
         // If compressed size is >= tag.size(), we don't want to compress.
         unsigned long blk_len = tag.size() - 1;
         unsigned char * blk = new unsigned char[blk_len];
-        stream.next_out = blk;
-        stream.avail_out = (uInt)blk_len;
+        deflate_zstream->next_out = blk;
+        deflate_zstream->avail_out = (uInt)blk_len;
 
-        err = deflate(&stream, Z_FINISH);
+        int err = deflate(deflate_zstream, Z_FINISH);
         if (err == Z_STREAM_END) {
             // If deflate succeeded, then the output was at least one byte
             // smaller than the input.
-            tag.assign(reinterpret_cast<const char *>(blk), stream.total_out);
+            tag.assign(reinterpret_cast<const char *>(blk), deflate_zstream->total_out);
             compressed = true;
-            err = deflateEnd(&stream);
-            if (err != Z_OK) {
-                string msg = "deflateEnd failed";
-                if (stream.msg) {
-                    msg += " (";
-                    msg += stream.msg;
-                    msg += ')';
-                }
-                throw Xapian::DatabaseError(msg);
-            }
         } else {
-            // Deflate failed - presumably the data wasn't compressible.
-            (void)deflateEnd(&stream);
         }
 
         delete [] blk;
@@ -1271 +1240 @@
 
     Bytef buf[8192];
 
-    z_stream stream;
-    stream.next_out = buf;
-    stream.avail_out = (uInt)sizeof(buf);
+    inflateReset(inflate_zstream);
 
-    stream.zalloc = reinterpret_cast<alloc_func>(0);
-    stream.zfree = reinterpret_cast<free_func>(0);
+    inflate_zstream->next_out = buf;
+    inflate_zstream->avail_out = (uInt)sizeof(buf);
 
-    stream.next_in = Z_NULL;
-    stream.avail_in = 0;
-
-    int err = inflateInit2(&stream, -15);
-    if (err != Z_OK) {
-        if (err == Z_MEM_ERROR) throw std::bad_alloc();
-        string msg = "inflateInit2 failed";
-        if (stream.msg) {
-            msg += " (";
-            msg += stream.msg;
-            msg += ')';
-        }
-        throw Xapian::DatabaseError(msg);
-    }
-
-    stream.next_in = (Bytef*)const_cast<char *>(tag->data());
-    stream.avail_in = (uInt)tag->size();
+    inflate_zstream->next_in = (Bytef*)const_cast<char *>(tag->data());
+    inflate_zstream->avail_in = (uInt)tag->size();
 
+    int err;
     while (err != Z_STREAM_END) {
-        stream.next_out = buf;
-        stream.avail_out = (uInt)sizeof(buf);
-        err = inflate(&stream, Z_SYNC_FLUSH);
-        if (err == Z_BUF_ERROR && stream.avail_in == 0) {
-            DEBUGLINE(DB, "Z_BUF_ERROR - faking checksum of " << stream.adler);
+        inflate_zstream->next_out = buf;
+        inflate_zstream->avail_out = (uInt)sizeof(buf);
+        err = inflate(inflate_zstream, Z_SYNC_FLUSH);
+        if (err == Z_BUF_ERROR && inflate_zstream->avail_in == 0) {
+            DEBUGLINE(DB, "Z_BUF_ERROR - faking checksum of " << inflate_zstream->adler);
             Bytef header2[4];
-            setint4(header2, 0, stream.adler);
-            stream.next_in = header2;
-            stream.avail_in = 4;
-            err = inflate(&stream, Z_SYNC_FLUSH);
+            setint4(header2, 0, inflate_zstream->adler);
+            inflate_zstream->next_in = header2;
+            inflate_zstream->avail_in = 4;
+            err = inflate(inflate_zstream, Z_SYNC_FLUSH);
             if (err == Z_STREAM_END) break;
         }
 
         if (err != Z_OK && err != Z_STREAM_END) {
             if (err == Z_MEM_ERROR) throw std::bad_alloc();
             string msg = "inflate failed";
-            if (stream.msg) {
+            if (inflate_zstream->msg) {
                 msg += " (";
-                msg += stream.msg;
+                msg += inflate_zstream->msg;
                 msg += ')';
             }
             throw Xapian::DatabaseError(msg);
         }
 
         utag.append(reinterpret_cast<const char *>(buf),
-                    stream.next_out - buf);
+                    inflate_zstream->next_out - buf);
     }
-    if (utag.size() != stream.total_out) {
+    if (utag.size() != inflate_zstream->total_out) {
         string msg = "compressed tag didn't expand to the expected size: ";
         msg += om_tostring(utag.size());
         msg += " != ";
         // OpenBSD's zlib.h uses off_t instead of uLong for total_out.
-        msg += om_tostring((size_t)stream.total_out);
+        msg += om_tostring((size_t)inflate_zstream->total_out);
         throw Xapian::DatabaseCorruptError(msg);
     }
 
-    err = inflateEnd(&stream);
-    if (err != Z_OK) abort();
-
     swap(*tag, utag);
 
     return false;
@@ -1612 +1562 @@
           writable(!readonly_),
           split_p(0),
           compress_strategy(compress_strategy_),
+          deflate_zstream(NULL),
+          inflate_zstream(NULL),
           lazy(lazy_)
 {
     DEBUGCALL(DB, void, "FlintTable::FlintTable", 
               tablename_ << "," << path_ << ", " << readonly_ << ", " <<
               compress_strategy_ << ", " << lazy_);
+
+    // We want to do this lazily, but the read functions are const so we can't :(
+    lazy_alloc_inflate_zstream();
+
+}
+
+
+void
+FlintTable::lazy_alloc_deflate_zstream() {
+    if (deflate_zstream)
+        return;
+
+    deflate_zstream = new z_stream;
+        
+    deflate_zstream->zalloc = reinterpret_cast<alloc_func>(0);
+    deflate_zstream->zfree = reinterpret_cast<free_func>(0);
+    deflate_zstream->opaque = (voidpf)0;
+
+    // -15 means raw deflate with 32K LZ77 window (largest)
+    // memLevel 9 is the highest (8 is default)
+    int err;
+    err = deflateInit2(deflate_zstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 9,
+                       compress_strategy);
+    if (err != Z_OK) {
+        if (err == Z_MEM_ERROR) throw std::bad_alloc();
+        string msg = "deflateInit2 failed. err = " + err;
+        if (deflate_zstream->msg) {
+            msg += " (";
+            msg += deflate_zstream->msg;
+            msg += ')';
+        }
+        throw Xapian::DatabaseError(msg);
+    }
+}
+
+
+void
+FlintTable::lazy_alloc_inflate_zstream() {
+    if (inflate_zstream)
+        return;
+
+    inflate_zstream = new z_stream;
+
+    inflate_zstream->zalloc = reinterpret_cast<alloc_func>(0);
+    inflate_zstream->zfree = reinterpret_cast<free_func>(0);
+
+    inflate_zstream->next_in = Z_NULL;
+    inflate_zstream->avail_in = 0;
+
+    int err = inflateInit2(inflate_zstream, -15);
+    if (err != Z_OK) {
+        if (err == Z_MEM_ERROR) throw std::bad_alloc();
+        string msg = "inflateInit2 failed";
+        if (inflate_zstream->msg) {
+            msg += " (";
+            msg += inflate_zstream->msg;
+            msg += ')';
+        }
+        throw Xapian::DatabaseError(msg);
+    }
 }
 
 bool
@@ -1699 +1711 @@
 FlintTable::~FlintTable() {
     DEBUGCALL(DB, void, "FlintTable::~FlintTable", "");
     FlintTable::close();
+
+    if (deflate_zstream) {
+        int err = deflateEnd(deflate_zstream);
+        if (err != Z_OK) {
+            string msg = "deflateEnd failed. err = " + err;
+            if (deflate_zstream->msg) {
+                msg += " (";
+                msg += deflate_zstream->msg;
+                msg += ')';
+            }
+            delete deflate_zstream;
+            throw Xapian::DatabaseError(msg);
+        }
+        delete deflate_zstream;        
+    }
+
+    if (inflate_zstream) {
+        int err = inflateEnd(inflate_zstream);
+        if (err != Z_OK) {
+            string msg = "inflateEnd failed. err = " + err;
+            if (inflate_zstream->msg) {
+                msg += " (";
+                msg += inflate_zstream->msg;
+                msg += ')';
+            }
+            delete inflate_zstream;
+            throw Xapian::DatabaseError(msg);
+        }
+        delete inflate_zstream;
+    }
 }
 
 void FlintTable::close(bool permanent) {

backends/flint/flint_table.h

@@ -587 +587 @@
            return (base_letter == 'A') ? 'B' : 'A';
         }
 
+    void lazy_alloc_deflate_zstream();
+    void lazy_alloc_inflate_zstream();
+
         /** revision number of the opened B-tree. */
         flint_revision_number_t revision_number;
 
@@ -705 +708 @@
          *  Z_RLE. */
         int compress_strategy;
 
+    /** Zlib state object */
+    z_stream *deflate_zstream;
+    z_stream *inflate_zstream;
+
+
         /// If true, don't create the table until it's needed.
         bool lazy;