Ticket #325: Refactor-zlib-usage-in-FlintTable-and-ChertTable-to-avoid-reallocat.3.patch

backends/chert/chert_table.h

@@ -300 +300 @@
          *  Any outstanding changes (ie, changes made without commit() having
          *  subsequently been called) will be lost.
          */
-        ~ChertTable();
+        virtual ~ChertTable();
 
         /** Close the Btree.  This closes and frees any of the btree
          *  structures which have been created and opened.
@@ -605 +605 @@
         /// The name of the table (used when writing changesets).
         const char * tablename;
 
+        void lazy_alloc_deflate_zstream() const;
+        void lazy_alloc_inflate_zstream() const;
+
         /** revision number of the opened B-tree. */
         chert_revision_number_t revision_number;
 
@@ -723 +726 @@
          *  Z_RLE. */
         int compress_strategy;
 
+        /** Zlib state object */
+        mutable z_stream *deflate_zstream;
+        mutable z_stream *inflate_zstream;
+
         /// If true, don't create the table until it's needed.
         bool lazy;
 

backends/chert/chert_table.cc

@@ -1058 +1058 @@
         CompileTimeAssert(DONT_COMPRESS != Z_RLE);
 #endif
 
-        z_stream stream;
+        lazy_alloc_deflate_zstream();
 
-        stream.zalloc = reinterpret_cast<alloc_func>(0);
-        stream.zfree = reinterpret_cast<free_func>(0);
-        stream.opaque = (voidpf)0;
+        deflate_zstream->next_in = (Bytef *)const_cast<char *>(tag.data());
+        deflate_zstream->avail_in = (uInt)tag.size();
 
-        // -15 means raw deflate with 32K LZ77 window (largest)
-        // memLevel 9 is the highest (8 is default)
-        int err;
-        err = deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 9,
-                           compress_strategy);
-        if (err != Z_OK) {
-            if (err == Z_MEM_ERROR) throw std::bad_alloc();
-            string msg = "deflateInit2 failed";
-            if (stream.msg) {
-                msg += " (";
-                msg += stream.msg;
-                msg += ')';
-            }
-            throw Xapian::DatabaseError(msg);
-        }
-
-        stream.next_in = (Bytef *)const_cast<char *>(tag.data());
-        stream.avail_in = (uInt)tag.size();
-
         // If compressed size is >= tag.size(), we don't want to compress.
         unsigned long blk_len = tag.size() - 1;
         unsigned char * blk = new unsigned char[blk_len];
-        stream.next_out = blk;
-        stream.avail_out = (uInt)blk_len;
+        deflate_zstream->next_out = blk;
+        deflate_zstream->avail_out = (uInt)blk_len;
 
-        err = deflate(&stream, Z_FINISH);
+        int err = deflate(deflate_zstream, Z_FINISH);
         if (err == Z_STREAM_END) {
             // If deflate succeeded, then the output was at least one byte
             // smaller than the input.
-            tag.assign(reinterpret_cast<const char *>(blk), stream.total_out);
+            tag.assign(reinterpret_cast<const char *>(blk), deflate_zstream->total_out);
             compressed = true;
-            err = deflateEnd(&stream);
-            if (err != Z_OK) {
-                string msg = "deflateEnd failed";
-                if (stream.msg) {
-                    msg += " (";
-                    msg += stream.msg;
-                    msg += ')';
-                }
-                throw Xapian::DatabaseError(msg);
-            }
-        } else {
-            // Deflate failed - presumably the data wasn't compressible.
-            (void)deflateEnd(&stream);
         }
 
         delete [] blk;
@@ -1281 +1248 @@
 
     Bytef buf[8192];
 
-    z_stream stream;
-    stream.next_out = buf;
-    stream.avail_out = (uInt)sizeof(buf);
+    lazy_alloc_inflate_zstream();
 
-    stream.zalloc = reinterpret_cast<alloc_func>(0);
-    stream.zfree = reinterpret_cast<free_func>(0);
+    inflate_zstream->next_out = buf;
+    inflate_zstream->avail_out = (uInt)sizeof(buf);
 
-    stream.next_in = Z_NULL;
-    stream.avail_in = 0;
+    inflate_zstream->next_in = (Bytef*)const_cast<char *>(tag->data());
+    inflate_zstream->avail_in = (uInt)tag->size();
 
-    int err = inflateInit2(&stream, -15);
-    if (err != Z_OK) {
-        if (err == Z_MEM_ERROR) throw std::bad_alloc();
-        string msg = "inflateInit2 failed";
-        if (stream.msg) {
-            msg += " (";
-            msg += stream.msg;
-            msg += ')';
-        }
-        throw Xapian::DatabaseError(msg);
-    }
-
-    stream.next_in = (Bytef*)const_cast<char *>(tag->data());
-    stream.avail_in = (uInt)tag->size();
-
+    int err = Z_OK;
     while (err != Z_STREAM_END) {
-        stream.next_out = buf;
-        stream.avail_out = (uInt)sizeof(buf);
-        err = inflate(&stream, Z_SYNC_FLUSH);
-        if (err == Z_BUF_ERROR && stream.avail_in == 0) {
-            LOGLINE(DB, "Z_BUF_ERROR - faking checksum of " << stream.adler);
+        inflate_zstream->next_out = buf;
+        inflate_zstream->avail_out = (uInt)sizeof(buf);
+        err = inflate(inflate_zstream, Z_SYNC_FLUSH);
+        if (err == Z_BUF_ERROR && inflate_zstream->avail_in == 0) {
+            LOGLINE(DB, "Z_BUF_ERROR - faking checksum of " << inflate_zstream->adler);
             Bytef header2[4];
-            setint4(header2, 0, stream.adler);
-            stream.next_in = header2;
-            stream.avail_in = 4;
-            err = inflate(&stream, Z_SYNC_FLUSH);
+            setint4(header2, 0, inflate_zstream->adler);
+            inflate_zstream->next_in = header2;
+            inflate_zstream->avail_in = 4;
+            err = inflate(inflate_zstream, Z_SYNC_FLUSH);
             if (err == Z_STREAM_END) break;
         }
 
         if (err != Z_OK && err != Z_STREAM_END) {
             if (err == Z_MEM_ERROR) throw std::bad_alloc();
             string msg = "inflate failed";
-            if (stream.msg) {
+            if (inflate_zstream->msg) {
                 msg += " (";
-                msg += stream.msg;
+                msg += inflate_zstream->msg;
                 msg += ')';
             }
             throw Xapian::DatabaseError(msg);
         }
 
         utag.append(reinterpret_cast<const char *>(buf),
-                    stream.next_out - buf);
+                    inflate_zstream->next_out - buf);
     }
-    if (utag.size() != stream.total_out) {
+    if (utag.size() != inflate_zstream->total_out) {
         string msg = "compressed tag didn't expand to the expected size: ";
         msg += om_tostring(utag.size());
         msg += " != ";
         // OpenBSD's zlib.h uses off_t instead of uLong for total_out.
-        msg += om_tostring((size_t)stream.total_out);
+        msg += om_tostring((size_t)inflate_zstream->total_out);
         throw Xapian::DatabaseCorruptError(msg);
     }
 
-    err = inflateEnd(&stream);
-    if (err != Z_OK) abort();
-
     swap(*tag, utag);
 
     RETURN(false);
@@ -1633 +1581 @@
           writable(!readonly_),
           split_p(0),
           compress_strategy(compress_strategy_),
+          deflate_zstream(NULL),
+          inflate_zstream(NULL),
           lazy(lazy_)
 {
     LOGCALL_CTOR(DB, "ChertTable",
@@ -1640 +1590 @@
                  compress_strategy_ << ", " << lazy_);
 }
 
+void
+ChertTable::lazy_alloc_deflate_zstream() const {
+    if (deflate_zstream) {
+        deflateReset(deflate_zstream);
+        return;
+    }
+
+    deflate_zstream = new z_stream;
+
+    deflate_zstream->zalloc = reinterpret_cast<alloc_func>(0);
+    deflate_zstream->zfree = reinterpret_cast<free_func>(0);
+    deflate_zstream->opaque = (voidpf)0;
+
+    // -15 means raw deflate with 32K LZ77 window (largest)
+    // memLevel 9 is the highest (8 is default)
+    int err;
+    err = deflateInit2(deflate_zstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 9,
+                       compress_strategy);
+    if (err != Z_OK) {
+        if (err == Z_MEM_ERROR) throw std::bad_alloc();
+        string msg = "deflateInit2 failed. err = " + err;
+        if (deflate_zstream->msg) {
+            msg += " (";
+            msg += deflate_zstream->msg;
+            msg += ')';
+        }
+        throw Xapian::DatabaseError(msg);
+    }
+}
+
+void
+ChertTable::lazy_alloc_inflate_zstream() const {
+    if (inflate_zstream) {
+        inflateReset(inflate_zstream);
+        return;
+    }
+
+    inflate_zstream = new z_stream;
+
+    inflate_zstream->zalloc = reinterpret_cast<alloc_func>(0);
+    inflate_zstream->zfree = reinterpret_cast<free_func>(0);
+
+    inflate_zstream->next_in = Z_NULL;
+    inflate_zstream->avail_in = 0;
+
+    int err = inflateInit2(inflate_zstream, -15);
+    if (err != Z_OK) {
+        if (err == Z_MEM_ERROR) throw std::bad_alloc();
+        string msg = "inflateInit2 failed";
+        if (inflate_zstream->msg) {
+            msg += " (";
+            msg += inflate_zstream->msg;
+            msg += ')';
+        }
+        throw Xapian::DatabaseError(msg);
+    }
+}
+
 bool
 ChertTable::exists() const {
     LOGCALL(DB, bool, "ChertTable::exists", "");
@@ -1724 +1732 @@
 ChertTable::~ChertTable() {
     LOGCALL_DTOR(DB, "ChertTable");
     ChertTable::close();
+
+    if (deflate_zstream) {
+        int err = deflateEnd(deflate_zstream);
+        if (err != Z_OK) {
+            // FIXME - display the warning message somewhere.
+            //string msg = "deflateEnd failed. err = " + err;
+            //if (deflate_zstream->msg) {
+            //    msg += " (";
+            //    msg += deflate_zstream->msg;
+            //    msg += ')';
+            //}
+            //
+            // Note - we mustn't throw an exception because we're in a
+            // destructor.  If the destructor was called due to stack
+            // unwinding for another exception, this will result in a
+            // call to terminate().
+        }
+        delete deflate_zstream;
+    }
+
+    if (inflate_zstream) {
+        int err = inflateEnd(inflate_zstream);
+        if (err != Z_OK) {
+            // FIXME - display the warning message somewhere.
+            //string msg = "inflateEnd failed. err = " + err;
+            //if (inflate_zstream->msg) {
+            //    msg += " (";
+            //    msg += inflate_zstream->msg;
+            //    msg += ')';
+            //}
+            //
+            // Note - we mustn't throw an exception because we're in a
+            // destructor.  If the destructor was called due to stack
+            // unwinding for another exception, this will result in a
+            // call to terminate().
+        }
+        delete inflate_zstream;
+    }
 }
 
 void ChertTable::close(bool permanent) {

backends/flint/flint_table.cc

@@ -1038 +1038 @@
         CompileTimeAssert(DONT_COMPRESS != Z_RLE);
 #endif
 
-        z_stream stream;
+        lazy_alloc_deflate_zstream();
 
-        stream.zalloc = reinterpret_cast<alloc_func>(0);
-        stream.zfree = reinterpret_cast<free_func>(0);
-        stream.opaque = (voidpf)0;
+        deflate_zstream->next_in = (Bytef *)const_cast<char *>(tag.data());
+        deflate_zstream->avail_in = (uInt)tag.size();
 
-        // -15 means raw deflate with 32K LZ77 window (largest)
-        // memLevel 9 is the highest (8 is default)
-        int err;
-        err = deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 9,
-                           compress_strategy);
-        if (err != Z_OK) {
-            if (err == Z_MEM_ERROR) throw std::bad_alloc();
-            string msg = "deflateInit2 failed";
-            if (stream.msg) {
-                msg += " (";
-                msg += stream.msg;
-                msg += ')';
-            }
-            throw Xapian::DatabaseError(msg);
-        }
-
-        stream.next_in = (Bytef *)const_cast<char *>(tag.data());
-        stream.avail_in = (uInt)tag.size();
-
         // If compressed size is >= tag.size(), we don't want to compress.
         unsigned long blk_len = tag.size() - 1;
         unsigned char * blk = new unsigned char[blk_len];
-        stream.next_out = blk;
-        stream.avail_out = (uInt)blk_len;
+        deflate_zstream->next_out = blk;
+        deflate_zstream->avail_out = (uInt)blk_len;
 
-        err = deflate(&stream, Z_FINISH);
+        int err = deflate(deflate_zstream, Z_FINISH);
         if (err == Z_STREAM_END) {
             // If deflate succeeded, then the output was at least one byte
             // smaller than the input.
-            tag.assign(reinterpret_cast<const char *>(blk), stream.total_out);
+            tag.assign(reinterpret_cast<const char *>(blk), deflate_zstream->total_out);
             compressed = true;
-            err = deflateEnd(&stream);
-            if (err != Z_OK) {
-                string msg = "deflateEnd failed";
-                if (stream.msg) {
-                    msg += " (";
-                    msg += stream.msg;
-                    msg += ')';
-                }
-                throw Xapian::DatabaseError(msg);
-            }
-        } else {
-            // Deflate failed - presumably the data wasn't compressible.
-            (void)deflateEnd(&stream);
         }
 
         delete [] blk;
@@ -1260 +1227 @@
 
     Bytef buf[8192];
 
-    z_stream stream;
-    stream.next_out = buf;
-    stream.avail_out = (uInt)sizeof(buf);
+    lazy_alloc_inflate_zstream();
 
-    stream.zalloc = reinterpret_cast<alloc_func>(0);
-    stream.zfree = reinterpret_cast<free_func>(0);
+    inflate_zstream->next_out = buf;
+    inflate_zstream->avail_out = (uInt)sizeof(buf);
 
-    stream.next_in = Z_NULL;
-    stream.avail_in = 0;
+    inflate_zstream->next_in = (Bytef*)const_cast<char *>(tag->data());
+    inflate_zstream->avail_in = (uInt)tag->size();
 
-    int err = inflateInit2(&stream, -15);
-    if (err != Z_OK) {
-        if (err == Z_MEM_ERROR) throw std::bad_alloc();
-        string msg = "inflateInit2 failed";
-        if (stream.msg) {
-            msg += " (";
-            msg += stream.msg;
-            msg += ')';
-        }
-        throw Xapian::DatabaseError(msg);
-    }
-
-    stream.next_in = (Bytef*)const_cast<char *>(tag->data());
-    stream.avail_in = (uInt)tag->size();
-
+    int err = Z_OK;
     while (err != Z_STREAM_END) {
-        stream.next_out = buf;
-        stream.avail_out = (uInt)sizeof(buf);
-        err = inflate(&stream, Z_SYNC_FLUSH);
-        if (err == Z_BUF_ERROR && stream.avail_in == 0) {
-            LOGLINE(DB, "Z_BUF_ERROR - faking checksum of " << stream.adler);
+        inflate_zstream->next_out = buf;
+        inflate_zstream->avail_out = (uInt)sizeof(buf);
+        err = inflate(inflate_zstream, Z_SYNC_FLUSH);
+        if (err == Z_BUF_ERROR && inflate_zstream->avail_in == 0) {
+            LOGLINE(DB, "Z_BUF_ERROR - faking checksum of " << inflate_zstream->adler);
             Bytef header2[4];
-            setint4(header2, 0, stream.adler);
-            stream.next_in = header2;
-            stream.avail_in = 4;
-            err = inflate(&stream, Z_SYNC_FLUSH);
+            setint4(header2, 0, inflate_zstream->adler);
+            inflate_zstream->next_in = header2;
+            inflate_zstream->avail_in = 4;
+            err = inflate(inflate_zstream, Z_SYNC_FLUSH);
             if (err == Z_STREAM_END) break;
         }
 
         if (err != Z_OK && err != Z_STREAM_END) {
             if (err == Z_MEM_ERROR) throw std::bad_alloc();
             string msg = "inflate failed";
-            if (stream.msg) {
+            if (inflate_zstream->msg) {
                 msg += " (";
-                msg += stream.msg;
+                msg += inflate_zstream->msg;
                 msg += ')';
             }
             throw Xapian::DatabaseError(msg);
         }
 
         utag.append(reinterpret_cast<const char *>(buf),
-                    stream.next_out - buf);
+                    inflate_zstream->next_out - buf);
     }
-    if (utag.size() != stream.total_out) {
+    if (utag.size() != inflate_zstream->total_out) {
         string msg = "compressed tag didn't expand to the expected size: ";
         msg += om_tostring(utag.size());
         msg += " != ";
         // OpenBSD's zlib.h uses off_t instead of uLong for total_out.
-        msg += om_tostring((size_t)stream.total_out);
+        msg += om_tostring((size_t)inflate_zstream->total_out);
         throw Xapian::DatabaseCorruptError(msg);
     }
 
-    err = inflateEnd(&stream);
-    if (err != Z_OK) abort();
-
     swap(*tag, utag);
 
     return false;
@@ -1607 +1555 @@
           writable(!readonly_),
           split_p(0),
           compress_strategy(compress_strategy_),
+          deflate_zstream(NULL),
+          inflate_zstream(NULL),
           lazy(lazy_)
 {
-    DEBUGCALL(DB, void, "FlintTable::FlintTable", 
+    DEBUGCALL(DB, void, "FlintTable::FlintTable",
               tablename_ << "," << path_ << ", " << readonly_ << ", " <<
               compress_strategy_ << ", " << lazy_);
 }
 
+void
+FlintTable::lazy_alloc_deflate_zstream() const {
+    if (deflate_zstream) {
+        deflateReset(deflate_zstream);
+        return;
+    }
+
+    deflate_zstream = new z_stream;
+
+    deflate_zstream->zalloc = reinterpret_cast<alloc_func>(0);
+    deflate_zstream->zfree = reinterpret_cast<free_func>(0);
+    deflate_zstream->opaque = (voidpf)0;
+
+    // -15 means raw deflate with 32K LZ77 window (largest)
+    // memLevel 9 is the highest (8 is default)
+    int err;
+    err = deflateInit2(deflate_zstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 9,
+                       compress_strategy);
+    if (err != Z_OK) {
+        if (err == Z_MEM_ERROR) throw std::bad_alloc();
+        string msg = "deflateInit2 failed. err = " + err;
+        if (deflate_zstream->msg) {
+            msg += " (";
+            msg += deflate_zstream->msg;
+            msg += ')';
+        }
+        throw Xapian::DatabaseError(msg);
+    }
+}
+
+void
+FlintTable::lazy_alloc_inflate_zstream() const {
+    if (inflate_zstream) {
+        inflateReset(inflate_zstream);
+        return;
+    }
+
+    inflate_zstream = new z_stream;
+
+    inflate_zstream->zalloc = reinterpret_cast<alloc_func>(0);
+    inflate_zstream->zfree = reinterpret_cast<free_func>(0);
+
+    inflate_zstream->next_in = Z_NULL;
+    inflate_zstream->avail_in = 0;
+
+    int err = inflateInit2(inflate_zstream, -15);
+    if (err != Z_OK) {
+        if (err == Z_MEM_ERROR) throw std::bad_alloc();
+        string msg = "inflateInit2 failed";
+        if (inflate_zstream->msg) {
+            msg += " (";
+            msg += inflate_zstream->msg;
+            msg += ')';
+        }
+        throw Xapian::DatabaseError(msg);
+    }
+}
+
 bool
 FlintTable::exists() const {
     DEBUGCALL(DB, bool, "FlintTable::exists", "");
@@ -1698 +1706 @@
 FlintTable::~FlintTable() {
     DEBUGCALL(DB, void, "FlintTable::~FlintTable", "");
     FlintTable::close();
+
+    if (deflate_zstream) {
+        int err = deflateEnd(deflate_zstream);
+        if (err != Z_OK) {
+            // FIXME - display the warning message somewhere.
+            //string msg = "deflateEnd failed. err = " + err;
+            //if (deflate_zstream->msg) {
+            //    msg += " (";
+            //    msg += deflate_zstream->msg;
+            //    msg += ')';
+            //}
+            //
+            // Note - we mustn't throw an exception because we're in a
+            // destructor.  If the destructor was called due to stack
+            // unwinding for another exception, this will result in a
+            // call to terminate().
+        }
+        delete deflate_zstream;
+    }
+
+    if (inflate_zstream) {
+        int err = inflateEnd(inflate_zstream);
+        if (err != Z_OK) {
+            // FIXME - display the warning message somewhere.
+            //string msg = "inflateEnd failed. err = " + err;
+            //if (inflate_zstream->msg) {
+            //    msg += " (";
+            //    msg += inflate_zstream->msg;
+            //    msg += ')';
+            //}
+            //
+            // Note - we mustn't throw an exception because we're in a
+            // destructor.  If the destructor was called due to stack
+            // unwinding for another exception, this will result in a
+            // call to terminate().
+        }
+        delete inflate_zstream;
+    }
 }
 
 void FlintTable::close(bool permanent) {

backends/flint/flint_table.h

@@ -290 +290 @@
          *  Any outstanding changes (ie, changes made without commit() having
          *  subsequently been called) will be lost.
          */
-        ~FlintTable();
+        virtual ~FlintTable();
 
         /** Close the Btree.  This closes and frees any of the btree
          *  structures which have been created and opened.
@@ -595 +595 @@
            return (base_letter == 'A') ? 'B' : 'A';
         }
 
+        void lazy_alloc_deflate_zstream() const;
+        void lazy_alloc_inflate_zstream() const;
+
         /** revision number of the opened B-tree. */
         flint_revision_number_t revision_number;
 
@@ -713 +716 @@
          *  Z_RLE. */
         int compress_strategy;
 
+        /** Zlib state object */
+        mutable z_stream *deflate_zstream;
+        mutable z_stream *inflate_zstream;
+
         /// If true, don't create the table until it's needed.
         bool lazy;