Index: backends/chert/chert_table.h =================================================================== --- backends/chert/chert_table.h (revision 11827) +++ backends/chert/chert_table.h (working copy) @@ -300,7 +300,7 @@ * Any outstanding changes (ie, changes made without commit() having * subsequently been called) will be lost. */ - ~ChertTable(); + virtual ~ChertTable(); /** Close the Btree. This closes and frees any of the btree * structures which have been created and opened. @@ -605,6 +605,9 @@ /// The name of the table (used when writing changesets). const char * tablename; + void lazy_alloc_deflate_zstream() const; + void lazy_alloc_inflate_zstream() const; + /** revision number of the opened B-tree. */ chert_revision_number_t revision_number; @@ -723,6 +726,10 @@ * Z_RLE. */ int compress_strategy; + /** Zlib state object */ + mutable z_stream *deflate_zstream; + mutable z_stream *inflate_zstream; + /// If true, don't create the table until it's needed. bool lazy; Index: backends/chert/chert_table.cc =================================================================== --- backends/chert/chert_table.cc (revision 11827) +++ backends/chert/chert_table.cc (working copy) @@ -1058,56 +1058,23 @@ CompileTimeAssert(DONT_COMPRESS != Z_RLE); #endif - z_stream stream; + lazy_alloc_deflate_zstream(); - stream.zalloc = reinterpret_cast(0); - stream.zfree = reinterpret_cast(0); - stream.opaque = (voidpf)0; + deflate_zstream->next_in = (Bytef *)const_cast(tag.data()); + deflate_zstream->avail_in = (uInt)tag.size(); - // -15 means raw deflate with 32K LZ77 window (largest) - // memLevel 9 is the highest (8 is default) - int err; - err = deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 9, - compress_strategy); - if (err != Z_OK) { - if (err == Z_MEM_ERROR) throw std::bad_alloc(); - string msg = "deflateInit2 failed"; - if (stream.msg) { - msg += " ("; - msg += stream.msg; - msg += ')'; - } - throw Xapian::DatabaseError(msg); - } - - stream.next_in = (Bytef *)const_cast(tag.data()); - stream.avail_in = (uInt)tag.size(); - // If compressed size is >= tag.size(), we don't want to compress. unsigned long blk_len = tag.size() - 1; unsigned char * blk = new unsigned char[blk_len]; - stream.next_out = blk; - stream.avail_out = (uInt)blk_len; + deflate_zstream->next_out = blk; + deflate_zstream->avail_out = (uInt)blk_len; - err = deflate(&stream, Z_FINISH); + int err = deflate(deflate_zstream, Z_FINISH); if (err == Z_STREAM_END) { // If deflate succeeded, then the output was at least one byte // smaller than the input. - tag.assign(reinterpret_cast(blk), stream.total_out); + tag.assign(reinterpret_cast(blk), deflate_zstream->total_out); compressed = true; - err = deflateEnd(&stream); - if (err != Z_OK) { - string msg = "deflateEnd failed"; - if (stream.msg) { - msg += " ("; - msg += stream.msg; - msg += ')'; - } - throw Xapian::DatabaseError(msg); - } - } else { - // Deflate failed - presumably the data wasn't compressible. - (void)deflateEnd(&stream); } delete [] blk; @@ -1281,71 +1248,52 @@ Bytef buf[8192]; - z_stream stream; - stream.next_out = buf; - stream.avail_out = (uInt)sizeof(buf); + lazy_alloc_inflate_zstream(); - stream.zalloc = reinterpret_cast(0); - stream.zfree = reinterpret_cast(0); + inflate_zstream->next_out = buf; + inflate_zstream->avail_out = (uInt)sizeof(buf); - stream.next_in = Z_NULL; - stream.avail_in = 0; + inflate_zstream->next_in = (Bytef*)const_cast(tag->data()); + inflate_zstream->avail_in = (uInt)tag->size(); - int err = inflateInit2(&stream, -15); - if (err != Z_OK) { - if (err == Z_MEM_ERROR) throw std::bad_alloc(); - string msg = "inflateInit2 failed"; - if (stream.msg) { - msg += " ("; - msg += stream.msg; - msg += ')'; - } - throw Xapian::DatabaseError(msg); - } - - stream.next_in = (Bytef*)const_cast(tag->data()); - stream.avail_in = (uInt)tag->size(); - + int err = Z_OK; while (err != Z_STREAM_END) { - stream.next_out = buf; - stream.avail_out = (uInt)sizeof(buf); - err = inflate(&stream, Z_SYNC_FLUSH); - if (err == Z_BUF_ERROR && stream.avail_in == 0) { - LOGLINE(DB, "Z_BUF_ERROR - faking checksum of " << stream.adler); + inflate_zstream->next_out = buf; + inflate_zstream->avail_out = (uInt)sizeof(buf); + err = inflate(inflate_zstream, Z_SYNC_FLUSH); + if (err == Z_BUF_ERROR && inflate_zstream->avail_in == 0) { + LOGLINE(DB, "Z_BUF_ERROR - faking checksum of " << inflate_zstream->adler); Bytef header2[4]; - setint4(header2, 0, stream.adler); - stream.next_in = header2; - stream.avail_in = 4; - err = inflate(&stream, Z_SYNC_FLUSH); + setint4(header2, 0, inflate_zstream->adler); + inflate_zstream->next_in = header2; + inflate_zstream->avail_in = 4; + err = inflate(inflate_zstream, Z_SYNC_FLUSH); if (err == Z_STREAM_END) break; } if (err != Z_OK && err != Z_STREAM_END) { if (err == Z_MEM_ERROR) throw std::bad_alloc(); string msg = "inflate failed"; - if (stream.msg) { + if (inflate_zstream->msg) { msg += " ("; - msg += stream.msg; + msg += inflate_zstream->msg; msg += ')'; } throw Xapian::DatabaseError(msg); } utag.append(reinterpret_cast(buf), - stream.next_out - buf); + inflate_zstream->next_out - buf); } - if (utag.size() != stream.total_out) { + if (utag.size() != inflate_zstream->total_out) { string msg = "compressed tag didn't expand to the expected size: "; msg += om_tostring(utag.size()); msg += " != "; // OpenBSD's zlib.h uses off_t instead of uLong for total_out. - msg += om_tostring((size_t)stream.total_out); + msg += om_tostring((size_t)inflate_zstream->total_out); throw Xapian::DatabaseCorruptError(msg); } - err = inflateEnd(&stream); - if (err != Z_OK) abort(); - swap(*tag, utag); RETURN(false); @@ -1633,6 +1581,8 @@ writable(!readonly_), split_p(0), compress_strategy(compress_strategy_), + deflate_zstream(NULL), + inflate_zstream(NULL), lazy(lazy_) { LOGCALL_CTOR(DB, "ChertTable", @@ -1640,6 +1590,64 @@ compress_strategy_ << ", " << lazy_); } +void +ChertTable::lazy_alloc_deflate_zstream() const { + if (deflate_zstream) { + deflateReset(deflate_zstream); + return; + } + + deflate_zstream = new z_stream; + + deflate_zstream->zalloc = reinterpret_cast(0); + deflate_zstream->zfree = reinterpret_cast(0); + deflate_zstream->opaque = (voidpf)0; + + // -15 means raw deflate with 32K LZ77 window (largest) + // memLevel 9 is the highest (8 is default) + int err; + err = deflateInit2(deflate_zstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 9, + compress_strategy); + if (err != Z_OK) { + if (err == Z_MEM_ERROR) throw std::bad_alloc(); + string msg = "deflateInit2 failed. err = " + err; + if (deflate_zstream->msg) { + msg += " ("; + msg += deflate_zstream->msg; + msg += ')'; + } + throw Xapian::DatabaseError(msg); + } +} + +void +ChertTable::lazy_alloc_inflate_zstream() const { + if (inflate_zstream) { + inflateReset(inflate_zstream); + return; + } + + inflate_zstream = new z_stream; + + inflate_zstream->zalloc = reinterpret_cast(0); + inflate_zstream->zfree = reinterpret_cast(0); + + inflate_zstream->next_in = Z_NULL; + inflate_zstream->avail_in = 0; + + int err = inflateInit2(inflate_zstream, -15); + if (err != Z_OK) { + if (err == Z_MEM_ERROR) throw std::bad_alloc(); + string msg = "inflateInit2 failed"; + if (inflate_zstream->msg) { + msg += " ("; + msg += inflate_zstream->msg; + msg += ')'; + } + throw Xapian::DatabaseError(msg); + } +} + bool ChertTable::exists() const { LOGCALL(DB, bool, "ChertTable::exists", ""); @@ -1724,6 +1732,44 @@ ChertTable::~ChertTable() { LOGCALL_DTOR(DB, "ChertTable"); ChertTable::close(); + + if (deflate_zstream) { + int err = deflateEnd(deflate_zstream); + if (err != Z_OK) { + // FIXME - display the warning message somewhere. + //string msg = "deflateEnd failed. err = " + err; + //if (deflate_zstream->msg) { + // msg += " ("; + // msg += deflate_zstream->msg; + // msg += ')'; + //} + // + // Note - we mustn't throw an exception because we're in a + // destructor. If the destructor was called due to stack + // unwinding for another exception, this will result in a + // call to terminate(). + } + delete deflate_zstream; + } + + if (inflate_zstream) { + int err = inflateEnd(inflate_zstream); + if (err != Z_OK) { + // FIXME - display the warning message somewhere. + //string msg = "inflateEnd failed. err = " + err; + //if (inflate_zstream->msg) { + // msg += " ("; + // msg += inflate_zstream->msg; + // msg += ')'; + //} + // + // Note - we mustn't throw an exception because we're in a + // destructor. If the destructor was called due to stack + // unwinding for another exception, this will result in a + // call to terminate(). + } + delete inflate_zstream; + } } void ChertTable::close(bool permanent) { Index: backends/flint/flint_table.cc =================================================================== --- backends/flint/flint_table.cc (revision 11827) +++ backends/flint/flint_table.cc (working copy) @@ -1038,56 +1038,23 @@ CompileTimeAssert(DONT_COMPRESS != Z_RLE); #endif - z_stream stream; + lazy_alloc_deflate_zstream(); - stream.zalloc = reinterpret_cast(0); - stream.zfree = reinterpret_cast(0); - stream.opaque = (voidpf)0; + deflate_zstream->next_in = (Bytef *)const_cast(tag.data()); + deflate_zstream->avail_in = (uInt)tag.size(); - // -15 means raw deflate with 32K LZ77 window (largest) - // memLevel 9 is the highest (8 is default) - int err; - err = deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 9, - compress_strategy); - if (err != Z_OK) { - if (err == Z_MEM_ERROR) throw std::bad_alloc(); - string msg = "deflateInit2 failed"; - if (stream.msg) { - msg += " ("; - msg += stream.msg; - msg += ')'; - } - throw Xapian::DatabaseError(msg); - } - - stream.next_in = (Bytef *)const_cast(tag.data()); - stream.avail_in = (uInt)tag.size(); - // If compressed size is >= tag.size(), we don't want to compress. unsigned long blk_len = tag.size() - 1; unsigned char * blk = new unsigned char[blk_len]; - stream.next_out = blk; - stream.avail_out = (uInt)blk_len; + deflate_zstream->next_out = blk; + deflate_zstream->avail_out = (uInt)blk_len; - err = deflate(&stream, Z_FINISH); + int err = deflate(deflate_zstream, Z_FINISH); if (err == Z_STREAM_END) { // If deflate succeeded, then the output was at least one byte // smaller than the input. - tag.assign(reinterpret_cast(blk), stream.total_out); + tag.assign(reinterpret_cast(blk), deflate_zstream->total_out); compressed = true; - err = deflateEnd(&stream); - if (err != Z_OK) { - string msg = "deflateEnd failed"; - if (stream.msg) { - msg += " ("; - msg += stream.msg; - msg += ')'; - } - throw Xapian::DatabaseError(msg); - } - } else { - // Deflate failed - presumably the data wasn't compressible. - (void)deflateEnd(&stream); } delete [] blk; @@ -1260,71 +1227,52 @@ Bytef buf[8192]; - z_stream stream; - stream.next_out = buf; - stream.avail_out = (uInt)sizeof(buf); + lazy_alloc_inflate_zstream(); - stream.zalloc = reinterpret_cast(0); - stream.zfree = reinterpret_cast(0); + inflate_zstream->next_out = buf; + inflate_zstream->avail_out = (uInt)sizeof(buf); - stream.next_in = Z_NULL; - stream.avail_in = 0; + inflate_zstream->next_in = (Bytef*)const_cast(tag->data()); + inflate_zstream->avail_in = (uInt)tag->size(); - int err = inflateInit2(&stream, -15); - if (err != Z_OK) { - if (err == Z_MEM_ERROR) throw std::bad_alloc(); - string msg = "inflateInit2 failed"; - if (stream.msg) { - msg += " ("; - msg += stream.msg; - msg += ')'; - } - throw Xapian::DatabaseError(msg); - } - - stream.next_in = (Bytef*)const_cast(tag->data()); - stream.avail_in = (uInt)tag->size(); - + int err = Z_OK; while (err != Z_STREAM_END) { - stream.next_out = buf; - stream.avail_out = (uInt)sizeof(buf); - err = inflate(&stream, Z_SYNC_FLUSH); - if (err == Z_BUF_ERROR && stream.avail_in == 0) { - LOGLINE(DB, "Z_BUF_ERROR - faking checksum of " << stream.adler); + inflate_zstream->next_out = buf; + inflate_zstream->avail_out = (uInt)sizeof(buf); + err = inflate(inflate_zstream, Z_SYNC_FLUSH); + if (err == Z_BUF_ERROR && inflate_zstream->avail_in == 0) { + LOGLINE(DB, "Z_BUF_ERROR - faking checksum of " << inflate_zstream->adler); Bytef header2[4]; - setint4(header2, 0, stream.adler); - stream.next_in = header2; - stream.avail_in = 4; - err = inflate(&stream, Z_SYNC_FLUSH); + setint4(header2, 0, inflate_zstream->adler); + inflate_zstream->next_in = header2; + inflate_zstream->avail_in = 4; + err = inflate(inflate_zstream, Z_SYNC_FLUSH); if (err == Z_STREAM_END) break; } if (err != Z_OK && err != Z_STREAM_END) { if (err == Z_MEM_ERROR) throw std::bad_alloc(); string msg = "inflate failed"; - if (stream.msg) { + if (inflate_zstream->msg) { msg += " ("; - msg += stream.msg; + msg += inflate_zstream->msg; msg += ')'; } throw Xapian::DatabaseError(msg); } utag.append(reinterpret_cast(buf), - stream.next_out - buf); + inflate_zstream->next_out - buf); } - if (utag.size() != stream.total_out) { + if (utag.size() != inflate_zstream->total_out) { string msg = "compressed tag didn't expand to the expected size: "; msg += om_tostring(utag.size()); msg += " != "; // OpenBSD's zlib.h uses off_t instead of uLong for total_out. - msg += om_tostring((size_t)stream.total_out); + msg += om_tostring((size_t)inflate_zstream->total_out); throw Xapian::DatabaseCorruptError(msg); } - err = inflateEnd(&stream); - if (err != Z_OK) abort(); - swap(*tag, utag); return false; @@ -1607,13 +1555,73 @@ writable(!readonly_), split_p(0), compress_strategy(compress_strategy_), + deflate_zstream(NULL), + inflate_zstream(NULL), lazy(lazy_) { - DEBUGCALL(DB, void, "FlintTable::FlintTable", + DEBUGCALL(DB, void, "FlintTable::FlintTable", tablename_ << "," << path_ << ", " << readonly_ << ", " << compress_strategy_ << ", " << lazy_); } +void +FlintTable::lazy_alloc_deflate_zstream() const { + if (deflate_zstream) { + deflateReset(deflate_zstream); + return; + } + + deflate_zstream = new z_stream; + + deflate_zstream->zalloc = reinterpret_cast(0); + deflate_zstream->zfree = reinterpret_cast(0); + deflate_zstream->opaque = (voidpf)0; + + // -15 means raw deflate with 32K LZ77 window (largest) + // memLevel 9 is the highest (8 is default) + int err; + err = deflateInit2(deflate_zstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 9, + compress_strategy); + if (err != Z_OK) { + if (err == Z_MEM_ERROR) throw std::bad_alloc(); + string msg = "deflateInit2 failed. err = " + err; + if (deflate_zstream->msg) { + msg += " ("; + msg += deflate_zstream->msg; + msg += ')'; + } + throw Xapian::DatabaseError(msg); + } +} + +void +FlintTable::lazy_alloc_inflate_zstream() const { + if (inflate_zstream) { + inflateReset(inflate_zstream); + return; + } + + inflate_zstream = new z_stream; + + inflate_zstream->zalloc = reinterpret_cast(0); + inflate_zstream->zfree = reinterpret_cast(0); + + inflate_zstream->next_in = Z_NULL; + inflate_zstream->avail_in = 0; + + int err = inflateInit2(inflate_zstream, -15); + if (err != Z_OK) { + if (err == Z_MEM_ERROR) throw std::bad_alloc(); + string msg = "inflateInit2 failed"; + if (inflate_zstream->msg) { + msg += " ("; + msg += inflate_zstream->msg; + msg += ')'; + } + throw Xapian::DatabaseError(msg); + } +} + bool FlintTable::exists() const { DEBUGCALL(DB, bool, "FlintTable::exists", ""); @@ -1698,6 +1706,44 @@ FlintTable::~FlintTable() { DEBUGCALL(DB, void, "FlintTable::~FlintTable", ""); FlintTable::close(); + + if (deflate_zstream) { + int err = deflateEnd(deflate_zstream); + if (err != Z_OK) { + // FIXME - display the warning message somewhere. + //string msg = "deflateEnd failed. err = " + err; + //if (deflate_zstream->msg) { + // msg += " ("; + // msg += deflate_zstream->msg; + // msg += ')'; + //} + // + // Note - we mustn't throw an exception because we're in a + // destructor. If the destructor was called due to stack + // unwinding for another exception, this will result in a + // call to terminate(). + } + delete deflate_zstream; + } + + if (inflate_zstream) { + int err = inflateEnd(inflate_zstream); + if (err != Z_OK) { + // FIXME - display the warning message somewhere. + //string msg = "inflateEnd failed. err = " + err; + //if (inflate_zstream->msg) { + // msg += " ("; + // msg += inflate_zstream->msg; + // msg += ')'; + //} + // + // Note - we mustn't throw an exception because we're in a + // destructor. If the destructor was called due to stack + // unwinding for another exception, this will result in a + // call to terminate(). + } + delete inflate_zstream; + } } void FlintTable::close(bool permanent) { Index: backends/flint/flint_table.h =================================================================== --- backends/flint/flint_table.h (revision 11827) +++ backends/flint/flint_table.h (working copy) @@ -290,7 +290,7 @@ * Any outstanding changes (ie, changes made without commit() having * subsequently been called) will be lost. */ - ~FlintTable(); + virtual ~FlintTable(); /** Close the Btree. This closes and frees any of the btree * structures which have been created and opened. @@ -595,6 +595,9 @@ return (base_letter == 'A') ? 'B' : 'A'; } + void lazy_alloc_deflate_zstream() const; + void lazy_alloc_inflate_zstream() const; + /** revision number of the opened B-tree. */ flint_revision_number_t revision_number; @@ -713,6 +716,10 @@ * Z_RLE. */ int compress_strategy; + /** Zlib state object */ + mutable z_stream *deflate_zstream; + mutable z_stream *inflate_zstream; + /// If true, don't create the table until it's needed. bool lazy;