Ticket #325: Refactor-zlib-usage-in-FlintTable-and-ChertTable-to-avoid-reallocat.3.patch
File Refactor-zlib-usage-in-FlintTable-and-ChertTable-to-avoid-reallocat.3.patch, 18.6 KB (added by , 16 years ago) |
---|
-
backends/chert/chert_table.h
300 300 * Any outstanding changes (ie, changes made without commit() having 301 301 * subsequently been called) will be lost. 302 302 */ 303 ~ChertTable();303 virtual ~ChertTable(); 304 304 305 305 /** Close the Btree. This closes and frees any of the btree 306 306 * structures which have been created and opened. … … 605 605 /// The name of the table (used when writing changesets). 606 606 const char * tablename; 607 607 608 void lazy_alloc_deflate_zstream() const; 609 void lazy_alloc_inflate_zstream() const; 610 608 611 /** revision number of the opened B-tree. */ 609 612 chert_revision_number_t revision_number; 610 613 … … 723 726 * Z_RLE. */ 724 727 int compress_strategy; 725 728 729 /** Zlib state object */ 730 mutable z_stream *deflate_zstream; 731 mutable z_stream *inflate_zstream; 732 726 733 /// If true, don't create the table until it's needed. 727 734 bool lazy; 728 735 -
backends/chert/chert_table.cc
1058 1058 CompileTimeAssert(DONT_COMPRESS != Z_RLE); 1059 1059 #endif 1060 1060 1061 z_stream stream;1061 lazy_alloc_deflate_zstream(); 1062 1062 1063 stream.zalloc = reinterpret_cast<alloc_func>(0); 1064 stream.zfree = reinterpret_cast<free_func>(0); 1065 stream.opaque = (voidpf)0; 1063 deflate_zstream->next_in = (Bytef *)const_cast<char *>(tag.data()); 1064 deflate_zstream->avail_in = (uInt)tag.size(); 1066 1065 1067 // -15 means raw deflate with 32K LZ77 window (largest)1068 // memLevel 9 is the highest (8 is default)1069 int err;1070 err = deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 9,1071 compress_strategy);1072 if (err != Z_OK) {1073 if (err == Z_MEM_ERROR) throw std::bad_alloc();1074 string msg = "deflateInit2 failed";1075 if (stream.msg) {1076 msg += " (";1077 msg += stream.msg;1078 msg += ')';1079 }1080 throw Xapian::DatabaseError(msg);1081 }1082 1083 stream.next_in = (Bytef *)const_cast<char *>(tag.data());1084 stream.avail_in = (uInt)tag.size();1085 1086 1066 // If compressed size is >= tag.size(), we don't want to compress. 1087 1067 unsigned long blk_len = tag.size() - 1; 1088 1068 unsigned char * blk = new unsigned char[blk_len]; 1089 stream.next_out = blk;1090 stream.avail_out = (uInt)blk_len;1069 deflate_zstream->next_out = blk; 1070 deflate_zstream->avail_out = (uInt)blk_len; 1091 1071 1092 err = deflate(&stream, Z_FINISH);1072 int err = deflate(deflate_zstream, Z_FINISH); 1093 1073 if (err == Z_STREAM_END) { 1094 1074 // If deflate succeeded, then the output was at least one byte 1095 1075 // smaller than the input. 1096 tag.assign(reinterpret_cast<const char *>(blk), stream.total_out);1076 tag.assign(reinterpret_cast<const char *>(blk), deflate_zstream->total_out); 1097 1077 compressed = true; 1098 err = deflateEnd(&stream);1099 if (err != Z_OK) {1100 string msg = "deflateEnd failed";1101 if (stream.msg) {1102 msg += " (";1103 msg += stream.msg;1104 msg += ')';1105 }1106 throw Xapian::DatabaseError(msg);1107 }1108 } else {1109 // Deflate failed - presumably the data wasn't compressible.1110 (void)deflateEnd(&stream);1111 1078 } 1112 1079 1113 1080 delete [] blk; … … 1281 1248 1282 1249 Bytef buf[8192]; 1283 1250 1284 z_stream stream; 1285 stream.next_out = buf; 1286 stream.avail_out = (uInt)sizeof(buf); 1251 lazy_alloc_inflate_zstream(); 1287 1252 1288 stream.zalloc = reinterpret_cast<alloc_func>(0);1289 stream.zfree = reinterpret_cast<free_func>(0);1253 inflate_zstream->next_out = buf; 1254 inflate_zstream->avail_out = (uInt)sizeof(buf); 1290 1255 1291 stream.next_in = Z_NULL;1292 stream.avail_in = 0;1256 inflate_zstream->next_in = (Bytef*)const_cast<char *>(tag->data()); 1257 inflate_zstream->avail_in = (uInt)tag->size(); 1293 1258 1294 int err = inflateInit2(&stream, -15); 1295 if (err != Z_OK) { 1296 if (err == Z_MEM_ERROR) throw std::bad_alloc(); 1297 string msg = "inflateInit2 failed"; 1298 if (stream.msg) { 1299 msg += " ("; 1300 msg += stream.msg; 1301 msg += ')'; 1302 } 1303 throw Xapian::DatabaseError(msg); 1304 } 1305 1306 stream.next_in = (Bytef*)const_cast<char *>(tag->data()); 1307 stream.avail_in = (uInt)tag->size(); 1308 1259 int err = Z_OK; 1309 1260 while (err != Z_STREAM_END) { 1310 stream.next_out = buf;1311 stream.avail_out = (uInt)sizeof(buf);1312 err = inflate( &stream, Z_SYNC_FLUSH);1313 if (err == Z_BUF_ERROR && stream.avail_in == 0) {1314 LOGLINE(DB, "Z_BUF_ERROR - faking checksum of " << stream.adler);1261 inflate_zstream->next_out = buf; 1262 inflate_zstream->avail_out = (uInt)sizeof(buf); 1263 err = inflate(inflate_zstream, Z_SYNC_FLUSH); 1264 if (err == Z_BUF_ERROR && inflate_zstream->avail_in == 0) { 1265 LOGLINE(DB, "Z_BUF_ERROR - faking checksum of " << inflate_zstream->adler); 1315 1266 Bytef header2[4]; 1316 setint4(header2, 0, stream.adler);1317 stream.next_in = header2;1318 stream.avail_in = 4;1319 err = inflate( &stream, Z_SYNC_FLUSH);1267 setint4(header2, 0, inflate_zstream->adler); 1268 inflate_zstream->next_in = header2; 1269 inflate_zstream->avail_in = 4; 1270 err = inflate(inflate_zstream, Z_SYNC_FLUSH); 1320 1271 if (err == Z_STREAM_END) break; 1321 1272 } 1322 1273 1323 1274 if (err != Z_OK && err != Z_STREAM_END) { 1324 1275 if (err == Z_MEM_ERROR) throw std::bad_alloc(); 1325 1276 string msg = "inflate failed"; 1326 if ( stream.msg) {1277 if (inflate_zstream->msg) { 1327 1278 msg += " ("; 1328 msg += stream.msg;1279 msg += inflate_zstream->msg; 1329 1280 msg += ')'; 1330 1281 } 1331 1282 throw Xapian::DatabaseError(msg); 1332 1283 } 1333 1284 1334 1285 utag.append(reinterpret_cast<const char *>(buf), 1335 stream.next_out - buf);1286 inflate_zstream->next_out - buf); 1336 1287 } 1337 if (utag.size() != stream.total_out) {1288 if (utag.size() != inflate_zstream->total_out) { 1338 1289 string msg = "compressed tag didn't expand to the expected size: "; 1339 1290 msg += om_tostring(utag.size()); 1340 1291 msg += " != "; 1341 1292 // OpenBSD's zlib.h uses off_t instead of uLong for total_out. 1342 msg += om_tostring((size_t) stream.total_out);1293 msg += om_tostring((size_t)inflate_zstream->total_out); 1343 1294 throw Xapian::DatabaseCorruptError(msg); 1344 1295 } 1345 1296 1346 err = inflateEnd(&stream);1347 if (err != Z_OK) abort();1348 1349 1297 swap(*tag, utag); 1350 1298 1351 1299 RETURN(false); … … 1633 1581 writable(!readonly_), 1634 1582 split_p(0), 1635 1583 compress_strategy(compress_strategy_), 1584 deflate_zstream(NULL), 1585 inflate_zstream(NULL), 1636 1586 lazy(lazy_) 1637 1587 { 1638 1588 LOGCALL_CTOR(DB, "ChertTable", … … 1640 1590 compress_strategy_ << ", " << lazy_); 1641 1591 } 1642 1592 1593 void 1594 ChertTable::lazy_alloc_deflate_zstream() const { 1595 if (deflate_zstream) { 1596 deflateReset(deflate_zstream); 1597 return; 1598 } 1599 1600 deflate_zstream = new z_stream; 1601 1602 deflate_zstream->zalloc = reinterpret_cast<alloc_func>(0); 1603 deflate_zstream->zfree = reinterpret_cast<free_func>(0); 1604 deflate_zstream->opaque = (voidpf)0; 1605 1606 // -15 means raw deflate with 32K LZ77 window (largest) 1607 // memLevel 9 is the highest (8 is default) 1608 int err; 1609 err = deflateInit2(deflate_zstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 9, 1610 compress_strategy); 1611 if (err != Z_OK) { 1612 if (err == Z_MEM_ERROR) throw std::bad_alloc(); 1613 string msg = "deflateInit2 failed. err = " + err; 1614 if (deflate_zstream->msg) { 1615 msg += " ("; 1616 msg += deflate_zstream->msg; 1617 msg += ')'; 1618 } 1619 throw Xapian::DatabaseError(msg); 1620 } 1621 } 1622 1623 void 1624 ChertTable::lazy_alloc_inflate_zstream() const { 1625 if (inflate_zstream) { 1626 inflateReset(inflate_zstream); 1627 return; 1628 } 1629 1630 inflate_zstream = new z_stream; 1631 1632 inflate_zstream->zalloc = reinterpret_cast<alloc_func>(0); 1633 inflate_zstream->zfree = reinterpret_cast<free_func>(0); 1634 1635 inflate_zstream->next_in = Z_NULL; 1636 inflate_zstream->avail_in = 0; 1637 1638 int err = inflateInit2(inflate_zstream, -15); 1639 if (err != Z_OK) { 1640 if (err == Z_MEM_ERROR) throw std::bad_alloc(); 1641 string msg = "inflateInit2 failed"; 1642 if (inflate_zstream->msg) { 1643 msg += " ("; 1644 msg += inflate_zstream->msg; 1645 msg += ')'; 1646 } 1647 throw Xapian::DatabaseError(msg); 1648 } 1649 } 1650 1643 1651 bool 1644 1652 ChertTable::exists() const { 1645 1653 LOGCALL(DB, bool, "ChertTable::exists", ""); … … 1724 1732 ChertTable::~ChertTable() { 1725 1733 LOGCALL_DTOR(DB, "ChertTable"); 1726 1734 ChertTable::close(); 1735 1736 if (deflate_zstream) { 1737 int err = deflateEnd(deflate_zstream); 1738 if (err != Z_OK) { 1739 // FIXME - display the warning message somewhere. 1740 //string msg = "deflateEnd failed. err = " + err; 1741 //if (deflate_zstream->msg) { 1742 // msg += " ("; 1743 // msg += deflate_zstream->msg; 1744 // msg += ')'; 1745 //} 1746 // 1747 // Note - we mustn't throw an exception because we're in a 1748 // destructor. If the destructor was called due to stack 1749 // unwinding for another exception, this will result in a 1750 // call to terminate(). 1751 } 1752 delete deflate_zstream; 1753 } 1754 1755 if (inflate_zstream) { 1756 int err = inflateEnd(inflate_zstream); 1757 if (err != Z_OK) { 1758 // FIXME - display the warning message somewhere. 1759 //string msg = "inflateEnd failed. err = " + err; 1760 //if (inflate_zstream->msg) { 1761 // msg += " ("; 1762 // msg += inflate_zstream->msg; 1763 // msg += ')'; 1764 //} 1765 // 1766 // Note - we mustn't throw an exception because we're in a 1767 // destructor. If the destructor was called due to stack 1768 // unwinding for another exception, this will result in a 1769 // call to terminate(). 1770 } 1771 delete inflate_zstream; 1772 } 1727 1773 } 1728 1774 1729 1775 void ChertTable::close(bool permanent) { -
backends/flint/flint_table.cc
1038 1038 CompileTimeAssert(DONT_COMPRESS != Z_RLE); 1039 1039 #endif 1040 1040 1041 z_stream stream;1041 lazy_alloc_deflate_zstream(); 1042 1042 1043 stream.zalloc = reinterpret_cast<alloc_func>(0); 1044 stream.zfree = reinterpret_cast<free_func>(0); 1045 stream.opaque = (voidpf)0; 1043 deflate_zstream->next_in = (Bytef *)const_cast<char *>(tag.data()); 1044 deflate_zstream->avail_in = (uInt)tag.size(); 1046 1045 1047 // -15 means raw deflate with 32K LZ77 window (largest)1048 // memLevel 9 is the highest (8 is default)1049 int err;1050 err = deflateInit2(&stream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 9,1051 compress_strategy);1052 if (err != Z_OK) {1053 if (err == Z_MEM_ERROR) throw std::bad_alloc();1054 string msg = "deflateInit2 failed";1055 if (stream.msg) {1056 msg += " (";1057 msg += stream.msg;1058 msg += ')';1059 }1060 throw Xapian::DatabaseError(msg);1061 }1062 1063 stream.next_in = (Bytef *)const_cast<char *>(tag.data());1064 stream.avail_in = (uInt)tag.size();1065 1066 1046 // If compressed size is >= tag.size(), we don't want to compress. 1067 1047 unsigned long blk_len = tag.size() - 1; 1068 1048 unsigned char * blk = new unsigned char[blk_len]; 1069 stream.next_out = blk;1070 stream.avail_out = (uInt)blk_len;1049 deflate_zstream->next_out = blk; 1050 deflate_zstream->avail_out = (uInt)blk_len; 1071 1051 1072 err = deflate(&stream, Z_FINISH);1052 int err = deflate(deflate_zstream, Z_FINISH); 1073 1053 if (err == Z_STREAM_END) { 1074 1054 // If deflate succeeded, then the output was at least one byte 1075 1055 // smaller than the input. 1076 tag.assign(reinterpret_cast<const char *>(blk), stream.total_out);1056 tag.assign(reinterpret_cast<const char *>(blk), deflate_zstream->total_out); 1077 1057 compressed = true; 1078 err = deflateEnd(&stream);1079 if (err != Z_OK) {1080 string msg = "deflateEnd failed";1081 if (stream.msg) {1082 msg += " (";1083 msg += stream.msg;1084 msg += ')';1085 }1086 throw Xapian::DatabaseError(msg);1087 }1088 } else {1089 // Deflate failed - presumably the data wasn't compressible.1090 (void)deflateEnd(&stream);1091 1058 } 1092 1059 1093 1060 delete [] blk; … … 1260 1227 1261 1228 Bytef buf[8192]; 1262 1229 1263 z_stream stream; 1264 stream.next_out = buf; 1265 stream.avail_out = (uInt)sizeof(buf); 1230 lazy_alloc_inflate_zstream(); 1266 1231 1267 stream.zalloc = reinterpret_cast<alloc_func>(0);1268 stream.zfree = reinterpret_cast<free_func>(0);1232 inflate_zstream->next_out = buf; 1233 inflate_zstream->avail_out = (uInt)sizeof(buf); 1269 1234 1270 stream.next_in = Z_NULL;1271 stream.avail_in = 0;1235 inflate_zstream->next_in = (Bytef*)const_cast<char *>(tag->data()); 1236 inflate_zstream->avail_in = (uInt)tag->size(); 1272 1237 1273 int err = inflateInit2(&stream, -15); 1274 if (err != Z_OK) { 1275 if (err == Z_MEM_ERROR) throw std::bad_alloc(); 1276 string msg = "inflateInit2 failed"; 1277 if (stream.msg) { 1278 msg += " ("; 1279 msg += stream.msg; 1280 msg += ')'; 1281 } 1282 throw Xapian::DatabaseError(msg); 1283 } 1284 1285 stream.next_in = (Bytef*)const_cast<char *>(tag->data()); 1286 stream.avail_in = (uInt)tag->size(); 1287 1238 int err = Z_OK; 1288 1239 while (err != Z_STREAM_END) { 1289 stream.next_out = buf;1290 stream.avail_out = (uInt)sizeof(buf);1291 err = inflate( &stream, Z_SYNC_FLUSH);1292 if (err == Z_BUF_ERROR && stream.avail_in == 0) {1293 LOGLINE(DB, "Z_BUF_ERROR - faking checksum of " << stream.adler);1240 inflate_zstream->next_out = buf; 1241 inflate_zstream->avail_out = (uInt)sizeof(buf); 1242 err = inflate(inflate_zstream, Z_SYNC_FLUSH); 1243 if (err == Z_BUF_ERROR && inflate_zstream->avail_in == 0) { 1244 LOGLINE(DB, "Z_BUF_ERROR - faking checksum of " << inflate_zstream->adler); 1294 1245 Bytef header2[4]; 1295 setint4(header2, 0, stream.adler);1296 stream.next_in = header2;1297 stream.avail_in = 4;1298 err = inflate( &stream, Z_SYNC_FLUSH);1246 setint4(header2, 0, inflate_zstream->adler); 1247 inflate_zstream->next_in = header2; 1248 inflate_zstream->avail_in = 4; 1249 err = inflate(inflate_zstream, Z_SYNC_FLUSH); 1299 1250 if (err == Z_STREAM_END) break; 1300 1251 } 1301 1252 1302 1253 if (err != Z_OK && err != Z_STREAM_END) { 1303 1254 if (err == Z_MEM_ERROR) throw std::bad_alloc(); 1304 1255 string msg = "inflate failed"; 1305 if ( stream.msg) {1256 if (inflate_zstream->msg) { 1306 1257 msg += " ("; 1307 msg += stream.msg;1258 msg += inflate_zstream->msg; 1308 1259 msg += ')'; 1309 1260 } 1310 1261 throw Xapian::DatabaseError(msg); 1311 1262 } 1312 1263 1313 1264 utag.append(reinterpret_cast<const char *>(buf), 1314 stream.next_out - buf);1265 inflate_zstream->next_out - buf); 1315 1266 } 1316 if (utag.size() != stream.total_out) {1267 if (utag.size() != inflate_zstream->total_out) { 1317 1268 string msg = "compressed tag didn't expand to the expected size: "; 1318 1269 msg += om_tostring(utag.size()); 1319 1270 msg += " != "; 1320 1271 // OpenBSD's zlib.h uses off_t instead of uLong for total_out. 1321 msg += om_tostring((size_t) stream.total_out);1272 msg += om_tostring((size_t)inflate_zstream->total_out); 1322 1273 throw Xapian::DatabaseCorruptError(msg); 1323 1274 } 1324 1275 1325 err = inflateEnd(&stream);1326 if (err != Z_OK) abort();1327 1328 1276 swap(*tag, utag); 1329 1277 1330 1278 return false; … … 1607 1555 writable(!readonly_), 1608 1556 split_p(0), 1609 1557 compress_strategy(compress_strategy_), 1558 deflate_zstream(NULL), 1559 inflate_zstream(NULL), 1610 1560 lazy(lazy_) 1611 1561 { 1612 DEBUGCALL(DB, void, "FlintTable::FlintTable", 1562 DEBUGCALL(DB, void, "FlintTable::FlintTable", 1613 1563 tablename_ << "," << path_ << ", " << readonly_ << ", " << 1614 1564 compress_strategy_ << ", " << lazy_); 1615 1565 } 1616 1566 1567 void 1568 FlintTable::lazy_alloc_deflate_zstream() const { 1569 if (deflate_zstream) { 1570 deflateReset(deflate_zstream); 1571 return; 1572 } 1573 1574 deflate_zstream = new z_stream; 1575 1576 deflate_zstream->zalloc = reinterpret_cast<alloc_func>(0); 1577 deflate_zstream->zfree = reinterpret_cast<free_func>(0); 1578 deflate_zstream->opaque = (voidpf)0; 1579 1580 // -15 means raw deflate with 32K LZ77 window (largest) 1581 // memLevel 9 is the highest (8 is default) 1582 int err; 1583 err = deflateInit2(deflate_zstream, Z_DEFAULT_COMPRESSION, Z_DEFLATED, -15, 9, 1584 compress_strategy); 1585 if (err != Z_OK) { 1586 if (err == Z_MEM_ERROR) throw std::bad_alloc(); 1587 string msg = "deflateInit2 failed. err = " + err; 1588 if (deflate_zstream->msg) { 1589 msg += " ("; 1590 msg += deflate_zstream->msg; 1591 msg += ')'; 1592 } 1593 throw Xapian::DatabaseError(msg); 1594 } 1595 } 1596 1597 void 1598 FlintTable::lazy_alloc_inflate_zstream() const { 1599 if (inflate_zstream) { 1600 inflateReset(inflate_zstream); 1601 return; 1602 } 1603 1604 inflate_zstream = new z_stream; 1605 1606 inflate_zstream->zalloc = reinterpret_cast<alloc_func>(0); 1607 inflate_zstream->zfree = reinterpret_cast<free_func>(0); 1608 1609 inflate_zstream->next_in = Z_NULL; 1610 inflate_zstream->avail_in = 0; 1611 1612 int err = inflateInit2(inflate_zstream, -15); 1613 if (err != Z_OK) { 1614 if (err == Z_MEM_ERROR) throw std::bad_alloc(); 1615 string msg = "inflateInit2 failed"; 1616 if (inflate_zstream->msg) { 1617 msg += " ("; 1618 msg += inflate_zstream->msg; 1619 msg += ')'; 1620 } 1621 throw Xapian::DatabaseError(msg); 1622 } 1623 } 1624 1617 1625 bool 1618 1626 FlintTable::exists() const { 1619 1627 DEBUGCALL(DB, bool, "FlintTable::exists", ""); … … 1698 1706 FlintTable::~FlintTable() { 1699 1707 DEBUGCALL(DB, void, "FlintTable::~FlintTable", ""); 1700 1708 FlintTable::close(); 1709 1710 if (deflate_zstream) { 1711 int err = deflateEnd(deflate_zstream); 1712 if (err != Z_OK) { 1713 // FIXME - display the warning message somewhere. 1714 //string msg = "deflateEnd failed. err = " + err; 1715 //if (deflate_zstream->msg) { 1716 // msg += " ("; 1717 // msg += deflate_zstream->msg; 1718 // msg += ')'; 1719 //} 1720 // 1721 // Note - we mustn't throw an exception because we're in a 1722 // destructor. If the destructor was called due to stack 1723 // unwinding for another exception, this will result in a 1724 // call to terminate(). 1725 } 1726 delete deflate_zstream; 1727 } 1728 1729 if (inflate_zstream) { 1730 int err = inflateEnd(inflate_zstream); 1731 if (err != Z_OK) { 1732 // FIXME - display the warning message somewhere. 1733 //string msg = "inflateEnd failed. err = " + err; 1734 //if (inflate_zstream->msg) { 1735 // msg += " ("; 1736 // msg += inflate_zstream->msg; 1737 // msg += ')'; 1738 //} 1739 // 1740 // Note - we mustn't throw an exception because we're in a 1741 // destructor. If the destructor was called due to stack 1742 // unwinding for another exception, this will result in a 1743 // call to terminate(). 1744 } 1745 delete inflate_zstream; 1746 } 1701 1747 } 1702 1748 1703 1749 void FlintTable::close(bool permanent) { -
backends/flint/flint_table.h
290 290 * Any outstanding changes (ie, changes made without commit() having 291 291 * subsequently been called) will be lost. 292 292 */ 293 ~FlintTable();293 virtual ~FlintTable(); 294 294 295 295 /** Close the Btree. This closes and frees any of the btree 296 296 * structures which have been created and opened. … … 595 595 return (base_letter == 'A') ? 'B' : 'A'; 596 596 } 597 597 598 void lazy_alloc_deflate_zstream() const; 599 void lazy_alloc_inflate_zstream() const; 600 598 601 /** revision number of the opened B-tree. */ 599 602 flint_revision_number_t revision_number; 600 603 … … 713 716 * Z_RLE. */ 714 717 int compress_strategy; 715 718 719 /** Zlib state object */ 720 mutable z_stream *deflate_zstream; 721 mutable z_stream *inflate_zstream; 722 716 723 /// If true, don't create the table until it's needed. 717 724 bool lazy; 718 725