Ticket #604: get_data_test.py

#!/usr/bin/env python

import sys
import logging
import getopt
import xapian
import os


def get_memory():
    return "memory used = %s" % os.popen('ps -p %d -o rss -o vsz | tail -1' % os.getpid()).read().strip()


def test(name, get_data):
    logging.info('version: %s', xapian.version_string())
    database = xapian.Database(name)
    enquiry = xapian.Enquire(database)
    enquiry.set_query(xapian.Query.MatchAll)
    matches = enquiry.get_mset(0, database.get_doccount(), None, None)
    total = matches.get_matches_estimated()
    logging.info('matches: %s  %s', total, get_memory())

    for index, match in enumerate(matches, 1):
        if get_data:
            match.document.get_data()

        if index % 10000 == 0:
            logging.info('read: %s  %s', index, get_memory())


def main(arguments):
    level = logging.INFO
    get_data = True
    options, arguments = getopt.getopt(arguments, "l:d")
    for option, argument in options:
        if option == '-l':
            level = getattr(logging, argument.upper())
        elif option == '-d':
            get_data = False

    logging.basicConfig(level=level)
    test(arguments[0], get_data)


if __name__ == '__main__':
    main(sys.argv[1:])