Ticket #431: valuestream_pythonic_iterators.patch

python/pythontest2.py

@@ -633 +633 @@
                          'Iterator has moved, and does not support random access',
                          getattr, postings[i], 'positer')
 
+def test_valuestream_iter():
+    """Test a valuestream iterator on Database.
+
+    """
+    db = setup_database()
+
+    # Check basic iteration
+    expect([(item.docid, item.value) for item in db.valuestream(0)],
+           [(3, '\xa4'), (4, '\xa2'), (5, '\xa4')])
+    expect([(item.docid, item.value) for item in db.valuestream(1)], [])
+    expect([(item.docid, item.value) for item in db.valuestream(5)],
+           [(5, "five")])
+    expect([(item.docid, item.value) for item in db.valuestream(9)],
+           [(5, "nine")])
+
+    # Test skip_to() on iterator with no values, and behaviours when called
+    # after already returning StopIteration.
+    i = db.valuestream(1)
+    expect_exception(StopIteration, "", i.skip_to, 1)
+    expect_exception(StopIteration, "", i.skip_to, 1)
+    i = db.valuestream(1)
+    expect_exception(StopIteration, "", i.skip_to, 1)
+    expect_exception(StopIteration, "", i.next)
+    i = db.valuestream(1)
+    expect_exception(StopIteration, "", i.next)
+    expect_exception(StopIteration, "", i.skip_to, 1)
+
+    # Test that skipping to a value works, and that skipping doesn't have to
+    # advance.
+    i = db.valuestream(0)
+    item = i.skip_to(4)
+    expect((item.docid, item.value), (4, '\xa2'))
+    item = i.skip_to(4)
+    expect((item.docid, item.value), (4, '\xa2'))
+    item = i.skip_to(1)
+    expect((item.docid, item.value), (4, '\xa2'))
+    item = i.skip_to(5)
+    expect((item.docid, item.value), (5, '\xa4'))
+    expect_exception(StopIteration, "", i.skip_to, 6)
+
+    # Test that alternating skip_to() and next() works.
+    i = db.valuestream(0)
+    item = i.next()
+    expect((item.docid, item.value), (3, '\xa4'))
+    item = i.skip_to(4)
+    expect((item.docid, item.value), (4, '\xa2'))
+    item = i.next()
+    expect((item.docid, item.value), (5, '\xa4'))
+    expect_exception(StopIteration, "", i.skip_to, 6)
+
+    # Test that next works correctly after skip_to() called with an earlier
+    # item.
+    i = db.valuestream(0)
+    item = i.skip_to(4)
+    expect((item.docid, item.value), (4, '\xa2'))
+    item = i.skip_to(1)
+    expect((item.docid, item.value), (4, '\xa2'))
+    item = i.next()
+    expect((item.docid, item.value), (5, '\xa4'))
+
+    # Test that next works correctly after skipping to last item
+    i = db.valuestream(0)
+    item = i.skip_to(5)
+    expect((item.docid, item.value), (5, '\xa4'))
+    expect_exception(StopIteration, "", i.next)
+
 def test_position_iter():
     """Test position iterator for a document in a database.
 

python/extra.i

@@ -1145 +1145 @@
     return ValueIter(self.values_begin(), self.values_end())
 Document.values = _document_gen_values_iter
 
+
+##########################################
+# Support for iteration of value streams #
+##########################################
+
+class ValueStreamItem(object):
+    """An item returned from iteration of the values in a document.
+
+    The item supports access to the following attributes:
+
+     - `docid`: The docid for the item.
+     - `value`: The contents of the value.
+
+    """
+
+    __slots__ = ('docid', 'value', )
+
+    def __init__(self, docid, value):
+        self.docid = docid
+        self.value = value
+
+class ValueStreamIter(object):
+    """An iterator over all the values stored in a document.
+
+    The iterator will return ValueStreamItem objects, in ascending order of value number.
+
+    """
+    def __init__(self, start, end):
+        self.iter = start
+        self.end = end
+        self.moved = True
+
+    def __iter__(self):
+        return self
+
+    # For Python2:
+    def next(self):
+        if not self.moved:
+            self.iter.next()
+            self.moved = True
+
+        if self.iter==self.end:
+            raise StopIteration
+        else:
+            self.moved = False
+            return ValueStreamItem(self.iter.get_docid(), self.iter.get_value())
+
+    # For Python3:
+    def __next__(self):
+        if not self.moved:
+            self.iter.next()
+            self.moved = True
+
+        if self.iter==self.end:
+            raise StopIteration
+        else:
+            self.moved = False
+            return ValueStreamItem(self.iter.get_docid(), self.iter.get_value())
+
+    def skip_to(self, docid):
+        """Skip the iterator forward.
+
+        The iterator is advanced to the first document with a document ID
+        which is greater than or equal to the supplied document ID.
+
+        If there are no such items, this will raise StopIteration.
+
+        This returns the item which the iterator is moved to.  The subsequent
+        item will be returned the next time that next() is called (unless
+        skip_to() is called again first).
+
+        """
+        if self.iter != self.end:
+            self.iter.skip_to(docid)
+        if self.iter == self.end:
+            self.moved = True
+            raise StopIteration
+        self.moved = False
+        return ValueStreamItem(self.iter.get_docid(), self.iter.get_value())
+
+# Modify Database to add a "valuestream()" method, and remove the
+# valuestream_begin() and valuestream_end() methods.
+def wrapper():
+    vs_begin = Database.valuestream_begin
+    vs_end = Database.valuestream_end
+    def _database_gen_valuestream_iter(self, slot):
+        """Get an iterator over all the values stored in a slot in the database.
+
+        The iterator will return ValueStreamItem objects, in ascending order of
+        document id.
+
+        """
+        return ValueStreamIter(vs_begin(self, slot), vs_end(self, slot))
+    return _database_gen_valuestream_iter
+Database.valuestream = wrapper()
+del wrapper
+del Database.valuestream_begin
+del Database.valuestream_end
+
 # Set the list of names which should be public.
 # Note that this needs to happen at the end of xapian.py.
 __all__ = []

xapian.i

@@ -183 +183 @@
         }
     }
 
-    Xapian::valueno get_valueno();
+    Xapian::docid get_docid() const;
+    Xapian::valueno get_valueno() const;
+    void skip_to(Xapian::docid docid_or_slot);
+    bool check(Xapian::docid docid);
     std::string get_description() const;
 };
 
@@ -519 +522 @@
         doccount get_value_freq(Xapian::valueno valno) const;
         string get_value_lower_bound(Xapian::valueno valno) const;
         string get_value_upper_bound(Xapian::valueno valno) const;
+        Xapian::termcount get_doclength_lower_bound() const;
+        Xapian::termcount get_doclength_upper_bound() const;
+        Xapian::termcount get_wdf_upper_bound(const std::string & term) const;
+        ValueIterator valuestream_begin(Xapian::valueno slot) const;
+        ValueIteratorEnd_ valuestream_end(Xapian::valueno) const;
         doclength get_doclength(docid docid) const;
         void keep_alive();
         Document get_document(docid did);