1.1 --- a/iixr.py Wed Sep 09 00:29:30 2009 +0200
1.2 +++ b/iixr.py Wed Sep 09 01:18:04 2009 +0200
1.3 @@ -26,6 +26,7 @@
1.4 from bisect import bisect_right # to find terms in the dictionary index
1.5 from bisect import insort_right # to maintain a sorted list of data for merging
1.6 import bz2, zlib # for field compression
1.7 +from itermerge import itermerge
1.8
1.9 try:
1.10 set
1.11 @@ -1311,11 +1312,9 @@
1.12 other_term, other_doc_positions, other_partition = entries[i]
1.13
1.14 # For such entries, merge the positions.
1.15 - # Since document positions should only appear in a single
1.16 - # partition, a simple update should be sufficient.
1.17
1.18 if other_term == term:
1.19 - doc_positions.update(other_doc_positions)
1.20 + doc_positions = itermerge(doc_positions, other_doc_positions)
1.21 to_update.append(other_partition)
1.22 i += 1
1.23 else:
2.1 --- /dev/null Thu Jan 01 00:00:00 1970 +0000
2.2 +++ b/itermerge.py Wed Sep 09 01:18:04 2009 +0200
2.3 @@ -0,0 +1,54 @@
2.4 +#!/usr/bin/env python
2.5 +
2.6 +def itermerge(seq1, seq2):
2.7 +
2.8 + "Merge 'seq1' and 'seq2' to produce an ordered, combined list of results."
2.9 +
2.10 + results = []
2.11 +
2.12 + iter1 = iter(seq1)
2.13 + iter2 = iter(seq2)
2.14 +
2.15 + t1 = None
2.16 + t2 = None
2.17 +
2.18 + t1 = _itermerge_next(iter1)
2.19 + if t1 is None:
2.20 + _itermerge_fill(iter2, results)
2.21 + return results
2.22 +
2.23 + while 1:
2.24 + if t1 is None:
2.25 + t1 = _itermerge_next(iter1)
2.26 + if t1 is None:
2.27 + results.append(t2)
2.28 + _itermerge_fill(iter2, results)
2.29 + return results
2.30 + else:
2.31 + t2 = _itermerge_next(iter2)
2.32 + if t2 is None:
2.33 + results.append(t1)
2.34 + _itermerge_fill(iter1, results)
2.35 + return results
2.36 +
2.37 + if t1 < t2:
2.38 + results.append(t1)
2.39 + t1 = None
2.40 + else:
2.41 + results.append(t2)
2.42 + t2 = None
2.43 +
2.44 +def _itermerge_next(iter):
2.45 + try:
2.46 + return iter.next()
2.47 + except StopIteration:
2.48 + return None
2.49 +
2.50 +def _itermerge_fill(iter, results):
2.51 + try:
2.52 + while 1:
2.53 + results.append(iter.next())
2.54 + except StopIteration:
2.55 + pass
2.56 +
2.57 +# vim: tabstop=4 expandtab shiftwidth=4