paul@44 | 1 | #!/usr/bin/env python |
paul@44 | 2 | |
paul@44 | 3 | """ |
paul@44 | 4 | Dictionary merging classes. |
paul@44 | 5 | |
paul@44 | 6 | Copyright (C) 2009 Paul Boddie <paul@boddie.org.uk> |
paul@44 | 7 | |
paul@44 | 8 | This program is free software; you can redistribute it and/or modify it under |
paul@44 | 9 | the terms of the GNU General Public License as published by the Free Software |
paul@44 | 10 | Foundation; either version 3 of the License, or (at your option) any later |
paul@44 | 11 | version. |
paul@44 | 12 | |
paul@44 | 13 | This program is distributed in the hope that it will be useful, but WITHOUT ANY |
paul@44 | 14 | WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A |
paul@44 | 15 | PARTICULAR PURPOSE. See the GNU General Public License for more details. |
paul@44 | 16 | |
paul@44 | 17 | You should have received a copy of the GNU General Public License along |
paul@44 | 18 | with this program. If not, see <http://www.gnu.org/licenses/>. |
paul@44 | 19 | """ |
paul@44 | 20 | |
paul@44 | 21 | from itermerge import itermerge |
paul@44 | 22 | |
paul@44 | 23 | class Merger: |
paul@44 | 24 | |
paul@44 | 25 | "Merge files." |
paul@44 | 26 | |
paul@44 | 27 | def __init__(self, writer, readers): |
paul@44 | 28 | self.writer = writer |
paul@44 | 29 | self.readers = readers |
paul@44 | 30 | |
paul@44 | 31 | def close(self): |
paul@44 | 32 | for reader in self.readers: |
paul@44 | 33 | reader.close() |
paul@44 | 34 | self.writer.close() |
paul@44 | 35 | |
paul@44 | 36 | class TermDictionaryMerger(Merger): |
paul@44 | 37 | |
paul@44 | 38 | "Merge term and position files." |
paul@44 | 39 | |
paul@44 | 40 | def merge(self): |
paul@44 | 41 | |
paul@44 | 42 | """ |
paul@44 | 43 | Merge terms and positions from the readers, sending them to the writer. |
paul@44 | 44 | """ |
paul@44 | 45 | |
paul@44 | 46 | last_term = None |
paul@44 | 47 | current_readers = [] |
paul@44 | 48 | |
paul@44 | 49 | for term, frequency, doc_frequency, positions in itermerge(self.readers): |
paul@44 | 50 | if term == last_term: |
paul@44 | 51 | current_readers.append(positions) |
paul@44 | 52 | else: |
paul@44 | 53 | if current_readers: |
paul@44 | 54 | self.writer.write_term_positions(last_term, itermerge(current_readers)) |
paul@44 | 55 | last_term = term |
paul@44 | 56 | current_readers = [positions] |
paul@44 | 57 | else: |
paul@44 | 58 | if current_readers: |
paul@44 | 59 | self.writer.write_term_positions(last_term, itermerge(current_readers)) |
paul@44 | 60 | |
paul@44 | 61 | class FieldDictionaryMerger(Merger): |
paul@44 | 62 | |
paul@44 | 63 | "Merge field files." |
paul@44 | 64 | |
paul@44 | 65 | def merge(self): |
paul@44 | 66 | |
paul@44 | 67 | """ |
paul@44 | 68 | Merge fields from the readers, sending them to the writer. |
paul@44 | 69 | """ |
paul@44 | 70 | |
paul@44 | 71 | for docnum, fields in itermerge(self.readers): |
paul@44 | 72 | self.writer.write_fields(docnum, fields) |
paul@44 | 73 | |
paul@44 | 74 | # vim: tabstop=4 expandtab shiftwidth=4 |