1 #!/usr/bin/env python 2 3 from parallel import start, Exchange 4 from Dict import Indexer, Searcher, Parser 5 6 def apply_parser(channel, filename): 7 print "Parsing", filename 8 parser = Parser(filename) 9 parser.send_entries(channel) 10 11 def get_searcher(filenames, window_size): 12 master_index = Indexer() 13 14 # Start indexing. 15 16 for i in range(0, len(filenames), window_size): 17 channels = [] 18 for filename in filenames[i:i + window_size]: 19 channels.append(start(apply_parser, filename)) 20 21 # Start listening for responses. 22 23 exchange = Exchange(channels) 24 while exchange.active(): 25 print "Waiting for %d channels..." % exchange.active() 26 for channel in exchange.ready(): 27 entry = channel.receive() 28 master_index.add_entry(entry) 29 30 # Provide a search interface. 31 32 return Searcher(master_index.get_index()) 33 34 if __name__ == "__main__": 35 import sys, os 36 window_size, directory = int(sys.argv[1]), sys.argv[2] 37 filenames = [os.path.join(directory, filename) for filename in os.listdir(directory)] 38 filenames = [filename for filename in filenames if os.path.isfile(filename)] 39 40 searcher = get_searcher(filenames, window_size) 41 while 1: 42 print "Pattern:", 43 pattern = raw_input() 44 print searcher.find(pattern) 45 46 # vim: tabstop=4 expandtab shiftwidth=4