1 #!/usr/bin/env python 2 3 from parallel import start, Exchange 4 from Dict import Indexer, Searcher, update 5 6 def apply_indexer(channel, filename): 7 print "Indexing", filename 8 indexer = Indexer(filename) 9 channel.send(indexer.get_index()) 10 11 def get_searcher(filenames, window_size): 12 master_index = {} 13 14 # Start indexing. 15 16 for i in range(0, len(filenames), window_size): 17 channels = [] 18 for filename in filenames[i:i + window_size]: 19 channels.append(start(apply_indexer, filename)) 20 21 # Start listening for responses. 22 23 exchange = Exchange(channels) 24 while exchange.active(): 25 print "Waiting for %d channels..." % exchange.active() 26 for channel in exchange.ready(): 27 index = channel.receive() 28 update(master_index, index) 29 exchange.remove(channel) 30 channel.close() 31 32 # Provide a search interface. 33 34 return Searcher(master_index) 35 36 if __name__ == "__main__": 37 import sys, os 38 window_size, directory = int(sys.argv[1]), sys.argv[2] 39 filenames = [os.path.join(directory, filename) for filename in os.listdir(directory)] 40 41 searcher = get_searcher(filenames, window_size) 42 while 1: 43 print "Pattern:", 44 pattern = raw_input() 45 print searcher.find(pattern) 46 47 # vim: tabstop=4 expandtab shiftwidth=4