1.1 --- a/tests/start_indexer.py Thu Sep 29 16:20:06 2005 +0000
1.2 +++ b/tests/start_indexer.py Thu Sep 29 16:21:30 2005 +0000
1.3 @@ -1,7 +1,8 @@
1.4 #!/usr/bin/env python
1.5
1.6 -from parallel import start, Exchange, waitall
1.7 +from parallel import start, Exchange
1.8 from Dict import Indexer, Searcher, Parser
1.9 +import os
1.10
1.11 def apply_parser(channel, filename):
1.12 print "Parsing", filename
1.13 @@ -9,31 +10,31 @@
1.14 parser.send_entries(channel)
1.15 print "Done", filename
1.16
1.17 +def fill_window(filenames, i, window_size, exchange):
1.18 + limit = min(len(filenames), i + window_size - len(exchange.active()))
1.19 + while i < limit:
1.20 + channel = start(apply_parser, filenames[i])
1.21 + exchange.add(channel)
1.22 + i = i + 1
1.23 + return i
1.24 +
1.25 def get_searcher(filenames, window_size):
1.26 master_index = Indexer()
1.27
1.28 - # Start indexing.
1.29 + # Start indexing by filling a window with channels.
1.30
1.31 - for i in range(0, len(filenames), window_size):
1.32 - channels = []
1.33 - for filename in filenames[i:i + window_size]:
1.34 - channels.append(start(apply_parser, filename))
1.35 + exchange = Exchange()
1.36 + i = fill_window(filenames, 0, window_size, exchange)
1.37
1.38 - # Start listening for responses.
1.39 + # Start listening for responses.
1.40
1.41 - exchange = Exchange(channels)
1.42 - while exchange.active():
1.43 - #print "Waiting for %d channels..." % len(exchange.active())
1.44 - for channel in exchange.ready():
1.45 - entry = channel.receive()
1.46 - master_index.add_entry(entry)
1.47 + while exchange.active():
1.48 + print "Waiting for %d channels..." % len(exchange.active())
1.49 + for channel in exchange.ready():
1.50 + entry = channel.receive()
1.51 + master_index.add_entry(entry)
1.52
1.53 - # Tidy up as we go.
1.54 -
1.55 - for channel in channels:
1.56 - channel.close()
1.57 -
1.58 - waitall()
1.59 + i = fill_window(filenames, i, window_size, exchange)
1.60
1.61 # Provide a search interface.
1.62
1.63 @@ -54,7 +55,6 @@
1.64 # Get a searcher using an index built in parallel.
1.65
1.66 searcher = get_searcher(filenames, window_size)
1.67 - waitall()
1.68
1.69 # Present a user interface.
1.70