paulb@40 | 1 | #!/usr/bin/env python |
paulb@40 | 2 | |
paulb@40 | 3 | """ |
paulb@40 | 4 | A simple parallel processing API for Python, inspired somewhat by the thread |
paulb@40 | 5 | module, slightly less by pypar, and slightly less still by pypvm. |
paulb@40 | 6 | |
paul@173 | 7 | Copyright (C) 2005, 2006, 2007, 2008, 2009, 2013, 2014, |
paul@173 | 8 | 2016 Paul Boddie <paul@boddie.org.uk> |
paul@166 | 9 | Copyright (C) 2013 Yaroslav Halchenko <debian@onerussian.com> |
paulb@41 | 10 | |
paulb@79 | 11 | This program is free software; you can redistribute it and/or modify it under |
paulb@79 | 12 | the terms of the GNU Lesser General Public License as published by the Free |
paulb@79 | 13 | Software Foundation; either version 3 of the License, or (at your option) any |
paulb@79 | 14 | later version. |
paulb@41 | 15 | |
paulb@79 | 16 | This program is distributed in the hope that it will be useful, but WITHOUT |
paulb@79 | 17 | ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS |
paulb@79 | 18 | FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more |
paulb@79 | 19 | details. |
paulb@41 | 20 | |
paulb@79 | 21 | You should have received a copy of the GNU Lesser General Public License along |
paulb@79 | 22 | with this program. If not, see <http://www.gnu.org/licenses/>. |
paulb@40 | 23 | """ |
paulb@40 | 24 | |
paul@166 | 25 | __version__ = "0.5.1" |
paulb@40 | 26 | |
paulb@40 | 27 | import os |
paulb@40 | 28 | import sys |
paulb@40 | 29 | import select |
paulb@40 | 30 | import socket |
paulb@150 | 31 | import platform |
paul@173 | 32 | import errno |
paulb@40 | 33 | |
paul@173 | 34 | from time import sleep |
paul@166 | 35 | from warnings import warn |
paul@166 | 36 | |
paulb@40 | 37 | try: |
paulb@40 | 38 | import cPickle as pickle |
paulb@40 | 39 | except ImportError: |
paulb@40 | 40 | import pickle |
paulb@40 | 41 | |
paulb@135 | 42 | try: |
paulb@135 | 43 | set |
paulb@135 | 44 | except NameError: |
paulb@135 | 45 | from sets import Set as set |
paulb@135 | 46 | |
paul@156 | 47 | # Special values. |
paul@156 | 48 | |
paul@156 | 49 | class Undefined: pass |
paul@156 | 50 | |
paulb@84 | 51 | # Communications. |
paulb@84 | 52 | |
paulb@40 | 53 | class AcknowledgementError(Exception): |
paulb@40 | 54 | pass |
paulb@40 | 55 | |
paulb@40 | 56 | class Channel: |
paulb@40 | 57 | |
paulb@40 | 58 | "A communications channel." |
paulb@40 | 59 | |
paulb@40 | 60 | def __init__(self, pid, read_pipe, write_pipe): |
paulb@40 | 61 | |
paulb@40 | 62 | """ |
paulb@40 | 63 | Initialise the channel with a process identifier 'pid', a 'read_pipe' |
paulb@40 | 64 | from which messages will be received, and a 'write_pipe' into which |
paulb@40 | 65 | messages will be sent. |
paulb@40 | 66 | """ |
paulb@40 | 67 | |
paulb@40 | 68 | self.pid = pid |
paulb@40 | 69 | self.read_pipe = read_pipe |
paulb@40 | 70 | self.write_pipe = write_pipe |
paulb@40 | 71 | |
paulb@40 | 72 | def __del__(self): |
paulb@40 | 73 | |
paulb@40 | 74 | # Since signals don't work well with I/O, we close pipes and wait for |
paulb@40 | 75 | # created processes upon finalisation. |
paulb@40 | 76 | |
paulb@40 | 77 | self.close() |
paulb@40 | 78 | |
paulb@40 | 79 | def close(self): |
paulb@40 | 80 | |
paulb@40 | 81 | "Explicitly close the channel." |
paulb@40 | 82 | |
paulb@140 | 83 | if self.read_pipe is not None: |
paulb@40 | 84 | self.read_pipe.close() |
paulb@140 | 85 | self.read_pipe = None |
paulb@140 | 86 | if self.write_pipe is not None: |
paulb@40 | 87 | self.write_pipe.close() |
paulb@140 | 88 | self.write_pipe = None |
paulb@140 | 89 | #self.wait(os.WNOHANG) |
paulb@40 | 90 | |
paulb@40 | 91 | def wait(self, options=0): |
paulb@40 | 92 | |
paulb@40 | 93 | "Wait for the created process, if any, to exit." |
paulb@40 | 94 | |
paulb@40 | 95 | if self.pid != 0: |
paulb@40 | 96 | try: |
paulb@40 | 97 | os.waitpid(self.pid, options) |
paulb@40 | 98 | except OSError: |
paulb@40 | 99 | pass |
paulb@40 | 100 | |
paulb@40 | 101 | def _send(self, obj): |
paulb@40 | 102 | |
paulb@40 | 103 | "Send the given object 'obj' through the channel." |
paulb@40 | 104 | |
paulb@40 | 105 | pickle.dump(obj, self.write_pipe) |
paulb@40 | 106 | self.write_pipe.flush() |
paulb@40 | 107 | |
paulb@40 | 108 | def send(self, obj): |
paulb@40 | 109 | |
paulb@40 | 110 | """ |
paulb@40 | 111 | Send the given object 'obj' through the channel. Then wait for an |
paulb@40 | 112 | acknowledgement. (The acknowledgement makes the caller wait, thus |
paulb@40 | 113 | preventing processes from exiting and disrupting the communications |
paulb@40 | 114 | channel and losing data.) |
paulb@40 | 115 | """ |
paulb@40 | 116 | |
paulb@40 | 117 | self._send(obj) |
paulb@40 | 118 | if self._receive() != "OK": |
paulb@40 | 119 | raise AcknowledgementError, obj |
paulb@40 | 120 | |
paulb@40 | 121 | def _receive(self): |
paulb@40 | 122 | |
paulb@40 | 123 | "Receive an object through the channel, returning the object." |
paulb@40 | 124 | |
paulb@40 | 125 | obj = pickle.load(self.read_pipe) |
paulb@40 | 126 | if isinstance(obj, Exception): |
paulb@40 | 127 | raise obj |
paulb@40 | 128 | else: |
paulb@40 | 129 | return obj |
paulb@40 | 130 | |
paulb@40 | 131 | def receive(self): |
paulb@40 | 132 | |
paulb@40 | 133 | """ |
paulb@40 | 134 | Receive an object through the channel, returning the object. Send an |
paulb@40 | 135 | acknowledgement of receipt. (The acknowledgement makes the sender wait, |
paulb@40 | 136 | thus preventing processes from exiting and disrupting the communications |
paulb@40 | 137 | channel and losing data.) |
paulb@40 | 138 | """ |
paulb@40 | 139 | |
paulb@40 | 140 | try: |
paulb@40 | 141 | obj = self._receive() |
paulb@40 | 142 | return obj |
paulb@40 | 143 | finally: |
paulb@40 | 144 | self._send("OK") |
paulb@40 | 145 | |
paulb@140 | 146 | class PersistentChannel(Channel): |
paulb@140 | 147 | |
paulb@140 | 148 | """ |
paulb@140 | 149 | A persistent communications channel which can handle peer disconnection, |
paulb@140 | 150 | acting as a server, meaning that this channel is associated with a specific |
paulb@140 | 151 | address which can be contacted by other processes. |
paulb@140 | 152 | """ |
paulb@140 | 153 | |
paulb@140 | 154 | def __init__(self, pid, endpoint, address): |
paulb@140 | 155 | Channel.__init__(self, pid, None, None) |
paulb@140 | 156 | self.endpoint = endpoint |
paulb@140 | 157 | self.address = address |
paulb@140 | 158 | self.poller = select.poll() |
paulb@142 | 159 | |
paulb@142 | 160 | # Listen for connections before this process is interested in |
paulb@142 | 161 | # communicating. It is not desirable to wait for connections at this |
paulb@142 | 162 | # point because this will block the process. |
paulb@142 | 163 | |
paulb@142 | 164 | self.endpoint.listen(1) |
paulb@140 | 165 | |
paulb@140 | 166 | def close(self): |
paulb@142 | 167 | |
paulb@142 | 168 | "Close the persistent channel and remove the socket file." |
paulb@142 | 169 | |
paulb@140 | 170 | Channel.close(self) |
paulb@140 | 171 | try: |
paulb@140 | 172 | os.unlink(self.address) |
paulb@140 | 173 | except OSError: |
paulb@140 | 174 | pass |
paulb@140 | 175 | |
paulb@140 | 176 | def _ensure_pipes(self): |
paulb@140 | 177 | |
paulb@140 | 178 | "Ensure that the channel is capable of communicating." |
paulb@140 | 179 | |
paulb@140 | 180 | if self.read_pipe is None or self.write_pipe is None: |
paulb@142 | 181 | |
paulb@142 | 182 | # Accept any incoming connections. |
paulb@142 | 183 | |
paulb@140 | 184 | endpoint, address = self.endpoint.accept() |
paulb@140 | 185 | self.read_pipe = endpoint.makefile("r", 0) |
paulb@140 | 186 | self.write_pipe = endpoint.makefile("w", 0) |
paulb@140 | 187 | |
paulb@140 | 188 | # Monitor the write pipe for error conditions. |
paulb@140 | 189 | |
paulb@140 | 190 | fileno = self.write_pipe.fileno() |
paulb@140 | 191 | self.poller.register(fileno, select.POLLOUT | select.POLLHUP | select.POLLNVAL | select.POLLERR) |
paulb@140 | 192 | |
paulb@140 | 193 | def _reset_pipes(self): |
paulb@142 | 194 | |
paulb@142 | 195 | "Discard the existing connection." |
paulb@142 | 196 | |
paulb@140 | 197 | fileno = self.write_pipe.fileno() |
paulb@140 | 198 | self.poller.unregister(fileno) |
paulb@140 | 199 | self.read_pipe = None |
paulb@140 | 200 | self.write_pipe = None |
paulb@142 | 201 | self.endpoint.listen(1) |
paulb@140 | 202 | |
paulb@140 | 203 | def _ensure_communication(self, timeout=None): |
paulb@140 | 204 | |
paulb@140 | 205 | "Ensure that sending and receiving are possible." |
paulb@140 | 206 | |
paulb@140 | 207 | while 1: |
paulb@142 | 208 | self._ensure_pipes() |
paulb@140 | 209 | fileno = self.write_pipe.fileno() |
paulb@140 | 210 | fds = self.poller.poll(timeout) |
paulb@140 | 211 | for fd, status in fds: |
paulb@140 | 212 | if fd != fileno: |
paulb@140 | 213 | continue |
paulb@140 | 214 | if status & (select.POLLHUP | select.POLLNVAL | select.POLLERR): |
paulb@142 | 215 | |
paulb@142 | 216 | # Broken connection: discard it and start all over again. |
paulb@142 | 217 | |
paulb@140 | 218 | self._reset_pipes() |
paulb@140 | 219 | break |
paulb@140 | 220 | else: |
paulb@140 | 221 | return |
paulb@140 | 222 | |
paulb@140 | 223 | def _send(self, obj): |
paulb@140 | 224 | |
paulb@140 | 225 | "Send the given object 'obj' through the channel." |
paulb@140 | 226 | |
paulb@140 | 227 | self._ensure_communication() |
paulb@140 | 228 | Channel._send(self, obj) |
paulb@140 | 229 | |
paulb@140 | 230 | def _receive(self): |
paulb@140 | 231 | |
paulb@140 | 232 | "Receive an object through the channel, returning the object." |
paulb@140 | 233 | |
paulb@140 | 234 | self._ensure_communication() |
paulb@140 | 235 | return Channel._receive(self) |
paulb@140 | 236 | |
paulb@84 | 237 | # Management of processes and communications. |
paulb@84 | 238 | |
paulb@40 | 239 | class Exchange: |
paulb@40 | 240 | |
paulb@40 | 241 | """ |
paulb@40 | 242 | A communications exchange that can be used to detect channels which are |
paulb@67 | 243 | ready to communicate. Subclasses of this class can define the 'store_data' |
paulb@67 | 244 | method in order to enable the 'add_wait', 'wait' and 'finish' methods. |
paul@158 | 245 | |
paul@158 | 246 | Once exchanges are populated with active channels, use of the principal |
paul@158 | 247 | methods of the exchange typically cause the 'store' method to be invoked, |
paul@158 | 248 | resulting in the processing of any incoming data. |
paulb@40 | 249 | """ |
paulb@40 | 250 | |
paul@157 | 251 | def __init__(self, channels=None, limit=None, reuse=0, continuous=0, autoclose=1): |
paulb@40 | 252 | |
paulb@40 | 253 | """ |
paulb@67 | 254 | Initialise the exchange with an optional list of 'channels'. |
paulb@67 | 255 | |
paulb@67 | 256 | If the optional 'limit' is specified, restrictions on the addition of |
paulb@67 | 257 | new channels can be enforced and observed through the 'add_wait', 'wait' |
paulb@67 | 258 | and 'finish' methods. To make use of these methods, create a subclass of |
paulb@67 | 259 | this class and define a working 'store_data' method. |
paulb@67 | 260 | |
paulb@116 | 261 | If the optional 'reuse' parameter is set to a true value, channels and |
paul@157 | 262 | processes will be reused for waiting computations, but the callable will |
paul@157 | 263 | be invoked for each computation. |
paul@157 | 264 | |
paul@157 | 265 | If the optional 'continuous' parameter is set to a true value, channels |
paul@157 | 266 | and processes will be retained after receiving data sent from such |
paul@157 | 267 | processes, since it will be assumed that they will communicate more |
paul@157 | 268 | data. |
paulb@116 | 269 | |
paulb@67 | 270 | If the optional 'autoclose' parameter is set to a false value, channels |
paulb@67 | 271 | will not be closed automatically when they are removed from the exchange |
paulb@67 | 272 | - by default they are closed when removed. |
paulb@40 | 273 | """ |
paulb@40 | 274 | |
paulb@67 | 275 | self.limit = limit |
paulb@116 | 276 | self.reuse = reuse |
paulb@116 | 277 | self.autoclose = autoclose |
paul@157 | 278 | self.continuous = continuous |
paul@157 | 279 | |
paulb@99 | 280 | self.waiting = [] |
paulb@40 | 281 | self.readables = {} |
paulb@58 | 282 | self.removed = [] |
paulb@40 | 283 | self.poller = select.poll() |
paul@157 | 284 | |
paulb@40 | 285 | for channel in channels or []: |
paulb@40 | 286 | self.add(channel) |
paulb@40 | 287 | |
paul@158 | 288 | # Core methods, registering and reporting on channels. |
paul@158 | 289 | |
paulb@40 | 290 | def add(self, channel): |
paulb@40 | 291 | |
paulb@40 | 292 | "Add the given 'channel' to the exchange." |
paulb@40 | 293 | |
paulb@140 | 294 | fileno = channel.read_pipe.fileno() |
paulb@140 | 295 | self.readables[fileno] = channel |
paulb@140 | 296 | self.poller.register(fileno, select.POLLIN | select.POLLHUP | select.POLLNVAL | select.POLLERR) |
paulb@40 | 297 | |
paulb@40 | 298 | def active(self): |
paulb@40 | 299 | |
paulb@40 | 300 | "Return a list of active channels." |
paulb@40 | 301 | |
paulb@40 | 302 | return self.readables.values() |
paulb@40 | 303 | |
paulb@40 | 304 | def ready(self, timeout=None): |
paulb@40 | 305 | |
paulb@40 | 306 | """ |
paul@158 | 307 | Wait for a period of time specified by the optional 'timeout' in |
paul@158 | 308 | milliseconds (or until communication is possible) and return a list of |
paul@158 | 309 | channels which are ready to be read from. |
paulb@40 | 310 | """ |
paulb@40 | 311 | |
paulb@40 | 312 | fds = self.poller.poll(timeout) |
paulb@40 | 313 | readables = [] |
paulb@58 | 314 | self.removed = [] |
paulb@58 | 315 | |
paulb@40 | 316 | for fd, status in fds: |
paulb@40 | 317 | channel = self.readables[fd] |
paulb@55 | 318 | removed = 0 |
paulb@40 | 319 | |
paulb@40 | 320 | # Remove ended/error channels. |
paulb@40 | 321 | |
paulb@40 | 322 | if status & (select.POLLHUP | select.POLLNVAL | select.POLLERR): |
paulb@40 | 323 | self.remove(channel) |
paulb@58 | 324 | self.removed.append(channel) |
paulb@55 | 325 | removed = 1 |
paulb@40 | 326 | |
paulb@40 | 327 | # Record readable channels. |
paulb@40 | 328 | |
paulb@55 | 329 | if status & select.POLLIN: |
paulb@55 | 330 | if not (removed and self.autoclose): |
paulb@55 | 331 | readables.append(channel) |
paulb@40 | 332 | |
paulb@40 | 333 | return readables |
paulb@40 | 334 | |
paulb@40 | 335 | def remove(self, channel): |
paulb@40 | 336 | |
paulb@40 | 337 | """ |
paulb@40 | 338 | Remove the given 'channel' from the exchange. |
paulb@40 | 339 | """ |
paulb@40 | 340 | |
paulb@142 | 341 | fileno = channel.read_pipe.fileno() |
paulb@142 | 342 | del self.readables[fileno] |
paulb@142 | 343 | self.poller.unregister(fileno) |
paulb@40 | 344 | if self.autoclose: |
paulb@40 | 345 | channel.close() |
paulb@40 | 346 | channel.wait() |
paulb@40 | 347 | |
paulb@67 | 348 | # Enhanced exchange methods involving channel limits. |
paulb@67 | 349 | |
paul@158 | 350 | def unfinished(self): |
paul@158 | 351 | |
paul@158 | 352 | "Return whether the exchange still has work scheduled or in progress." |
paul@158 | 353 | |
paul@158 | 354 | return self.active() or self.waiting |
paul@158 | 355 | |
paul@158 | 356 | def busy(self): |
paul@158 | 357 | |
paul@158 | 358 | "Return whether the exchange uses as many channels as it is allowed to." |
paul@158 | 359 | |
paul@158 | 360 | return self.limit is not None and len(self.active()) >= self.limit |
paul@158 | 361 | |
paulb@67 | 362 | def add_wait(self, channel): |
paulb@67 | 363 | |
paulb@67 | 364 | """ |
paulb@67 | 365 | Add the given 'channel' to the exchange, waiting if the limit on active |
paulb@67 | 366 | channels would be exceeded by adding the channel. |
paulb@67 | 367 | """ |
paulb@67 | 368 | |
paulb@67 | 369 | self.wait() |
paulb@67 | 370 | self.add(channel) |
paulb@67 | 371 | |
paulb@67 | 372 | def wait(self): |
paulb@67 | 373 | |
paulb@67 | 374 | """ |
paulb@67 | 375 | Test for the limit on channels, blocking and reading incoming data until |
paulb@67 | 376 | the number of channels is below the limit. |
paulb@67 | 377 | """ |
paulb@67 | 378 | |
paulb@67 | 379 | # If limited, block until channels have been closed. |
paulb@67 | 380 | |
paul@158 | 381 | while self.busy(): |
paul@158 | 382 | self.store() |
paul@158 | 383 | |
paul@158 | 384 | def finish(self): |
paul@158 | 385 | |
paul@158 | 386 | """ |
paul@158 | 387 | Finish the use of the exchange by waiting for all channels to complete. |
paul@158 | 388 | """ |
paul@158 | 389 | |
paul@158 | 390 | while self.unfinished(): |
paulb@67 | 391 | self.store() |
paulb@67 | 392 | |
paul@158 | 393 | def store(self, timeout=None): |
paul@158 | 394 | |
paul@158 | 395 | """ |
paul@158 | 396 | For each ready channel, process the incoming data. If the optional |
paul@158 | 397 | 'timeout' parameter (a duration in milliseconds) is specified, wait only |
paul@158 | 398 | for the specified duration if no channels are ready to provide data. |
paul@158 | 399 | """ |
paul@158 | 400 | |
paul@158 | 401 | # Either process input from active channels. |
paul@158 | 402 | |
paul@158 | 403 | if self.active(): |
paul@158 | 404 | for channel in self.ready(timeout): |
paul@166 | 405 | try: |
paul@166 | 406 | self.store_data(channel) |
paul@166 | 407 | self.start_waiting(channel) |
paul@173 | 408 | except (IOError, OSError), exc: |
paul@166 | 409 | self.remove(channel) |
paul@173 | 410 | warn("Removed channel %r due to exception: %s" % (channel, exc)) |
paul@158 | 411 | |
paul@158 | 412 | # Or schedule new processes and channels. |
paul@158 | 413 | |
paul@158 | 414 | else: |
paul@158 | 415 | while self.waiting and not self.busy(): |
paul@173 | 416 | details = self.waiting.pop() |
paul@173 | 417 | |
paul@173 | 418 | # Stop actively scheduling if resources are exhausted. |
paul@173 | 419 | |
paul@173 | 420 | if not self.start_new_waiting(details): |
paul@173 | 421 | if not self.active(): |
paul@173 | 422 | sleep(1) |
paul@173 | 423 | break |
paul@158 | 424 | |
paul@158 | 425 | def store_data(self, channel): |
paul@158 | 426 | |
paul@158 | 427 | """ |
paul@158 | 428 | Store incoming data from the specified 'channel'. In subclasses of this |
paul@158 | 429 | class, such data could be stored using instance attributes. |
paul@158 | 430 | """ |
paul@158 | 431 | |
paul@158 | 432 | raise NotImplementedError, "store_data" |
paul@158 | 433 | |
paul@158 | 434 | # Support for the convenience methods. |
paul@158 | 435 | |
paulb@140 | 436 | def _get_waiting(self, channel): |
paulb@99 | 437 | |
paulb@99 | 438 | """ |
paulb@140 | 439 | Get waiting callable and argument information for new processes, given |
paulb@140 | 440 | the reception of data on the given 'channel'. |
paulb@99 | 441 | """ |
paulb@99 | 442 | |
paul@158 | 443 | # For continuous channels, no scheduling is requested. |
paul@158 | 444 | |
paul@158 | 445 | if self.waiting and not self.continuous: |
paul@158 | 446 | |
paul@158 | 447 | # Schedule this callable and arguments. |
paul@158 | 448 | |
paulb@99 | 449 | callable, args, kw = self.waiting.pop() |
paulb@116 | 450 | |
paulb@116 | 451 | # Try and reuse existing channels if possible. |
paulb@116 | 452 | |
paulb@116 | 453 | if self.reuse: |
paulb@119 | 454 | |
paulb@119 | 455 | # Re-add the channel - this may update information related to |
paulb@119 | 456 | # the channel in subclasses. |
paulb@119 | 457 | |
paulb@119 | 458 | self.add(channel) |
paulb@116 | 459 | channel.send((args, kw)) |
paul@157 | 460 | |
paul@173 | 461 | # Return the details for a new channel. |
paul@173 | 462 | |
paulb@116 | 463 | else: |
paulb@140 | 464 | return callable, args, kw |
paulb@116 | 465 | |
paulb@116 | 466 | # Where channels are being reused, but where no processes are waiting |
paulb@116 | 467 | # any more, send a special value to tell them to quit. |
paulb@116 | 468 | |
paulb@116 | 469 | elif self.reuse: |
paulb@116 | 470 | channel.send(None) |
paulb@99 | 471 | |
paulb@140 | 472 | return None |
paulb@140 | 473 | |
paulb@140 | 474 | def _set_waiting(self, callable, args, kw): |
paulb@140 | 475 | |
paulb@140 | 476 | """ |
paulb@140 | 477 | Support process creation by returning whether the given 'callable' has |
paulb@140 | 478 | been queued for later invocation. |
paulb@140 | 479 | """ |
paulb@140 | 480 | |
paul@158 | 481 | if self.busy(): |
paulb@140 | 482 | self.waiting.insert(0, (callable, args, kw)) |
paulb@140 | 483 | return 1 |
paulb@140 | 484 | else: |
paulb@140 | 485 | return 0 |
paulb@140 | 486 | |
paulb@140 | 487 | def _get_channel_for_process(self, channel): |
paulb@140 | 488 | |
paulb@140 | 489 | """ |
paulb@140 | 490 | Support process creation by returning the given 'channel' to the |
paulb@140 | 491 | creating process, and None to the created process. |
paulb@140 | 492 | """ |
paulb@140 | 493 | |
paulb@140 | 494 | if channel.pid == 0: |
paulb@140 | 495 | return channel |
paulb@140 | 496 | else: |
paulb@140 | 497 | self.add_wait(channel) |
paulb@140 | 498 | return None |
paulb@140 | 499 | |
paulb@140 | 500 | # Methods for overriding, related to the convenience methods. |
paulb@140 | 501 | |
paulb@140 | 502 | def start_waiting(self, channel): |
paulb@140 | 503 | |
paulb@140 | 504 | """ |
paulb@140 | 505 | Start a waiting process given the reception of data on the given |
paulb@140 | 506 | 'channel'. |
paulb@140 | 507 | """ |
paulb@140 | 508 | |
paulb@140 | 509 | details = self._get_waiting(channel) |
paul@173 | 510 | |
paulb@140 | 511 | if details is not None: |
paul@173 | 512 | self.start_new_waiting(details) |
paul@173 | 513 | |
paul@173 | 514 | def start_new_waiting(self, details): |
paul@173 | 515 | |
paul@173 | 516 | """ |
paul@173 | 517 | Start a waiting process with the given 'details', obtaining a new |
paul@173 | 518 | channel. |
paul@173 | 519 | """ |
paul@173 | 520 | |
paul@173 | 521 | callable, args, kw = details |
paul@173 | 522 | channel = self._start(callable, *args, **kw) |
paul@173 | 523 | |
paul@173 | 524 | # Monitor any newly-created process. |
paul@173 | 525 | |
paul@173 | 526 | if channel: |
paul@173 | 527 | self.add(channel) |
paul@173 | 528 | return True |
paul@173 | 529 | |
paul@173 | 530 | # Push the details back onto the end of the waiting list. |
paul@173 | 531 | |
paul@173 | 532 | else: |
paul@173 | 533 | self.waiting.append(details) |
paul@173 | 534 | return False |
paulb@140 | 535 | |
paulb@79 | 536 | # Convenience methods. |
paulb@79 | 537 | |
paulb@84 | 538 | def start(self, callable, *args, **kw): |
paulb@79 | 539 | |
paulb@79 | 540 | """ |
paulb@140 | 541 | Create a new process for the given 'callable' using any additional |
paulb@140 | 542 | arguments provided. Then, monitor the channel created between this |
paulb@140 | 543 | process and the created process. |
paulb@79 | 544 | """ |
paulb@79 | 545 | |
paulb@140 | 546 | if self._set_waiting(callable, args, kw): |
paul@173 | 547 | return False |
paul@173 | 548 | |
paul@173 | 549 | channel = self._start(callable, *args, **kw) |
paul@173 | 550 | |
paul@173 | 551 | # Monitor any newly-created process. |
paulb@99 | 552 | |
paul@173 | 553 | if channel: |
paul@173 | 554 | self.add_wait(channel) |
paul@173 | 555 | return True |
paul@173 | 556 | |
paul@173 | 557 | # Otherwise, add the details to the waiting list unconditionally. |
paul@173 | 558 | |
paul@173 | 559 | else: |
paul@173 | 560 | self.waiting.insert(0, (callable, args, kw)) |
paul@173 | 561 | return False |
paulb@84 | 562 | |
paulb@97 | 563 | def create(self): |
paulb@97 | 564 | |
paulb@97 | 565 | """ |
paulb@140 | 566 | Create a new process and return the created communications channel to |
paulb@140 | 567 | the created process. In the creating process, return None - the channel |
paulb@140 | 568 | receiving data from the created process will be automatically managed by |
paulb@140 | 569 | this exchange. |
paulb@97 | 570 | """ |
paulb@97 | 571 | |
paulb@97 | 572 | channel = create() |
paulb@140 | 573 | return self._get_channel_for_process(channel) |
paulb@97 | 574 | |
paulb@84 | 575 | def manage(self, callable): |
paulb@84 | 576 | |
paulb@84 | 577 | """ |
paulb@84 | 578 | Wrap the given 'callable' in an object which can then be called in the |
paulb@84 | 579 | same way as 'callable', but with new processes and communications |
paulb@84 | 580 | managed automatically. |
paulb@84 | 581 | """ |
paulb@84 | 582 | |
paulb@84 | 583 | return ManagedCallable(callable, self) |
paulb@84 | 584 | |
paul@173 | 585 | def _start(self, callable, *args, **kw): |
paul@173 | 586 | |
paul@173 | 587 | """ |
paul@173 | 588 | Create a new process for the given 'callable' using any additional |
paul@173 | 589 | arguments provided. Return any successfully created channel or None if |
paul@173 | 590 | no process could be created at the present time. |
paul@173 | 591 | """ |
paul@173 | 592 | |
paul@173 | 593 | try: |
paul@173 | 594 | return start(callable, *args, **kw) |
paul@173 | 595 | except OSError, exc: |
paul@173 | 596 | if exc.errno != errno.EAGAIN: |
paul@173 | 597 | raise |
paul@173 | 598 | else: |
paul@173 | 599 | return None |
paul@173 | 600 | |
paulb@140 | 601 | class Persistent: |
paulb@140 | 602 | |
paulb@140 | 603 | """ |
paulb@140 | 604 | A mix-in class providing methods to exchanges for the management of |
paulb@140 | 605 | persistent communications. |
paulb@140 | 606 | """ |
paulb@140 | 607 | |
paulb@140 | 608 | def start_waiting(self, channel): |
paulb@140 | 609 | |
paulb@140 | 610 | """ |
paulb@140 | 611 | Start a waiting process given the reception of data on the given |
paulb@140 | 612 | 'channel'. |
paulb@140 | 613 | """ |
paulb@140 | 614 | |
paulb@140 | 615 | details = self._get_waiting(channel) |
paulb@140 | 616 | if details is not None: |
paulb@140 | 617 | callable, args, kw = details |
paulb@140 | 618 | self.add(start_persistent(channel.address, callable, *args, **kw)) |
paulb@140 | 619 | |
paulb@140 | 620 | def start(self, address, callable, *args, **kw): |
paulb@140 | 621 | |
paulb@140 | 622 | """ |
paulb@140 | 623 | Create a new process, located at the given 'address', for the given |
paulb@140 | 624 | 'callable' using any additional arguments provided. Then, monitor the |
paulb@140 | 625 | channel created between this process and the created process. |
paulb@140 | 626 | """ |
paulb@140 | 627 | |
paulb@140 | 628 | if self._set_waiting(callable, args, kw): |
paulb@140 | 629 | return |
paulb@140 | 630 | |
paulb@140 | 631 | start_persistent(address, callable, *args, **kw) |
paulb@140 | 632 | |
paulb@140 | 633 | def create(self, address): |
paulb@140 | 634 | |
paulb@140 | 635 | """ |
paulb@140 | 636 | Create a new process, located at the given 'address', and return the |
paulb@140 | 637 | created communications channel to the created process. In the creating |
paulb@140 | 638 | process, return None - the channel receiving data from the created |
paulb@140 | 639 | process will be automatically managed by this exchange. |
paulb@140 | 640 | """ |
paulb@140 | 641 | |
paulb@140 | 642 | channel = create_persistent(address) |
paulb@140 | 643 | return self._get_channel_for_process(channel) |
paulb@140 | 644 | |
paulb@140 | 645 | def manage(self, address, callable): |
paulb@140 | 646 | |
paulb@140 | 647 | """ |
paulb@140 | 648 | Using the given 'address', publish the given 'callable' in an object |
paulb@140 | 649 | which can then be called in the same way as 'callable', but with new |
paulb@140 | 650 | processes and communications managed automatically. |
paulb@140 | 651 | """ |
paulb@140 | 652 | |
paulb@140 | 653 | return PersistentCallable(address, callable, self) |
paulb@140 | 654 | |
paulb@140 | 655 | def connect(self, address): |
paulb@140 | 656 | |
paulb@140 | 657 | "Connect to a process which is contactable via the given 'address'." |
paulb@140 | 658 | |
paulb@140 | 659 | channel = connect_persistent(address) |
paulb@140 | 660 | self.add_wait(channel) |
paulb@140 | 661 | |
paulb@84 | 662 | class ManagedCallable: |
paulb@84 | 663 | |
paulb@84 | 664 | "A callable managed by an exchange." |
paulb@84 | 665 | |
paulb@84 | 666 | def __init__(self, callable, exchange): |
paulb@84 | 667 | |
paulb@84 | 668 | """ |
paulb@84 | 669 | Wrap the given 'callable', using the given 'exchange' to monitor the |
paulb@84 | 670 | channels created for communications between this and the created |
paulb@94 | 671 | processes. Note that the 'callable' must be parallel-aware (that is, |
paulb@94 | 672 | have a 'channel' parameter). Use the MakeParallel class to wrap other |
paulb@94 | 673 | kinds of callable objects. |
paulb@84 | 674 | """ |
paulb@84 | 675 | |
paulb@84 | 676 | self.callable = callable |
paulb@84 | 677 | self.exchange = exchange |
paulb@84 | 678 | |
paulb@84 | 679 | def __call__(self, *args, **kw): |
paulb@84 | 680 | |
paulb@84 | 681 | "Invoke the callable with the supplied arguments." |
paulb@84 | 682 | |
paulb@84 | 683 | self.exchange.start(self.callable, *args, **kw) |
paulb@84 | 684 | |
paulb@140 | 685 | class PersistentCallable: |
paulb@140 | 686 | |
paulb@140 | 687 | "A callable which sets up a persistent communications channel." |
paulb@140 | 688 | |
paulb@140 | 689 | def __init__(self, address, callable, exchange): |
paulb@140 | 690 | |
paulb@140 | 691 | """ |
paulb@140 | 692 | Using the given 'address', wrap the given 'callable', using the given |
paulb@140 | 693 | 'exchange' to monitor the channels created for communications between |
paulb@140 | 694 | this and the created processes, so that when it is called, a background |
paulb@140 | 695 | process is started within which the 'callable' will run. Note that the |
paulb@140 | 696 | 'callable' must be parallel-aware (that is, have a 'channel' parameter). |
paulb@140 | 697 | Use the MakeParallel class to wrap other kinds of callable objects. |
paulb@140 | 698 | """ |
paulb@140 | 699 | |
paulb@140 | 700 | self.callable = callable |
paulb@140 | 701 | self.exchange = exchange |
paulb@140 | 702 | self.address = address |
paulb@140 | 703 | |
paulb@140 | 704 | def __call__(self, *args, **kw): |
paulb@140 | 705 | |
paulb@142 | 706 | "Invoke the callable with the supplied arguments." |
paulb@140 | 707 | |
paulb@140 | 708 | self.exchange.start(self.address, self.callable, *args, **kw) |
paulb@140 | 709 | |
paulb@142 | 710 | class BackgroundCallable: |
paulb@142 | 711 | |
paulb@142 | 712 | """ |
paulb@142 | 713 | A callable which sets up a persistent communications channel, but is |
paulb@142 | 714 | unmanaged by an exchange. |
paulb@142 | 715 | """ |
paulb@142 | 716 | |
paulb@142 | 717 | def __init__(self, address, callable): |
paulb@142 | 718 | |
paulb@142 | 719 | """ |
paulb@142 | 720 | Using the given 'address', wrap the given 'callable'. This object can |
paulb@142 | 721 | then be invoked, but the wrapped callable will be run in a background |
paulb@142 | 722 | process. Note that the 'callable' must be parallel-aware (that is, have |
paulb@142 | 723 | a 'channel' parameter). Use the MakeParallel class to wrap other kinds |
paulb@142 | 724 | of callable objects. |
paulb@142 | 725 | """ |
paulb@142 | 726 | |
paulb@142 | 727 | self.callable = callable |
paulb@142 | 728 | self.address = address |
paulb@142 | 729 | |
paulb@142 | 730 | def __call__(self, *args, **kw): |
paulb@142 | 731 | |
paulb@142 | 732 | "Invoke the callable with the supplied arguments." |
paulb@142 | 733 | |
paulb@142 | 734 | start_persistent(self.address, self.callable, *args, **kw) |
paulb@142 | 735 | |
paulb@84 | 736 | # Abstractions and utilities. |
paulb@84 | 737 | |
paulb@84 | 738 | class Map(Exchange): |
paulb@84 | 739 | |
paulb@84 | 740 | "An exchange which can be used like the built-in 'map' function." |
paulb@84 | 741 | |
paulb@107 | 742 | def __init__(self, *args, **kw): |
paulb@107 | 743 | Exchange.__init__(self, *args, **kw) |
paulb@107 | 744 | self.init() |
paulb@107 | 745 | |
paulb@107 | 746 | def init(self): |
paulb@107 | 747 | |
paulb@107 | 748 | "Remember the channel addition order to order output." |
paulb@107 | 749 | |
paulb@107 | 750 | self.channel_number = 0 |
paulb@107 | 751 | self.channels = {} |
paulb@107 | 752 | self.results = [] |
paul@156 | 753 | self.current_index = 0 |
paulb@107 | 754 | |
paulb@84 | 755 | def add(self, channel): |
paulb@84 | 756 | |
paulb@84 | 757 | "Add the given 'channel' to the exchange." |
paulb@84 | 758 | |
paulb@84 | 759 | Exchange.add(self, channel) |
paulb@92 | 760 | self.channels[channel] = self.channel_number |
paulb@92 | 761 | self.channel_number += 1 |
paulb@84 | 762 | |
paulb@107 | 763 | def start(self, callable, *args, **kw): |
paulb@107 | 764 | |
paulb@107 | 765 | """ |
paulb@140 | 766 | Create a new process for the given 'callable' using any additional |
paulb@140 | 767 | arguments provided. Then, monitor the channel created between this |
paulb@140 | 768 | process and the created process. |
paulb@107 | 769 | """ |
paulb@107 | 770 | |
paul@156 | 771 | self.results.append(Undefined) # placeholder |
paulb@107 | 772 | Exchange.start(self, callable, *args, **kw) |
paulb@107 | 773 | |
paulb@110 | 774 | def create(self): |
paulb@110 | 775 | |
paulb@110 | 776 | """ |
paulb@140 | 777 | Create a new process and return the created communications channel to |
paulb@140 | 778 | the created process. In the creating process, return None - the channel |
paulb@140 | 779 | receiving data from the created process will be automatically managed by |
paulb@140 | 780 | this exchange. |
paulb@110 | 781 | """ |
paulb@110 | 782 | |
paul@156 | 783 | self.results.append(Undefined) # placeholder |
paulb@110 | 784 | return Exchange.create(self) |
paulb@110 | 785 | |
paulb@84 | 786 | def __call__(self, callable, sequence): |
paulb@84 | 787 | |
paulb@89 | 788 | "Wrap and invoke 'callable' for each element in the 'sequence'." |
paulb@89 | 789 | |
paulb@92 | 790 | if not isinstance(callable, MakeParallel): |
paulb@92 | 791 | wrapped = MakeParallel(callable) |
paulb@92 | 792 | else: |
paulb@92 | 793 | wrapped = callable |
paulb@84 | 794 | |
paulb@107 | 795 | self.init() |
paulb@84 | 796 | |
paulb@107 | 797 | # Start processes for each element in the sequence. |
paulb@84 | 798 | |
paulb@84 | 799 | for i in sequence: |
paulb@92 | 800 | self.start(wrapped, i) |
paulb@107 | 801 | |
paulb@107 | 802 | # Access to the results occurs through this object. |
paulb@107 | 803 | |
paulb@107 | 804 | return self |
paulb@84 | 805 | |
paulb@84 | 806 | def store_data(self, channel): |
paulb@84 | 807 | |
paulb@84 | 808 | "Accumulate the incoming data, associating results with channels." |
paulb@84 | 809 | |
paulb@84 | 810 | data = channel.receive() |
paulb@92 | 811 | self.results[self.channels[channel]] = data |
paulb@92 | 812 | del self.channels[channel] |
paulb@84 | 813 | |
paul@156 | 814 | def __iter__(self): |
paul@156 | 815 | return self |
paul@156 | 816 | |
paul@156 | 817 | def next(self): |
paul@156 | 818 | |
paul@156 | 819 | "Return the next element in the map." |
paul@156 | 820 | |
paul@156 | 821 | try: |
paul@156 | 822 | return self._next() |
paul@156 | 823 | except IndexError: |
paul@156 | 824 | pass |
paul@156 | 825 | |
paul@158 | 826 | while self.unfinished(): |
paul@156 | 827 | self.store() |
paul@156 | 828 | try: |
paul@156 | 829 | return self._next() |
paul@156 | 830 | except IndexError: |
paul@156 | 831 | pass |
paul@156 | 832 | else: |
paul@156 | 833 | raise StopIteration |
paul@156 | 834 | |
paul@156 | 835 | def __getitem__(self, i): |
paul@156 | 836 | |
paul@156 | 837 | "Return element 'i' from the map." |
paul@156 | 838 | |
paul@156 | 839 | try: |
paul@156 | 840 | return self._get(i) |
paul@156 | 841 | except IndexError: |
paul@156 | 842 | pass |
paul@156 | 843 | |
paul@158 | 844 | while self.unfinished(): |
paul@156 | 845 | self.store() |
paul@156 | 846 | try: |
paul@156 | 847 | return self._get(i) |
paul@156 | 848 | except IndexError: |
paul@156 | 849 | pass |
paul@156 | 850 | else: |
paul@156 | 851 | raise IndexError, i |
paul@156 | 852 | |
paul@156 | 853 | # Helper methods for the above access methods. |
paul@156 | 854 | |
paul@156 | 855 | def _next(self): |
paul@156 | 856 | result = self._get(self.current_index) |
paul@156 | 857 | self.current_index += 1 |
paul@156 | 858 | return result |
paul@156 | 859 | |
paul@156 | 860 | def _get(self, i): |
paul@156 | 861 | result = self.results[i] |
paul@156 | 862 | if result is Undefined or isinstance(i, slice) and Undefined in result: |
paul@156 | 863 | raise IndexError, i |
paul@156 | 864 | return result |
paul@156 | 865 | |
paulb@97 | 866 | class Queue(Exchange): |
paulb@97 | 867 | |
paulb@97 | 868 | """ |
paulb@97 | 869 | An exchange acting as a queue, making data from created processes available |
paulb@97 | 870 | in the order in which it is received. |
paulb@97 | 871 | """ |
paulb@97 | 872 | |
paulb@97 | 873 | def __init__(self, *args, **kw): |
paulb@97 | 874 | Exchange.__init__(self, *args, **kw) |
paulb@97 | 875 | self.queue = [] |
paulb@97 | 876 | |
paulb@97 | 877 | def store_data(self, channel): |
paulb@97 | 878 | |
paulb@97 | 879 | "Accumulate the incoming data, associating results with channels." |
paulb@97 | 880 | |
paulb@97 | 881 | data = channel.receive() |
paulb@97 | 882 | self.queue.insert(0, data) |
paulb@97 | 883 | |
paulb@97 | 884 | def __iter__(self): |
paulb@97 | 885 | return self |
paulb@97 | 886 | |
paulb@97 | 887 | def next(self): |
paulb@97 | 888 | |
paulb@97 | 889 | "Return the next element in the queue." |
paulb@97 | 890 | |
paulb@97 | 891 | if self.queue: |
paulb@97 | 892 | return self.queue.pop() |
paul@158 | 893 | |
paul@158 | 894 | while self.unfinished(): |
paulb@97 | 895 | self.store() |
paulb@97 | 896 | if self.queue: |
paulb@97 | 897 | return self.queue.pop() |
paulb@97 | 898 | else: |
paulb@97 | 899 | raise StopIteration |
paulb@97 | 900 | |
paul@158 | 901 | def __len__(self): |
paul@158 | 902 | |
paul@158 | 903 | "Return the current length of the queue." |
paul@158 | 904 | |
paul@158 | 905 | return len(self.queue) |
paul@158 | 906 | |
paulb@84 | 907 | class MakeParallel: |
paulb@84 | 908 | |
paulb@84 | 909 | "A wrapper around functions making them able to communicate results." |
paulb@84 | 910 | |
paulb@84 | 911 | def __init__(self, callable): |
paulb@84 | 912 | |
paulb@94 | 913 | """ |
paulb@94 | 914 | Initialise the wrapper with the given 'callable'. This object will then |
paulb@94 | 915 | be able to accept a 'channel' parameter when invoked, and to forward the |
paulb@94 | 916 | result of the given 'callable' via the channel provided back to the |
paulb@94 | 917 | invoking process. |
paulb@94 | 918 | """ |
paulb@84 | 919 | |
paulb@84 | 920 | self.callable = callable |
paulb@84 | 921 | |
paulb@84 | 922 | def __call__(self, channel, *args, **kw): |
paulb@84 | 923 | |
paulb@84 | 924 | "Invoke the callable and return its result via the given 'channel'." |
paulb@84 | 925 | |
paulb@84 | 926 | channel.send(self.callable(*args, **kw)) |
paulb@84 | 927 | |
paulb@119 | 928 | class MakeReusable(MakeParallel): |
paulb@119 | 929 | |
paulb@119 | 930 | """ |
paulb@119 | 931 | A wrapper around functions making them able to communicate results in a |
paulb@119 | 932 | reusable fashion. |
paulb@119 | 933 | """ |
paulb@119 | 934 | |
paulb@119 | 935 | def __call__(self, channel, *args, **kw): |
paulb@119 | 936 | |
paulb@119 | 937 | "Invoke the callable and return its result via the given 'channel'." |
paulb@119 | 938 | |
paulb@119 | 939 | channel.send(self.callable(*args, **kw)) |
paulb@119 | 940 | t = channel.receive() |
paulb@119 | 941 | while t is not None: |
paulb@119 | 942 | args, kw = t |
paulb@119 | 943 | channel.send(self.callable(*args, **kw)) |
paulb@119 | 944 | t = channel.receive() |
paulb@119 | 945 | |
paulb@140 | 946 | # Persistent variants. |
paulb@140 | 947 | |
paulb@140 | 948 | class PersistentExchange(Persistent, Exchange): |
paulb@140 | 949 | |
paulb@140 | 950 | "An exchange which manages persistent communications." |
paulb@140 | 951 | |
paulb@140 | 952 | pass |
paulb@140 | 953 | |
paulb@140 | 954 | class PersistentQueue(Persistent, Queue): |
paulb@140 | 955 | |
paulb@140 | 956 | "A queue which manages persistent communications." |
paulb@140 | 957 | |
paulb@140 | 958 | pass |
paulb@140 | 959 | |
paulb@142 | 960 | # Convenience functions. |
paulb@142 | 961 | |
paulb@142 | 962 | def BackgroundQueue(address): |
paulb@142 | 963 | |
paulb@142 | 964 | """ |
paulb@142 | 965 | Connect to a process reachable via the given 'address', making the results |
paulb@142 | 966 | of which accessible via a queue. |
paulb@142 | 967 | """ |
paulb@142 | 968 | |
paulb@142 | 969 | queue = PersistentQueue(limit=1) |
paulb@142 | 970 | queue.connect(address) |
paulb@142 | 971 | return queue |
paulb@142 | 972 | |
paulb@142 | 973 | def pmap(callable, sequence, limit=None): |
paulb@142 | 974 | |
paulb@142 | 975 | """ |
paulb@142 | 976 | A parallel version of the built-in map function with an optional process |
paulb@142 | 977 | 'limit'. The given 'callable' should not be parallel-aware (that is, have a |
paulb@142 | 978 | 'channel' parameter) since it will be wrapped for parallel communications |
paulb@142 | 979 | before being invoked. |
paulb@142 | 980 | |
paulb@142 | 981 | Return the processed 'sequence' where each element in the sequence is |
paulb@142 | 982 | processed by a different process. |
paulb@142 | 983 | """ |
paulb@142 | 984 | |
paulb@142 | 985 | mymap = Map(limit=limit) |
paulb@142 | 986 | return mymap(callable, sequence) |
paulb@142 | 987 | |
paulb@84 | 988 | # Utility functions. |
paulb@79 | 989 | |
paul@155 | 990 | _cpuinfo_fields = "processor", "physical id", "core id" |
paulb@135 | 991 | |
paul@152 | 992 | def _get_number_of_cores(): |
paulb@135 | 993 | |
paulb@135 | 994 | """ |
paulb@135 | 995 | Return the number of distinct, genuine processor cores. If the platform is |
paulb@135 | 996 | not supported by this function, None is returned. |
paulb@135 | 997 | """ |
paulb@135 | 998 | |
paulb@135 | 999 | try: |
paulb@135 | 1000 | f = open("/proc/cpuinfo") |
paulb@135 | 1001 | try: |
paulb@135 | 1002 | processors = set() |
paul@155 | 1003 | |
paul@155 | 1004 | # Use the _cpuinfo_field values as "digits" in a larger unique |
paul@155 | 1005 | # core identifier. |
paul@155 | 1006 | |
paul@155 | 1007 | processor = [None, None, None] |
paulb@135 | 1008 | |
paulb@135 | 1009 | for line in f.xreadlines(): |
paulb@135 | 1010 | for i, field in enumerate(_cpuinfo_fields): |
paul@155 | 1011 | |
paul@155 | 1012 | # Where the field is found, insert the value into the |
paul@155 | 1013 | # appropriate location in the processor identifier. |
paul@155 | 1014 | |
paulb@135 | 1015 | if line.startswith(field): |
paulb@135 | 1016 | t = line.split(":") |
paulb@135 | 1017 | processor[i] = int(t[1].strip()) |
paulb@135 | 1018 | break |
paul@155 | 1019 | |
paul@155 | 1020 | # Where a new processor description is started, record the |
paul@155 | 1021 | # identifier. |
paul@155 | 1022 | |
paul@155 | 1023 | if line.startswith("processor") and processor[0] is not None: |
paul@155 | 1024 | processors.add(tuple(processor)) |
paul@155 | 1025 | processor = [None, None, None] |
paul@155 | 1026 | |
paul@155 | 1027 | # At the end of reading the file, add any unrecorded processors. |
paulb@135 | 1028 | |
paulb@135 | 1029 | if processor[0] is not None: |
paulb@135 | 1030 | processors.add(tuple(processor)) |
paulb@135 | 1031 | |
paulb@135 | 1032 | return len(processors) |
paulb@135 | 1033 | |
paulb@135 | 1034 | finally: |
paulb@135 | 1035 | f.close() |
paulb@135 | 1036 | |
paulb@135 | 1037 | except OSError: |
paulb@135 | 1038 | return None |
paulb@135 | 1039 | |
paul@152 | 1040 | def _get_number_of_cores_solaris(): |
paul@152 | 1041 | |
paul@152 | 1042 | """ |
paul@152 | 1043 | Return the number of cores for OpenSolaris 2008.05 and possibly other |
paul@152 | 1044 | editions of Solaris. |
paul@152 | 1045 | """ |
paul@152 | 1046 | |
paul@152 | 1047 | f = os.popen("psrinfo -p") |
paul@152 | 1048 | try: |
paul@152 | 1049 | return int(f.read().strip()) |
paul@152 | 1050 | finally: |
paul@152 | 1051 | f.close() |
paul@152 | 1052 | |
paulb@142 | 1053 | # Low-level functions. |
paulb@142 | 1054 | |
paulb@150 | 1055 | def create_socketpair(): |
paulb@40 | 1056 | |
paulb@40 | 1057 | """ |
paulb@40 | 1058 | Create a new process, returning a communications channel to both the |
paulb@40 | 1059 | creating process and the created process. |
paulb@40 | 1060 | """ |
paulb@40 | 1061 | |
paulb@40 | 1062 | parent, child = socket.socketpair() |
paulb@40 | 1063 | for s in [parent, child]: |
paulb@40 | 1064 | s.setblocking(1) |
paulb@40 | 1065 | |
paulb@40 | 1066 | pid = os.fork() |
paulb@40 | 1067 | if pid == 0: |
paulb@40 | 1068 | parent.close() |
paulb@73 | 1069 | return Channel(pid, child.makefile("r", 0), child.makefile("w", 0)) |
paulb@40 | 1070 | else: |
paulb@40 | 1071 | child.close() |
paulb@73 | 1072 | return Channel(pid, parent.makefile("r", 0), parent.makefile("w", 0)) |
paulb@40 | 1073 | |
paulb@150 | 1074 | def create_pipes(): |
paulb@150 | 1075 | |
paulb@150 | 1076 | """ |
paulb@150 | 1077 | Create a new process, returning a communications channel to both the |
paulb@150 | 1078 | creating process and the created process. |
paulb@150 | 1079 | |
paulb@150 | 1080 | This function uses pipes instead of a socket pair, since some platforms |
paulb@150 | 1081 | seem to have problems with poll and such socket pairs. |
paulb@150 | 1082 | """ |
paulb@150 | 1083 | |
paulb@150 | 1084 | pr, cw = os.pipe() |
paulb@150 | 1085 | cr, pw = os.pipe() |
paulb@150 | 1086 | |
paulb@150 | 1087 | pid = os.fork() |
paulb@150 | 1088 | if pid == 0: |
paulb@151 | 1089 | os.close(pr) |
paulb@151 | 1090 | os.close(pw) |
paulb@150 | 1091 | return Channel(pid, os.fdopen(cr, "r", 0), os.fdopen(cw, "w", 0)) |
paulb@150 | 1092 | else: |
paulb@151 | 1093 | os.close(cr) |
paulb@151 | 1094 | os.close(cw) |
paulb@150 | 1095 | return Channel(pid, os.fdopen(pr, "r", 0), os.fdopen(pw, "w", 0)) |
paulb@150 | 1096 | |
paulb@150 | 1097 | if platform.system() == "SunOS": |
paulb@150 | 1098 | create = create_pipes |
paul@152 | 1099 | get_number_of_cores = _get_number_of_cores_solaris |
paulb@150 | 1100 | else: |
paulb@150 | 1101 | create = create_socketpair |
paul@152 | 1102 | get_number_of_cores = _get_number_of_cores |
paulb@150 | 1103 | |
paulb@140 | 1104 | def create_persistent(address): |
paulb@140 | 1105 | |
paulb@140 | 1106 | """ |
paulb@140 | 1107 | Create a new process, returning a persistent communications channel between |
paulb@140 | 1108 | the creating process and the created process. This channel can be |
paulb@140 | 1109 | disconnected from the creating process and connected to another process, and |
paulb@140 | 1110 | thus can be used to collect results from daemon processes. |
paulb@140 | 1111 | |
paulb@140 | 1112 | In order to be able to reconnect to created processes, the 'address' of the |
paulb@140 | 1113 | communications endpoint for the created process needs to be provided. This |
paulb@140 | 1114 | should be a filename. |
paulb@140 | 1115 | """ |
paulb@140 | 1116 | |
paulb@140 | 1117 | parent = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) |
paulb@140 | 1118 | child = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) |
paulb@140 | 1119 | child.bind(address) |
paulb@140 | 1120 | |
paulb@140 | 1121 | for s in [parent, child]: |
paulb@140 | 1122 | s.setblocking(1) |
paulb@140 | 1123 | |
paulb@140 | 1124 | pid = os.fork() |
paulb@140 | 1125 | if pid == 0: |
paulb@140 | 1126 | parent.close() |
paulb@140 | 1127 | return PersistentChannel(pid, child, address) |
paulb@140 | 1128 | else: |
paulb@140 | 1129 | child.close() |
paulb@140 | 1130 | #parent.connect(address) |
paulb@140 | 1131 | return Channel(pid, parent.makefile("r", 0), parent.makefile("w", 0)) |
paulb@140 | 1132 | |
paulb@140 | 1133 | def connect_persistent(address): |
paulb@140 | 1134 | |
paulb@140 | 1135 | """ |
paulb@140 | 1136 | Connect via a persistent channel to an existing created process, reachable |
paulb@140 | 1137 | at the given 'address'. |
paulb@140 | 1138 | """ |
paulb@140 | 1139 | |
paulb@140 | 1140 | parent = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) |
paulb@140 | 1141 | parent.setblocking(1) |
paulb@140 | 1142 | parent.connect(address) |
paulb@140 | 1143 | return Channel(0, parent.makefile("r", 0), parent.makefile("w", 0)) |
paulb@140 | 1144 | |
paulb@97 | 1145 | def exit(channel): |
paulb@97 | 1146 | |
paulb@97 | 1147 | """ |
paulb@97 | 1148 | Terminate a created process, closing the given 'channel'. |
paulb@97 | 1149 | """ |
paulb@97 | 1150 | |
paulb@97 | 1151 | channel.close() |
paulb@97 | 1152 | os._exit(0) |
paulb@97 | 1153 | |
paulb@84 | 1154 | def start(callable, *args, **kw): |
paulb@40 | 1155 | |
paulb@40 | 1156 | """ |
paulb@40 | 1157 | Create a new process which shall start running in the given 'callable'. |
paulb@94 | 1158 | Additional arguments to the 'callable' can be given as additional arguments |
paulb@94 | 1159 | to this function. |
paulb@94 | 1160 | |
paulb@94 | 1161 | Return a communications channel to the creating process. For the created |
paulb@94 | 1162 | process, supply a channel as the 'channel' parameter in the given 'callable' |
paulb@94 | 1163 | so that it may send data back to the creating process. |
paulb@40 | 1164 | """ |
paulb@40 | 1165 | |
paulb@40 | 1166 | channel = create() |
paulb@40 | 1167 | if channel.pid == 0: |
paulb@40 | 1168 | try: |
paulb@40 | 1169 | try: |
paulb@84 | 1170 | callable(channel, *args, **kw) |
paulb@40 | 1171 | except: |
paulb@40 | 1172 | exc_type, exc_value, exc_traceback = sys.exc_info() |
paulb@40 | 1173 | channel.send(exc_value) |
paulb@40 | 1174 | finally: |
paulb@99 | 1175 | exit(channel) |
paulb@40 | 1176 | else: |
paulb@40 | 1177 | return channel |
paulb@40 | 1178 | |
paulb@140 | 1179 | def start_persistent(address, callable, *args, **kw): |
paulb@140 | 1180 | |
paulb@140 | 1181 | """ |
paulb@140 | 1182 | Create a new process which shall be reachable using the given 'address' and |
paulb@140 | 1183 | which will start running in the given 'callable'. Additional arguments to |
paulb@140 | 1184 | the 'callable' can be given as additional arguments to this function. |
paulb@140 | 1185 | |
paulb@140 | 1186 | Return a communications channel to the creating process. For the created |
paulb@140 | 1187 | process, supply a channel as the 'channel' parameter in the given 'callable' |
paulb@140 | 1188 | so that it may send data back to the creating process. |
paulb@140 | 1189 | |
paulb@140 | 1190 | Note that the created process employs a channel which is persistent: it can |
paulb@140 | 1191 | withstand disconnection from the creating process and subsequent connections |
paulb@140 | 1192 | from other processes. |
paulb@140 | 1193 | """ |
paulb@140 | 1194 | |
paulb@140 | 1195 | channel = create_persistent(address) |
paulb@140 | 1196 | if channel.pid == 0: |
paulb@140 | 1197 | close_streams() |
paulb@140 | 1198 | try: |
paulb@140 | 1199 | try: |
paulb@140 | 1200 | callable(channel, *args, **kw) |
paulb@140 | 1201 | except: |
paulb@140 | 1202 | exc_type, exc_value, exc_traceback = sys.exc_info() |
paulb@140 | 1203 | channel.send(exc_value) |
paulb@140 | 1204 | finally: |
paulb@140 | 1205 | exit(channel) |
paulb@140 | 1206 | else: |
paulb@140 | 1207 | return channel |
paulb@140 | 1208 | |
paulb@140 | 1209 | def close_streams(): |
paulb@140 | 1210 | |
paulb@140 | 1211 | """ |
paulb@140 | 1212 | Close streams which keep the current process attached to any creating |
paulb@140 | 1213 | processes. |
paulb@140 | 1214 | """ |
paulb@140 | 1215 | |
paulb@140 | 1216 | os.close(sys.stdin.fileno()) |
paulb@140 | 1217 | os.close(sys.stdout.fileno()) |
paulb@140 | 1218 | os.close(sys.stderr.fileno()) |
paulb@140 | 1219 | |
paulb@40 | 1220 | def waitall(): |
paulb@40 | 1221 | |
paulb@40 | 1222 | "Wait for all created processes to terminate." |
paulb@40 | 1223 | |
paulb@40 | 1224 | try: |
paulb@40 | 1225 | while 1: |
paulb@40 | 1226 | os.wait() |
paulb@40 | 1227 | except OSError: |
paulb@40 | 1228 | pass |
paulb@40 | 1229 | |
paulb@40 | 1230 | # vim: tabstop=4 expandtab shiftwidth=4 |