1 #!/usr/bin/env python 2 3 """ 4 Import logic. 5 6 Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 7 2014, 2015, 2016, 2017 Paul Boddie <paul@boddie.org.uk> 8 9 This program is free software; you can redistribute it and/or modify it under 10 the terms of the GNU General Public License as published by the Free Software 11 Foundation; either version 3 of the License, or (at your option) any later 12 version. 13 14 This program is distributed in the hope that it will be useful, but WITHOUT 15 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 16 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 17 details. 18 19 You should have received a copy of the GNU General Public License along with 20 this program. If not, see <http://www.gnu.org/licenses/>. 21 """ 22 23 from errors import ProgramError 24 from os.path import exists, extsep, getmtime, join 25 from os import listdir, makedirs, remove 26 from common import init_item, readfile, writefile 27 from modules import CachedModule 28 from referencing import Reference 29 import inspector 30 import sys 31 32 class Importer: 33 34 "An import machine, searching for and loading modules." 35 36 special_attributes = ("__args__", "__file__", "__fn__", "__name__", "__parent__") 37 38 def __init__(self, path, cache=None, verbose=False, warnings=None): 39 40 """ 41 Initialise the importer with the given search 'path' - a list of 42 directories to search for Python modules. 43 44 The optional 'cache' should be the name of a directory used to store 45 cached module information. 46 47 The optional 'verbose' parameter causes output concerning the activities 48 of the object to be produced if set to a true value (not the default). 49 50 The optional 'warnings' parameter may indicate classes of warnings to be 51 produced. 52 """ 53 54 self.path = path 55 self.cache = cache 56 self.verbose = verbose 57 self.warnings = warnings 58 59 # Module importing queue, required modules, removed modules and active 60 # modules in the final program. 61 62 self.to_import = set() 63 self.required = set(["__main__"]) 64 self.removed = {} 65 self.modules = {} 66 67 # Module relationships and invalidated cached modules. 68 69 self.accessing_modules = {} 70 self.invalidated = set() 71 72 # Object relationships and dependencies. 73 74 self.depends = {} 75 self.module_depends = {} 76 77 # Basic program information. 78 79 self.objects = {} 80 self.classes = {} 81 self.function_parameters = {} 82 self.function_defaults = {} 83 self.function_locals = {} 84 85 # Unresolved names. 86 87 self.missing = set() 88 89 # Derived information. 90 91 self.subclasses = {} 92 93 # Attributes of different object types. 94 95 self.all_class_attrs = {} 96 self.all_instance_attrs = {} 97 self.all_instance_attr_constants = {} 98 self.all_combined_attrs = {} 99 self.all_module_attrs = {} 100 self.all_shadowed_attrs = {} 101 102 # References to external names and aliases within program units. 103 104 self.all_name_references = {} 105 self.all_initialised_names = {} 106 self.all_aliased_names = {} 107 108 # General attribute accesses. 109 110 self.all_attr_accesses = {} 111 self.all_const_accesses = {} 112 self.all_attr_access_modifiers = {} 113 114 # Constant literals and values. 115 116 self.all_constants = {} 117 self.all_constant_values = {} 118 119 # Return values. 120 121 self.all_return_values = {} 122 123 self.make_cache() 124 125 def give_warning(self, name): 126 127 "Return whether the indicated warning 'name' should be given." 128 129 return self.warnings and (name in self.warnings or "all" in self.warnings) 130 131 def make_cache(self): 132 133 "Make a cache directory if it does not already exist." 134 135 if self.cache and not exists(self.cache): 136 makedirs(self.cache) 137 138 def check_cache(self, details): 139 140 """ 141 Check whether the cache applies for the given 'details', invalidating it 142 if it does not. 143 """ 144 145 recorded_details = self.get_cache_details() 146 147 if recorded_details != details: 148 self.remove_cache() 149 150 writefile(self.get_cache_details_filename(), details) 151 152 def get_cache_details_filename(self): 153 154 "Return the filename for the cache details." 155 156 return join(self.cache, "$details") 157 158 def get_cache_details(self): 159 160 "Return details of the cache." 161 162 details_filename = self.get_cache_details_filename() 163 164 if not exists(details_filename): 165 return None 166 else: 167 return readfile(details_filename) 168 169 def remove_cache(self): 170 171 "Remove the contents of the cache." 172 173 for filename in listdir(self.cache): 174 remove(join(self.cache, filename)) 175 176 def to_cache(self): 177 178 "Write modules to the cache." 179 180 if self.cache: 181 for module_name, module in self.modules.items(): 182 module.to_cache(join(self.cache, module_name)) 183 184 # Object retrieval and storage. 185 186 def get_object(self, name): 187 188 """ 189 Return a reference for the given 'name' or None if no such object 190 exists. 191 """ 192 193 return self.objects.get(name) 194 195 def set_object(self, name, value=None): 196 197 "Set the object with the given 'name' and the given 'value'." 198 199 if isinstance(value, Reference): 200 ref = value.alias(name) 201 else: 202 ref = Reference(value, name) 203 204 self.objects[name] = ref 205 206 # Identification of both stored object names and name references. 207 208 def identify(self, name): 209 210 "Identify 'name' using stored object and external name records." 211 212 return self.objects.get(name) or self.all_name_references.get(name) 213 214 # Indirect object retrieval. 215 216 def get_attributes(self, ref, attrname): 217 218 """ 219 Return attributes provided by 'ref' for 'attrname'. Class attributes 220 may be provided by instances. 221 """ 222 223 kind = ref.get_kind() 224 if kind == "<class>": 225 ref = self.get_class_attribute(ref.get_origin(), attrname) 226 return ref and set([ref]) or set() 227 elif kind == "<instance>": 228 return self.get_combined_attributes(ref.get_origin(), attrname) 229 elif kind == "<module>": 230 ref = self.get_module_attribute(ref.get_origin(), attrname) 231 return ref and set([ref]) or set() 232 else: 233 return set() 234 235 def get_class_attribute(self, object_type, attrname): 236 237 "Return from 'object_type' the details of class attribute 'attrname'." 238 239 attrs = self.all_class_attrs.get(object_type) 240 attr = attrs and attrs.get(attrname) 241 return attr and self.get_object(attr) 242 243 def get_instance_attributes(self, object_type, attrname): 244 245 """ 246 Return from 'object_type' the details of instance attribute 'attrname'. 247 """ 248 249 consts = self.all_instance_attr_constants.get(object_type) 250 attrs = set() 251 for attr in self.all_instance_attrs[object_type].get(attrname, []): 252 attrs.add(consts and consts.get(attrname) or Reference("<var>", attr)) 253 return attrs 254 255 def get_combined_attributes(self, object_type, attrname): 256 257 """ 258 Return from 'object_type' the details of class or instance attribute 259 'attrname'. 260 """ 261 262 ref = self.get_class_attribute(object_type, attrname) 263 refs = ref and set([ref]) or set() 264 refs.update(self.get_instance_attributes(object_type, attrname)) 265 return refs 266 267 def get_module_attribute(self, object_type, attrname): 268 269 "Return from 'object_type' the details of module attribute 'attrname'." 270 271 if attrname in self.all_module_attrs[object_type]: 272 return self.get_object("%s.%s" % (object_type, attrname)) 273 else: 274 return None 275 276 # Convenience methods for deducing which kind of object provided an 277 # attribute. 278 279 def get_attribute_provider(self, ref, attrname): 280 281 """ 282 Return the kind of provider of the attribute accessed via 'ref' using 283 'attrname'. 284 """ 285 286 kind = ref.get_kind() 287 288 if kind in ["<class>", "<module>"]: 289 return kind 290 else: 291 return self.get_instance_attribute_provider(ref.get_origin(), attrname) 292 293 def get_instance_attribute_provider(self, object_type, attrname): 294 295 """ 296 Return the kind of provider of the attribute accessed via an instance of 297 'object_type' using 'attrname'. 298 """ 299 300 if self.get_class_attribute(object_type, attrname): 301 return "<class>" 302 else: 303 return "<instance>" 304 305 # Module management. 306 307 def queue_module(self, name, accessor, required=False): 308 309 """ 310 Queue the module with the given 'name' for import from the given 311 'accessor' module. If 'required' is true (it is false by default), the 312 module will be required in the final program. 313 """ 314 315 if not self.modules.has_key(name): 316 self.to_import.add(name) 317 318 if required: 319 self.required.add(name) 320 321 init_item(self.accessing_modules, name, set) 322 self.accessing_modules[name].add(accessor.name) 323 324 def get_modules(self): 325 326 "Return all modules known to the importer." 327 328 return self.modules.values() 329 330 def get_module(self, name): 331 332 "Return the module with the given 'name'." 333 334 if not self.modules.has_key(name): 335 return None 336 337 return self.modules[name] 338 339 # Program operations. 340 341 def initialise(self, filename, reset=False): 342 343 """ 344 Initialise a program whose main module is 'filename', resetting the 345 cache if 'reset' is true. Return the main module. 346 """ 347 348 if reset: 349 self.remove_cache() 350 self.check_cache(filename) 351 352 # Load the program itself. 353 354 m = self.load_from_file(filename) 355 356 # Load any queued modules. 357 358 while self.to_import: 359 for name in list(self.to_import): # avoid mutation issue 360 self.load(name) 361 362 # Resolve dependencies between modules. 363 364 self.resolve() 365 366 # Record the type of all classes. 367 368 self.type_ref = self.get_object("__builtins__.type") 369 370 # Resolve dependencies within the program. 371 372 for module in self.modules.values(): 373 module.complete() 374 375 # Remove unneeded modules. 376 377 all_modules = self.modules.items() 378 379 for name, module in all_modules: 380 if name not in self.required: 381 module.unpropagate() 382 del self.modules[name] 383 self.removed[name] = module 384 385 # Collect redundant objects. 386 387 for module in self.removed.values(): 388 module.collect() 389 390 # Assert module objects where aliases have been removed. 391 392 for name in self.required: 393 if not self.objects.has_key(name): 394 self.objects[name] = Reference("<module>", name) 395 396 return m 397 398 def finalise(self): 399 400 """ 401 Finalise the inspected program, returning whether the program could be 402 finalised. 403 """ 404 405 self.finalise_classes() 406 self.add_init_dependencies() 407 self.to_cache() 408 409 if self.missing: 410 return False 411 412 self.set_class_types() 413 self.define_instantiators() 414 self.collect_constants() 415 416 return True 417 418 # Supporting operations. 419 420 def resolve(self): 421 422 "Resolve dependencies between modules." 423 424 self.waiting = {} 425 426 for module in self.modules.values(): 427 428 # Resolve all deferred references in each module. 429 430 original_deferred = [] 431 432 for ref in module.deferred: 433 434 # Retain original references for caching. 435 436 original_deferred.append(ref.copy()) 437 438 # Update references throughout the program. 439 440 found = self.find_dependency(ref) 441 if not found: 442 self.missing.add((module.name, ref.get_origin())) 443 444 # Record the resolved names and identify required modules. 445 446 else: 447 # Find the providing module of this reference. 448 # Where definitive details of the origin cannot be found, 449 # identify the provider using the deferred reference. 450 # NOTE: This may need to test for static origins. 451 452 provider = self.get_module_provider(found.unresolved() and ref or found) 453 ref.mutate(found) 454 455 # Record any external dependency. 456 457 if provider and provider != module.name: 458 459 # Handle built-in modules accidentally referenced by 460 # names. 461 462 if provider == "__builtins__" and found.has_kind("<module>"): 463 raise ProgramError("Name %s, used by %s, refers to module %s." % 464 (found.leaf(), module.name, found.get_origin())) 465 466 # Record the provider dependency. 467 468 module.required.add(provider) 469 self.accessing_modules[provider].add(module.name) 470 471 # Postpone any inclusion of the provider until this 472 # module becomes required. 473 474 if module.name not in self.required: 475 init_item(self.waiting, module.name, set) 476 self.waiting[module.name].add(provider) 477 if self.verbose: 478 print >>sys.stderr, "Noting", provider, "for", ref, "from", module.name 479 480 # Make this module required in the accessing module. 481 482 elif provider not in self.required: 483 self.required.add(provider) 484 if self.verbose: 485 print >>sys.stderr, "Requiring", provider, "for", ref, "from", module.name 486 487 # Record a module ordering dependency. 488 489 if not found.static() or self.is_dynamic_class(found) or self.is_dynamic_callable(found): 490 self.add_module_dependency(module.name, provider) 491 492 # Restore the original references so that they may be read back in 493 # and produce the same results. 494 495 module.deferred = original_deferred 496 497 # Check modules again to see if they are now required and should now 498 # cause the inclusion of other modules providing objects to the program. 499 500 for module_name in self.waiting.keys(): 501 self.require_providers(module_name) 502 503 self.add_special_dependencies() 504 self.add_module_dependencies() 505 506 def require_providers(self, module_name): 507 508 """ 509 Test if 'module_name' is itself required and, if so, require modules 510 containing objects provided to the module. 511 """ 512 513 if module_name in self.required and self.waiting.has_key(module_name): 514 for provider in self.waiting[module_name]: 515 if provider not in self.required: 516 self.required.add(provider) 517 if self.verbose: 518 print >>sys.stderr, "Requiring", provider 519 self.require_providers(provider) 520 521 def add_special_dependencies(self): 522 523 "Add dependencies due to the use of special names in namespaces." 524 525 for module in self.modules.values(): 526 for ref, paths in module.special.values(): 527 for path in paths: 528 self.add_dependency(path, ref.get_origin()) 529 530 def add_init_dependencies(self): 531 532 "Add dependencies related to object initialisation." 533 534 for name in self.classes.keys(): 535 if self.is_dynamic_class(name): 536 537 # Make subclasses depend on any class with non-static 538 # attributes, plus its module for the initialisation. 539 540 for subclass in self.subclasses[name]: 541 ref = Reference("<class>", subclass) 542 self.add_dependency(subclass, name) 543 self.add_dependency(subclass, self.get_module_provider(ref)) 544 545 # Also make the class dependent on its module for 546 # initialisation. 547 548 ref = Reference("<class>", name) 549 self.add_dependency(name, self.get_module_provider(ref)) 550 551 for name in self.function_defaults.keys(): 552 if self.is_dynamic_callable(name): 553 554 # Make functions with defaults requiring initialisation depend 555 # on the parent scope, if a function, or the module scope. 556 557 ref = Reference("<function>", name) 558 parent_ref = self.get_object(ref.parent()) 559 560 # Function no longer present in the program. 561 562 if not parent_ref: 563 continue 564 565 if parent_ref.has_kind("<class>"): 566 parent = self.get_module_provider(parent_ref) 567 else: 568 parent = parent_ref.get_origin() 569 570 self.add_dependency(name, parent) 571 572 def add_module_dependencies(self): 573 574 "Record module-based dependencies." 575 576 for module_name, providers in self.module_depends.items(): 577 if self.modules.has_key(module_name): 578 for provider in providers: 579 if self.modules.has_key(provider): 580 self.add_dependency(module_name, provider) 581 582 def add_dependency(self, path, origin): 583 584 "Add dependency details for 'path' involving 'origin'." 585 586 if origin and not origin.startswith("%s." % path): 587 init_item(self.depends, path, set) 588 self.depends[path].add(origin) 589 590 def add_module_dependency(self, module_name, provider): 591 592 "Add dependency details for 'module_name' involving 'provider'." 593 594 if provider: 595 init_item(self.module_depends, module_name, set) 596 self.module_depends[module_name].add(provider) 597 598 def condense_dependencies(self): 599 600 """ 601 Condense the dependencies by removing all functions that do not need 602 initialisation. 603 """ 604 605 d = {} 606 for path, depends in self.depends.items(): 607 d[path] = {} 608 d[path] = self.condense_dependency_entry(depends, d) 609 610 self.depends = {} 611 612 for path, depends in d.items(): 613 if depends: 614 self.depends[path] = depends 615 616 def condense_dependency_entry(self, depends, d): 617 l = set() 618 for depend in depends: 619 if self.modules.has_key(depend) or self.classes.has_key(depend) or \ 620 self.is_dynamic_callable(depend): 621 622 l.add(depend) 623 else: 624 deps = d.get(depend) 625 if deps: 626 l.update(self.condense_dependency_entry(deps, d)) 627 return l 628 629 def is_dynamic(self, ref): 630 return not ref or not ref.static() and not ref.is_constant_alias() and not ref.is_predefined_value() 631 632 def is_dynamic_class(self, name): 633 634 """ 635 Return whether 'name' refers to a class with members that must be 636 initialised dynamically. 637 """ 638 639 attrs = self.all_class_attrs.get(name) 640 641 if not attrs: 642 return False 643 644 for attrname, attr in attrs.items(): 645 if attrname in self.special_attributes: 646 continue 647 ref = attr and self.get_object(attr) 648 if self.is_dynamic(ref): 649 return True 650 651 return False 652 653 def is_dynamic_callable(self, name): 654 655 """ 656 Return whether 'name' refers to a callable employing defaults that may 657 need initialising before the callable can be used. 658 """ 659 660 # Find any defaults for the function or method. 661 662 defaults = self.function_defaults.get(name) 663 if not defaults: 664 return False 665 666 # Identify non-constant defaults. 667 668 for name, ref in defaults: 669 if self.is_dynamic(ref): 670 return True 671 672 return False 673 674 def order_objects(self): 675 676 "Produce an object initialisation ordering." 677 678 self.condense_dependencies() 679 680 # Record the number of modules using or depending on each module. 681 682 usage = {} 683 684 # Record path-based dependencies. 685 686 for path in self.depends.keys(): 687 usage[path] = set() 688 689 for path, depends in self.depends.items(): 690 for origin in depends: 691 init_item(usage, origin, set) 692 usage[origin].add(path) 693 694 # Produce an ordering by obtaining exposed modules (required by modules 695 # already processed) and putting them at the start of the list. 696 697 ordered = [] 698 699 while usage: 700 have_next = False 701 702 for path, n in usage.items(): 703 if not n: 704 ordered.insert(0, path) 705 depends = self.depends.get(path) 706 707 # Reduce usage of the referenced objects. 708 709 if depends: 710 for origin in depends: 711 usage[origin].remove(path) 712 713 del usage[path] 714 have_next = True 715 716 if not have_next: 717 raise ProgramError("Modules with unresolvable dependencies exist: %s" % ", ".join(usage.keys())) 718 719 if "__main__" in ordered: 720 ordered.remove("__main__") 721 722 ordered.append("__main__") 723 return ordered 724 725 def order_modules(self): 726 727 "Produce a module initialisation ordering." 728 729 ordered = self.order_objects() 730 filtered = [] 731 732 for module_name in self.modules.keys(): 733 if module_name not in ordered: 734 filtered.append(module_name) 735 736 for path in ordered: 737 if self.modules.has_key(path): 738 filtered.append(path) 739 740 return filtered 741 742 def find_dependency(self, ref): 743 744 "Find the ultimate dependency for 'ref'." 745 746 found = set() 747 while ref and ref.has_kind("<depends>") and not ref in found: 748 found.add(ref) 749 ref = self.identify(ref.get_origin()) 750 return ref 751 752 def get_module_provider(self, ref): 753 754 "Identify the provider of the given 'ref'." 755 756 for ancestor in ref.ancestors(): 757 if self.modules.has_key(ancestor): 758 return ancestor 759 return None 760 761 def finalise_classes(self): 762 763 "Finalise the class relationships and attributes." 764 765 self.derive_inherited_attrs() 766 self.derive_subclasses() 767 self.derive_shadowed_attrs() 768 769 def derive_inherited_attrs(self): 770 771 "Derive inherited attributes for classes throughout the program." 772 773 for name in self.classes.keys(): 774 self.propagate_attrs_for_class(name) 775 776 def propagate_attrs_for_class(self, name, visited=None): 777 778 "Propagate inherited attributes for class 'name'." 779 780 # Visit classes only once. 781 782 if self.all_combined_attrs.has_key(name): 783 return 784 785 visited = visited or [] 786 787 if name in visited: 788 raise ProgramError, "Class %s may not inherit from itself: %s -> %s." % (name, " -> ".join(visited), name) 789 790 visited.append(name) 791 792 class_attrs = {} 793 instance_attrs = {} 794 795 # Aggregate the attributes from base classes, recording the origins of 796 # applicable attributes. 797 798 for base in self.classes[name][::-1]: 799 800 # Get the identity of the class from the reference. 801 802 base = base.get_origin() 803 804 # Define the base class completely before continuing with this 805 # class. 806 807 self.propagate_attrs_for_class(base, visited) 808 class_attrs.update(self.all_class_attrs[base]) 809 810 # Instance attribute origins are combined if different. 811 812 for key, values in self.all_instance_attrs[base].items(): 813 init_item(instance_attrs, key, set) 814 instance_attrs[key].update(values) 815 816 # Class attributes override those defined earlier in the hierarchy. 817 818 class_attrs.update(self.all_class_attrs.get(name, {})) 819 820 # Instance attributes are merely added if not already defined. 821 822 for key in self.all_instance_attrs.get(name, []): 823 if not instance_attrs.has_key(key): 824 instance_attrs[key] = set(["%s.%s" % (name, key)]) 825 826 self.all_class_attrs[name] = class_attrs 827 self.all_instance_attrs[name] = instance_attrs 828 self.all_combined_attrs[name] = set(class_attrs.keys()).union(instance_attrs.keys()) 829 830 def derive_subclasses(self): 831 832 "Derive subclass details for classes." 833 834 for name, bases in self.classes.items(): 835 for base in bases: 836 837 # Get the identity of the class from the reference. 838 839 base = base.get_origin() 840 self.subclasses[base].add(name) 841 842 def derive_shadowed_attrs(self): 843 844 "Derive shadowed attributes for classes." 845 846 for name, attrs in self.all_instance_attrs.items(): 847 attrs = set(attrs.keys()).intersection(self.all_class_attrs[name].keys()) 848 if attrs: 849 self.all_shadowed_attrs[name] = attrs 850 851 def set_class_types(self): 852 853 "Set the type of each class." 854 855 for attrs in self.all_class_attrs.values(): 856 attrs["__class__"] = self.type_ref.get_origin() 857 858 def define_instantiators(self): 859 860 """ 861 Consolidate parameter and default details, incorporating initialiser 862 details to define instantiator signatures. 863 """ 864 865 for cls, attrs in self.all_class_attrs.items(): 866 initialiser = attrs["__init__"] 867 self.function_parameters[cls] = self.function_parameters[initialiser] 868 self.function_defaults[cls] = self.function_defaults[initialiser] 869 870 def collect_constants(self): 871 872 "Get constants from all active modules." 873 874 for module in self.modules.values(): 875 self.all_constants.update(module.constants) 876 877 # Import methods. 878 879 def find_in_path(self, name): 880 881 """ 882 Find the given module 'name' in the search path, returning None where no 883 such module could be found, or a 2-tuple from the 'find' method 884 otherwise. 885 """ 886 887 for d in self.path: 888 m = self.find(d, name) 889 if m: return m 890 return None 891 892 def find(self, d, name): 893 894 """ 895 In the directory 'd', find the given module 'name', where 'name' can 896 either refer to a single file module or to a package. Return None if the 897 'name' cannot be associated with either a file or a package directory, 898 or a 2-tuple from '_find_package' or '_find_module' otherwise. 899 """ 900 901 m = self._find_package(d, name) 902 if m: return m 903 m = self._find_module(d, name) 904 if m: return m 905 return None 906 907 def _find_module(self, d, name): 908 909 """ 910 In the directory 'd', find the given module 'name', returning None where 911 no suitable file exists in the directory, or a 2-tuple consisting of 912 None (indicating that no package directory is involved) and a filename 913 indicating the location of the module. 914 """ 915 916 name_py = name + extsep + "py" 917 filename = self._find_file(d, name_py) 918 if filename: 919 return None, filename 920 return None 921 922 def _find_package(self, d, name): 923 924 """ 925 In the directory 'd', find the given package 'name', returning None 926 where no suitable package directory exists, or a 2-tuple consisting of 927 a directory (indicating the location of the package directory itself) 928 and a filename indicating the location of the __init__.py module which 929 declares the package's top-level contents. 930 """ 931 932 filename = self._find_file(d, name) 933 if filename: 934 init_py = "__init__" + extsep + "py" 935 init_py_filename = self._find_file(filename, init_py) 936 if init_py_filename: 937 return filename, init_py_filename 938 return None 939 940 def _find_file(self, d, filename): 941 942 """ 943 Return the filename obtained when searching the directory 'd' for the 944 given 'filename', or None if no actual file exists for the filename. 945 """ 946 947 filename = join(d, filename) 948 if exists(filename): 949 return filename 950 else: 951 return None 952 953 def load(self, name): 954 955 """ 956 Load the module or package with the given 'name'. Return an object 957 referencing the loaded module or package, or None if no such module or 958 package exists. 959 """ 960 961 # Loaded modules are returned immediately. 962 # Modules may be known but not yet loading (having been registered as 963 # submodules), loading, loaded, or completely unknown. 964 965 module = self.get_module(name) 966 967 if module: 968 return self.modules[name] 969 970 # Otherwise, modules are loaded. 971 972 # Split the name into path components, and try to find the uppermost in 973 # the search path. 974 975 path = name.split(".") 976 path_so_far = [] 977 module = None 978 979 for p in path: 980 981 # Get the module's filesystem details. 982 983 if not path_so_far: 984 m = self.find_in_path(p) 985 elif d: 986 m = self.find(d, p) 987 else: 988 m = None 989 990 path_so_far.append(p) 991 module_name = ".".join(path_so_far) 992 993 # Raise an exception if the module could not be located. 994 995 if not m: 996 raise ProgramError("Module not found: %s" % name) 997 998 # Get the directory and module filename. 999 1000 d, filename = m 1001 1002 # Get the module itself. 1003 1004 return self.load_from_file(filename, module_name) 1005 1006 def load_from_file(self, filename, module_name=None): 1007 1008 "Load the module from the given 'filename'." 1009 1010 if module_name is None: 1011 module_name = "__main__" 1012 1013 module = self.modules.get(module_name) 1014 1015 if not module: 1016 1017 # Try to load from cache. 1018 1019 module = self.load_from_cache(filename, module_name) 1020 if module: 1021 return module 1022 1023 # If no cache entry exists, load from file. 1024 1025 module = inspector.InspectedModule(module_name, self) 1026 self.add_module(module_name, module) 1027 self.update_cache_validity(module) 1028 1029 self._load(module, module_name, lambda m: m.parse, filename) 1030 1031 return module 1032 1033 def update_cache_validity(self, module): 1034 1035 "Make 'module' valid in the cache, but invalidate accessing modules." 1036 1037 accessing = self.accessing_modules.get(module.name) 1038 if accessing: 1039 self.invalidated.update(accessing) 1040 if module.name in self.invalidated: 1041 self.invalidated.remove(module.name) 1042 1043 def source_is_new(self, filename, module_name): 1044 1045 "Return whether 'filename' is newer than the cached 'module_name'." 1046 1047 if self.cache: 1048 cache_filename = join(self.cache, module_name) 1049 return not exists(cache_filename) or \ 1050 getmtime(filename) > getmtime(cache_filename) or \ 1051 module_name in self.invalidated 1052 else: 1053 return True 1054 1055 def load_from_cache(self, filename, module_name): 1056 1057 "Return a module residing in the cache." 1058 1059 module = self.modules.get(module_name) 1060 1061 if not module and not self.source_is_new(filename, module_name): 1062 module = CachedModule(module_name, self) 1063 self.add_module(module_name, module) 1064 1065 filename = join(self.cache, module_name) 1066 self._load(module, module_name, lambda m: m.from_cache, filename) 1067 1068 return module 1069 1070 def _load(self, module, module_name, fn, filename): 1071 1072 """ 1073 Load 'module' for the given 'module_name', and with 'fn' performing an 1074 invocation on the module with the given 'filename'. 1075 """ 1076 1077 # Load the module. 1078 1079 if self.verbose: 1080 print >>sys.stderr, module_name in self.required and "Required" or "Loading", module_name, "from", filename 1081 fn(module)(filename) 1082 1083 # Add the module object if not already defined. 1084 1085 if not self.objects.has_key(module_name): 1086 self.objects[module_name] = Reference("<module>", module_name) 1087 1088 def add_module(self, module_name, module): 1089 1090 """ 1091 Return the module with the given 'module_name', adding a new module 1092 object if one does not already exist. 1093 """ 1094 1095 self.modules[module_name] = module 1096 if module_name in self.to_import: 1097 self.to_import.remove(module_name) 1098 1099 # vim: tabstop=4 expandtab shiftwidth=4