Lichen

Annotated importer.py

225:dfbc750d3fce
2016-11-23 Paul Boddie Produce a proper error when too many arguments are given for an invocation.
paul@0 1
#!/usr/bin/env python
paul@0 2
paul@0 3
"""
paul@0 4
Import logic.
paul@0 5
paul@0 6
Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013,
paul@0 7
              2014, 2015, 2016 Paul Boddie <paul@boddie.org.uk>
paul@0 8
paul@0 9
This program is free software; you can redistribute it and/or modify it under
paul@0 10
the terms of the GNU General Public License as published by the Free Software
paul@0 11
Foundation; either version 3 of the License, or (at your option) any later
paul@0 12
version.
paul@0 13
paul@0 14
This program is distributed in the hope that it will be useful, but WITHOUT
paul@0 15
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
paul@0 16
FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
paul@0 17
details.
paul@0 18
paul@0 19
You should have received a copy of the GNU General Public License along with
paul@0 20
this program.  If not, see <http://www.gnu.org/licenses/>.
paul@0 21
"""
paul@0 22
paul@0 23
from errors import ProgramError
paul@0 24
from os.path import exists, extsep, getmtime, join
paul@0 25
from os import listdir, makedirs, remove
paul@0 26
from common import init_item, readfile, writefile
paul@13 27
from modules import CachedModule
paul@0 28
from referencing import Reference
paul@0 29
import inspector
paul@0 30
import sys
paul@0 31
paul@0 32
class Importer:
paul@0 33
paul@0 34
    "An import machine, searching for and loading modules."
paul@0 35
paul@0 36
    def __init__(self, path, cache=None, verbose=False):
paul@0 37
paul@0 38
        """
paul@0 39
        Initialise the importer with the given search 'path' - a list of
paul@0 40
        directories to search for Python modules.
paul@0 41
paul@0 42
        The optional 'cache' should be the name of a directory used to store
paul@0 43
        cached module information.
paul@0 44
paul@0 45
        The optional 'verbose' parameter causes output concerning the activities
paul@0 46
        of the object to be produced if set to a true value (not the default).
paul@0 47
        """
paul@0 48
paul@0 49
        self.path = path
paul@0 50
        self.cache = cache
paul@0 51
        self.verbose = verbose
paul@0 52
paul@41 53
        # Module importing queue, required modules, removed modules and active
paul@41 54
        # modules in the final program.
paul@41 55
paul@12 56
        self.to_import = set()
paul@16 57
        self.required = set(["__main__"])
paul@24 58
        self.removed = {}
paul@41 59
        self.modules = {}
paul@12 60
paul@41 61
        # Module relationships and invalidated cached modules.
paul@41 62
paul@12 63
        self.accessing_modules = {}
paul@0 64
        self.invalidated = set()
paul@0 65
paul@41 66
        # Basic program information.
paul@41 67
paul@0 68
        self.objects = {}
paul@0 69
        self.classes = {}
paul@0 70
        self.function_parameters = {}
paul@0 71
        self.function_defaults = {}
paul@109 72
        self.function_locals = {}
paul@0 73
        self.function_targets = {}
paul@0 74
        self.function_arguments = {}
paul@0 75
paul@41 76
        # Unresolved names.
paul@41 77
paul@41 78
        self.missing = set()
paul@41 79
paul@0 80
        # Derived information.
paul@0 81
paul@0 82
        self.subclasses = {}
paul@0 83
paul@0 84
        # Attributes of different object types.
paul@0 85
paul@0 86
        self.all_class_attrs = {}
paul@0 87
        self.all_instance_attrs = {}
paul@0 88
        self.all_instance_attr_constants = {}
paul@0 89
        self.all_combined_attrs = {}
paul@0 90
        self.all_module_attrs = {}
paul@0 91
        self.all_shadowed_attrs = {}
paul@0 92
paul@0 93
        # References to external names and aliases within program units.
paul@0 94
paul@0 95
        self.all_name_references = {}
paul@0 96
        self.all_initialised_names = {}
paul@0 97
        self.all_aliased_names = {}
paul@0 98
paul@0 99
        # General attribute accesses.
paul@0 100
paul@0 101
        self.all_attr_accesses = {}
paul@0 102
        self.all_const_accesses = {}
paul@0 103
        self.all_attr_access_modifiers = {}
paul@0 104
paul@0 105
        # Constant literals and values.
paul@0 106
paul@0 107
        self.all_constants = {}
paul@0 108
        self.all_constant_values = {}
paul@0 109
paul@0 110
        self.make_cache()
paul@0 111
paul@0 112
    def make_cache(self):
paul@0 113
        if self.cache and not exists(self.cache):
paul@0 114
            makedirs(self.cache)
paul@0 115
paul@0 116
    def check_cache(self, details):
paul@0 117
paul@0 118
        """
paul@0 119
        Check whether the cache applies for the given 'details', invalidating it
paul@0 120
        if it does not.
paul@0 121
        """
paul@0 122
paul@0 123
        recorded_details = self.get_cache_details()
paul@0 124
paul@0 125
        if recorded_details != details:
paul@0 126
            self.remove_cache()
paul@0 127
paul@0 128
        writefile(self.get_cache_details_filename(), details)
paul@0 129
paul@0 130
    def get_cache_details_filename(self):
paul@0 131
paul@0 132
        "Return the filename for the cache details."
paul@0 133
paul@0 134
        return join(self.cache, "$details")
paul@0 135
paul@0 136
    def get_cache_details(self):
paul@0 137
paul@0 138
        "Return details of the cache."
paul@0 139
paul@0 140
        details_filename = self.get_cache_details_filename()
paul@0 141
paul@0 142
        if not exists(details_filename):
paul@0 143
            return None
paul@0 144
        else:
paul@0 145
            return readfile(details_filename)
paul@0 146
paul@0 147
    def remove_cache(self):
paul@0 148
paul@0 149
        "Remove the contents of the cache."
paul@0 150
paul@0 151
        for filename in listdir(self.cache):
paul@0 152
            remove(join(self.cache, filename))
paul@0 153
paul@0 154
    def to_cache(self):
paul@0 155
paul@0 156
        "Write modules to the cache."
paul@0 157
paul@0 158
        if self.cache:
paul@0 159
            for module_name, module in self.modules.items():
paul@0 160
                module.to_cache(join(self.cache, module_name))
paul@0 161
paul@0 162
    # Object retrieval and storage.
paul@0 163
paul@0 164
    def get_object(self, name):
paul@0 165
paul@0 166
        """
paul@0 167
        Return a reference for the given 'name' or None if no such object
paul@0 168
        exists.
paul@0 169
        """
paul@0 170
paul@0 171
        return self.objects.get(name)
paul@0 172
paul@0 173
    def set_object(self, name, value=None):
paul@0 174
paul@0 175
        "Set the object with the given 'name' and the given 'value'."
paul@0 176
paul@0 177
        if isinstance(value, Reference):
paul@0 178
            ref = value.alias(name)
paul@0 179
        else:
paul@0 180
            ref = Reference(value, name)
paul@0 181
paul@0 182
        self.objects[name] = ref
paul@0 183
paul@27 184
    # Identification of both stored object names and name references.
paul@27 185
paul@27 186
    def identify(self, name):
paul@27 187
paul@27 188
        "Identify 'name' using stored object and external name records."
paul@27 189
paul@27 190
        return self.objects.get(name) or self.all_name_references.get(name)
paul@27 191
paul@0 192
    # Indirect object retrieval.
paul@0 193
paul@0 194
    def get_attributes(self, ref, attrname):
paul@0 195
paul@0 196
        """
paul@0 197
        Return attributes provided by 'ref' for 'attrname'. Class attributes
paul@0 198
        may be provided by instances.
paul@0 199
        """
paul@0 200
paul@0 201
        kind = ref.get_kind()
paul@0 202
        if kind == "<class>":
paul@0 203
            ref = self.get_class_attribute(ref.get_origin(), attrname)
paul@0 204
            return ref and set([ref]) or set()
paul@0 205
        elif kind == "<instance>":
paul@0 206
            return self.get_combined_attributes(ref.get_origin(), attrname)
paul@0 207
        elif kind == "<module>":
paul@0 208
            ref = self.get_module_attribute(ref.get_origin(), attrname)
paul@0 209
            return ref and set([ref]) or set()
paul@0 210
        else:
paul@0 211
            return set()
paul@0 212
paul@0 213
    def get_class_attribute(self, object_type, attrname):
paul@0 214
paul@0 215
        "Return from 'object_type' the details of class attribute 'attrname'."
paul@0 216
paul@0 217
        attr = self.all_class_attrs[object_type].get(attrname)
paul@0 218
        return attr and self.get_object(attr)
paul@0 219
paul@0 220
    def get_instance_attributes(self, object_type, attrname):
paul@0 221
paul@0 222
        """
paul@0 223
        Return from 'object_type' the details of instance attribute 'attrname'.
paul@0 224
        """
paul@0 225
paul@0 226
        consts = self.all_instance_attr_constants.get(object_type)
paul@0 227
        attrs = set()
paul@0 228
        for attr in self.all_instance_attrs[object_type].get(attrname, []):
paul@0 229
            attrs.add(consts and consts.get(attrname) or Reference("<var>", attr))
paul@0 230
        return attrs
paul@0 231
paul@0 232
    def get_combined_attributes(self, object_type, attrname):
paul@0 233
paul@0 234
        """
paul@0 235
        Return from 'object_type' the details of class or instance attribute
paul@0 236
        'attrname'.
paul@0 237
        """
paul@0 238
paul@0 239
        ref = self.get_class_attribute(object_type, attrname)
paul@0 240
        refs = ref and set([ref]) or set()
paul@0 241
        refs.update(self.get_instance_attributes(object_type, attrname))
paul@0 242
        return refs
paul@0 243
paul@0 244
    def get_module_attribute(self, object_type, attrname):
paul@0 245
paul@0 246
        "Return from 'object_type' the details of module attribute 'attrname'."
paul@0 247
paul@0 248
        if attrname in self.all_module_attrs[object_type]:
paul@0 249
            return self.get_object("%s.%s" % (object_type, attrname))
paul@0 250
        else:
paul@0 251
            return None
paul@0 252
paul@96 253
    # Convenience methods for deducing which kind of object provided an
paul@96 254
    # attribute.
paul@96 255
paul@96 256
    def get_attribute_provider(self, ref, attrname):
paul@96 257
paul@96 258
        """
paul@96 259
        Return the kind of provider of the attribute accessed via 'ref' using
paul@96 260
        'attrname'.
paul@96 261
        """
paul@96 262
paul@96 263
        kind = ref.get_kind()
paul@96 264
paul@96 265
        if kind in ["<class>", "<module>"]:
paul@96 266
            return kind
paul@96 267
        else:
paul@96 268
            return self.get_instance_attribute_provider(ref.get_origin(), attrname)
paul@96 269
paul@96 270
    def get_instance_attribute_provider(self, object_type, attrname):
paul@96 271
paul@96 272
        """
paul@96 273
        Return the kind of provider of the attribute accessed via an instance of
paul@96 274
        'object_type' using 'attrname'.
paul@96 275
        """
paul@96 276
paul@96 277
        if self.get_class_attribute(object_type, attrname):
paul@96 278
            return "<class>"
paul@96 279
        else:
paul@96 280
            return "<instance>"
paul@96 281
paul@0 282
    # Module management.
paul@0 283
paul@16 284
    def queue_module(self, name, accessor, required=False):
paul@12 285
paul@12 286
        """
paul@12 287
        Queue the module with the given 'name' for import from the given
paul@16 288
        'accessor' module. If 'required' is true (it is false by default), the
paul@16 289
        module will be required in the final program.
paul@12 290
        """
paul@12 291
paul@12 292
        if not self.modules.has_key(name):
paul@12 293
            self.to_import.add(name)
paul@12 294
paul@16 295
        if required:
paul@16 296
            self.required.add(name)
paul@16 297
paul@12 298
        init_item(self.accessing_modules, name, set)
paul@16 299
        self.accessing_modules[name].add(accessor.name)
paul@12 300
paul@0 301
    def get_modules(self):
paul@0 302
paul@0 303
        "Return all modules known to the importer."
paul@0 304
paul@0 305
        return self.modules.values()
paul@0 306
paul@12 307
    def get_module(self, name):
paul@0 308
paul@0 309
        "Return the module with the given 'name'."
paul@0 310
paul@0 311
        if not self.modules.has_key(name):
paul@0 312
            return None
paul@0 313
paul@12 314
        return self.modules[name]
paul@0 315
paul@0 316
    # Program operations.
paul@0 317
paul@0 318
    def initialise(self, filename, reset=False):
paul@0 319
paul@0 320
        """
paul@0 321
        Initialise a program whose main module is 'filename', resetting the
paul@0 322
        cache if 'reset' is true. Return the main module.
paul@0 323
        """
paul@0 324
paul@0 325
        if reset:
paul@0 326
            self.remove_cache()
paul@0 327
        self.check_cache(filename)
paul@0 328
paul@0 329
        # Load the program itself.
paul@0 330
paul@0 331
        m = self.load_from_file(filename)
paul@0 332
paul@12 333
        # Load any queued modules.
paul@12 334
paul@12 335
        while self.to_import:
paul@12 336
            for name in list(self.to_import): # avoid mutation issue
paul@12 337
                self.load(name)
paul@12 338
paul@12 339
        # Resolve dependencies between modules.
paul@12 340
paul@12 341
        self.resolve()
paul@12 342
paul@16 343
        # Record the type of all classes.
paul@16 344
paul@16 345
        self.type_ref = self.get_object("__builtins__.type")
paul@16 346
paul@0 347
        # Resolve dependencies within the program.
paul@0 348
paul@12 349
        for module in self.modules.values():
paul@12 350
            module.complete()
paul@0 351
paul@16 352
        # Remove unneeded modules.
paul@16 353
paul@16 354
        all_modules = self.modules.items()
paul@16 355
paul@16 356
        for name, module in all_modules:
paul@16 357
            if name not in self.required:
paul@16 358
                module.unpropagate()
paul@16 359
                del self.modules[name]
paul@24 360
                self.removed[name] = module
paul@16 361
paul@68 362
        # Collect redundant objects.
paul@68 363
paul@68 364
        for module in self.removed.values():
paul@68 365
            module.collect()
paul@68 366
paul@68 367
        # Assert module objects where aliases have been removed.
paul@68 368
paul@68 369
        for name in self.required:
paul@68 370
            if not self.objects.has_key(name):
paul@68 371
                self.objects[name] = Reference("<module>", name)
paul@68 372
paul@0 373
        return m
paul@0 374
paul@0 375
    def finalise(self):
paul@0 376
paul@41 377
        """
paul@41 378
        Finalise the inspected program, returning whether the program could be
paul@41 379
        finalised.
paul@41 380
        """
paul@41 381
paul@41 382
        if self.missing:
paul@41 383
            return False
paul@0 384
paul@0 385
        self.finalise_classes()
paul@0 386
        self.to_cache()
paul@0 387
        self.set_class_types()
paul@0 388
        self.define_instantiators()
paul@0 389
        self.collect_constants()
paul@0 390
paul@41 391
        return True
paul@41 392
paul@12 393
    # Supporting operations.
paul@12 394
paul@12 395
    def resolve(self):
paul@12 396
paul@12 397
        "Resolve dependencies between modules."
paul@12 398
paul@35 399
        self.waiting = {}
paul@186 400
        self.depends = {}
paul@35 401
paul@35 402
        for module in self.modules.values():
paul@35 403
paul@35 404
            # Resolve all deferred references in each module.
paul@12 405
paul@35 406
            for ref in module.deferred:
paul@35 407
                found = self.find_dependency(ref)
paul@35 408
                if not found:
paul@41 409
                    self.missing.add((module.name, ref.get_origin()))
paul@35 410
paul@35 411
                # Record the resolved names and identify required modules.
paul@12 412
paul@35 413
                else:
paul@170 414
                    # Find the providing module of this reference.
paul@170 415
                    # Where definitive details of the origin cannot be found,
paul@170 416
                    # identify the provider using the deferred reference.
paul@170 417
                    # NOTE: This may need to test for static origins.
paul@170 418
paul@170 419
                    provider = self.get_module_provider(found.unresolved() and ref or found)
paul@35 420
                    ref.mutate(found)
paul@35 421
paul@186 422
                    # Record any external dependency.
paul@186 423
paul@186 424
                    if provider and provider != module.name:
paul@186 425
paul@186 426
                        # Record the provider dependency.
paul@16 427
paul@35 428
                        module.required.add(provider)
paul@35 429
                        self.accessing_modules[provider].add(module.name)
paul@35 430
paul@35 431
                        # Postpone any inclusion of the provider until this
paul@35 432
                        # module becomes required.
paul@12 433
paul@35 434
                        if module.name not in self.required:
paul@35 435
                            init_item(self.waiting, module.name, set)
paul@35 436
                            self.waiting[module.name].add(provider)
paul@35 437
paul@35 438
                        # Make this module required in the accessing module.
paul@32 439
paul@53 440
                        elif provider not in self.required:
paul@35 441
                            self.required.add(provider)
paul@53 442
                            if self.verbose:
paul@53 443
                                print >>sys.stderr, "Requiring", provider, "for", ref
paul@35 444
paul@186 445
                        # Record a module ordering dependency.
paul@186 446
paul@186 447
                        if not found.static():
paul@186 448
                            init_item(self.depends, module.name, set)
paul@186 449
                            self.depends[module.name].add(provider)
paul@186 450
paul@38 451
        # Check modules again to see if they are now required and should now
paul@38 452
        # cause the inclusion of other modules providing objects to the program.
paul@38 453
paul@35 454
        for module_name in self.waiting.keys():
paul@35 455
            self.require_providers(module_name)
paul@16 456
paul@35 457
    def require_providers(self, module_name):
paul@38 458
paul@38 459
        """
paul@38 460
        Test if 'module_name' is itself required and, if so, require modules
paul@38 461
        containing objects provided to the module.
paul@38 462
        """
paul@38 463
paul@35 464
        if module_name in self.required and self.waiting.has_key(module_name):
paul@35 465
            for provider in self.waiting[module_name]:
paul@35 466
                if provider not in self.required:
paul@35 467
                    self.required.add(provider)
paul@53 468
                    if self.verbose:
paul@53 469
                        print >>sys.stderr, "Requiring", provider
paul@35 470
                    self.require_providers(provider)
paul@32 471
paul@186 472
    def order_modules(self):
paul@186 473
paul@186 474
        "Produce a module initialisation ordering."
paul@186 475
paul@186 476
        self.check_ordering()
paul@186 477
        return self.order_modules_for_module("__main__", set())
paul@186 478
paul@186 479
    def order_modules_for_module(self, module_name, visited):
paul@186 480
paul@186 481
        """
paul@186 482
        Order the modules required by 'module_name', using 'visited' to track
paul@186 483
        visited modules.
paul@186 484
        """
paul@186 485
paul@186 486
        if module_name in visited:
paul@186 487
            return []
paul@186 488
paul@186 489
        module = self.modules[module_name]
paul@186 490
        visited.add(module_name)
paul@186 491
        ordered = [module_name]
paul@186 492
paul@186 493
        for module_name in module.required:
paul@186 494
            modules = self.order_modules_for_module(module_name, visited)
paul@186 495
            ordered[:0] = modules
paul@186 496
            visited.update(modules)
paul@186 497
paul@186 498
        return ordered
paul@186 499
paul@186 500
    def check_ordering(self):
paul@186 501
paul@186 502
        "Check the ordering dependencies."
paul@186 503
paul@186 504
        for module_name, modules in self.depends.items():
paul@186 505
            for provider in modules:
paul@186 506
                if self.depends.has_key(provider) and module_name in self.depends[provider]:
paul@186 507
                    raise ProgramError, "Modules %s and %s may not depend on each other for non-static objects." % (module_name, provider)
paul@186 508
paul@12 509
    def find_dependency(self, ref):
paul@12 510
paul@12 511
        "Find the ultimate dependency for 'ref'."
paul@12 512
paul@12 513
        found = set()
paul@12 514
        while ref and ref.has_kind("<depends>") and not ref in found:
paul@12 515
            found.add(ref)
paul@35 516
            ref = self.identify(ref.get_origin())
paul@12 517
        return ref
paul@12 518
paul@16 519
    def get_module_provider(self, ref):
paul@16 520
paul@16 521
        "Identify the provider of the given 'ref'."
paul@16 522
paul@16 523
        for ancestor in ref.ancestors():
paul@16 524
            if self.modules.has_key(ancestor):
paul@16 525
                return ancestor
paul@16 526
        return None
paul@16 527
paul@0 528
    def finalise_classes(self):
paul@0 529
paul@0 530
        "Finalise the class relationships and attributes."
paul@0 531
paul@0 532
        self.derive_inherited_attrs()
paul@0 533
        self.derive_subclasses()
paul@0 534
        self.derive_shadowed_attrs()
paul@0 535
paul@0 536
    def derive_inherited_attrs(self):
paul@0 537
paul@0 538
        "Derive inherited attributes for classes throughout the program."
paul@0 539
paul@0 540
        for name in self.classes.keys():
paul@0 541
            self.propagate_attrs_for_class(name)
paul@0 542
paul@0 543
    def propagate_attrs_for_class(self, name, visited=None):
paul@0 544
paul@0 545
        "Propagate inherited attributes for class 'name'."
paul@0 546
paul@0 547
        # Visit classes only once.
paul@0 548
paul@0 549
        if self.all_combined_attrs.has_key(name):
paul@0 550
            return
paul@0 551
paul@0 552
        visited = visited or []
paul@0 553
paul@0 554
        if name in visited:
paul@0 555
            raise ProgramError, "Class %s may not inherit from itself: %s -> %s." % (name, " -> ".join(visited), name)
paul@0 556
paul@0 557
        visited.append(name)
paul@0 558
paul@0 559
        class_attrs = {}
paul@0 560
        instance_attrs = {}
paul@0 561
paul@0 562
        # Aggregate the attributes from base classes, recording the origins of
paul@0 563
        # applicable attributes.
paul@0 564
paul@0 565
        for base in self.classes[name][::-1]:
paul@0 566
paul@0 567
            # Get the identity of the class from the reference.
paul@0 568
paul@0 569
            base = base.get_origin()
paul@0 570
paul@0 571
            # Define the base class completely before continuing with this
paul@0 572
            # class.
paul@0 573
paul@0 574
            self.propagate_attrs_for_class(base, visited)
paul@0 575
            class_attrs.update(self.all_class_attrs[base])
paul@0 576
paul@0 577
            # Instance attribute origins are combined if different.
paul@0 578
paul@0 579
            for key, values in self.all_instance_attrs[base].items():
paul@0 580
                init_item(instance_attrs, key, set)
paul@0 581
                instance_attrs[key].update(values)
paul@0 582
paul@0 583
        # Class attributes override those defined earlier in the hierarchy.
paul@0 584
paul@0 585
        class_attrs.update(self.all_class_attrs.get(name, {}))
paul@0 586
paul@0 587
        # Instance attributes are merely added if not already defined.
paul@0 588
paul@0 589
        for key in self.all_instance_attrs.get(name, []):
paul@0 590
            if not instance_attrs.has_key(key):
paul@0 591
                instance_attrs[key] = set(["%s.%s" % (name, key)])
paul@0 592
paul@0 593
        self.all_class_attrs[name] = class_attrs
paul@0 594
        self.all_instance_attrs[name] = instance_attrs
paul@0 595
        self.all_combined_attrs[name] = set(class_attrs.keys()).union(instance_attrs.keys())
paul@0 596
paul@0 597
    def derive_subclasses(self):
paul@0 598
paul@0 599
        "Derive subclass details for classes."
paul@0 600
paul@0 601
        for name, bases in self.classes.items():
paul@0 602
            for base in bases:
paul@0 603
paul@0 604
                # Get the identity of the class from the reference.
paul@0 605
paul@0 606
                base = base.get_origin()
paul@0 607
                self.subclasses[base].add(name)
paul@0 608
paul@0 609
    def derive_shadowed_attrs(self):
paul@0 610
paul@0 611
        "Derive shadowed attributes for classes."
paul@0 612
paul@0 613
        for name, attrs in self.all_instance_attrs.items():
paul@0 614
            attrs = set(attrs.keys()).intersection(self.all_class_attrs[name].keys())
paul@0 615
            if attrs:
paul@0 616
                self.all_shadowed_attrs[name] = attrs
paul@0 617
paul@0 618
    def set_class_types(self):
paul@0 619
paul@0 620
        "Set the type of each class."
paul@0 621
paul@0 622
        for attrs in self.all_class_attrs.values():
paul@16 623
            attrs["__class__"] = self.type_ref.get_origin()
paul@0 624
paul@0 625
    def define_instantiators(self):
paul@0 626
paul@0 627
        """
paul@0 628
        Consolidate parameter and default details, incorporating initialiser
paul@0 629
        details to define instantiator signatures.
paul@0 630
        """
paul@0 631
paul@0 632
        for cls, attrs in self.all_class_attrs.items():
paul@0 633
            initialiser = attrs["__init__"]
paul@119 634
            self.function_parameters[cls] = self.function_parameters[initialiser]
paul@0 635
            self.function_defaults[cls] = self.function_defaults[initialiser]
paul@0 636
paul@0 637
    def collect_constants(self):
paul@0 638
paul@0 639
        "Get constants from all active modules."
paul@0 640
paul@0 641
        for module in self.modules.values():
paul@0 642
            self.all_constants.update(module.constants)
paul@0 643
paul@0 644
    # Import methods.
paul@0 645
paul@0 646
    def find_in_path(self, name):
paul@0 647
paul@0 648
        """
paul@0 649
        Find the given module 'name' in the search path, returning None where no
paul@0 650
        such module could be found, or a 2-tuple from the 'find' method
paul@0 651
        otherwise.
paul@0 652
        """
paul@0 653
paul@0 654
        for d in self.path:
paul@0 655
            m = self.find(d, name)
paul@0 656
            if m: return m
paul@0 657
        return None
paul@0 658
paul@0 659
    def find(self, d, name):
paul@0 660
paul@0 661
        """
paul@0 662
        In the directory 'd', find the given module 'name', where 'name' can
paul@0 663
        either refer to a single file module or to a package. Return None if the
paul@0 664
        'name' cannot be associated with either a file or a package directory,
paul@0 665
        or a 2-tuple from '_find_package' or '_find_module' otherwise.
paul@0 666
        """
paul@0 667
paul@0 668
        m = self._find_package(d, name)
paul@0 669
        if m: return m
paul@0 670
        m = self._find_module(d, name)
paul@0 671
        if m: return m
paul@0 672
        return None
paul@0 673
paul@0 674
    def _find_module(self, d, name):
paul@0 675
paul@0 676
        """
paul@0 677
        In the directory 'd', find the given module 'name', returning None where
paul@0 678
        no suitable file exists in the directory, or a 2-tuple consisting of
paul@0 679
        None (indicating that no package directory is involved) and a filename
paul@0 680
        indicating the location of the module.
paul@0 681
        """
paul@0 682
paul@0 683
        name_py = name + extsep + "py"
paul@0 684
        filename = self._find_file(d, name_py)
paul@0 685
        if filename:
paul@0 686
            return None, filename
paul@0 687
        return None
paul@0 688
paul@0 689
    def _find_package(self, d, name):
paul@0 690
paul@0 691
        """
paul@0 692
        In the directory 'd', find the given package 'name', returning None
paul@0 693
        where no suitable package directory exists, or a 2-tuple consisting of
paul@0 694
        a directory (indicating the location of the package directory itself)
paul@0 695
        and a filename indicating the location of the __init__.py module which
paul@0 696
        declares the package's top-level contents.
paul@0 697
        """
paul@0 698
paul@0 699
        filename = self._find_file(d, name)
paul@0 700
        if filename:
paul@0 701
            init_py = "__init__" + extsep + "py"
paul@0 702
            init_py_filename = self._find_file(filename, init_py)
paul@0 703
            if init_py_filename:
paul@0 704
                return filename, init_py_filename
paul@0 705
        return None
paul@0 706
paul@0 707
    def _find_file(self, d, filename):
paul@0 708
paul@0 709
        """
paul@0 710
        Return the filename obtained when searching the directory 'd' for the
paul@0 711
        given 'filename', or None if no actual file exists for the filename.
paul@0 712
        """
paul@0 713
paul@0 714
        filename = join(d, filename)
paul@0 715
        if exists(filename):
paul@0 716
            return filename
paul@0 717
        else:
paul@0 718
            return None
paul@0 719
paul@12 720
    def load(self, name):
paul@0 721
paul@0 722
        """
paul@0 723
        Load the module or package with the given 'name'. Return an object
paul@0 724
        referencing the loaded module or package, or None if no such module or
paul@0 725
        package exists.
paul@0 726
        """
paul@0 727
paul@0 728
        # Loaded modules are returned immediately.
paul@0 729
        # Modules may be known but not yet loading (having been registered as
paul@0 730
        # submodules), loading, loaded, or completely unknown.
paul@0 731
paul@12 732
        module = self.get_module(name)
paul@0 733
paul@0 734
        if module:
paul@12 735
            return self.modules[name]
paul@0 736
paul@0 737
        # Otherwise, modules are loaded.
paul@0 738
paul@0 739
        # Split the name into path components, and try to find the uppermost in
paul@0 740
        # the search path.
paul@0 741
paul@0 742
        path = name.split(".")
paul@0 743
        path_so_far = []
paul@12 744
        module = None
paul@0 745
paul@0 746
        for p in path:
paul@0 747
paul@0 748
            # Get the module's filesystem details.
paul@0 749
paul@0 750
            if not path_so_far:
paul@0 751
                m = self.find_in_path(p)
paul@0 752
            elif d:
paul@0 753
                m = self.find(d, p)
paul@0 754
            else:
paul@0 755
                m = None
paul@0 756
paul@0 757
            path_so_far.append(p)
paul@0 758
            module_name = ".".join(path_so_far)
paul@0 759
paul@0 760
            if not m:
paul@0 761
                if self.verbose:
paul@0 762
                    print >>sys.stderr, "Not found (%s)" % name
paul@0 763
paul@0 764
                return None # NOTE: Import error.
paul@0 765
paul@0 766
            # Get the module itself.
paul@0 767
paul@0 768
            d, filename = m
paul@12 769
            module = self.load_from_file(filename, module_name)
paul@0 770
paul@12 771
        return module
paul@0 772
paul@12 773
    def load_from_file(self, filename, module_name=None):
paul@0 774
paul@0 775
        "Load the module from the given 'filename'."
paul@0 776
paul@0 777
        if module_name is None:
paul@0 778
            module_name = "__main__"
paul@0 779
paul@0 780
        module = self.modules.get(module_name)
paul@0 781
paul@0 782
        if not module:
paul@0 783
paul@0 784
            # Try to load from cache.
paul@0 785
paul@12 786
            module = self.load_from_cache(filename, module_name)
paul@0 787
            if module:
paul@0 788
                return module
paul@0 789
paul@0 790
            # If no cache entry exists, load from file.
paul@0 791
paul@0 792
            module = inspector.InspectedModule(module_name, self)
paul@0 793
            self.add_module(module_name, module)
paul@0 794
            self.update_cache_validity(module)
paul@0 795
paul@12 796
            self._load(module, module_name, lambda m: m.parse, filename)
paul@0 797
paul@0 798
        return module
paul@0 799
paul@0 800
    def update_cache_validity(self, module):
paul@0 801
paul@0 802
        "Make 'module' valid in the cache, but invalidate accessing modules."
paul@0 803
paul@12 804
        accessing = self.accessing_modules.get(module.name)
paul@12 805
        if accessing:
paul@12 806
            self.invalidated.update(accessing)
paul@0 807
        if module.name in self.invalidated:
paul@0 808
            self.invalidated.remove(module.name)
paul@0 809
paul@0 810
    def source_is_new(self, filename, module_name):
paul@0 811
paul@0 812
        "Return whether 'filename' is newer than the cached 'module_name'."
paul@0 813
paul@0 814
        if self.cache:
paul@0 815
            cache_filename = join(self.cache, module_name)
paul@0 816
            return not exists(cache_filename) or \
paul@0 817
                getmtime(filename) > getmtime(cache_filename) or \
paul@0 818
                module_name in self.invalidated
paul@0 819
        else:
paul@0 820
            return True
paul@0 821
paul@12 822
    def load_from_cache(self, filename, module_name):
paul@0 823
paul@0 824
        "Return a module residing in the cache."
paul@0 825
paul@0 826
        module = self.modules.get(module_name)
paul@0 827
paul@12 828
        if not module and not self.source_is_new(filename, module_name):
paul@13 829
            module = CachedModule(module_name, self)
paul@12 830
            self.add_module(module_name, module)
paul@0 831
paul@12 832
            filename = join(self.cache, module_name)
paul@12 833
            self._load(module, module_name, lambda m: m.from_cache, filename)
paul@0 834
paul@0 835
        return module
paul@0 836
paul@12 837
    def _load(self, module, module_name, fn, filename):
paul@0 838
paul@0 839
        """
paul@12 840
        Load 'module' for the given 'module_name', and with 'fn' performing an
paul@12 841
        invocation on the module with the given 'filename'.
paul@0 842
        """
paul@0 843
paul@12 844
        # Load the module.
paul@0 845
paul@0 846
        if self.verbose:
paul@53 847
            print >>sys.stderr, module_name in self.required and "Required" or "Loading", module_name, "from", filename
paul@0 848
        fn(module)(filename)
paul@0 849
paul@54 850
        # Add the module object if not already defined.
paul@54 851
paul@54 852
        if not self.objects.has_key(module_name):
paul@54 853
            self.objects[module_name] = Reference("<module>", module_name)
paul@54 854
paul@0 855
    def add_module(self, module_name, module):
paul@0 856
paul@0 857
        """
paul@0 858
        Return the module with the given 'module_name', adding a new module
paul@0 859
        object if one does not already exist.
paul@0 860
        """
paul@0 861
paul@0 862
        self.modules[module_name] = module
paul@12 863
        if module_name in self.to_import:
paul@12 864
            self.to_import.remove(module_name)
paul@0 865
paul@0 866
# vim: tabstop=4 expandtab shiftwidth=4