Lichen

Annotated modules.py

132:637f2b13210a
2016-10-25 Paul Boddie Added parameter table code and position constant generation. Removed the parameter list adjustments for instantiators. Fixed method and function structure generation. Fixed argument limit calculations. Fixed class type position attribute generation for non-classes. Fixed bound method and parameter table references. Updated generated instantiators to update the initial context argument. Added CFLAGS to the Makefile.
paul@0 1
#!/usr/bin/env python
paul@0 2
paul@0 3
"""
paul@0 4
Module abstractions.
paul@0 5
paul@0 6
Copyright (C) 2007, 2008, 2009, 2010, 2011, 2012, 2013,
paul@0 7
              2014, 2015, 2016 Paul Boddie <paul@boddie.org.uk>
paul@0 8
paul@0 9
This program is free software; you can redistribute it and/or modify it under
paul@0 10
the terms of the GNU General Public License as published by the Free Software
paul@0 11
Foundation; either version 3 of the License, or (at your option) any later
paul@0 12
version.
paul@0 13
paul@0 14
This program is distributed in the hope that it will be useful, but WITHOUT
paul@0 15
ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
paul@0 16
FOR A PARTICULAR PURPOSE.  See the GNU General Public License for more
paul@0 17
details.
paul@0 18
paul@0 19
You should have received a copy of the GNU General Public License along with
paul@0 20
this program.  If not, see <http://www.gnu.org/licenses/>.
paul@0 21
"""
paul@0 22
paul@16 23
from common import init_item, remove_items, CommonModule
paul@88 24
from encoders import decode_modifier_term, decode_usage, encode_modifiers, encode_usage
paul@0 25
from referencing import decode_reference, Reference
paul@12 26
from results import ResolvedNameRef
paul@0 27
import sys
paul@0 28
paul@0 29
class BasicModule(CommonModule):
paul@0 30
paul@0 31
    "The basic module information."
paul@0 32
paul@0 33
    def __init__(self, name, importer):
paul@0 34
        CommonModule.__init__(self, name, importer)
paul@0 35
paul@18 36
        # Import details, primarily for cache output.
paul@18 37
paul@18 38
        self.imports = set()
paul@18 39
        self.required = set()
paul@35 40
        self.deferred = []
paul@18 41
paul@0 42
        # Global name information.
paul@0 43
paul@0 44
        self.objects = {}
paul@0 45
        self.special = {}
paul@0 46
paul@0 47
        # Class relationships.
paul@0 48
paul@0 49
        self.classes = {}
paul@0 50
paul@0 51
        # Attributes.
paul@0 52
paul@0 53
        self.class_attrs = {}
paul@0 54
        self.instance_attrs = {}
paul@0 55
        self.instance_attr_constants = {}
paul@0 56
        self.module_attrs = set()
paul@0 57
paul@40 58
        # Names used in each namespace.
paul@0 59
paul@0 60
        self.names_used = {}
paul@0 61
paul@0 62
        # Function details.
paul@0 63
paul@0 64
        self.function_parameters = {}
paul@0 65
        self.function_defaults = {}
paul@0 66
        self.function_locals = {}
paul@0 67
        self.scope_globals = {}
paul@0 68
paul@0 69
        # Invocation details.
paul@0 70
paul@0 71
        self.function_targets = {}
paul@0 72
        self.function_arguments = {}
paul@0 73
paul@0 74
        # Attribute usage at module and function levels.
paul@0 75
paul@0 76
        self.attr_usage = {}
paul@0 77
        self.name_initialisers = {}
paul@0 78
paul@0 79
        # General attribute access expressions.
paul@0 80
paul@0 81
        self.attr_accesses = {}
paul@0 82
        self.const_accesses = {}
paul@0 83
paul@0 84
        # Attribute accessor definition details.
paul@0 85
paul@0 86
        self.attr_accessors = {}
paul@0 87
paul@0 88
        # Assignment details for accesses.
paul@0 89
paul@0 90
        self.attr_access_modifiers = {}
paul@0 91
paul@13 92
        # Name resolution details.
paul@13 93
paul@13 94
        self.name_references = {} # references to globals
paul@13 95
paul@13 96
        # Initialisation-related details.
paul@13 97
paul@13 98
        self.initialised_names = {}
paul@13 99
        self.aliased_names = {}
paul@13 100
paul@0 101
    def __repr__(self):
paul@0 102
        return "BasicModule(%r, %r)" % (self.name, self.importer)
paul@0 103
paul@0 104
    # Derived information methods.
paul@0 105
paul@0 106
    def propagate(self):
paul@0 107
paul@0 108
        "Finalise and propagate module information."
paul@0 109
paul@0 110
        self.propagate_attrs()
paul@0 111
        self.propagate_name_references()
paul@0 112
        self.propagate_attr_accesses()
paul@0 113
        self.propagate_constants()
paul@0 114
paul@0 115
    def unpropagate(self):
paul@0 116
paul@0 117
        """
paul@0 118
        Retract information from the importer including information about this
paul@0 119
        module derived by the importer.
paul@0 120
        """
paul@0 121
paul@0 122
        del self.importer.all_module_attrs[self.name]
paul@0 123
paul@0 124
        for name in self.classes.keys():
paul@68 125
            del self.importer.classes[name]
paul@0 126
            del self.importer.all_class_attrs[name]
paul@0 127
            del self.importer.all_instance_attrs[name]
paul@0 128
            del self.importer.all_instance_attr_constants[name]
paul@0 129
paul@0 130
            for name, bases in self.classes.items():
paul@0 131
                for base in bases:
paul@0 132
paul@0 133
                    # Get the identity of the class from the reference.
paul@0 134
paul@0 135
                    base = base.get_origin()
paul@0 136
paul@0 137
                    try:
paul@0 138
                        self.importer.subclasses[base].remove(name)
paul@0 139
                    except (KeyError, ValueError):
paul@0 140
                        pass
paul@0 141
paul@0 142
        remove_items(self.importer.all_name_references, self.name_references)
paul@0 143
        remove_items(self.importer.all_initialised_names, self.initialised_names)
paul@0 144
        remove_items(self.importer.all_aliased_names, self.aliased_names)
paul@0 145
        remove_items(self.importer.all_attr_accesses, self.attr_accesses)
paul@0 146
        remove_items(self.importer.all_const_accesses, self.const_accesses)
paul@0 147
        remove_items(self.importer.all_attr_access_modifiers, self.attr_access_modifiers)
paul@0 148
        remove_items(self.importer.all_constants, self.constants)
paul@0 149
        remove_items(self.importer.all_constant_values, self.constant_values)
paul@0 150
paul@0 151
        # Remove this module's objects from the importer. Objects are
paul@0 152
        # automatically propagated when defined.
paul@0 153
paul@68 154
        ref = self.importer.objects.get(self.name)
paul@68 155
        if ref and ref.has_kind("<module>"):
paul@68 156
            del self.importer.objects[self.name]
paul@68 157
paul@0 158
        for name, ref in self.objects.items():
paul@20 159
            if not ref.has_kind("<module>"):
paul@20 160
                del self.importer.objects[name]
paul@0 161
paul@68 162
    def collect(self):
paul@68 163
paul@68 164
        "Collect removed objects."
paul@68 165
paul@68 166
        for name, ref in self.objects.items():
paul@68 167
            if not self.importer.objects.has_key(ref.get_origin()) and self.importer.objects.has_key(name):
paul@68 168
                del self.importer.objects[name]
paul@68 169
paul@0 170
    def propagate_attrs(self):
paul@0 171
paul@0 172
        "Derive attributes from the class and module member details."
paul@0 173
paul@0 174
        # Initialise class attribute records for all classes.
paul@0 175
paul@0 176
        for name in self.classes.keys():
paul@0 177
            self.importer.all_class_attrs[name] = self.class_attrs[name] = {}
paul@0 178
paul@0 179
        # Separate the objects into module and class attributes.
paul@0 180
paul@0 181
        for name in self.objects.keys():
paul@0 182
            if "." in name:
paul@0 183
                parent, attrname = name.rsplit(".", 1)
paul@0 184
                if self.classes.has_key(parent):
paul@0 185
                    self.class_attrs[parent][attrname] = name
paul@0 186
                elif parent == self.name:
paul@0 187
                    self.module_attrs.add(attrname)
paul@0 188
paul@0 189
        # Propagate the module attributes.
paul@0 190
paul@0 191
        self.importer.all_module_attrs[self.name] = self.module_attrs
paul@0 192
paul@0 193
    def propagate_name_references(self):
paul@0 194
paul@0 195
        "Propagate name references for the module."
paul@0 196
paul@0 197
        self.importer.all_initialised_names.update(self.initialised_names)
paul@0 198
        self.importer.all_aliased_names.update(self.aliased_names)
paul@0 199
paul@0 200
    def propagate_attr_accesses(self):
paul@0 201
paul@0 202
        "Propagate attribute accesses for the module."
paul@0 203
paul@0 204
        self.importer.all_attr_accesses.update(self.attr_accesses)
paul@0 205
        self.importer.all_const_accesses.update(self.const_accesses)
paul@0 206
        self.importer.all_attr_access_modifiers.update(self.attr_access_modifiers)
paul@0 207
paul@0 208
    def propagate_constants(self):
paul@0 209
paul@0 210
        "Propagate constant values and aliases for the module."
paul@0 211
paul@0 212
        self.importer.all_constants.update(self.constants)
paul@0 213
        self.importer.all_constant_values.update(self.constant_values)
paul@0 214
paul@0 215
        for name in self.classes.keys():
paul@0 216
            self.importer.all_instance_attrs[name] = self.instance_attrs.get(name) or {}
paul@0 217
            self.importer.all_instance_attr_constants[name] = self.instance_attr_constants.get(name) or {}
paul@0 218
paul@26 219
    def set_object(self, name, value=None):
paul@26 220
paul@26 221
        "Set an object with the given 'name' and the given 'value'."
paul@26 222
paul@35 223
        # Decode any string value, with a new reference being returned even
paul@35 224
        # given a provided reference.
paul@35 225
paul@26 226
        ref = decode_reference(value, name)
paul@35 227
        self.add_deferred(ref)
paul@35 228
        self._set_object(name, ref)
paul@35 229
paul@35 230
    def _set_object(self, name, ref):
paul@35 231
paul@35 232
        # Determine how the object properties will be defined.
paul@35 233
paul@26 234
        multiple = self.objects.has_key(name) and self.objects[name].get_kind() != ref.get_kind()
paul@26 235
        self.importer.objects[name] = self.objects[name] = multiple and ref.as_var() or ref
paul@26 236
paul@26 237
    def queue_module(self, name, required=False):
paul@26 238
paul@26 239
        """
paul@26 240
        Queue the module with the given 'name'. If 'required' is true (it is
paul@26 241
        false by default), indicate that the module is required in the final
paul@26 242
        program.
paul@26 243
        """
paul@26 244
paul@26 245
        self.importer.queue_module(name, self, required)
paul@26 246
        if required:
paul@26 247
            self.required.add(name)
paul@26 248
        self.imports.add(name)
paul@26 249
paul@26 250
class InspectionNaming:
paul@26 251
paul@26 252
    "Name operations related to inspection."
paul@26 253
paul@0 254
    # Module-relative naming.
paul@0 255
paul@0 256
    def is_global(self, name):
paul@0 257
paul@0 258
        """
paul@0 259
        Return whether 'name' is registered as a global in the current
paul@0 260
        namespace.
paul@0 261
        """
paul@0 262
paul@0 263
        path = self.get_namespace_path()
paul@0 264
        return name in self.scope_globals.get(path, [])
paul@0 265
paul@0 266
    def get_global(self, name):
paul@0 267
paul@0 268
        """
paul@0 269
        Get the global of the given 'name' from this module, returning a
paul@0 270
        reference incorporating the original definition details.
paul@0 271
        """
paul@0 272
paul@0 273
        path = self.get_global_path(name)
paul@0 274
        return self.objects.get(path)
paul@0 275
paul@0 276
    # Name definition discovery.
paul@0 277
paul@0 278
    def get_global_or_builtin(self, name):
paul@0 279
paul@0 280
        """
paul@12 281
        Return a reference for the given 'name' found in this module or in the
paul@12 282
        __builtins__.
paul@0 283
        """
paul@0 284
paul@0 285
        return self.get_global(name) or self.get_builtin(name)
paul@0 286
paul@0 287
    def get_builtin(self, name):
paul@0 288
paul@12 289
        "Return a reference to the built-in with the given 'name'."
paul@0 290
paul@18 291
        self.queue_module("__builtins__")
paul@35 292
        ref = Reference("<depends>", "__builtins__.%s" % name)
paul@35 293
        self.deferred.append(ref)
paul@35 294
        return ref
paul@0 295
paul@12 296
    def get_builtin_class(self, name):
paul@0 297
paul@12 298
        "Return a reference to the actual object providing 'name'."
paul@0 299
paul@12 300
        # NOTE: This makes assumptions about the __builtins__ structure.
paul@0 301
paul@30 302
        module_name = "__builtins__.%s" % name
paul@30 303
        if self.name != module_name:
paul@30 304
            self.queue_module(module_name, True)
paul@12 305
        return Reference("<class>", "__builtins__.%s.%s" % (name, name))
paul@0 306
paul@0 307
    def get_object(self, path):
paul@0 308
paul@0 309
        """
paul@12 310
        Get the details of an object with the given 'path'. Where the object
paul@12 311
        cannot be resolved, an unresolved reference is returned.
paul@0 312
        """
paul@0 313
paul@0 314
        if self.objects.has_key(path):
paul@0 315
            return self.objects[path]
paul@0 316
        else:
paul@35 317
            ref = Reference("<depends>", path)
paul@35 318
            self.deferred.append(ref)
paul@35 319
            return ref
paul@0 320
paul@12 321
    def import_name_from_module(self, name, module_name):
paul@12 322
paul@12 323
        "Import 'name' from the module having the given 'module_name'."
paul@12 324
paul@12 325
        if module_name != self.name:
paul@18 326
            self.queue_module(module_name)
paul@12 327
        return Reference("<depends>", "%s.%s" % (module_name, name))
paul@12 328
paul@35 329
    def add_deferred(self, ref):
paul@35 330
paul@35 331
        "Record 'ref' as a deferred reference."
paul@35 332
paul@35 333
        if ref.has_kind("<depends>"):
paul@35 334
            self.deferred.append(ref)
paul@35 335
paul@0 336
class CachedModule(BasicModule):
paul@0 337
paul@0 338
    "A cached module."
paul@0 339
paul@0 340
    def __repr__(self):
paul@0 341
        return "CachedModule(%r, %r)" % (self.name, self.importer)
paul@0 342
paul@35 343
    def set_object(self, name, value=None):
paul@35 344
paul@35 345
        "Set an object with the given 'name' and the given 'value'."
paul@35 346
paul@35 347
        # Decode any string value, with a new reference being returned even
paul@35 348
        # given a provided reference.
paul@35 349
paul@35 350
        ref = decode_reference(value, name)
paul@35 351
        self._set_object(name, ref)
paul@35 352
paul@0 353
    def to_cache(self, filename):
paul@0 354
paul@0 355
        "Not actually writing the module back to 'filename'."
paul@0 356
paul@0 357
        pass
paul@0 358
paul@0 359
    def from_cache(self, filename):
paul@0 360
paul@0 361
        """
paul@0 362
        Read a module's details from the file with the given 'filename' as
paul@0 363
        described in the to_cache method of InspectedModule.
paul@0 364
        """
paul@0 365
paul@0 366
        f = open(filename)
paul@0 367
        try:
paul@0 368
            self.filename = f.readline().rstrip()
paul@0 369
paul@0 370
            f.readline() # (empty line)
paul@0 371
paul@18 372
            self._get_imports(f)
paul@0 373
            self._get_members(f)
paul@0 374
            self._get_class_relationships(f)
paul@0 375
            self._get_instance_attrs(f)
paul@0 376
            self._get_instance_attr_constants(f)
paul@0 377
            self.from_lines(f, self.names_used)     # "names used:"
paul@0 378
            self._get_name_references(f)
paul@0 379
            self._get_initialised_names(f)
paul@0 380
            self._get_aliased_names(f)
paul@0 381
            self._get_function_parameters(f)
paul@0 382
            self._get_function_defaults(f)
paul@0 383
            self._get_function_locals(f)
paul@0 384
            self.from_lines(f, self.scope_globals)  # "scope globals:"
paul@0 385
            self._get_function_targets(f)
paul@0 386
            self._get_function_arguments(f)
paul@0 387
            self._get_attribute_usage(f)
paul@0 388
            self._get_attr_accesses(f)
paul@0 389
            self._get_const_accesses(f)
paul@0 390
            self._get_attr_accessors(f)
paul@0 391
            self._get_attr_access_modifiers(f)
paul@0 392
            self._get_constant_literals(f)
paul@0 393
            self._get_constant_values(f)
paul@0 394
paul@0 395
        finally:
paul@0 396
            f.close()
paul@0 397
paul@12 398
    def complete(self):
paul@0 399
        self.propagate()
paul@0 400
paul@18 401
    def _get_imports(self, f):
paul@18 402
        f.readline() # "imports:"
paul@18 403
        line = f.readline().strip()
paul@18 404
        self.required = line != "{}" and set(line.split(", ")) or set()
paul@18 405
        line = f.readline().strip()
paul@18 406
        self.imports = line != "{}" and set(line.split(", ")) or set()
paul@18 407
        f.readline()
paul@18 408
paul@18 409
        for name in self.required:
paul@18 410
            self.queue_module(name, True)
paul@18 411
        for name in self.imports:
paul@18 412
            self.queue_module(name)
paul@18 413
paul@0 414
    def _get_members(self, f):
paul@0 415
        f.readline() # "members:"
paul@0 416
        line = f.readline().rstrip()
paul@0 417
        while line:
paul@0 418
            name, ref = line.split(" ", 1)
paul@0 419
            self.set_object(name, ref)
paul@0 420
            line = f.readline().rstrip()
paul@0 421
paul@0 422
    def _get_class_relationships(self, f):
paul@0 423
        f.readline() # "class relationships:"
paul@0 424
        line = f.readline().rstrip()
paul@0 425
        while line:
paul@0 426
            name, value = self._get_fields(line)
paul@0 427
            values = value and value.split(", ") or []
paul@0 428
            self.importer.classes[name] = self.classes[name] = map(decode_reference, values)
paul@0 429
            self.importer.subclasses[name] = set()
paul@0 430
            line = f.readline().rstrip()
paul@0 431
paul@0 432
    def _get_instance_attrs(self, f):
paul@0 433
        f.readline() # "instance attributes:"
paul@0 434
        line = f.readline().rstrip()
paul@0 435
        while line:
paul@0 436
            name, value = self._get_fields(line)
paul@0 437
            self.importer.all_instance_attrs[name] = self.instance_attrs[name] = set(value and value.split(", ") or [])
paul@0 438
            line = f.readline().rstrip()
paul@0 439
paul@0 440
    def _get_instance_attr_constants(self, f):
paul@0 441
        f.readline() # "instance attribute constants:"
paul@0 442
        line = f.readline().rstrip()
paul@0 443
        while line:
paul@0 444
            name, attrname, ref = self._get_fields(line, 3)
paul@0 445
            init_item(self.instance_attr_constants, name, dict)
paul@0 446
            self.instance_attr_constants[name][attrname] = decode_reference(ref)
paul@0 447
            line = f.readline().rstrip()
paul@0 448
paul@0 449
    def _get_name_references(self, f):
paul@0 450
        f.readline() # "name references:"
paul@0 451
        line = f.readline().rstrip()
paul@0 452
        while line:
paul@37 453
            name, ref = self._get_fields(line)
paul@37 454
            self.importer.all_name_references[name] = self.name_references[name] = decode_reference(ref)
paul@0 455
            line = f.readline().rstrip()
paul@0 456
paul@0 457
    def _get_initialised_names(self, f):
paul@0 458
        f.readline() # "initialised names:"
paul@0 459
        line = f.readline().rstrip()
paul@0 460
        while line:
paul@0 461
            name, version, value = self._get_fields(line, 3)
paul@0 462
            init_item(self.initialised_names, name, dict)
paul@0 463
            self.initialised_names[name][int(version)] = decode_reference(value)
paul@0 464
            line = f.readline().rstrip()
paul@0 465
paul@0 466
    def _get_aliased_names(self, f):
paul@0 467
        f.readline() # "aliased names:"
paul@0 468
        line = f.readline().rstrip()
paul@0 469
        while line:
paul@0 470
            name, version, original_name, attrnames, number = self._get_fields(line, 5)
paul@0 471
            init_item(self.aliased_names, name, dict)
paul@0 472
            if number == "{}": number = None
paul@0 473
            else: number = int(number)
paul@0 474
            self.aliased_names[name][int(version)] = (original_name, attrnames != "{}" and attrnames or None, number)
paul@0 475
            line = f.readline().rstrip()
paul@0 476
paul@0 477
    def _get_function_parameters(self, f):
paul@0 478
        f.readline() # "function parameters:"
paul@0 479
        line = f.readline().rstrip()
paul@0 480
        while line:
paul@0 481
            function, names = self._get_fields(line)
paul@0 482
            self.importer.function_parameters[function] = \
paul@49 483
                self.function_parameters[function] = names != "{}" and names.split(", ") or []
paul@0 484
            line = f.readline().rstrip()
paul@0 485
paul@0 486
    def _get_function_defaults(self, f):
paul@0 487
        f.readline() # "function default parameters:"
paul@0 488
        line = f.readline().rstrip()
paul@0 489
        while line:
paul@0 490
            function, defaults = self._get_fields(line)
paul@0 491
            self.importer.function_defaults[function] = \
paul@0 492
                self.function_defaults[function] = l = []
paul@0 493
            if defaults != "{}":
paul@0 494
                for value in defaults.split(", "):
paul@0 495
                    name, default = value.split("=")
paul@0 496
                    default = decode_reference(default)
paul@0 497
                    l.append((name, default))
paul@0 498
            line = f.readline().rstrip()
paul@0 499
paul@0 500
    def _get_function_locals(self, f):
paul@0 501
        f.readline() # "function locals:"
paul@0 502
        line = f.readline().rstrip()
paul@0 503
        while line:
paul@0 504
            function, name, value = self._get_fields(line, 3)
paul@0 505
            init_item(self.function_locals, function, dict)
paul@115 506
            init_item(self.importer.function_locals, function, dict)
paul@0 507
            if name != "{}":
paul@115 508
                self.importer.function_locals[function][name] = \
paul@109 509
                    self.function_locals[function][name] = decode_reference(value)
paul@0 510
            line = f.readline().rstrip()
paul@0 511
paul@0 512
    def _get_function_targets(self, f):
paul@0 513
        f.readline() # "function targets:"
paul@0 514
        line = f.readline().rstrip()
paul@0 515
        while line:
paul@0 516
            function, n = self._get_fields(line)
paul@0 517
            self.importer.function_targets[function] = \
paul@0 518
                self.function_targets[function] = int(n)
paul@0 519
            line = f.readline().rstrip()
paul@0 520
paul@0 521
    def _get_function_arguments(self, f):
paul@0 522
        f.readline() # "function arguments:"
paul@0 523
        line = f.readline().rstrip()
paul@0 524
        while line:
paul@0 525
            function, n = self._get_fields(line)
paul@0 526
            self.importer.function_arguments[function] = \
paul@0 527
                self.function_arguments[function] = int(n)
paul@0 528
            line = f.readline().rstrip()
paul@0 529
paul@0 530
    def _get_attribute_usage(self, f):
paul@0 531
        f.readline() # "attribute usage:"
paul@0 532
        line = f.readline().rstrip()
paul@0 533
        while line:
paul@0 534
            unit, value = self._get_fields(line)
paul@0 535
            init_item(self.attr_usage, unit, dict)
paul@0 536
            self.usage_from_cache(value, self.attr_usage[unit])
paul@0 537
            line = f.readline().rstrip()
paul@0 538
paul@0 539
    def _get_attr_accesses(self, f):
paul@0 540
        f.readline() # "attribute accesses:"
paul@0 541
        line = f.readline().rstrip()
paul@0 542
        while line:
paul@0 543
            name, value = self._get_fields(line)
paul@0 544
            self.attr_accesses[name] = set(value.split(", "))
paul@0 545
            line = f.readline().rstrip()
paul@0 546
paul@0 547
    def _get_const_accesses(self, f):
paul@0 548
        f.readline() # "constant accesses:"
paul@0 549
        line = f.readline().rstrip()
paul@0 550
        while line:
paul@0 551
            name, original_name, attrnames, objpath, ref, remaining = self._get_fields(line, 6)
paul@0 552
            if attrnames == "{}": attrnames = None
paul@0 553
            init_item(self.const_accesses, name, dict)
paul@0 554
            self.const_accesses[name][(original_name, attrnames)] = (objpath, decode_reference(ref), remaining != "{}" and remaining or "")
paul@0 555
            line = f.readline().rstrip()
paul@0 556
paul@0 557
    def _get_attr_accessors(self, f):
paul@0 558
        f.readline() # "attribute access usage:"
paul@0 559
        line = f.readline().rstrip()
paul@0 560
        while line:
paul@0 561
            objpath, name, attrname, value = self._get_fields(line, 4)
paul@0 562
            if attrname == "{}": attrname = None
paul@0 563
            access = name, attrname
paul@0 564
            init_item(self.attr_accessors, objpath, dict)
paul@0 565
            init_item(self.attr_accessors[objpath], access, list)
paul@0 566
            positions = map(int, value.split(", "))
paul@0 567
            self.attr_accessors[objpath][access].append(positions)
paul@0 568
            line = f.readline().rstrip()
paul@0 569
paul@0 570
    def _get_attr_access_modifiers(self, f):
paul@0 571
        f.readline() # "attribute access modifiers:"
paul@0 572
        line = f.readline().rstrip()
paul@0 573
        while line:
paul@0 574
            objpath, name, attrnames, value = self._get_fields(line, 4)
paul@0 575
            if name == "{}": name = None
paul@0 576
            if attrnames == "{}": attrnames = None
paul@0 577
            access = name, attrnames
paul@0 578
            init_item(self.attr_access_modifiers, objpath, dict)
paul@0 579
            init_item(self.attr_access_modifiers[objpath], access, list)
paul@0 580
            modifiers = [decode_modifier_term(s) for s in value]
paul@0 581
            self.attr_access_modifiers[objpath][access] = modifiers
paul@0 582
            line = f.readline().rstrip()
paul@0 583
paul@0 584
    def _get_constant_literals(self, f):
paul@0 585
        f.readline() # "constant literals:"
paul@0 586
        line = f.readline().rstrip()
paul@0 587
        last_path = None
paul@0 588
        n = None
paul@0 589
        while line:
paul@0 590
            path, constant = self._get_fields(line)
paul@0 591
            if path != last_path:
paul@0 592
                n = 0
paul@0 593
                last_path = path
paul@0 594
            else:
paul@0 595
                n += 1
paul@0 596
            init_item(self.constants, path, dict)
paul@0 597
            self.constants[path][eval(constant)] = n
paul@0 598
            line = f.readline().rstrip()
paul@0 599
paul@0 600
    def _get_constant_values(self, f):
paul@0 601
        f.readline() # "constant values:"
paul@0 602
        line = f.readline().rstrip()
paul@0 603
        while line:
paul@0 604
            name, value_type, value = self._get_fields(line, 3)
paul@0 605
            self.constant_values[name] = eval(value), value_type
paul@0 606
            line = f.readline().rstrip()
paul@0 607
paul@0 608
    # Generic parsing methods.
paul@0 609
paul@0 610
    def from_lines(self, f, d):
paul@0 611
paul@0 612
        "Read lines from 'f', populating 'd'."
paul@0 613
paul@0 614
        f.readline() # section heading
paul@0 615
        line = f.readline().rstrip()
paul@0 616
        while line:
paul@0 617
            name, value = self._get_fields(line)
paul@0 618
            d[name] = set(value and value.split(", ") or [])
paul@0 619
            line = f.readline().rstrip()
paul@0 620
paul@0 621
    def usage_from_cache(self, value, mapping):
paul@0 622
paul@0 623
        """
paul@0 624
        Interpret the given 'value' containing name and usage information,
paul@0 625
        storing the information in the given 'mapping'.
paul@0 626
        """
paul@0 627
paul@0 628
        local, usage = self._get_fields(value)
paul@0 629
        init_item(mapping, local, list)
paul@0 630
        self._usage_from_cache(mapping[local], usage)
paul@0 631
paul@0 632
    def _usage_from_cache(self, d, usage):
paul@0 633
paul@0 634
        # Interpret descriptions of each version of the name.
paul@0 635
paul@0 636
        all_usages = set()
paul@0 637
        for attrnames in usage.split("; "):
paul@0 638
            if attrnames == "{}":
paul@0 639
                all_attrnames = ()
paul@0 640
            else:
paul@88 641
                all_attrnames = decode_usage(attrnames)
paul@88 642
            all_usages.add(all_attrnames)
paul@0 643
paul@0 644
        d.append(all_usages)
paul@0 645
paul@0 646
    def _get_fields(self, s, n=2):
paul@0 647
        result = s.split(" ", n-1)
paul@0 648
        if len(result) == n:
paul@0 649
            return result
paul@0 650
        else:
paul@0 651
            return tuple(result) + tuple([""] * (n - len(result)))
paul@0 652
paul@0 653
class CacheWritingModule:
paul@0 654
paul@0 655
    """
paul@0 656
    A mix-in providing cache-writing support, to be combined with BasicModule.
paul@0 657
    """
paul@0 658
paul@0 659
    def to_cache(self, filename):
paul@0 660
paul@0 661
        """
paul@0 662
        Write a cached representation of the inspected module with the following
paul@0 663
        format to the file having the given 'filename':
paul@0 664
paul@0 665
        filename
paul@0 666
        (empty line)
paul@18 667
        "imports:"
paul@18 668
        required module names
paul@18 669
        possibly required module names
paul@0 670
        "members:"
paul@0 671
        zero or more: qualified name " " reference
paul@0 672
        (empty line)
paul@0 673
        "class relationships:"
paul@0 674
        zero or more: qualified class name " " base class references
paul@0 675
        (empty line)
paul@0 676
        "instance attributes:"
paul@0 677
        zero or more: qualified class name " " instance attribute names
paul@0 678
        (empty line)
paul@0 679
        "instance attribute constants:"
paul@0 680
        zero or more: qualified class name " " attribute name " " reference
paul@0 681
        (empty line)
paul@0 682
        "names used:"
paul@0 683
        zero or more: qualified class/function/module name " " names
paul@0 684
        (empty line)
paul@0 685
        "names missing:"
paul@0 686
        zero or more: qualified class/function/module name " " names
paul@0 687
        (empty line)
paul@0 688
        "name references:"
paul@0 689
        zero or more: qualified name " " reference
paul@0 690
        (empty line)
paul@0 691
        "initialised names:"
paul@0 692
        zero or more: qualified name " " definition version " " reference
paul@0 693
        (empty line)
paul@0 694
        "aliased names:"
paul@0 695
        zero or more: qualified name " " definition version " " original name " " attribute names " " access number
paul@0 696
        (empty line)
paul@0 697
        "function parameters:"
paul@0 698
        zero or more: qualified function name " " parameter names
paul@0 699
        (empty line)
paul@0 700
        "function default parameters:"
paul@0 701
        zero or more: qualified function name " " parameter names with defaults
paul@0 702
        (empty line)
paul@0 703
        "function locals:"
paul@0 704
        zero or more: qualified function name " " local variable name " " reference
paul@0 705
        (empty line)
paul@0 706
        "scope globals:"
paul@0 707
        zero or more: qualified function name " " global variable names
paul@0 708
        (empty line)
paul@0 709
        "function targets:"
paul@0 710
        zero or more: qualified function name " " maximum number of targets allocated
paul@0 711
        (empty line)
paul@0 712
        "function arguments:"
paul@0 713
        zero or more: qualified function name " " maximum number of arguments allocated
paul@0 714
        (empty line)
paul@0 715
        "attribute usage:"
paul@0 716
        zero or more: qualified scope name " " local/global/qualified variable name " " usages
paul@0 717
        (empty line)
paul@0 718
        "attribute accesses:"
paul@0 719
        zero or more: qualified scope name " " attribute-chains
paul@0 720
        (empty line)
paul@0 721
        "constant accesses:"
paul@0 722
        zero or more: qualified function name " " attribute-chain " " reference " " remaining attribute-chain
paul@0 723
        (empty line)
paul@0 724
        "attribute access usage:"
paul@0 725
        zero or more: qualified function name " " local/global variable name " " attribute name " " definition versions
paul@0 726
        (empty line)
paul@0 727
        "attribute access modifiers:"
paul@0 728
        zero or more: qualified function name " " local/global variable name " " attribute name " " access modifiers
paul@0 729
        "constant literals:"
paul@0 730
        zero or more: qualified scope name " " constant literal
paul@0 731
        "constant values:"
paul@0 732
        zero or more: qualified name " " value type " " constant literal
paul@0 733
paul@0 734
        All collections of names are separated by ", " characters.
paul@0 735
paul@0 736
        References can be "<var>", a module name, or one of "<class>" or
paul@0 737
        "<function>" followed optionally by a ":" character and a qualified
paul@0 738
        name.
paul@0 739
paul@0 740
        Parameter names with defaults are separated by ", " characters, with
paul@0 741
        each name followed by "=" and then followed by a reference. If "{}" is
paul@0 742
        indicated, no defaults are defined for the function. Similarly, function
paul@0 743
        locals may be indicated as "{}" meaning that there are no locals.
paul@0 744
paul@0 745
        All usages (attribute usage sets) are separated by "; " characters, with
paul@0 746
        the special string "{}" representing an empty set.
paul@0 747
paul@0 748
        Each usage is a collection of names separated by ", " characters, with
paul@106 749
        invoked attribute names suffixed with a "!" character.
paul@0 750
paul@0 751
        Definition versions are separated by ", " characters and indicate the
paul@0 752
        name definition version associated with the access.
paul@0 753
paul@0 754
        Access modifiers are separated by ", " characters and indicate features
paul@0 755
        of each access, with multiple accesses described on a single line.
paul@0 756
        """
paul@0 757
paul@0 758
        f = open(filename, "w")
paul@0 759
        try:
paul@0 760
            print >>f, self.filename
paul@0 761
paul@0 762
            print >>f
paul@18 763
            print >>f, "imports:"
paul@18 764
            required = list(self.required)
paul@18 765
            required.sort()
paul@18 766
            print >>f, required and ", ".join(required) or "{}"
paul@18 767
            imports = list(self.imports)
paul@18 768
            imports.sort()
paul@18 769
            print >>f, imports and ", ".join(imports) or "{}"
paul@18 770
paul@18 771
            print >>f
paul@0 772
            print >>f, "members:"
paul@0 773
            objects = self.objects.keys()
paul@0 774
            objects.sort()
paul@0 775
            for name in objects:
paul@0 776
                print >>f, name, self.objects[name]
paul@0 777
paul@0 778
            print >>f
paul@0 779
            print >>f, "class relationships:"
paul@0 780
            classes = self.classes.keys()
paul@0 781
            classes.sort()
paul@0 782
            for class_ in classes:
paul@0 783
                bases = self.classes[class_]
paul@0 784
                if bases:
paul@0 785
                    print >>f, class_, ", ".join(map(str, bases))
paul@0 786
                else:
paul@0 787
                    print >>f, class_
paul@0 788
paul@0 789
            self.to_lines(f, "instance attributes:", self.instance_attrs)
paul@0 790
paul@0 791
            print >>f
paul@0 792
            print >>f, "instance attribute constants:"
paul@0 793
            classes = self.instance_attr_constants.items()
paul@0 794
            classes.sort()
paul@0 795
            for name, attrs in classes:
paul@0 796
                attrs = attrs.items()
paul@0 797
                attrs.sort()
paul@0 798
                for attrname, ref in attrs:
paul@0 799
                    print >>f, name, attrname, ref
paul@0 800
paul@0 801
            self.to_lines(f, "names used:", self.names_used)
paul@0 802
paul@0 803
            print >>f
paul@0 804
            print >>f, "name references:"
paul@0 805
            refs = self.name_references.items()
paul@0 806
            refs.sort()
paul@0 807
            for name, ref in refs:
paul@0 808
                print >>f, name, ref
paul@0 809
paul@0 810
            print >>f
paul@0 811
            print >>f, "initialised names:"
paul@0 812
            assignments = self.initialised_names.items()
paul@0 813
            assignments.sort()
paul@0 814
            for name, refs in assignments:
paul@0 815
                versions = refs.items()
paul@0 816
                versions.sort()
paul@0 817
                for version, ref in versions:
paul@0 818
                    print >>f, name, version, ref
paul@0 819
paul@0 820
            print >>f
paul@0 821
            print >>f, "aliased names:"
paul@0 822
            assignments = self.aliased_names.items()
paul@0 823
            assignments.sort()
paul@0 824
            for name, aliases in assignments:
paul@0 825
                versions = aliases.items()
paul@0 826
                versions.sort()
paul@0 827
                for version, alias in versions:
paul@0 828
                    original_name, attrnames, number = alias
paul@0 829
                    print >>f, name, version, original_name, attrnames or "{}", number is None and "{}" or number
paul@0 830
paul@0 831
            print >>f
paul@0 832
            print >>f, "function parameters:"
paul@0 833
            functions = self.function_parameters.keys()
paul@0 834
            functions.sort()
paul@0 835
            for function in functions:
paul@49 836
                parameters = self.function_parameters[function]
paul@49 837
                if parameters:
paul@49 838
                    print >>f, function, ", ".join(parameters)
paul@49 839
                else:
paul@49 840
                    print >>f, function, "{}"
paul@0 841
paul@0 842
            print >>f
paul@0 843
            print >>f, "function default parameters:"
paul@0 844
            functions = self.function_defaults.keys()
paul@0 845
            functions.sort()
paul@0 846
            for function in functions:
paul@0 847
                parameters = self.function_defaults[function]
paul@0 848
                if parameters:
paul@0 849
                    print >>f, function, ", ".join([("%s=%s" % (name, default)) for (name, default) in parameters])
paul@0 850
                else:
paul@0 851
                    print >>f, function, "{}"
paul@0 852
paul@0 853
            print >>f
paul@0 854
            print >>f, "function locals:"
paul@0 855
            functions = self.function_locals.keys()
paul@0 856
            functions.sort()
paul@0 857
            for function in functions:
paul@0 858
                names = self.function_locals[function].items()
paul@0 859
                if names:
paul@0 860
                    names.sort()
paul@0 861
                    for name, value in names:
paul@0 862
                        print >>f, function, name, value
paul@0 863
                else:
paul@0 864
                    print >>f, function, "{}"
paul@0 865
paul@0 866
            self.to_lines(f, "scope globals:", self.scope_globals)
paul@0 867
paul@0 868
            print >>f
paul@0 869
            print >>f, "function targets:"
paul@0 870
            functions = self.function_targets.keys()
paul@0 871
            functions.sort()
paul@0 872
            for function in functions:
paul@0 873
                print >>f, function, self.function_targets[function]
paul@0 874
paul@0 875
            print >>f
paul@0 876
            print >>f, "function arguments:"
paul@0 877
            functions = self.function_arguments.keys()
paul@0 878
            functions.sort()
paul@0 879
            for function in functions:
paul@0 880
                print >>f, function, self.function_arguments[function]
paul@0 881
paul@0 882
            print >>f
paul@0 883
            print >>f, "attribute usage:"
paul@0 884
            units = self.attr_usage.keys()
paul@0 885
            units.sort()
paul@0 886
            for unit in units:
paul@0 887
                d = self.attr_usage[unit]
paul@0 888
                self.usage_to_cache(d, f, unit)
paul@0 889
paul@0 890
            print >>f
paul@0 891
            print >>f, "attribute accesses:"
paul@0 892
            paths = self.attr_accesses.keys()
paul@0 893
            paths.sort()
paul@0 894
            for path in paths:
paul@0 895
                accesses = list(self.attr_accesses[path])
paul@0 896
                accesses.sort()
paul@0 897
                print >>f, path, ", ".join(accesses)
paul@0 898
paul@0 899
            print >>f
paul@0 900
            print >>f, "constant accesses:"
paul@0 901
            paths = self.const_accesses.keys()
paul@0 902
            paths.sort()
paul@0 903
            for path in paths:
paul@0 904
                accesses = self.const_accesses[path].items()
paul@0 905
                accesses.sort()
paul@0 906
                for (original_name, attrnames), (objpath, ref, remaining_attrnames) in accesses:
paul@0 907
                    print >>f, path, original_name, attrnames, objpath, ref, remaining_attrnames or "{}"
paul@0 908
paul@0 909
            print >>f
paul@0 910
            print >>f, "attribute access usage:"
paul@0 911
            paths = self.attr_accessors.keys()
paul@0 912
            paths.sort()
paul@0 913
            for path in paths:
paul@0 914
                all_accesses = self.attr_accessors[path].items()
paul@0 915
                all_accesses.sort()
paul@0 916
                for (name, attrname), accesses in all_accesses:
paul@0 917
                    for positions in accesses:
paul@0 918
                        positions = map(str, positions)
paul@0 919
                        print >>f, path, name, attrname or "{}", ", ".join(positions)
paul@0 920
paul@0 921
            print >>f
paul@0 922
            print >>f, "attribute access modifiers:"
paul@0 923
            paths = self.attr_access_modifiers.keys()
paul@0 924
            paths.sort()
paul@0 925
            for path in paths:
paul@0 926
                all_accesses = self.attr_access_modifiers[path].items()
paul@0 927
                all_accesses.sort()
paul@0 928
                for (name, attrnames), modifiers in all_accesses:
paul@0 929
                    print >>f, path, name or "{}", attrnames or "{}", encode_modifiers(modifiers)
paul@0 930
paul@0 931
            print >>f
paul@0 932
            print >>f, "constant literals:"
paul@0 933
            paths = self.constants.keys()
paul@0 934
            paths.sort()
paul@0 935
            for path in paths:
paul@0 936
                constants = [(v, k) for (k, v) in self.constants[path].items()]
paul@0 937
                constants.sort()
paul@0 938
                for n, constant in constants:
paul@0 939
                    print >>f, path, repr(constant)
paul@0 940
paul@0 941
            print >>f
paul@0 942
            print >>f, "constant values:"
paul@0 943
            names = self.constant_values.keys()
paul@0 944
            names.sort()
paul@0 945
            for name in names:
paul@0 946
                value, value_type = self.constant_values[name]
paul@0 947
                print >>f, name, value_type, repr(value)
paul@0 948
paul@0 949
        finally:
paul@0 950
            f.close()
paul@0 951
paul@0 952
    def to_lines(self, f, heading, d):
paul@0 953
paul@0 954
        "Write lines to 'f' with the given 'heading', using 'd'."
paul@0 955
paul@0 956
        print >>f
paul@0 957
        print >>f, heading
paul@0 958
        keys = d.keys()
paul@0 959
        keys.sort()
paul@0 960
        for key in keys:
paul@0 961
            attrs = list(d[key])
paul@0 962
            if attrs:
paul@0 963
                attrs.sort()
paul@0 964
                print >>f, key, ", ".join(attrs)
paul@0 965
paul@0 966
    def usage_to_cache(self, details, f, prefix):
paul@0 967
paul@0 968
        "Write the given namespace usage details to the cache."
paul@0 969
paul@0 970
        names = list(details.keys())
paul@0 971
        if names:
paul@0 972
            names.sort()
paul@0 973
            for name in names:
paul@0 974
                if details[name]:
paul@0 975
paul@0 976
                    # Produce descriptions for each version of the name.
paul@0 977
paul@0 978
                    for version in details[name]:
paul@0 979
                        all_usages = []
paul@0 980
                        for usage in version:
paul@0 981
                            all_usages.append(encode_usage(usage))
paul@0 982
paul@0 983
                        print >>f, "%s %s %s" % (prefix, name, "; ".join(all_usages))
paul@0 984
paul@0 985
# vim: tabstop=4 expandtab shiftwidth=4