1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 To generate programs, the above importer should be supplied in the 33 initialisation of a program instance, and then various methods are called: 34 35 program = Program(importer) 36 image = program.get_raw_image() 37 38 Such importer and program objects are the most convenient mechanism through 39 which the functionality of the micropython package may be accessed. 40 """ 41 42 from micropython.common import ProcessingError, TableGenerationError 43 import micropython.ast 44 import micropython.data 45 import micropython.opt 46 import micropython.inspect 47 import micropython.table 48 import os 49 50 try: 51 set 52 except NameError: 53 from sets import Set as set 54 55 class Program: 56 57 "This class supports the generation of a program image." 58 59 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 60 61 def __init__(self, importer, optimisations=None): 62 63 """ 64 Initialise the program representation with an 'importer' which is able 65 to locate and load Python modules. 66 67 The optional 'optimisations' cause certain techniques to be used in 68 reducing program size and improving program efficiency. 69 """ 70 71 self.importer = importer 72 self.optimisations = optimisations or set() 73 74 # Remember the tables once generated. 75 76 self.objtable = None 77 self.paramtable = None 78 79 # Main program information. 80 81 self.code = None 82 self.code_location = None 83 84 def get_importer(self): 85 return self.importer 86 87 # Access to finalised program information. 88 89 def get_image(self, with_builtins=0): 90 91 """ 92 Return the program image including built-in objects if 'with_builtins' 93 is specified and set to a true value. 94 """ 95 96 if self.code is not None: 97 return self.code 98 99 objtable = self.get_object_table() 100 paramtable = self.get_parameter_table() 101 self.importer.finalise() 102 103 self.code = [] 104 105 # Append constants to the image. 106 107 for const in self.importer.constants(): 108 self.code.append(const) 109 110 last_module = self.importer.modules_ordered[-1] 111 112 for module in self.importer.modules_ordered: 113 suppress_builtins = not with_builtins and module.name == "__builtins__" 114 115 # Position the module in the image and make a translation. 116 117 trans = micropython.ast.Translation(module, self) 118 119 # Add header details. 120 121 self.code.append(module) 122 123 # Append module attributes to the image. 124 125 attributes = module.module_attributes() 126 self.code += module.attributes_as_list() 127 128 # Append classes and functions to the image. 129 130 for obj in module.all_objects: 131 if isinstance(obj, micropython.inspect.Class): 132 133 # Add header details. 134 135 self.code.append(obj) 136 137 # Append class attributes to the image. 138 139 attributes = obj.class_attributes() 140 self.code += obj.attributes_as_list() 141 142 # Omit built-in function code where requested. 143 144 if suppress_builtins and obj.astnode.doc is None: 145 continue 146 147 # Generate the instantiator/initialiser. 148 # Append the function code to the image. 149 150 instantiator = obj.get_instantiator() 151 code = trans.get_instantiator_code(obj) 152 self.code += code 153 154 # Class-level code is generated separately at the module 155 # level, and the code location is set within the code 156 # generation process for the module. 157 158 elif isinstance(obj, micropython.inspect.Function): 159 160 # Add header details. 161 162 self.code.append(obj) 163 164 # Append any default values to the image. 165 # Only do this for named functions (not lambdas). 166 167 if obj.name is not None: 168 self.code += obj.default_attrs 169 170 # Omit built-in function code where requested. 171 172 if suppress_builtins and obj.astnode.doc is None: 173 pass 174 175 # Append the function code to the image. 176 177 else: 178 code = trans.get_code(obj) 179 self.code += code 180 181 # Omit built-in module code where requested. 182 183 if suppress_builtins: 184 pass 185 186 # Append the module top-level code to the image. 187 188 else: 189 code = trans.get_module_code() 190 self.code += code 191 192 return self.code 193 194 def get_raw_image(self, architecture=None, with_builtins=0): 195 196 "Return the raw image representation of the program." 197 198 architecture = architecture or micropython.rsvp 199 200 self.get_image(with_builtins) 201 202 objtable = self.get_object_table() 203 paramtable = self.get_parameter_table() 204 205 # Position the objects. 206 207 pos = 0 208 209 for item in self.code: 210 arch_item = architecture.get_object(item) 211 212 # Get the raw version for the architecture. 213 214 if arch_item is not None: 215 pos = arch_item.set_location(pos, with_builtins) 216 else: 217 pos += 1 218 219 # Generate the raw code. 220 221 self.raw_code = [] 222 223 for item in self.code: 224 arch_item = architecture.get_object(item) 225 226 # Get the raw version for the architecture. 227 228 if arch_item is not None: 229 self.raw_code += arch_item.as_raw(objtable, paramtable, with_builtins) 230 arch_item.finalise_location(with_builtins) 231 else: 232 self.raw_code.append(item) 233 234 # Fix the module locations. 235 236 for module in self.importer.modules_ordered: 237 238 if not with_builtins and module.name == "__builtins__": 239 continue 240 241 module.code_location = module.blocks[0].location 242 243 self.code_location = self.importer.modules["__main__"].code_location 244 return self.raw_code 245 246 def get_object_table(self): 247 248 "Return a table with details of attributes for classes and modules." 249 250 if self.objtable is None: 251 self.importer.vacuum() 252 253 t = self.objtable = micropython.table.ObjectTable() 254 for module in self.importer.get_modules(): 255 t.add(module.full_name(), module.module_attributes()) 256 257 # Add class and instance attributes for all classes, together 258 # with descendant information. 259 260 for obj in module.all_objects: 261 if isinstance(obj, micropython.inspect.Class): 262 263 # Prevent ambiguous classes. 264 265 full_name = obj.full_name() 266 name = obj.name 267 268 if module.has_key(name) and module[name].defines_ambiguous_class(): 269 raise TableGenerationError, "Class %r in module %r is ambiguously defined." % (name, module.full_name()) 270 271 # Define a table entry for the class. 272 273 attributes = {full_name : obj} 274 attributes.update(obj.all_attributes()) 275 attributes.update(obj.all_descendants()) 276 t.add(full_name, attributes) 277 278 return self.objtable 279 280 def get_parameter_table(self): 281 282 "Return a table with details of parameters for functions and methods." 283 284 # Need the object table to get at class details. 285 286 objtable = self.get_object_table() 287 288 if self.paramtable is None: 289 t = self.paramtable = micropython.table.ParameterTable() 290 291 # Visit each module, getting function and method details. 292 293 for module in self.importer.get_modules(): 294 for obj in module.all_objects: 295 if isinstance(obj, micropython.inspect.Function): 296 t.add(obj.full_name(), obj.parameters()) 297 298 # Classes are callable, too. 299 # Take details of the appropriate __init__ method to make an 300 # entry for an instantiation function for the class. 301 302 elif isinstance(obj, micropython.inspect.Class): 303 t.add(obj.get_instantiator().full_name(), obj.get_instantiator().parameters()) 304 305 # Filter out all parameter table entries not referenced by keyword 306 # arguments. 307 308 keyword_names = set() 309 310 for module in self.importer.get_modules(): 311 keyword_names.update(module.keyword_names) 312 313 for function_name, parameters in t.table.items(): 314 for name in parameters.keys(): 315 if name in keyword_names: 316 break 317 else: 318 del t.table[function_name] 319 320 return self.paramtable 321 322 class Importer: 323 324 "An import machine, searching for and loading modules." 325 326 predefined_constants = { 327 "None" : None, 328 "True" : True, 329 "False" : False, 330 #"Ellipsis" : Ellipsis, 331 "NotImplemented" : NotImplemented 332 } 333 334 def __init__(self, path=None, verbose=0, optimisations=None): 335 336 """ 337 Initialise the importer with the given search 'path' - a list of 338 directories to search for Python modules. 339 340 The optional 'verbose' parameter causes output concerning the activities 341 of the object to be produced if set to a true value (not the default). 342 343 The optional 'optimisations' cause certain techniques to be used in 344 reducing program size and improving program efficiency. 345 """ 346 347 self.path = path or [os.getcwd()] 348 self.verbose = verbose 349 self.optimisations = optimisations or set() 350 351 self.modules = {} 352 self.modules_ordered = [] 353 self.loading = set() 354 355 # Constant records. 356 357 self.constant_values = {} 358 self.constant_list = None # cache for constants 359 self.init_predefined_constants() 360 361 # Name records (used to track actual use of names). 362 # Include names which may not be explicitly used in programs. 363 # NOTE: Potentially declare these when inspecting. 364 365 self.names_used = None 366 self.name_references = {} 367 368 # Status information. 369 370 self.vacuumed = 0 371 self.finalised = 0 372 373 def get_modules(self): 374 375 "Return all modules known to the importer." 376 377 return self.modules.values() 378 379 def get_module(self, name): 380 381 "Return the module with the given 'name'." 382 383 return self.modules[name] 384 385 # General maintenance. 386 387 def vacuum(self): 388 389 "Tidy up the modules." 390 391 if self.vacuumed: 392 return 393 394 for name, module in self.modules.items(): 395 if module.loaded: 396 module.vacuum() 397 else: 398 del self.modules[name] 399 400 self.vacuumed = 1 401 402 def finalise(self): 403 404 "Finalise the program." 405 406 if self.finalised: 407 return 408 409 self.vacuum() 410 411 for module in self.get_modules(): 412 module.finalise() 413 414 self.finalised = 1 415 416 # Name accounting. 417 418 def use_name(self, name, from_name): 419 420 """ 421 Register the given 'name' as being used in the program from within an 422 object with the specified 'from_name'. 423 """ 424 425 if not self.name_references.has_key(from_name): 426 self.name_references[from_name] = set() 427 self.name_references[from_name].add(name) 428 429 def uses_name(self, name): 430 431 "Return whether the given 'name' is used." 432 433 if self.names_used is None: 434 self.names_used = set(["bool", "__init__", "__call__", "__bool__", "__builtins__", "__main__"]) 435 self._collect_names("__builtins__") 436 self._collect_names("__main__") 437 438 return name in self.names_used 439 440 def _collect_names(self, from_name): 441 for name in self.name_references.get(from_name, []): 442 if name not in self.names_used: 443 self.names_used.add(name) 444 self._collect_names(name) 445 446 # Constant accounting. 447 448 def init_predefined_constants(self): 449 450 "Ensure the predefined constants." 451 452 for name, value in self.predefined_constants.items(): 453 self.make_constant(value) 454 455 def get_predefined_constant(self, name): 456 457 "Return the predefined constant for the given 'name'." 458 459 return self.make_constant(self.predefined_constants[name]) 460 461 def get_constant(self, value): 462 463 "Return a constant for the given 'value'." 464 465 const = micropython.data.Const(value) 466 return self.constant_values[const] 467 468 def get_constant_type_name(self, value): 469 return value.__class__.__name__ 470 471 def make_constant(self, value): 472 473 "Make and return a constant for the given 'value'." 474 475 # Ensure the presence of the constant's type. 476 477 name = self.get_constant_type_name(value) 478 if self.modules.has_key("__builtins__"): 479 attr = self.modules["__builtins__"].get(name) 480 481 # Make a constant object and return it. 482 483 const = micropython.data.Const(value) 484 if not self.constant_values.has_key(const): 485 self.constant_values[const] = const 486 return self.constant_values[const] 487 488 def constants(self): 489 490 "Return a list of constants." 491 492 if self.constant_list is None: 493 self.constant_list = list(self.constant_values.values()) 494 495 return self.constant_list 496 497 # Import methods. 498 499 def find_in_path(self, name): 500 501 """ 502 Find the given module 'name' in the search path, returning None where no 503 such module could be found, or a 2-tuple from the 'find' method 504 otherwise. 505 """ 506 507 for d in self.path: 508 m = self.find(d, name) 509 if m: return m 510 return None 511 512 def find(self, d, name): 513 514 """ 515 In the directory 'd', find the given module 'name', where 'name' can 516 either refer to a single file module or to a package. Return None if the 517 'name' cannot be associated with either a file or a package directory, 518 or a 2-tuple from '_find_package' or '_find_module' otherwise. 519 """ 520 521 m = self._find_package(d, name) 522 if m: return m 523 m = self._find_module(d, name) 524 if m: return m 525 return None 526 527 def _find_module(self, d, name): 528 529 """ 530 In the directory 'd', find the given module 'name', returning None where 531 no suitable file exists in the directory, or a 2-tuple consisting of 532 None (indicating that no package directory is involved) and a filename 533 indicating the location of the module. 534 """ 535 536 name_py = name + os.extsep + "py" 537 filename = self._find_file(d, name_py) 538 if filename: 539 return None, filename 540 return None 541 542 def _find_package(self, d, name): 543 544 """ 545 In the directory 'd', find the given package 'name', returning None 546 where no suitable package directory exists, or a 2-tuple consisting of 547 a directory (indicating the location of the package directory itself) 548 and a filename indicating the location of the __init__.py module which 549 declares the package's top-level contents. 550 """ 551 552 filename = self._find_file(d, name) 553 if filename: 554 init_py = "__init__" + os.path.extsep + "py" 555 init_py_filename = self._find_file(filename, init_py) 556 if init_py_filename: 557 return filename, init_py_filename 558 return None 559 560 def _find_file(self, d, filename): 561 562 """ 563 Return the filename obtained when searching the directory 'd' for the 564 given 'filename', or None if no actual file exists for the filename. 565 """ 566 567 filename = os.path.join(d, filename) 568 if os.path.exists(filename): 569 return filename 570 else: 571 return None 572 573 def load(self, name, return_leaf=0): 574 575 """ 576 Load the module or package with the given 'name'. Return an object 577 referencing the loaded module or package, or None if no such module or 578 package exists. 579 """ 580 581 if self.modules.has_key(name) and self.modules[name].loaded: 582 #print "Cached (%s)" % name 583 return self.modules[name] 584 if self.verbose: 585 print "Loading", name 586 587 # Split the name into path components, and try to find the uppermost in 588 # the search path. 589 590 path = name.split(".") 591 m = self.find_in_path(path[0]) 592 if not m: 593 if self.verbose: 594 print "Not found (%s)" % path[0] 595 return None # NOTE: Import error. 596 d, filename = m 597 598 # Either acquire a reference to an already-imported module, or load the 599 # module from a file. 600 601 top = module = self.load_from_file(filename, path[0]) 602 603 # For hierarchical names, traverse each path component... 604 605 if len(path) > 1: 606 if not d: 607 if self.verbose: 608 print "No package (%s)" % filename 609 return None # NOTE: Import error (package not found). 610 else: 611 self.add_submodules(d, module) 612 613 path_so_far = path[:1] 614 for p in path[1:]: 615 path_so_far.append(p) 616 617 # Find the package or module concerned. 618 619 m = self.find(d, p) 620 if not m: 621 if self.verbose: 622 print "Not found (%s)" % p 623 return None # NOTE: Import error. 624 d, filename = m 625 module_name = ".".join(path_so_far) 626 627 # Either reference an imported module or load one from a file. 628 629 submodule = self.load_from_file(filename, module_name) 630 631 if d: 632 self.add_submodules(d, module) 633 634 # Store the submodule within its parent module. 635 636 module.set_module(p, submodule) 637 module = submodule 638 639 # Return either the deepest or the uppermost module. 640 641 if return_leaf: 642 return module 643 else: 644 return top 645 646 def load_from_file(self, name, module_name=None): 647 648 """ 649 Load the module with the given 'name' (which may be a full module path). 650 """ 651 652 if module_name is None: 653 module_name = "__main__" 654 655 module = self.add_module(module_name) 656 if not module.loaded and module not in self.loading: 657 self.loading.add(module) 658 #print "Parsing", name 659 module.parse(name) 660 #print "Done", name 661 self.loading.remove(module) 662 module.loaded = 1 663 664 # Record the module. 665 666 #print "Loaded", module_name, "with namespace", module.namespace.keys() 667 return module 668 669 def add_module(self, module_name): 670 671 """ 672 Return the module with the given 'module_name', adding a new module 673 object if one does not already exist. 674 """ 675 676 if not self.modules.has_key(module_name): 677 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 678 self.modules_ordered.append(module) 679 else: 680 module = self.modules[module_name] 681 return module 682 683 def add_submodules(self, pathname, module): 684 685 """ 686 Work around insufficient __all__ declarations and examine the directory 687 with the given 'pathname', adding submodules to the given 'module'. 688 """ 689 690 for filename in os.listdir(pathname): 691 submodule, ext = os.path.splitext(filename) 692 if ext not in ("", ".py"): 693 continue 694 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 695 696 # vim: tabstop=4 expandtab shiftwidth=4