1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 To generate programs, the above importer should be supplied in the 33 initialisation of a program instance, and then various methods are called: 34 35 program = Program(importer) 36 image = program.get_raw_image() 37 38 Such importer and program objects are the most convenient mechanism through 39 which the functionality of the micropython package may be accessed. 40 """ 41 42 from micropython.common import * 43 from micropython.program import Block 44 import micropython.ast 45 import micropython.data 46 import micropython.opt 47 import micropython.inspect 48 import micropython.table 49 import os 50 try: 51 set 52 except NameError: 53 from sets import Set as set 54 55 class Program: 56 57 "This class supports the generation of a program image." 58 59 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 60 61 def __init__(self, importer, optimisations=None): 62 63 """ 64 Initialise the program representation with an 'importer' which is able 65 to locate and load Python modules. 66 67 The optional 'optimisations' cause certain techniques to be used in 68 reducing program size and improving program efficiency. 69 """ 70 71 self.importer = importer 72 self.optimisations = optimisations or set() 73 74 # Remember the tables once generated. 75 76 self.objtable = None 77 self.paramtable = None 78 79 # Main program information. 80 81 self.code = None 82 self.code_location = None 83 84 def get_importer(self): 85 return self.importer 86 87 # Access to finalised program information. 88 89 def get_image(self, with_builtins=0): 90 91 "Return a dictionary mapping modules to structures." 92 93 if self.code is not None: 94 return self.code 95 96 objtable = self.get_object_table() 97 paramtable = self.get_parameter_table() 98 self.importer.finalise() 99 100 self.code = [] 101 102 # Append constants to the image. 103 104 for pos, const in enumerate(self.importer.constants()): 105 self.code.append(const) 106 107 last_module = self.importer.modules_ordered[-1] 108 109 for module in self.importer.modules_ordered: 110 pos = len(self.code) 111 suppress_builtins = not with_builtins and module.name == "__builtins__" 112 113 # Position the module in the image and make a translation. 114 115 trans = micropython.ast.Translation(module, self) 116 117 # Add header details. 118 119 self.code.append(module) 120 pos += 1 121 122 # Append module attributes to the image. 123 124 attributes = module.module_attributes() 125 self.code += module.attributes_as_list() 126 pos += len(attributes.keys()) 127 128 # Append classes and functions to the image. 129 130 for obj in module.all_objects: 131 if isinstance(obj, micropython.inspect.Class): 132 133 # Add header details. 134 135 self.code.append(obj) 136 pos += 1 137 138 # Append class attributes to the image. 139 140 attributes = obj.class_attributes() 141 self.code += obj.attributes_as_list() 142 pos += len(attributes.keys()) 143 144 # Omit built-in function code where requested. 145 146 if suppress_builtins: 147 continue 148 149 # Generate the instantiator/initialiser. 150 # Append the function code to the image. 151 152 instantiator = obj.get_instantiator() 153 code = trans.get_instantiator_code(obj) 154 self.code += code 155 pos += len(code) 156 157 # Class-level code is generated separately at the module 158 # level, and the code location is set within the code 159 # generation process for the module. 160 161 elif isinstance(obj, micropython.inspect.Function): 162 163 # Add header details. 164 165 self.code.append(obj) 166 pos += 1 167 168 # Append any default values to the image. 169 # Only do this for named functions (not lambdas). 170 171 if obj.name is not None: 172 self.code += obj.default_attrs 173 pos += len(obj.default_attrs) 174 175 # Omit built-in function code where requested. 176 177 if suppress_builtins: 178 pass 179 180 # Append the function code to the image. 181 182 else: 183 code = trans.get_code(obj) 184 self.code += code 185 pos += len(code) 186 187 # Omit built-in module code where requested. 188 189 if suppress_builtins: 190 pass 191 192 # Append the module top-level code to the image. 193 194 else: 195 code = trans.get_module_code(final=(module is last_module)) 196 self.code += code 197 pos += len(code) 198 199 return self.code 200 201 def get_raw_image(self, with_builtins=0): 202 203 "Return the raw image representation of the program." 204 205 self.get_image(with_builtins) 206 207 objtable = self.get_object_table() 208 paramtable = self.get_parameter_table() 209 210 # Position the objects. 211 212 pos = 0 213 for item in self.code: 214 215 # Blocks are positioned leaving space for their expansion. 216 217 if isinstance(item, Block): 218 item.location = pos 219 pos += len(item.code) 220 221 # Other multi-location objects. 222 223 elif isinstance(item, ( 224 micropython.data.Class, micropython.data.Const, 225 micropython.data.Function, micropython.data.Module 226 )): 227 228 if isinstance(item, micropython.data.Class): 229 230 # Append a template of an instance for use when 231 # instantiating classes. 232 233 item.instance_template_location = pos 234 pos += 1 235 236 # Record the location of the object. 237 238 item.location = pos 239 pos += 1 240 241 # Code and details are associated with certain objects. 242 243 if isinstance(item, micropython.data.Function): 244 245 # Set the code location only where the code has been 246 # generated. 247 248 if not with_builtins and item.module.name == "__builtins__": 249 item.code_location = item.full_name() 250 251 # Skip any defaults. 252 253 else: 254 item.code_location = pos + len(item.defaults) 255 256 elif isinstance(item, micropython.data.Const): 257 pos += len(item.raw_data()) 258 259 else: 260 pos += 1 261 262 # Generate the raw code. 263 264 self.raw_code = [] 265 266 for item in self.code: 267 268 if isinstance(item, micropython.data.Attr): 269 self.raw_code += item.as_raw(objtable, paramtable) 270 271 elif isinstance(item, Block): 272 assert item.location == len(self.raw_code) 273 self.raw_code += item.as_raw(objtable, paramtable) 274 275 # Using classcode, attrcode, codeaddr, codedetails, instance. 276 277 elif isinstance(item, micropython.data.Class): 278 assert item.instance_template_location == len(self.raw_code) 279 self.raw_code += item.as_raw(objtable, paramtable, with_builtins or item.module.name != "__builtins__") 280 assert item.location == len(self.raw_code) - 1 281 282 elif isinstance(item, micropython.data.Const): 283 assert item.location == len(self.raw_code) 284 self.raw_code += item.as_raw(objtable, paramtable) 285 286 elif isinstance(item, micropython.data.Function): 287 assert item.location == len(self.raw_code) 288 self.raw_code += item.as_raw(objtable, paramtable) 289 290 # Check the code location only where the code has been generated. 291 292 assert (not with_builtins and item.module.name == "__builtins__") or \ 293 item.code_location == len(self.raw_code) + len(item.defaults) 294 295 elif isinstance(item, micropython.data.Module): 296 assert item.location == len(self.raw_code) 297 self.raw_code += item.as_raw(objtable, paramtable) 298 299 else: 300 self.raw_code.append(item) 301 302 # Fix the module locations. 303 304 for module in self.importer.modules_ordered: 305 306 if not with_builtins and module.name == "__builtins__": 307 continue 308 309 module.code_location = module.blocks[0].location 310 311 self.code_location = self.importer.modules["__main__"].code_location 312 return self.raw_code 313 314 def get_object_table(self): 315 316 "Return a table with details of attributes for classes and modules." 317 318 if self.objtable is None: 319 self.importer.vacuum() 320 321 t = self.objtable = micropython.table.ObjectTable() 322 for module in self.importer.get_modules(): 323 t.add(module.full_name(), module.module_attributes()) 324 325 # Add class and instance attributes for all classes, together 326 # with descendant information. 327 328 for obj in module.all_objects: 329 if isinstance(obj, micropython.inspect.Class): 330 attributes = {obj.full_name() : obj} 331 attributes.update(obj.all_attributes()) 332 attributes.update(obj.all_descendants()) 333 t.add(obj.full_name(), attributes) 334 335 return self.objtable 336 337 def get_parameter_table(self): 338 339 "Return a table with details of parameters for functions and methods." 340 341 # Need the object table to get at class details. 342 343 objtable = self.get_object_table() 344 345 if self.paramtable is None: 346 t = self.paramtable = micropython.table.ParameterTable() 347 348 # Visit each module, getting function and method details. 349 350 for module in self.importer.get_modules(): 351 for obj in module.all_objects: 352 if isinstance(obj, micropython.inspect.Function): 353 t.add(obj.full_name(), obj.parameters()) 354 355 # Classes are callable, too. 356 # Take details of the appropriate __init__ method to make an 357 # entry for an instantiation function for the class. 358 359 elif isinstance(obj, micropython.inspect.Class): 360 t.add(obj.full_name(), obj.get_instantiator().parameters()) 361 362 # Filter out all parameter table entries not referenced by keyword 363 # arguments. 364 365 keyword_names = set() 366 367 for module in self.importer.get_modules(): 368 keyword_names.update(module.keyword_names) 369 370 for function_name, parameters in t.table.items(): 371 for name in parameters.keys(): 372 if name in keyword_names: 373 break 374 else: 375 del t.table[function_name] 376 377 return self.paramtable 378 379 class Importer: 380 381 "An import machine, searching for and loading modules." 382 383 predefined_constants = { 384 "None" : None, 385 "True" : True, 386 "False" : False, 387 #"Ellipsis" : Ellipsis, 388 "NotImplemented" : NotImplemented 389 } 390 391 def __init__(self, path=None, verbose=0, optimisations=None): 392 393 """ 394 Initialise the importer with the given search 'path' - a list of 395 directories to search for Python modules. 396 397 The optional 'verbose' parameter causes output concerning the activities 398 of the object to be produced if set to a true value (not the default). 399 400 The optional 'optimisations' cause certain techniques to be used in 401 reducing program size and improving program efficiency. 402 """ 403 404 self.path = path or [os.getcwd()] 405 self.verbose = verbose 406 self.optimisations = optimisations or set() 407 408 self.modules = {} 409 self.modules_ordered = [] 410 self.loading = set() 411 412 # Constant records. 413 414 self.constant_values = {} 415 self.constant_list = None # cache for constants 416 self.init_predefined_constants() 417 418 # Name records (used to track actual use of names). 419 # Include names which may not be explicitly used in programs. 420 # NOTE: Potentially declare these when inspecting. 421 422 self.names_used = set(["__init__", "__call__", "__bool__"]) 423 424 # Status information. 425 426 self.vacuumed = 0 427 self.finalised = 0 428 429 def get_modules(self): 430 431 "Return all modules known to the importer." 432 433 return self.modules.values() 434 435 # General maintenance. 436 437 def vacuum(self): 438 439 "Tidy up the modules." 440 441 if self.vacuumed: 442 return 443 444 for name, module in self.modules.items(): 445 if module.loaded: 446 module.vacuum() 447 else: 448 del self.modules[name] 449 450 self.vacuumed = 1 451 452 def finalise(self): 453 454 "Finalise the program." 455 456 if self.finalised: 457 return 458 459 self.vacuum() 460 461 for module in self.get_modules(): 462 module.finalise() 463 464 self.finalised = 1 465 466 # Name accounting. 467 468 def use_name(self, name): 469 470 "Register the given 'name' as being used in the program." 471 472 self.names_used.add(name) 473 474 # Constant accounting. 475 476 def init_predefined_constants(self): 477 478 "Ensure the predefined constants." 479 480 for name, value in self.predefined_constants.items(): 481 self.make_constant(value) 482 483 def get_predefined_constant(self, name): 484 485 "Return the predefined constant for the given 'name'." 486 487 return self.make_constant(self.predefined_constants[name]) 488 489 def get_constant(self, value): 490 491 "Return a constant for the given 'value'." 492 493 const = micropython.data.Const(value) 494 return self.constant_values[const] 495 496 def make_constant(self, value): 497 498 "Make and return a constant for the given 'value'." 499 500 const = micropython.data.Const(value) 501 if not self.constant_values.has_key(const): 502 self.constant_values[const] = const 503 return self.constant_values[const] 504 505 def constants(self): 506 507 "Return a list of constants." 508 509 if self.constant_list is None: 510 self.constant_list = list(self.constant_values.values()) 511 512 return self.constant_list 513 514 # Import methods. 515 516 def find_in_path(self, name): 517 518 """ 519 Find the given module 'name' in the search path, returning None where no 520 such module could be found, or a 2-tuple from the 'find' method 521 otherwise. 522 """ 523 524 for d in self.path: 525 m = self.find(d, name) 526 if m: return m 527 return None 528 529 def find(self, d, name): 530 531 """ 532 In the directory 'd', find the given module 'name', where 'name' can 533 either refer to a single file module or to a package. Return None if the 534 'name' cannot be associated with either a file or a package directory, 535 or a 2-tuple from '_find_package' or '_find_module' otherwise. 536 """ 537 538 m = self._find_package(d, name) 539 if m: return m 540 m = self._find_module(d, name) 541 if m: return m 542 return None 543 544 def _find_module(self, d, name): 545 546 """ 547 In the directory 'd', find the given module 'name', returning None where 548 no suitable file exists in the directory, or a 2-tuple consisting of 549 None (indicating that no package directory is involved) and a filename 550 indicating the location of the module. 551 """ 552 553 name_py = name + os.extsep + "py" 554 filename = self._find_file(d, name_py) 555 if filename: 556 return None, filename 557 return None 558 559 def _find_package(self, d, name): 560 561 """ 562 In the directory 'd', find the given package 'name', returning None 563 where no suitable package directory exists, or a 2-tuple consisting of 564 a directory (indicating the location of the package directory itself) 565 and a filename indicating the location of the __init__.py module which 566 declares the package's top-level contents. 567 """ 568 569 filename = self._find_file(d, name) 570 if filename: 571 init_py = "__init__" + os.path.extsep + "py" 572 init_py_filename = self._find_file(filename, init_py) 573 if init_py_filename: 574 return filename, init_py_filename 575 return None 576 577 def _find_file(self, d, filename): 578 579 """ 580 Return the filename obtained when searching the directory 'd' for the 581 given 'filename', or None if no actual file exists for the filename. 582 """ 583 584 filename = os.path.join(d, filename) 585 if os.path.exists(filename): 586 return filename 587 else: 588 return None 589 590 def load(self, name, return_leaf=0): 591 592 """ 593 Load the module or package with the given 'name'. Return an object 594 referencing the loaded module or package, or None if no such module or 595 package exists. 596 """ 597 598 if self.modules.has_key(name) and self.modules[name].loaded: 599 #print "Cached (%s)" % name 600 return self.modules[name] 601 if self.verbose: 602 print "Loading", name 603 604 # Split the name into path components, and try to find the uppermost in 605 # the search path. 606 607 path = name.split(".") 608 m = self.find_in_path(path[0]) 609 if not m: 610 if self.verbose: 611 print "Not found (%s)" % path[0] 612 return None # NOTE: Import error. 613 d, filename = m 614 615 # Either acquire a reference to an already-imported module, or load the 616 # module from a file. 617 618 top = module = self.load_from_file(filename, path[0]) 619 620 # For hierarchical names, traverse each path component... 621 622 if len(path) > 1: 623 if not d: 624 if self.verbose: 625 print "No package (%s)" % filename 626 return None # NOTE: Import error (package not found). 627 else: 628 self.add_submodules(d, module) 629 630 path_so_far = path[:1] 631 for p in path[1:]: 632 path_so_far.append(p) 633 634 # Find the package or module concerned. 635 636 m = self.find(d, p) 637 if not m: 638 if self.verbose: 639 print "Not found (%s)" % p 640 return None # NOTE: Import error. 641 d, filename = m 642 module_name = ".".join(path_so_far) 643 644 # Either reference an imported module or load one from a file. 645 646 submodule = self.load_from_file(filename, module_name) 647 648 if d: 649 self.add_submodules(d, module) 650 651 # Store the submodule within its parent module. 652 653 module.set_module(p, submodule) 654 module = submodule 655 656 # Return either the deepest or the uppermost module. 657 658 if return_leaf: 659 return module 660 else: 661 return top 662 663 def load_from_file(self, name, module_name=None): 664 665 """ 666 Load the module with the given 'name' (which may be a full module path). 667 """ 668 669 if module_name is None: 670 module_name = "__main__" 671 672 module = self.add_module(module_name) 673 if not module.loaded and module not in self.loading: 674 self.loading.add(module) 675 #print "Parsing", name 676 module.parse(name) 677 #print "Done", name 678 self.loading.remove(module) 679 module.loaded = 1 680 681 # Record the module. 682 683 #print "Loaded", module_name, "with namespace", module.namespace.keys() 684 return module 685 686 def add_module(self, module_name): 687 688 """ 689 Return the module with the given 'module_name', adding a new module 690 object if one does not already exist. 691 """ 692 693 if not self.modules.has_key(module_name): 694 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 695 self.modules_ordered.append(module) 696 else: 697 module = self.modules[module_name] 698 return module 699 700 def add_submodules(self, pathname, module): 701 702 """ 703 Work around insufficient __all__ declarations and examine the directory 704 with the given 'pathname', adding submodules to the given 'module'. 705 """ 706 707 for filename in os.listdir(pathname): 708 submodule, ext = os.path.splitext(filename) 709 if ext not in ("", ".py"): 710 continue 711 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 712 713 # vim: tabstop=4 expandtab shiftwidth=4