1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008, 2009 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 To generate programs, the above importer should be supplied in the 33 initialisation of a program instance, and then various methods are called: 34 35 program = Program(importer) 36 image = program.get_raw_image() 37 38 Such importer and program objects are the most convenient mechanism through 39 which the functionality of the micropython package may be accessed. 40 """ 41 42 from micropython.common import * 43 import micropython.ast 44 import micropython.data 45 import micropython.opt 46 import micropython.inspect 47 import micropython.table 48 import os 49 try: 50 set 51 except NameError: 52 from sets import Set as set 53 54 class Program: 55 56 "This class supports the generation of a program image." 57 58 supported_optimisations = micropython.opt.Optimiser.supported_optimisations 59 60 def __init__(self, importer, optimisations=None): 61 62 """ 63 Initialise the program representation with an 'importer' which is able 64 to locate and load Python modules. 65 66 The optional 'optimisations' cause certain techniques to be used in 67 reducing program size and improving program efficiency. 68 """ 69 70 self.importer = importer 71 self.optimisations = optimisations or set() 72 73 # Remember the tables once generated. 74 75 self.objtable = None 76 self.paramtable = None 77 self.clstable = None 78 79 # Main program information. 80 81 self.code = None 82 self.code_location = None 83 84 def get_importer(self): 85 return self.importer 86 87 # Access to finalised program information. 88 89 def get_image(self, with_builtins=0): 90 91 "Return a dictionary mapping modules to structures." 92 93 if self.code is not None: 94 return self.code 95 96 objtable = self.get_object_table() 97 paramtable = self.get_parameter_table() 98 self.importer.finalise() 99 100 self.code = [] 101 102 # Append constants to the image. 103 104 for pos, const in enumerate(self.importer.constants()): 105 self.code.append(const) 106 107 last_module = self.importer.modules_ordered[-1] 108 109 for module in self.importer.modules_ordered: 110 pos = len(self.code) 111 112 # Position the module in the image and make a translation. 113 114 trans = micropython.ast.Translation(module, self) 115 116 # Add header details. 117 118 self.code.append(module) 119 pos += 1 120 121 # Append module attributes to the image. 122 123 attributes = module.module_attributes() 124 self.code += module.attributes_as_list() 125 pos += len(attributes.keys()) 126 127 # Append classes and functions to the image. 128 129 for obj in module.all_objects: 130 if isinstance(obj, micropython.inspect.Class): 131 132 # Add header details. 133 134 self.code.append(obj) 135 pos += 1 136 137 # Append class attributes to the image. 138 139 attributes = obj.class_attributes() 140 self.code += obj.attributes_as_list() 141 pos += len(attributes.keys()) 142 143 # Generate the instantiator/initialiser. 144 # Append the function code to the image. 145 146 instantiator = obj.get_instantiator() 147 code = trans.get_instantiator_code(obj) 148 self.code += code 149 pos += len(code) 150 151 # Class-level code is generated separately at the module 152 # level, and the code location is set within the code 153 # generation process for the module. 154 155 elif isinstance(obj, micropython.inspect.Function): 156 157 # Add header details. 158 159 self.code.append(obj) 160 pos += 1 161 162 # Append any default values to the image. 163 # Only do this for named functions (not lambdas). 164 165 if obj.name is not None: 166 self.code += obj.default_attrs 167 pos += len(obj.default_attrs) 168 169 # Omit built-in function code where requested. 170 171 if not with_builtins and module.name == "__builtins__": 172 pass 173 174 # Append the function code to the image. 175 176 else: 177 code = trans.get_code(obj) 178 self.code += code 179 pos += len(code) 180 181 # Omit built-in module code where requested. 182 183 if not with_builtins and module.name == "__builtins__": 184 pass 185 186 # Append the module top-level code to the image. 187 188 else: 189 code = trans.get_module_code(final=(module is last_module)) 190 self.code += code 191 pos += len(code) 192 193 return self.code 194 195 def get_raw_image(self, with_builtins=0): 196 197 "Return the raw image representation of the program." 198 199 self.get_image(with_builtins) 200 201 objtable = self.get_object_table() 202 paramtable = self.get_parameter_table() 203 204 # Position the objects. 205 206 pos = 0 207 for item in self.code: 208 209 # Blocks are positioned leaving space for their expansion. 210 211 if isinstance(item, Block): 212 item.location = pos 213 pos += len(item.code) 214 215 # Other multi-location objects. 216 217 elif isinstance(item, ( 218 micropython.data.Class, micropython.data.Const, 219 micropython.data.Function, micropython.data.Module 220 )): 221 222 if isinstance(item, micropython.data.Class): 223 224 # Append a template of an instance for use when 225 # instantiating classes. 226 227 item.instance_template_location = pos 228 pos += 1 229 230 # Record the location of the object. 231 232 item.location = pos 233 pos += 1 234 235 # Code and details are associated with certain objects. 236 237 if isinstance(item, micropython.data.Function): 238 239 # Set the code location only where the code has been 240 # generated. 241 242 if not with_builtins and item.module.name == "__builtins__": 243 item.code_location = item.full_name() 244 245 # Skip any defaults. 246 247 else: 248 item.code_location = pos + len(item.defaults) 249 250 elif isinstance(item, micropython.data.Const): 251 pos += len(item.raw_data()) 252 253 else: 254 pos += 1 255 256 # Generate the raw code. 257 258 self.raw_code = [] 259 260 for item in self.code: 261 262 if isinstance(item, micropython.data.Attr): 263 self.raw_code += item.as_raw(objtable, paramtable) 264 265 elif isinstance(item, Block): 266 assert item.location == len(self.raw_code) 267 self.raw_code += item.as_raw(objtable, paramtable) 268 269 # Using classcode, attrcode, codeaddr, codedetails, instance. 270 271 elif isinstance(item, micropython.data.Class): 272 assert item.instance_template_location == len(self.raw_code) 273 self.raw_code += item.as_raw(objtable, paramtable) 274 assert item.location == len(self.raw_code) - 1 275 276 elif isinstance(item, micropython.data.Const): 277 assert item.location == len(self.raw_code) 278 self.raw_code += item.as_raw(objtable, paramtable) 279 280 elif isinstance(item, micropython.data.Function): 281 assert item.location == len(self.raw_code) 282 self.raw_code += item.as_raw(objtable, paramtable) 283 284 # Check the code location only where the code has been generated. 285 286 assert (not with_builtins and item.module.name == "__builtins__") or \ 287 item.code_location == len(self.raw_code) + len(item.defaults) 288 289 elif isinstance(item, micropython.data.Module): 290 assert item.location == len(self.raw_code) 291 self.raw_code += item.as_raw(objtable, paramtable) 292 293 else: 294 self.raw_code.append(item) 295 296 # Fix the module locations. 297 298 for module in self.importer.modules_ordered: 299 300 if not with_builtins and module.name == "__builtins__": 301 continue 302 303 module.code_location = module.blocks[0].location 304 305 self.code_location = self.importer.modules["__main__"].code_location 306 return self.raw_code 307 308 def get_object_table(self): 309 310 "Return a table with details of attributes for classes and modules." 311 312 if self.objtable is None: 313 self.importer.vacuum() 314 315 t = self.objtable = micropython.table.ObjectTable() 316 for module in self.importer.get_modules(): 317 t.add(module.full_name(), module.module_attributes()) 318 319 # Add class and instance attributes for all classes, together 320 # with descendant information. 321 322 for obj in module.all_objects: 323 if isinstance(obj, micropython.inspect.Class): 324 attributes = {obj.full_name() : obj} 325 attributes.update(obj.all_attributes()) 326 attributes.update(obj.all_descendants()) 327 t.add(obj.full_name(), attributes) 328 329 return self.objtable 330 331 def get_parameter_table(self): 332 333 "Return a table with details of parameters for functions and methods." 334 335 # Need the object table to get at class details. 336 337 objtable = self.get_object_table() 338 339 if self.paramtable is None: 340 t = self.paramtable = micropython.table.ParameterTable() 341 342 # Visit each module, getting function and method details. 343 344 for module in self.importer.get_modules(): 345 for obj in module.all_objects: 346 if isinstance(obj, micropython.inspect.Function): 347 t.add(obj.full_name(), obj.parameters()) 348 349 # Classes are callable, too. 350 # Take details of the appropriate __init__ method to make an 351 # entry for an instantiation function for the class. 352 353 elif isinstance(obj, micropython.inspect.Class): 354 t.add(obj.full_name(), obj.get_instantiator().parameters()) 355 356 # Filter out all parameter table entries not referenced by keyword 357 # arguments. 358 359 keyword_names = set() 360 361 for module in self.importer.get_modules(): 362 keyword_names.update(module.keyword_names) 363 364 for function_name, parameters in t.table.items(): 365 for name in parameters.keys(): 366 if name in keyword_names: 367 break 368 else: 369 del t.table[function_name] 370 371 return self.paramtable 372 373 class Importer: 374 375 "An import machine, searching for and loading modules." 376 377 predefined_constants = { 378 "None" : None, 379 "True" : True, 380 "False" : False, 381 "Ellipsis" : Ellipsis, 382 "NotImplemented" : NotImplemented 383 } 384 385 def __init__(self, path=None, verbose=0, optimisations=None): 386 387 """ 388 Initialise the importer with the given search 'path' - a list of 389 directories to search for Python modules. 390 391 The optional 'verbose' parameter causes output concerning the activities 392 of the object to be produced if set to a true value (not the default). 393 394 The optional 'optimisations' cause certain techniques to be used in 395 reducing program size and improving program efficiency. 396 """ 397 398 self.path = path or [os.getcwd()] 399 self.verbose = verbose 400 self.optimisations = optimisations or set() 401 402 self.modules = {} 403 self.modules_ordered = [] 404 self.loading = set() 405 406 # Constant records. 407 408 self.constant_values = {} 409 self.constant_list = None # cache for constants 410 self.init_predefined_constants() 411 412 # Name records (used to track actual use of names). 413 # Include names which may not be explicitly used in programs. 414 415 self.names_used = set(["__init__", "__call__", "__getitem__", "__bool__"]) 416 417 # Status information. 418 419 self.vacuumed = 0 420 self.finalised = 0 421 422 def get_modules(self): 423 424 "Return all modules known to the importer." 425 426 return self.modules.values() 427 428 # General maintenance. 429 430 def vacuum(self): 431 432 "Tidy up the modules." 433 434 if self.vacuumed: 435 return 436 437 for name, module in self.modules.items(): 438 if module.loaded: 439 module.vacuum() 440 else: 441 del self.modules[name] 442 443 self.vacuumed = 1 444 445 def finalise(self): 446 447 "Finalise the program." 448 449 if self.finalised: 450 return 451 452 self.vacuum() 453 454 for module in self.get_modules(): 455 module.finalise() 456 457 self.finalised = 1 458 459 # Name accounting. 460 461 def use_name(self, name): 462 463 "Register the given 'name' as being used in the program." 464 465 self.names_used.add(name) 466 467 # Constant accounting. 468 469 def init_predefined_constants(self): 470 471 "Ensure the predefined constants." 472 473 for name, value in self.predefined_constants.items(): 474 self.make_constant(value) 475 476 def get_predefined_constant(self, name): 477 478 "Return the predefined constant for the given 'name'." 479 480 return self.make_constant(self.predefined_constants[name]) 481 482 def get_constant(self, value): 483 484 "Return a constant for the given 'value'." 485 486 const = micropython.data.Const(value) 487 return self.constant_values[const] 488 489 def make_constant(self, value): 490 491 "Make and return a constant for the given 'value'." 492 493 const = micropython.data.Const(value) 494 if not self.constant_values.has_key(const): 495 self.constant_values[const] = const 496 return self.constant_values[const] 497 498 def constants(self): 499 500 "Return a list of constants." 501 502 if self.constant_list is None: 503 self.constant_list = list(self.constant_values.values()) 504 505 return self.constant_list 506 507 # Import methods. 508 509 def find_in_path(self, name): 510 511 """ 512 Find the given module 'name' in the search path, returning None where no 513 such module could be found, or a 2-tuple from the 'find' method 514 otherwise. 515 """ 516 517 for d in self.path: 518 m = self.find(d, name) 519 if m: return m 520 return None 521 522 def find(self, d, name): 523 524 """ 525 In the directory 'd', find the given module 'name', where 'name' can 526 either refer to a single file module or to a package. Return None if the 527 'name' cannot be associated with either a file or a package directory, 528 or a 2-tuple from '_find_package' or '_find_module' otherwise. 529 """ 530 531 m = self._find_package(d, name) 532 if m: return m 533 m = self._find_module(d, name) 534 if m: return m 535 return None 536 537 def _find_module(self, d, name): 538 539 """ 540 In the directory 'd', find the given module 'name', returning None where 541 no suitable file exists in the directory, or a 2-tuple consisting of 542 None (indicating that no package directory is involved) and a filename 543 indicating the location of the module. 544 """ 545 546 name_py = name + os.extsep + "py" 547 filename = self._find_file(d, name_py) 548 if filename: 549 return None, filename 550 return None 551 552 def _find_package(self, d, name): 553 554 """ 555 In the directory 'd', find the given package 'name', returning None 556 where no suitable package directory exists, or a 2-tuple consisting of 557 a directory (indicating the location of the package directory itself) 558 and a filename indicating the location of the __init__.py module which 559 declares the package's top-level contents. 560 """ 561 562 filename = self._find_file(d, name) 563 if filename: 564 init_py = "__init__" + os.path.extsep + "py" 565 init_py_filename = self._find_file(filename, init_py) 566 if init_py_filename: 567 return filename, init_py_filename 568 return None 569 570 def _find_file(self, d, filename): 571 572 """ 573 Return the filename obtained when searching the directory 'd' for the 574 given 'filename', or None if no actual file exists for the filename. 575 """ 576 577 filename = os.path.join(d, filename) 578 if os.path.exists(filename): 579 return filename 580 else: 581 return None 582 583 def load(self, name, return_leaf=0): 584 585 """ 586 Load the module or package with the given 'name'. Return an object 587 referencing the loaded module or package, or None if no such module or 588 package exists. 589 """ 590 591 if self.modules.has_key(name) and self.modules[name].loaded: 592 #print "Cached (%s)" % name 593 return self.modules[name] 594 if self.verbose: 595 print "Loading", name 596 597 # Split the name into path components, and try to find the uppermost in 598 # the search path. 599 600 path = name.split(".") 601 m = self.find_in_path(path[0]) 602 if not m: 603 if self.verbose: 604 print "Not found (%s)" % path[0] 605 return None # NOTE: Import error. 606 d, filename = m 607 608 # Either acquire a reference to an already-imported module, or load the 609 # module from a file. 610 611 top = module = self.load_from_file(filename, path[0]) 612 613 # For hierarchical names, traverse each path component... 614 615 if len(path) > 1: 616 if not d: 617 if self.verbose: 618 print "No package (%s)" % filename 619 return None # NOTE: Import error (package not found). 620 else: 621 self.add_submodules(d, module) 622 623 path_so_far = path[:1] 624 for p in path[1:]: 625 path_so_far.append(p) 626 627 # Find the package or module concerned. 628 629 m = self.find(d, p) 630 if not m: 631 if self.verbose: 632 print "Not found (%s)" % p 633 return None # NOTE: Import error. 634 d, filename = m 635 module_name = ".".join(path_so_far) 636 637 # Either reference an imported module or load one from a file. 638 639 submodule = self.load_from_file(filename, module_name) 640 641 if d: 642 self.add_submodules(d, module) 643 644 # Store the submodule within its parent module. 645 646 module.set_module(p, submodule) 647 module = submodule 648 649 # Return either the deepest or the uppermost module. 650 651 if return_leaf: 652 return module 653 else: 654 return top 655 656 def load_from_file(self, name, module_name=None): 657 658 """ 659 Load the module with the given 'name' (which may be a full module path). 660 """ 661 662 if module_name is None: 663 module_name = "__main__" 664 665 module = self.add_module(module_name) 666 if not module.loaded and module not in self.loading: 667 self.loading.add(module) 668 #print "Parsing", name 669 module.parse(name) 670 #print "Done", name 671 self.loading.remove(module) 672 module.loaded = 1 673 674 # Record the module. 675 676 #print "Loaded", module_name, "with namespace", module.namespace.keys() 677 return module 678 679 def add_module(self, module_name): 680 681 """ 682 Return the module with the given 'module_name', adding a new module 683 object if one does not already exist. 684 """ 685 686 if not self.modules.has_key(module_name): 687 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 688 self.modules_ordered.append(module) 689 else: 690 module = self.modules[module_name] 691 return module 692 693 def add_submodules(self, pathname, module): 694 695 """ 696 Work around insufficient __all__ declarations and examine the directory 697 with the given 'pathname', adding submodules to the given 'module'. 698 """ 699 700 for filename in os.listdir(pathname): 701 submodule, ext = os.path.splitext(filename) 702 if ext not in ("", ".py"): 703 continue 704 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 705 706 # vim: tabstop=4 expandtab shiftwidth=4