1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 Such importer objects are the most convenient mechanism through which the 33 functionality of the micropython package may be accessed. 34 """ 35 36 from micropython.common import * 37 import micropython.ast 38 import micropython.inspect 39 import micropython.table 40 import os, sys 41 try: 42 set 43 except NameError: 44 from sets import Set as set 45 46 class Importer: 47 48 """ 49 An import machine, searching for and loading modules. In addition, this 50 class supports the generation of a program image. 51 """ 52 53 supported_optimisations = micropython.ast.Optimiser.supported_optimisations 54 55 def __init__(self, path=None, verbose=0): 56 57 """ 58 Initialise the importer with the given search 'path' - a list of 59 directories to search for Python modules. 60 61 The optional 'verbose' parameter causes output concerning the activities 62 of the object to be produced if set to a true value (not the default). 63 """ 64 65 self.path = path or [os.getcwd()] 66 self.verbose = verbose 67 self.modules = {} 68 self.modules_ordered = [] 69 self.loading = set() 70 71 # Remember the tables once generated. 72 73 self.objtable = None 74 self.paramtable = None 75 self.clstable = None 76 77 # Constant records. 78 79 self.constant_values = {} 80 self.constant_list = None # cache for constants 81 82 # Main program information. 83 84 self.code = None 85 self.code_location = None 86 87 def constants(self): 88 89 "Return a list of constants." 90 91 if self.constant_list is None: 92 self.constant_list = list(self.constant_values.values()) 93 94 return self.constant_list 95 96 def vacuum(self): 97 98 "Tidy up the modules." 99 100 for name, module in self.modules.items(): 101 if module.loaded: 102 module.vacuum() 103 else: 104 del self.modules[name] 105 106 def finalise(self): 107 108 "Finalise the program." 109 110 for module in self.get_modules(): 111 112 # Fix the attributes. 113 114 module.finalise_attributes() 115 116 for obj in module.all_objects: 117 if isinstance(obj, (micropython.inspect.Class, micropython.inspect.Function)): 118 obj.finalise_attributes() 119 120 def get_modules(self): 121 122 "Return all modules known to the importer." 123 124 return self.modules.values() 125 126 def get_image(self, with_builtins=0, optimisations=None): 127 128 "Return a dictionary mapping modules to structures." 129 130 if self.code is not None: 131 return self.code 132 133 objtable = self.get_object_table() 134 paramtable = self.get_parameter_table() 135 self.finalise() 136 137 image = [] 138 139 # Append constants to the image. 140 141 for pos, const in enumerate(self.constants()): 142 const.location = pos 143 image.append(const) 144 145 last_module = self.modules_ordered[-1] 146 147 for module in self.modules_ordered: 148 149 if not with_builtins and module.name == "__builtins__": 150 continue 151 152 pos = len(image) 153 154 # Position the module in the image and make a translation. 155 156 module.location = pos 157 trans = micropython.ast.Translation(module, self, optimisations) 158 159 # Add header details. 160 161 image.append(module) 162 pos += 1 163 164 # Append module attributes to the image. 165 166 attributes = module.module_attributes() 167 image += module.attributes_as_list() 168 pos += len(attributes.keys()) 169 170 # Append classes and functions to the image. 171 172 for obj in module.all_objects: 173 if isinstance(obj, micropython.inspect.Class): 174 175 # Position the class in the image. 176 177 obj.location = pos 178 179 # Add header details. 180 181 image.append(obj) 182 pos += 1 183 184 # Append class attributes to the image. 185 186 attributes = obj.class_attributes() 187 image += obj.attributes_as_list() 188 pos += len(attributes.keys()) 189 190 # Generate the instantiator/initialiser. 191 # Append the function code to the image. 192 193 instantiator = obj.get_instantiator() 194 instantiator.code_location = pos 195 code = trans.get_instantiator_code(obj) 196 image += code 197 pos += len(code) 198 199 # Class-level code is generated separately at the module 200 # level, and the code location is set within the code 201 # generation process for the module. 202 203 elif isinstance(obj, micropython.inspect.Function): 204 205 # Position the function in the image. 206 207 obj.location = pos 208 209 # Add header details. 210 211 image.append(obj) 212 pos += 1 213 214 # Append any default values to the image. 215 # Only do this for named functions (not lambdas). 216 217 if obj.name is not None: 218 image += obj.default_attrs 219 pos += len(obj.default_attrs) 220 221 # Append the function code to the image. 222 223 obj.code_location = pos 224 code = trans.get_code(obj) 225 image += code 226 pos += len(code) 227 228 # Remember the position of the module code. 229 230 module.code_location = pos 231 232 # Append the module top-level code to the image. 233 234 code = trans.get_module_code(final=(module is last_module)) 235 image += code 236 pos += len(code) 237 238 # Remember the generated code and the location of the first instruction. 239 240 self.code = image 241 self.code_location = self.modules["__main__"].code_location 242 return image 243 244 def get_object_table(self): 245 246 "Return a table with details of attributes for classes and modules." 247 248 if self.objtable is None: 249 t = self.objtable = micropython.table.ObjectTable() 250 for module in self.get_modules(): 251 t.add(module.full_name(), module.module_attributes()) 252 253 # Add class and instance attributes for all classes, together 254 # with descendant information. 255 256 for obj in module.all_objects: 257 if isinstance(obj, micropython.inspect.Class): 258 attributes = {obj.full_name() : obj} 259 attributes.update(obj.all_attributes()) 260 attributes.update(obj.all_descendants()) 261 t.add(obj.full_name(), attributes) 262 263 return self.objtable 264 265 def get_parameter_table(self): 266 267 "Return a table with details of parameters for functions and methods." 268 269 # Need the object table to get at class details. 270 271 objtable = self.get_object_table() 272 273 if self.paramtable is None: 274 t = self.paramtable = micropython.table.ParameterTable() 275 276 # Visit each module, getting function and method details. 277 278 for module in self.get_modules(): 279 for obj in module.all_objects: 280 if isinstance(obj, micropython.inspect.Function): 281 t.add(obj.full_name(), obj.parameters()) 282 283 # Classes are callable, too. 284 # Take details of the appropriate __init__ method to make an 285 # entry for an instantiation function for the class. 286 287 elif isinstance(obj, micropython.inspect.Class): 288 t.add(obj.full_name(), obj.get_instantiator().parameters()) 289 290 # Filter out all parameter table entries not referenced by keyword 291 # arguments. 292 293 keyword_names = set() 294 295 for module in self.get_modules(): 296 keyword_names.update(module.keyword_names) 297 298 for function_name, parameters in t.table.items(): 299 for name in parameters.keys(): 300 if name in keyword_names: 301 break 302 else: 303 del t.table[function_name] 304 305 return self.paramtable 306 307 def get_graph(self, out=None, with_builtins=0): 308 out = out or sys.stdout 309 print >>out, 'digraph G {' 310 print >>out, ' ratio=auto;' 311 print >>out, ' center=true;' 312 print >>out, ' rankdir=LR;' 313 for module in self.modules_ordered: 314 if not with_builtins and module.name == "__builtins__": 315 continue 316 317 print >>out, ' subgraph "%s" {' % module.full_name() 318 print >>out, ' label="%s";' % module.full_name() 319 for obj in module.all_objects: 320 321 if isinstance(obj, micropython.inspect.Class): 322 print >>out, ' "%s" [' % obj.full_name(), 323 print >>out, 'shape=record,', 324 print >>out, 'label="{<%s> %s|{' % (obj.full_name(), obj.full_name()), 325 first = 1 326 for attr in obj.all_attributes().values(): 327 if not first: 328 print >>out, '|', 329 print >>out, '<%s> %s' % (attr.name, attr.name), 330 first = 0 331 print >>out, '}}"];' 332 333 elif isinstance(obj, micropython.inspect.Function): 334 print >>out, ' "%s" [' % obj.full_name(), 335 print >>out, 'shape=record,', 336 print >>out, 'label="{<%s> %s|{' % (obj.full_name(), obj.full_name()), 337 first = 1 338 for attr in obj.all_locals().values(): 339 if not first: 340 print >>out, '|', 341 print >>out, '<%s> %s' % (attr.name, attr.name), 342 first = 0 343 print >>out, '}}"];' 344 345 print >>out, ' }' 346 347 for module in self.modules_ordered: 348 if not with_builtins and module.name == "__builtins__": 349 continue 350 351 print >>out, ' {' 352 for obj in module.all_objects: 353 if isinstance(obj, micropython.inspect.Class): 354 for attr in obj.all_attributes().values(): 355 if attr.value is not None: 356 print >>out, ' "%s":%s -> "%s";' % (obj.full_name(), attr.name, attr.value.full_name()) 357 elif isinstance(obj, micropython.inspect.Function): 358 for attr in obj.all_locals().values(): 359 if attr.value is not None: 360 print >>out, ' "%s":%s -> "%s";' % (obj.full_name(), attr.name, attr.value.full_name()) 361 print >>out, ' }' 362 363 print >>out, "}" 364 365 def _get_graph_name(self, obj): 366 return obj.full_name().replace(".", "___") 367 368 # Import methods. 369 370 def find_in_path(self, name): 371 372 """ 373 Find the given module 'name' in the search path, returning None where no 374 such module could be found, or a 2-tuple from the 'find' method 375 otherwise. 376 """ 377 378 for d in self.path: 379 m = self.find(d, name) 380 if m: return m 381 return None 382 383 def find(self, d, name): 384 385 """ 386 In the directory 'd', find the given module 'name', where 'name' can 387 either refer to a single file module or to a package. Return None if the 388 'name' cannot be associated with either a file or a package directory, 389 or a 2-tuple from '_find_package' or '_find_module' otherwise. 390 """ 391 392 m = self._find_package(d, name) 393 if m: return m 394 m = self._find_module(d, name) 395 if m: return m 396 return None 397 398 def _find_module(self, d, name): 399 400 """ 401 In the directory 'd', find the given module 'name', returning None where 402 no suitable file exists in the directory, or a 2-tuple consisting of 403 None (indicating that no package directory is involved) and a filename 404 indicating the location of the module. 405 """ 406 407 name_py = name + os.extsep + "py" 408 filename = self._find_file(d, name_py) 409 if filename: 410 return None, filename 411 return None 412 413 def _find_package(self, d, name): 414 415 """ 416 In the directory 'd', find the given package 'name', returning None 417 where no suitable package directory exists, or a 2-tuple consisting of 418 a directory (indicating the location of the package directory itself) 419 and a filename indicating the location of the __init__.py module which 420 declares the package's top-level contents. 421 """ 422 423 filename = self._find_file(d, name) 424 if filename: 425 init_py = "__init__" + os.path.extsep + "py" 426 init_py_filename = self._find_file(filename, init_py) 427 if init_py_filename: 428 return filename, init_py_filename 429 return None 430 431 def _find_file(self, d, filename): 432 433 """ 434 Return the filename obtained when searching the directory 'd' for the 435 given 'filename', or None if no actual file exists for the filename. 436 """ 437 438 filename = os.path.join(d, filename) 439 if os.path.exists(filename): 440 return filename 441 else: 442 return None 443 444 def load(self, name, return_leaf=0): 445 446 """ 447 Load the module or package with the given 'name'. Return an object 448 referencing the loaded module or package, or None if no such module or 449 package exists. 450 """ 451 452 if self.modules.has_key(name) and self.modules[name].loaded: 453 #print "Cached (%s)" % name 454 return self.modules[name] 455 if self.verbose: 456 print "Loading", name 457 458 # Split the name into path components, and try to find the uppermost in 459 # the search path. 460 461 path = name.split(".") 462 m = self.find_in_path(path[0]) 463 if not m: 464 if self.verbose: 465 print "Not found (%s)" % path[0] 466 return None # NOTE: Import error. 467 d, filename = m 468 469 # Either acquire a reference to an already-imported module, or load the 470 # module from a file. 471 472 top = module = self.load_from_file(filename, path[0]) 473 474 # For hierarchical names, traverse each path component... 475 476 if len(path) > 1: 477 if not d: 478 if self.verbose: 479 print "No package (%s)" % filename 480 return None # NOTE: Import error (package not found). 481 else: 482 self.add_submodules(d, module) 483 484 path_so_far = path[:1] 485 for p in path[1:]: 486 path_so_far.append(p) 487 488 # Find the package or module concerned. 489 490 m = self.find(d, p) 491 if not m: 492 if self.verbose: 493 print "Not found (%s)" % p 494 return None # NOTE: Import error. 495 d, filename = m 496 module_name = ".".join(path_so_far) 497 498 # Either reference an imported module or load one from a file. 499 500 submodule = self.load_from_file(filename, module_name) 501 502 if d: 503 self.add_submodules(d, module) 504 505 # Store the submodule within its parent module. 506 507 module.set_module(p, submodule) 508 module = submodule 509 510 # Return either the deepest or the uppermost module. 511 512 if return_leaf: 513 return module 514 else: 515 return top 516 517 def load_from_file(self, name, module_name=None): 518 519 """ 520 Load the module with the given 'name' (which may be a full module path). 521 """ 522 523 if module_name is None: 524 module_name = "__main__" 525 526 module = self.add_module(module_name) 527 if not module.loaded and module not in self.loading: 528 self.loading.add(module) 529 #print "Parsing", name 530 module.parse(name) 531 #print "Done", name 532 self.loading.remove(module) 533 module.loaded = 1 534 535 # Record the module. 536 537 #print "Loaded", module_name, "with namespace", module.namespace.keys() 538 return module 539 540 def add_module(self, module_name): 541 542 """ 543 Return the module with the given 'module_name', adding a new module 544 object if one does not already exist. 545 """ 546 547 if not self.modules.has_key(module_name): 548 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 549 self.modules_ordered.append(module) 550 else: 551 module = self.modules[module_name] 552 return module 553 554 def add_submodules(self, pathname, module): 555 556 """ 557 Work around insufficient __all__ declarations and examine the directory 558 with the given 'pathname', adding submodules to the given 'module'. 559 """ 560 561 for filename in os.listdir(pathname): 562 submodule, ext = os.path.splitext(filename) 563 if ext not in ("", ".py"): 564 continue 565 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 566 567 # vim: tabstop=4 expandtab shiftwidth=4