1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 Such importer objects are the most convenient mechanism through which the 33 functionality of the micropython package may be accessed. 34 """ 35 36 from micropython.common import * 37 import micropython.ast 38 import micropython.inspect 39 import micropython.table 40 import os 41 try: 42 set 43 except NameError: 44 from sets import Set as set 45 46 class Importer: 47 48 """ 49 An import machine, searching for and loading modules. In addition, this 50 class supports the generation of a program image. 51 """ 52 53 supported_optimisations = micropython.ast.Translation.supported_optimisations 54 55 def __init__(self, path=None, verbose=0): 56 57 """ 58 Initialise the importer with the given search 'path' - a list of 59 directories to search for Python modules. 60 61 The optional 'verbose' parameter causes output concerning the activities 62 of the object to be produced if set to a true value (not the default). 63 """ 64 65 self.path = path or [os.getcwd()] 66 self.verbose = verbose 67 self.modules = {} 68 self.modules_ordered = [] 69 self.loading = set() 70 71 # Remember the tables once generated. 72 73 self.objtable = None 74 self.paramtable = None 75 self.clstable = None 76 77 # Constant records. 78 79 self.constant_values = {} 80 self.constant_list = None # cache for constants 81 82 # Main program information. 83 84 self.code = None 85 self.code_location = None 86 87 def constants(self): 88 89 "Return a list of constants." 90 91 if self.constant_list is None: 92 self.constant_list = list(self.constant_values.values()) 93 94 return self.constant_list 95 96 def vacuum(self): 97 98 "Tidy up the modules." 99 100 for name, module in self.modules.items(): 101 if module.loaded: 102 module.vacuum() 103 else: 104 del self.modules[name] 105 106 def finalise(self): 107 108 "Finalise the program." 109 110 for module in self.get_modules(): 111 112 # Fix the attributes. 113 114 module.finalise_attributes() 115 116 for obj in module.all_objects: 117 if isinstance(obj, (micropython.inspect.Class, micropython.inspect.Function)): 118 obj.finalise_attributes() 119 120 def get_modules(self): 121 122 "Return all modules known to the importer." 123 124 return self.modules.values() 125 126 def get_image(self, with_builtins=0, optimisations=None): 127 128 "Return a dictionary mapping modules to structures." 129 130 if self.code is not None: 131 return self.code 132 133 objtable = self.get_object_table() 134 paramtable = self.get_parameter_table() 135 self.finalise() 136 137 image = [] 138 139 # Append constants to the image. 140 141 for pos, const in enumerate(self.constants()): 142 const.location = pos 143 image.append(const) 144 145 for module in self.modules_ordered: 146 147 if not with_builtins and module.name == "__builtins__": 148 continue 149 150 pos = len(image) 151 152 # Position the module in the image and make a translation. 153 154 module.location = pos 155 trans = micropython.ast.Translation(module, self, optimisations) 156 157 # Add header details. 158 159 image.append(module) 160 pos += 1 161 162 # Append module attributes to the image. 163 164 attributes = module.module_attributes() 165 image += module.attributes_as_list() 166 pos += len(attributes.keys()) 167 168 # Append classes and functions to the image. 169 170 for obj in module.all_objects: 171 if isinstance(obj, micropython.inspect.Class): 172 173 # Position the class in the image. 174 175 obj.location = pos 176 177 # Add header details. 178 179 image.append(obj) 180 pos += 1 181 182 # Append class attributes to the image. 183 184 attributes = obj.class_attributes() 185 image += obj.attributes_as_list() 186 pos += len(attributes.keys()) 187 188 # Generate the instantiator/initialiser. 189 # Append the function code to the image. 190 191 instantiator = obj.get_instantiator() 192 instantiator.code_location = pos 193 code = trans.get_instantiator_code(obj) 194 image += code 195 pos += len(code) 196 197 # Class-level code is generated separately at the module 198 # level, and the code location is set within the code 199 # generation process for the module. 200 201 elif isinstance(obj, micropython.inspect.Function): 202 203 # Position the function in the image. 204 205 obj.location = pos 206 207 # Add header details. 208 209 image.append(obj) 210 pos += 1 211 212 # Append any default values to the image. 213 # Only do this for named functions (not lambdas). 214 215 if obj.name is not None: 216 image += obj.default_attrs 217 pos += len(obj.default_attrs) 218 219 # Append the function code to the image. 220 221 obj.code_location = pos 222 code = trans.get_code(obj) 223 image += code 224 pos += len(code) 225 226 # Remember the position of the module code. 227 228 module.code_location = pos 229 230 # Append the module top-level code to the image. 231 232 code = trans.get_module_code() 233 image += code 234 pos += len(code) 235 236 self.code = image 237 self.code_location = self.modules["__main__"].code_location 238 return image 239 240 def get_object_table(self): 241 242 "Return a table with details of attributes for classes and modules." 243 244 if self.objtable is None: 245 t = self.objtable = micropython.table.ObjectTable() 246 for module in self.get_modules(): 247 t.add(module.full_name(), module.module_attributes()) 248 249 # Add class and instance attributes for all classes, together 250 # with descendant information. 251 252 for obj in module.all_objects: 253 if isinstance(obj, micropython.inspect.Class): 254 attributes = {obj.full_name() : obj} 255 attributes.update(obj.all_attributes()) 256 attributes.update(obj.all_descendants()) 257 t.add(obj.full_name(), attributes) 258 259 return self.objtable 260 261 def get_parameter_table(self): 262 263 "Return a table with details of parameters for functions and methods." 264 265 # Need the object table to get at class details. 266 267 objtable = self.get_object_table() 268 269 if self.paramtable is None: 270 t = self.paramtable = micropython.table.ParameterTable() 271 272 # Visit each module, getting function and method details. 273 274 for module in self.get_modules(): 275 for obj in module.all_objects: 276 if isinstance(obj, micropython.inspect.Function): 277 t.add(obj.full_name(), obj.parameters()) 278 279 # Classes are callable, too. 280 # Take details of the appropriate __init__ method to make an 281 # entry for an instantiation function for the class. 282 283 elif isinstance(obj, micropython.inspect.Class): 284 t.add(obj.full_name(), obj.get_instantiator().parameters()) 285 286 # Filter out all parameter table entries not referenced by keyword 287 # arguments. 288 289 keyword_names = set() 290 291 for module in self.get_modules(): 292 keyword_names.update(module.keyword_names) 293 294 for function_name, parameters in t.table.items(): 295 for name in parameters.keys(): 296 if name in keyword_names: 297 break 298 else: 299 del t.table[function_name] 300 301 return self.paramtable 302 303 # Import methods. 304 305 def find_in_path(self, name): 306 307 """ 308 Find the given module 'name' in the search path, returning None where no 309 such module could be found, or a 2-tuple from the 'find' method 310 otherwise. 311 """ 312 313 for d in self.path: 314 m = self.find(d, name) 315 if m: return m 316 return None 317 318 def find(self, d, name): 319 320 """ 321 In the directory 'd', find the given module 'name', where 'name' can 322 either refer to a single file module or to a package. Return None if the 323 'name' cannot be associated with either a file or a package directory, 324 or a 2-tuple from '_find_package' or '_find_module' otherwise. 325 """ 326 327 m = self._find_package(d, name) 328 if m: return m 329 m = self._find_module(d, name) 330 if m: return m 331 return None 332 333 def _find_module(self, d, name): 334 335 """ 336 In the directory 'd', find the given module 'name', returning None where 337 no suitable file exists in the directory, or a 2-tuple consisting of 338 None (indicating that no package directory is involved) and a filename 339 indicating the location of the module. 340 """ 341 342 name_py = name + os.extsep + "py" 343 filename = self._find_file(d, name_py) 344 if filename: 345 return None, filename 346 return None 347 348 def _find_package(self, d, name): 349 350 """ 351 In the directory 'd', find the given package 'name', returning None 352 where no suitable package directory exists, or a 2-tuple consisting of 353 a directory (indicating the location of the package directory itself) 354 and a filename indicating the location of the __init__.py module which 355 declares the package's top-level contents. 356 """ 357 358 filename = self._find_file(d, name) 359 if filename: 360 init_py = "__init__" + os.path.extsep + "py" 361 init_py_filename = self._find_file(filename, init_py) 362 if init_py_filename: 363 return filename, init_py_filename 364 return None 365 366 def _find_file(self, d, filename): 367 368 """ 369 Return the filename obtained when searching the directory 'd' for the 370 given 'filename', or None if no actual file exists for the filename. 371 """ 372 373 filename = os.path.join(d, filename) 374 if os.path.exists(filename): 375 return filename 376 else: 377 return None 378 379 def load(self, name, return_leaf=0): 380 381 """ 382 Load the module or package with the given 'name'. Return an object 383 referencing the loaded module or package, or None if no such module or 384 package exists. 385 """ 386 387 if self.modules.has_key(name) and self.modules[name].loaded: 388 #print "Cached (%s)" % name 389 return self.modules[name] 390 if self.verbose: 391 print "Loading", name 392 393 # Split the name into path components, and try to find the uppermost in 394 # the search path. 395 396 path = name.split(".") 397 m = self.find_in_path(path[0]) 398 if not m: 399 if self.verbose: 400 print "Not found (%s)" % path[0] 401 return None # NOTE: Import error. 402 d, filename = m 403 404 # Either acquire a reference to an already-imported module, or load the 405 # module from a file. 406 407 top = module = self.load_from_file(filename, path[0]) 408 409 # For hierarchical names, traverse each path component... 410 411 if len(path) > 1: 412 if not d: 413 if self.verbose: 414 print "No package (%s)" % filename 415 return None # NOTE: Import error (package not found). 416 else: 417 self.add_submodules(d, module) 418 419 path_so_far = path[:1] 420 for p in path[1:]: 421 path_so_far.append(p) 422 423 # Find the package or module concerned. 424 425 m = self.find(d, p) 426 if not m: 427 if self.verbose: 428 print "Not found (%s)" % p 429 return None # NOTE: Import error. 430 d, filename = m 431 module_name = ".".join(path_so_far) 432 433 # Either reference an imported module or load one from a file. 434 435 submodule = self.load_from_file(filename, module_name) 436 437 if d: 438 self.add_submodules(d, module) 439 440 # Store the submodule within its parent module. 441 442 module.set_module(p, submodule) 443 module = submodule 444 445 # Return either the deepest or the uppermost module. 446 447 if return_leaf: 448 return module 449 else: 450 return top 451 452 def load_from_file(self, name, module_name=None): 453 454 """ 455 Load the module with the given 'name' (which may be a full module path). 456 """ 457 458 if module_name is None: 459 module_name = "__main__" 460 461 module = self.add_module(module_name) 462 if not module.loaded and module not in self.loading: 463 self.loading.add(module) 464 #print "Parsing", name 465 module.parse(name) 466 #print "Done", name 467 self.loading.remove(module) 468 module.loaded = 1 469 470 # Record the module. 471 472 #print "Loaded", module_name, "with namespace", module.namespace.keys() 473 return module 474 475 def add_module(self, module_name): 476 477 """ 478 Return the module with the given 'module_name', adding a new module 479 object if one does not already exist. 480 """ 481 482 if not self.modules.has_key(module_name): 483 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 484 self.modules_ordered.append(module) 485 else: 486 module = self.modules[module_name] 487 return module 488 489 def add_submodules(self, pathname, module): 490 491 """ 492 Work around insufficient __all__ declarations and examine the directory 493 with the given 'pathname', adding submodules to the given 'module'. 494 """ 495 496 for filename in os.listdir(pathname): 497 submodule, ext = os.path.splitext(filename) 498 if ext not in ("", ".py"): 499 continue 500 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 501 502 # vim: tabstop=4 expandtab shiftwidth=4