1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 Such importer objects are the most convenient mechanism through which the 33 functionality of the micropython package may be accessed. 34 """ 35 36 from micropython.common import * 37 import micropython.ast 38 import micropython.inspect 39 import micropython.table 40 import os 41 try: 42 set 43 except NameError: 44 from sets import Set as set 45 46 class Importer: 47 48 """ 49 An import machine, searching for and loading modules. In addition, this 50 class supports the generation of a program image. 51 """ 52 53 supported_optimisations = micropython.ast.Translation.supported_optimisations 54 55 def __init__(self, path=None, verbose=0): 56 57 """ 58 Initialise the importer with the given search 'path' - a list of 59 directories to search for Python modules. 60 61 The optional 'verbose' parameter causes output concerning the activities 62 of the object to be produced if set to a true value (not the default). 63 """ 64 65 self.path = path or [os.getcwd()] 66 self.verbose = verbose 67 self.modules = {} 68 self.loading = set() 69 70 # Remember the tables once generated. 71 72 self.objtable = None 73 self.paramtable = None 74 self.clstable = None 75 76 # Constant records. 77 78 self.constant_values = {} 79 self.constant_list = None # cache for constants 80 81 # Main program information. 82 83 self.code = None 84 self.code_location = None 85 86 def constants(self): 87 88 "Return a list of constants." 89 90 if self.constant_list is None: 91 self.constant_list = list(self.constant_values.values()) 92 93 return self.constant_list 94 95 def vacuum(self): 96 97 "Tidy up the modules." 98 99 for name, module in self.modules.items(): 100 if module.loaded: 101 module.vacuum() 102 else: 103 del self.modules[name] 104 105 def get_modules(self): 106 107 "Return all modules known to the importer." 108 109 return self.modules.values() 110 111 def get_image(self, with_builtins=0, optimisations=None): 112 113 "Return a dictionary mapping modules to structures." 114 115 objtable = self.get_object_table() 116 paramtable = self.get_parameter_table() 117 clstable = self.get_class_table() 118 119 image = [] 120 121 # Append constants to the image. 122 123 for pos, const in enumerate(self.constants()): 124 const.location = pos 125 image.append(const) 126 127 for module_name, module in self.modules.items(): 128 if not with_builtins and module_name == "__builtins__": 129 continue 130 131 # Fix the attributes. 132 133 module.finalise_attributes() 134 135 pos = len(image) 136 137 # Position the module in the image and make a translation. 138 139 module.location = pos 140 trans = micropython.ast.Translation(module, self, optimisations) 141 142 # Add header details. 143 144 image.append(module) 145 pos += 1 146 147 # Append module attributes to the image. 148 149 attributes = module.module_attributes() 150 image += module.attributes_as_list() 151 pos += len(attributes.keys()) 152 153 # Append classes and functions to the image. 154 155 for obj in module.all_objects: 156 if isinstance(obj, micropython.inspect.Class): 157 158 # Fix the attributes. 159 160 obj.finalise_attributes() 161 162 # Position the class in the image. 163 164 obj.location = pos 165 166 # Add header details. 167 168 image.append(obj) 169 pos += 1 170 171 # Append class attributes to the image. 172 173 attributes = obj.class_attributes() 174 image += obj.attributes_as_list() 175 pos += len(attributes.keys()) 176 177 # Class-level code is generated separately. 178 # The code location is set within the code generation 179 # process for the module. 180 181 # NOTE: Generate module and function code here. 182 183 elif isinstance(obj, micropython.inspect.Function): 184 185 # Fix the attributes. 186 187 obj.finalise_attributes() 188 189 # Position the function in the image. 190 191 obj.location = pos 192 193 # Add header details. 194 195 image.append(obj) 196 pos += 1 197 198 # Append any default values to the image. 199 # Only do this for named functions (not lambdas). 200 201 if obj.name is not None: 202 image += obj.default_attrs 203 pos += len(obj.default_attrs) 204 205 # Append the function code to the image. 206 207 obj.code_location = pos 208 code = trans.get_code(obj) 209 image += code 210 pos += len(code) 211 212 # Remember the position of the module code. 213 214 module.code_location = pos 215 216 # Append the module top-level code to the image. 217 218 code = trans.get_module_code() 219 image += code 220 pos += len(code) 221 222 self.code = image 223 self.code_location = self.modules["__main__"].code_location 224 return image 225 226 def get_object_table(self): 227 228 "Return a table with details of attributes for classes and modules." 229 230 if self.objtable is None: 231 t = self.objtable = micropython.table.Table() 232 for module in self.get_modules(): 233 t.add(module.full_name(), module.module_attributes()) 234 for obj in module.all_objects: 235 if isinstance(obj, micropython.inspect.Class): 236 t.add(obj.full_name(), obj.all_attributes()) 237 238 return self.objtable 239 240 def get_class_table(self): 241 242 "Return a table with details of class compatibility." 243 244 if self.clstable is None: 245 t = self.clstable = micropython.table.Table() 246 for module in self.get_modules(): 247 for obj in module.all_objects: 248 if isinstance(obj, micropython.inspect.Class): 249 compatible = [obj] + list(obj.descendants) 250 compatible_dict = dict([(cls.full_name(), cls) for cls in compatible]) 251 t.add(obj.full_name(), compatible_dict) 252 253 return self.clstable 254 255 def get_parameter_table(self): 256 257 "Return a table with details of parameters for functions and methods." 258 259 # Need the object table to get at class details. 260 261 objtable = self.get_object_table() 262 263 if self.paramtable is None: 264 t = self.paramtable = micropython.table.Table() 265 266 # Visit each module, getting function and method details. 267 268 for module in self.get_modules(): 269 for obj in module.all_objects: 270 if isinstance(obj, micropython.inspect.Function): 271 t.add(obj.full_name(), obj.parameters()) 272 273 # Classes are callable, too. 274 # Take details of the appropriate __init__ method to make an 275 # entry for an instantiation function for the class. 276 277 elif isinstance(obj, micropython.inspect.Class): 278 t.add(obj.full_name(), obj.get_instantiator().parameters()) 279 280 # Filter out all parameter table entries not referenced by keyword 281 # arguments. 282 283 keyword_names = set() 284 285 for module in self.get_modules(): 286 keyword_names.update(module.keyword_names) 287 288 for function_name, parameters in t.table.items(): 289 for name in parameters.keys(): 290 if name in keyword_names: 291 break 292 else: 293 del t.table[function_name] 294 295 return self.paramtable 296 297 # Import methods. 298 299 def find_in_path(self, name): 300 301 """ 302 Find the given module 'name' in the search path, returning None where no 303 such module could be found, or a 2-tuple from the 'find' method 304 otherwise. 305 """ 306 307 for d in self.path: 308 m = self.find(d, name) 309 if m: return m 310 return None 311 312 def find(self, d, name): 313 314 """ 315 In the directory 'd', find the given module 'name', where 'name' can 316 either refer to a single file module or to a package. Return None if the 317 'name' cannot be associated with either a file or a package directory, 318 or a 2-tuple from '_find_package' or '_find_module' otherwise. 319 """ 320 321 m = self._find_package(d, name) 322 if m: return m 323 m = self._find_module(d, name) 324 if m: return m 325 return None 326 327 def _find_module(self, d, name): 328 329 """ 330 In the directory 'd', find the given module 'name', returning None where 331 no suitable file exists in the directory, or a 2-tuple consisting of 332 None (indicating that no package directory is involved) and a filename 333 indicating the location of the module. 334 """ 335 336 name_py = name + os.extsep + "py" 337 filename = self._find_file(d, name_py) 338 if filename: 339 return None, filename 340 return None 341 342 def _find_package(self, d, name): 343 344 """ 345 In the directory 'd', find the given package 'name', returning None 346 where no suitable package directory exists, or a 2-tuple consisting of 347 a directory (indicating the location of the package directory itself) 348 and a filename indicating the location of the __init__.py module which 349 declares the package's top-level contents. 350 """ 351 352 filename = self._find_file(d, name) 353 if filename: 354 init_py = "__init__" + os.path.extsep + "py" 355 init_py_filename = self._find_file(filename, init_py) 356 if init_py_filename: 357 return filename, init_py_filename 358 return None 359 360 def _find_file(self, d, filename): 361 362 """ 363 Return the filename obtained when searching the directory 'd' for the 364 given 'filename', or None if no actual file exists for the filename. 365 """ 366 367 filename = os.path.join(d, filename) 368 if os.path.exists(filename): 369 return filename 370 else: 371 return None 372 373 def load(self, name, return_leaf=0): 374 375 """ 376 Load the module or package with the given 'name'. Return an object 377 referencing the loaded module or package, or None if no such module or 378 package exists. 379 """ 380 381 if self.modules.has_key(name) and self.modules[name].loaded: 382 #print "Cached (%s)" % name 383 return self.modules[name] 384 if self.verbose: 385 print "Loading", name 386 387 # Split the name into path components, and try to find the uppermost in 388 # the search path. 389 390 path = name.split(".") 391 m = self.find_in_path(path[0]) 392 if not m: 393 if self.verbose: 394 print "Not found (%s)" % path[0] 395 return None # NOTE: Import error. 396 d, filename = m 397 398 # Either acquire a reference to an already-imported module, or load the 399 # module from a file. 400 401 top = module = self.load_from_file(filename, path[0]) 402 403 # For hierarchical names, traverse each path component... 404 405 if len(path) > 1: 406 if not d: 407 if self.verbose: 408 print "No package (%s)" % filename 409 return None # NOTE: Import error (package not found). 410 else: 411 self.add_submodules(d, module) 412 413 path_so_far = path[:1] 414 for p in path[1:]: 415 path_so_far.append(p) 416 417 # Find the package or module concerned. 418 419 m = self.find(d, p) 420 if not m: 421 if self.verbose: 422 print "Not found (%s)" % p 423 return None # NOTE: Import error. 424 d, filename = m 425 module_name = ".".join(path_so_far) 426 427 # Either reference an imported module or load one from a file. 428 429 submodule = self.load_from_file(filename, module_name) 430 431 if d: 432 self.add_submodules(d, module) 433 434 # Store the submodule within its parent module. 435 436 module.set_module(p, submodule) 437 module = submodule 438 439 # Return either the deepest or the uppermost module. 440 441 if return_leaf: 442 return module 443 else: 444 return top 445 446 def load_from_file(self, name, module_name=None): 447 448 """ 449 Load the module with the given 'name' (which may be a full module path). 450 """ 451 452 if module_name is None: 453 module_name = "__main__" 454 455 module = self.add_module(module_name) 456 if not module.loaded and module not in self.loading: 457 self.loading.add(module) 458 #print "Parsing", name 459 module.parse(name) 460 #print "Done", name 461 self.loading.remove(module) 462 module.loaded = 1 463 464 # Record the module. 465 466 #print "Loaded", module_name, "with namespace", module.namespace.keys() 467 return module 468 469 def add_module(self, module_name): 470 471 """ 472 Return the module with the given 'module_name', adding a new module 473 object if one does not already exist. 474 """ 475 476 if not self.modules.has_key(module_name): 477 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 478 else: 479 module = self.modules[module_name] 480 return module 481 482 def add_submodules(self, pathname, module): 483 484 """ 485 Work around insufficient __all__ declarations and examine the directory 486 with the given 'pathname', adding submodules to the given 'module'. 487 """ 488 489 for filename in os.listdir(pathname): 490 submodule, ext = os.path.splitext(filename) 491 if ext not in ("", ".py"): 492 continue 493 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 494 495 # vim: tabstop=4 expandtab shiftwidth=4