1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 Such importer objects are the most convenient mechanism through which the 33 functionality of the micropython package may be accessed. 34 """ 35 36 from micropython.common import * 37 import micropython.ast 38 import micropython.inspect 39 import micropython.table 40 import os 41 try: 42 set 43 except NameError: 44 from sets import Set as set 45 46 class Importer: 47 48 """ 49 An import machine, searching for and loading modules. In addition, this 50 class supports the generation of a program image. 51 """ 52 53 supported_optimisations = micropython.ast.Translation.supported_optimisations 54 55 def __init__(self, path=None, verbose=0): 56 57 """ 58 Initialise the importer with the given search 'path' - a list of 59 directories to search for Python modules. 60 61 The optional 'verbose' parameter causes output concerning the activities 62 of the object to be produced if set to a true value (not the default). 63 """ 64 65 self.path = path or [os.getcwd()] 66 self.verbose = verbose 67 self.modules = {} 68 self.modules_ordered = [] 69 self.loading = set() 70 71 # Remember the tables once generated. 72 73 self.objtable = None 74 self.paramtable = None 75 self.clstable = None 76 77 # Constant records. 78 79 self.constant_values = {} 80 self.constant_list = None # cache for constants 81 82 # Main program information. 83 84 self.code = None 85 self.code_location = None 86 87 def constants(self): 88 89 "Return a list of constants." 90 91 if self.constant_list is None: 92 self.constant_list = list(self.constant_values.values()) 93 94 return self.constant_list 95 96 def vacuum(self): 97 98 "Tidy up the modules." 99 100 for name, module in self.modules.items(): 101 if module.loaded: 102 module.vacuum() 103 else: 104 del self.modules[name] 105 106 def get_modules(self): 107 108 "Return all modules known to the importer." 109 110 return self.modules.values() 111 112 def get_image(self, with_builtins=0, optimisations=None): 113 114 "Return a dictionary mapping modules to structures." 115 116 objtable = self.get_object_table() 117 paramtable = self.get_parameter_table() 118 clstable = self.get_class_table() 119 120 image = [] 121 122 # Append constants to the image. 123 124 for pos, const in enumerate(self.constants()): 125 const.location = pos 126 image.append(const) 127 128 for module in self.modules_ordered: 129 130 if not with_builtins and module.name == "__builtins__": 131 continue 132 133 # Fix the attributes. 134 135 module.finalise_attributes() 136 137 pos = len(image) 138 139 # Position the module in the image and make a translation. 140 141 module.location = pos 142 trans = micropython.ast.Translation(module, self, optimisations) 143 144 # Add header details. 145 146 image.append(module) 147 pos += 1 148 149 # Append module attributes to the image. 150 151 attributes = module.module_attributes() 152 image += module.attributes_as_list() 153 pos += len(attributes.keys()) 154 155 # Append classes and functions to the image. 156 157 for obj in module.all_objects: 158 if isinstance(obj, micropython.inspect.Class): 159 160 # Fix the attributes. 161 162 obj.finalise_attributes() 163 164 # Position the class in the image. 165 166 obj.location = pos 167 168 # Add header details. 169 170 image.append(obj) 171 pos += 1 172 173 # Append class attributes to the image. 174 175 attributes = obj.class_attributes() 176 image += obj.attributes_as_list() 177 pos += len(attributes.keys()) 178 179 # Class-level code is generated separately. 180 # The code location is set within the code generation 181 # process for the module. 182 183 # NOTE: Generate module and function code here. 184 185 elif isinstance(obj, micropython.inspect.Function): 186 187 # Fix the attributes. 188 189 obj.finalise_attributes() 190 191 # Position the function in the image. 192 193 obj.location = pos 194 195 # Add header details. 196 197 image.append(obj) 198 pos += 1 199 200 # Append any default values to the image. 201 # Only do this for named functions (not lambdas). 202 203 if obj.name is not None: 204 image += obj.default_attrs 205 pos += len(obj.default_attrs) 206 207 # Append the function code to the image. 208 209 obj.code_location = pos 210 code = trans.get_code(obj) 211 image += code 212 pos += len(code) 213 214 # Remember the position of the module code. 215 216 module.code_location = pos 217 218 # Append the module top-level code to the image. 219 220 code = trans.get_module_code() 221 image += code 222 pos += len(code) 223 224 self.code = image 225 self.code_location = self.modules["__main__"].code_location 226 return image 227 228 def get_object_table(self): 229 230 "Return a table with details of attributes for classes and modules." 231 232 if self.objtable is None: 233 t = self.objtable = micropython.table.Table() 234 for module in self.get_modules(): 235 t.add(module.full_name(), module.module_attributes()) 236 for obj in module.all_objects: 237 if isinstance(obj, micropython.inspect.Class): 238 t.add(obj.full_name(), obj.all_attributes()) 239 240 return self.objtable 241 242 def get_class_table(self): 243 244 "Return a table with details of class compatibility." 245 246 if self.clstable is None: 247 t = self.clstable = micropython.table.Table() 248 for module in self.get_modules(): 249 for obj in module.all_objects: 250 if isinstance(obj, micropython.inspect.Class): 251 compatible = [obj] + list(obj.descendants) 252 compatible_dict = dict([(cls.full_name(), cls) for cls in compatible]) 253 t.add(obj.full_name(), compatible_dict) 254 255 return self.clstable 256 257 def get_parameter_table(self): 258 259 "Return a table with details of parameters for functions and methods." 260 261 # Need the object table to get at class details. 262 263 objtable = self.get_object_table() 264 265 if self.paramtable is None: 266 t = self.paramtable = micropython.table.Table() 267 268 # Visit each module, getting function and method details. 269 270 for module in self.get_modules(): 271 for obj in module.all_objects: 272 if isinstance(obj, micropython.inspect.Function): 273 t.add(obj.full_name(), obj.parameters()) 274 275 # Classes are callable, too. 276 # Take details of the appropriate __init__ method to make an 277 # entry for an instantiation function for the class. 278 279 elif isinstance(obj, micropython.inspect.Class): 280 t.add(obj.full_name(), obj.get_instantiator().parameters()) 281 282 # Filter out all parameter table entries not referenced by keyword 283 # arguments. 284 285 keyword_names = set() 286 287 for module in self.get_modules(): 288 keyword_names.update(module.keyword_names) 289 290 for function_name, parameters in t.table.items(): 291 for name in parameters.keys(): 292 if name in keyword_names: 293 break 294 else: 295 del t.table[function_name] 296 297 return self.paramtable 298 299 # Import methods. 300 301 def find_in_path(self, name): 302 303 """ 304 Find the given module 'name' in the search path, returning None where no 305 such module could be found, or a 2-tuple from the 'find' method 306 otherwise. 307 """ 308 309 for d in self.path: 310 m = self.find(d, name) 311 if m: return m 312 return None 313 314 def find(self, d, name): 315 316 """ 317 In the directory 'd', find the given module 'name', where 'name' can 318 either refer to a single file module or to a package. Return None if the 319 'name' cannot be associated with either a file or a package directory, 320 or a 2-tuple from '_find_package' or '_find_module' otherwise. 321 """ 322 323 m = self._find_package(d, name) 324 if m: return m 325 m = self._find_module(d, name) 326 if m: return m 327 return None 328 329 def _find_module(self, d, name): 330 331 """ 332 In the directory 'd', find the given module 'name', returning None where 333 no suitable file exists in the directory, or a 2-tuple consisting of 334 None (indicating that no package directory is involved) and a filename 335 indicating the location of the module. 336 """ 337 338 name_py = name + os.extsep + "py" 339 filename = self._find_file(d, name_py) 340 if filename: 341 return None, filename 342 return None 343 344 def _find_package(self, d, name): 345 346 """ 347 In the directory 'd', find the given package 'name', returning None 348 where no suitable package directory exists, or a 2-tuple consisting of 349 a directory (indicating the location of the package directory itself) 350 and a filename indicating the location of the __init__.py module which 351 declares the package's top-level contents. 352 """ 353 354 filename = self._find_file(d, name) 355 if filename: 356 init_py = "__init__" + os.path.extsep + "py" 357 init_py_filename = self._find_file(filename, init_py) 358 if init_py_filename: 359 return filename, init_py_filename 360 return None 361 362 def _find_file(self, d, filename): 363 364 """ 365 Return the filename obtained when searching the directory 'd' for the 366 given 'filename', or None if no actual file exists for the filename. 367 """ 368 369 filename = os.path.join(d, filename) 370 if os.path.exists(filename): 371 return filename 372 else: 373 return None 374 375 def load(self, name, return_leaf=0): 376 377 """ 378 Load the module or package with the given 'name'. Return an object 379 referencing the loaded module or package, or None if no such module or 380 package exists. 381 """ 382 383 if self.modules.has_key(name) and self.modules[name].loaded: 384 #print "Cached (%s)" % name 385 return self.modules[name] 386 if self.verbose: 387 print "Loading", name 388 389 # Split the name into path components, and try to find the uppermost in 390 # the search path. 391 392 path = name.split(".") 393 m = self.find_in_path(path[0]) 394 if not m: 395 if self.verbose: 396 print "Not found (%s)" % path[0] 397 return None # NOTE: Import error. 398 d, filename = m 399 400 # Either acquire a reference to an already-imported module, or load the 401 # module from a file. 402 403 top = module = self.load_from_file(filename, path[0]) 404 405 # For hierarchical names, traverse each path component... 406 407 if len(path) > 1: 408 if not d: 409 if self.verbose: 410 print "No package (%s)" % filename 411 return None # NOTE: Import error (package not found). 412 else: 413 self.add_submodules(d, module) 414 415 path_so_far = path[:1] 416 for p in path[1:]: 417 path_so_far.append(p) 418 419 # Find the package or module concerned. 420 421 m = self.find(d, p) 422 if not m: 423 if self.verbose: 424 print "Not found (%s)" % p 425 return None # NOTE: Import error. 426 d, filename = m 427 module_name = ".".join(path_so_far) 428 429 # Either reference an imported module or load one from a file. 430 431 submodule = self.load_from_file(filename, module_name) 432 433 if d: 434 self.add_submodules(d, module) 435 436 # Store the submodule within its parent module. 437 438 module.set_module(p, submodule) 439 module = submodule 440 441 # Return either the deepest or the uppermost module. 442 443 if return_leaf: 444 return module 445 else: 446 return top 447 448 def load_from_file(self, name, module_name=None): 449 450 """ 451 Load the module with the given 'name' (which may be a full module path). 452 """ 453 454 if module_name is None: 455 module_name = "__main__" 456 457 module = self.add_module(module_name) 458 if not module.loaded and module not in self.loading: 459 self.loading.add(module) 460 #print "Parsing", name 461 module.parse(name) 462 #print "Done", name 463 self.loading.remove(module) 464 module.loaded = 1 465 466 # Record the module. 467 468 #print "Loaded", module_name, "with namespace", module.namespace.keys() 469 return module 470 471 def add_module(self, module_name): 472 473 """ 474 Return the module with the given 'module_name', adding a new module 475 object if one does not already exist. 476 """ 477 478 if not self.modules.has_key(module_name): 479 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 480 self.modules_ordered.append(module) 481 else: 482 module = self.modules[module_name] 483 return module 484 485 def add_submodules(self, pathname, module): 486 487 """ 488 Work around insufficient __all__ declarations and examine the directory 489 with the given 'pathname', adding submodules to the given 'module'. 490 """ 491 492 for filename in os.listdir(pathname): 493 submodule, ext = os.path.splitext(filename) 494 if ext not in ("", ".py"): 495 continue 496 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 497 498 # vim: tabstop=4 expandtab shiftwidth=4