1 #!/usr/bin/env python 2 3 """ 4 The micropython package for processing Python source code. The code originates 5 from the simplify package but has had various details related to that package 6 removed. 7 8 Copyright (C) 2006, 2007, 2008 Paul Boddie <paul@boddie.org.uk> 9 10 This program is free software; you can redistribute it and/or modify it under 11 the terms of the GNU General Public License as published by the Free Software 12 Foundation; either version 3 of the License, or (at your option) any later 13 version. 14 15 This program is distributed in the hope that it will be useful, but WITHOUT 16 ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 17 FOR A PARTICULAR PURPOSE. See the GNU General Public License for more 18 details. 19 20 You should have received a copy of the GNU General Public License along with 21 this program. If not, see <http://www.gnu.org/licenses/>. 22 23 -------- 24 25 To use this module, an importer should be constructed and the load_from_file 26 method used. Here, the standard path for module searching is employed: 27 28 importer = Importer(sys.path) 29 importer.load_from_file(filename) 30 importer.vacuum() 31 32 Such importer objects are the most convenient mechanism through which the 33 functionality of the micropython package may be accessed. 34 """ 35 36 from micropython.common import * 37 import micropython.ast 38 import micropython.inspect 39 import micropython.table 40 import os 41 try: 42 set 43 except NameError: 44 from sets import Set as set 45 46 class Importer: 47 48 """ 49 An import machine, searching for and loading modules. In addition, this 50 class supports the generation of a program image. 51 """ 52 53 supported_optimisations = micropython.ast.Translation.supported_optimisations 54 55 def __init__(self, path=None, verbose=0): 56 57 """ 58 Initialise the importer with the given search 'path' - a list of 59 directories to search for Python modules. 60 61 The optional 'verbose' parameter causes output concerning the activities 62 of the object to be produced if set to a true value (not the default). 63 """ 64 65 self.path = path or [os.getcwd()] 66 self.verbose = verbose 67 self.modules = {} 68 self.modules_ordered = [] 69 self.loading = set() 70 71 # Remember the tables once generated. 72 73 self.objtable = None 74 self.paramtable = None 75 self.clstable = None 76 77 # Constant records. 78 79 self.constant_values = {} 80 self.constant_list = None # cache for constants 81 82 # Main program information. 83 84 self.code = None 85 self.code_location = None 86 87 def constants(self): 88 89 "Return a list of constants." 90 91 if self.constant_list is None: 92 self.constant_list = list(self.constant_values.values()) 93 94 return self.constant_list 95 96 def vacuum(self): 97 98 "Tidy up the modules." 99 100 for name, module in self.modules.items(): 101 if module.loaded: 102 module.vacuum() 103 else: 104 del self.modules[name] 105 106 def get_modules(self): 107 108 "Return all modules known to the importer." 109 110 return self.modules.values() 111 112 def get_image(self, with_builtins=0, optimisations=None): 113 114 "Return a dictionary mapping modules to structures." 115 116 objtable = self.get_object_table() 117 paramtable = self.get_parameter_table() 118 119 image = [] 120 121 # Append constants to the image. 122 123 for pos, const in enumerate(self.constants()): 124 const.location = pos 125 image.append(const) 126 127 for module in self.modules_ordered: 128 129 if not with_builtins and module.name == "__builtins__": 130 continue 131 132 # Fix the attributes. 133 134 module.finalise_attributes() 135 136 pos = len(image) 137 138 # Position the module in the image and make a translation. 139 140 module.location = pos 141 trans = micropython.ast.Translation(module, self, optimisations) 142 143 # Add header details. 144 145 image.append(module) 146 pos += 1 147 148 # Append module attributes to the image. 149 150 attributes = module.module_attributes() 151 image += module.attributes_as_list() 152 pos += len(attributes.keys()) 153 154 # Append classes and functions to the image. 155 156 for obj in module.all_objects: 157 if isinstance(obj, micropython.inspect.Class): 158 159 # Fix the attributes. 160 161 obj.finalise_attributes() 162 163 # Position the class in the image. 164 165 obj.location = pos 166 167 # Add header details. 168 169 image.append(obj) 170 pos += 1 171 172 # Append class attributes to the image. 173 174 attributes = obj.class_attributes() 175 image += obj.attributes_as_list() 176 pos += len(attributes.keys()) 177 178 # Class-level code is generated separately. 179 # The code location is set within the code generation 180 # process for the module. 181 182 # NOTE: Generate module and function code here. 183 184 elif isinstance(obj, micropython.inspect.Function): 185 186 # Fix the attributes. 187 188 obj.finalise_attributes() 189 190 # Position the function in the image. 191 192 obj.location = pos 193 194 # Add header details. 195 196 image.append(obj) 197 pos += 1 198 199 # Append any default values to the image. 200 # Only do this for named functions (not lambdas). 201 202 if obj.name is not None: 203 image += obj.default_attrs 204 pos += len(obj.default_attrs) 205 206 # Append the function code to the image. 207 208 obj.code_location = pos 209 code = trans.get_code(obj) 210 image += code 211 pos += len(code) 212 213 # Remember the position of the module code. 214 215 module.code_location = pos 216 217 # Append the module top-level code to the image. 218 219 code = trans.get_module_code() 220 image += code 221 pos += len(code) 222 223 self.code = image 224 self.code_location = self.modules["__main__"].code_location 225 return image 226 227 def get_object_table(self): 228 229 "Return a table with details of attributes for classes and modules." 230 231 if self.objtable is None: 232 t = self.objtable = micropython.table.ObjectTable() 233 for module in self.get_modules(): 234 t.add(module.full_name(), module.module_attributes()) 235 for obj in module.all_objects: 236 if isinstance(obj, micropython.inspect.Class): 237 t.add(obj.full_name(), obj.all_attributes()) 238 239 return self.objtable 240 241 def get_parameter_table(self): 242 243 "Return a table with details of parameters for functions and methods." 244 245 # Need the object table to get at class details. 246 247 objtable = self.get_object_table() 248 249 if self.paramtable is None: 250 t = self.paramtable = micropython.table.ParameterTable() 251 252 # Visit each module, getting function and method details. 253 254 for module in self.get_modules(): 255 for obj in module.all_objects: 256 if isinstance(obj, micropython.inspect.Function): 257 t.add(obj.full_name(), obj.parameters()) 258 259 # Classes are callable, too. 260 # Take details of the appropriate __init__ method to make an 261 # entry for an instantiation function for the class. 262 263 elif isinstance(obj, micropython.inspect.Class): 264 t.add(obj.full_name(), obj.get_instantiator().parameters()) 265 266 # Filter out all parameter table entries not referenced by keyword 267 # arguments. 268 269 keyword_names = set() 270 271 for module in self.get_modules(): 272 keyword_names.update(module.keyword_names) 273 274 for function_name, parameters in t.table.items(): 275 for name in parameters.keys(): 276 if name in keyword_names: 277 break 278 else: 279 del t.table[function_name] 280 281 return self.paramtable 282 283 # Import methods. 284 285 def find_in_path(self, name): 286 287 """ 288 Find the given module 'name' in the search path, returning None where no 289 such module could be found, or a 2-tuple from the 'find' method 290 otherwise. 291 """ 292 293 for d in self.path: 294 m = self.find(d, name) 295 if m: return m 296 return None 297 298 def find(self, d, name): 299 300 """ 301 In the directory 'd', find the given module 'name', where 'name' can 302 either refer to a single file module or to a package. Return None if the 303 'name' cannot be associated with either a file or a package directory, 304 or a 2-tuple from '_find_package' or '_find_module' otherwise. 305 """ 306 307 m = self._find_package(d, name) 308 if m: return m 309 m = self._find_module(d, name) 310 if m: return m 311 return None 312 313 def _find_module(self, d, name): 314 315 """ 316 In the directory 'd', find the given module 'name', returning None where 317 no suitable file exists in the directory, or a 2-tuple consisting of 318 None (indicating that no package directory is involved) and a filename 319 indicating the location of the module. 320 """ 321 322 name_py = name + os.extsep + "py" 323 filename = self._find_file(d, name_py) 324 if filename: 325 return None, filename 326 return None 327 328 def _find_package(self, d, name): 329 330 """ 331 In the directory 'd', find the given package 'name', returning None 332 where no suitable package directory exists, or a 2-tuple consisting of 333 a directory (indicating the location of the package directory itself) 334 and a filename indicating the location of the __init__.py module which 335 declares the package's top-level contents. 336 """ 337 338 filename = self._find_file(d, name) 339 if filename: 340 init_py = "__init__" + os.path.extsep + "py" 341 init_py_filename = self._find_file(filename, init_py) 342 if init_py_filename: 343 return filename, init_py_filename 344 return None 345 346 def _find_file(self, d, filename): 347 348 """ 349 Return the filename obtained when searching the directory 'd' for the 350 given 'filename', or None if no actual file exists for the filename. 351 """ 352 353 filename = os.path.join(d, filename) 354 if os.path.exists(filename): 355 return filename 356 else: 357 return None 358 359 def load(self, name, return_leaf=0): 360 361 """ 362 Load the module or package with the given 'name'. Return an object 363 referencing the loaded module or package, or None if no such module or 364 package exists. 365 """ 366 367 if self.modules.has_key(name) and self.modules[name].loaded: 368 #print "Cached (%s)" % name 369 return self.modules[name] 370 if self.verbose: 371 print "Loading", name 372 373 # Split the name into path components, and try to find the uppermost in 374 # the search path. 375 376 path = name.split(".") 377 m = self.find_in_path(path[0]) 378 if not m: 379 if self.verbose: 380 print "Not found (%s)" % path[0] 381 return None # NOTE: Import error. 382 d, filename = m 383 384 # Either acquire a reference to an already-imported module, or load the 385 # module from a file. 386 387 top = module = self.load_from_file(filename, path[0]) 388 389 # For hierarchical names, traverse each path component... 390 391 if len(path) > 1: 392 if not d: 393 if self.verbose: 394 print "No package (%s)" % filename 395 return None # NOTE: Import error (package not found). 396 else: 397 self.add_submodules(d, module) 398 399 path_so_far = path[:1] 400 for p in path[1:]: 401 path_so_far.append(p) 402 403 # Find the package or module concerned. 404 405 m = self.find(d, p) 406 if not m: 407 if self.verbose: 408 print "Not found (%s)" % p 409 return None # NOTE: Import error. 410 d, filename = m 411 module_name = ".".join(path_so_far) 412 413 # Either reference an imported module or load one from a file. 414 415 submodule = self.load_from_file(filename, module_name) 416 417 if d: 418 self.add_submodules(d, module) 419 420 # Store the submodule within its parent module. 421 422 module.set_module(p, submodule) 423 module = submodule 424 425 # Return either the deepest or the uppermost module. 426 427 if return_leaf: 428 return module 429 else: 430 return top 431 432 def load_from_file(self, name, module_name=None): 433 434 """ 435 Load the module with the given 'name' (which may be a full module path). 436 """ 437 438 if module_name is None: 439 module_name = "__main__" 440 441 module = self.add_module(module_name) 442 if not module.loaded and module not in self.loading: 443 self.loading.add(module) 444 #print "Parsing", name 445 module.parse(name) 446 #print "Done", name 447 self.loading.remove(module) 448 module.loaded = 1 449 450 # Record the module. 451 452 #print "Loaded", module_name, "with namespace", module.namespace.keys() 453 return module 454 455 def add_module(self, module_name): 456 457 """ 458 Return the module with the given 'module_name', adding a new module 459 object if one does not already exist. 460 """ 461 462 if not self.modules.has_key(module_name): 463 self.modules[module_name] = module = micropython.inspect.InspectedModule(module_name, self) 464 self.modules_ordered.append(module) 465 else: 466 module = self.modules[module_name] 467 return module 468 469 def add_submodules(self, pathname, module): 470 471 """ 472 Work around insufficient __all__ declarations and examine the directory 473 with the given 'pathname', adding submodules to the given 'module'. 474 """ 475 476 for filename in os.listdir(pathname): 477 submodule, ext = os.path.splitext(filename) 478 if ext not in ("", ".py"): 479 continue 480 module.set_module(submodule, self.add_module(module.name + "." + submodule)) 481 482 # vim: tabstop=4 expandtab shiftwidth=4