# HG changeset patch
# User Paul Boddie <paul@boddie.org.uk>
# Date 1281139575 -7200
# Node ID 34ebed37aecc2779b7c0a06c0f2523f9cc242d9d
# Parent  67d7ed58ac5003cebb68bf267afac2ff423d9dd5
Added some support for HTML reports of program modules, separating the program
finalisation out into a separate method of the Program class, in order to avoid
obtaining an actual program image, yet removing unneeded parts of a program.
Moved code to test for the usage of program sections into the common module,
since this is also useful when generating reports.
Added AST node annotation documentation.

diff -r 67d7ed58ac50 -r 34ebed37aecc docs/annotations.txt
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/docs/annotations.txt	Sat Aug 07 02:06:15 2010 +0200
@@ -0,0 +1,9 @@
+AST Node Annotations
+====================
+
+_attrnames      defines a dictionary mapping local names to sets of attribute
+                names found to be used with those names
+_attrusers      defines a dictionary mapping local names to sets of nodes
+                defining those names
+_def            refers to a micropython Class or Function instance
+_scope          set as "constant", "local", "global" or "builtins"
diff -r 67d7ed58ac50 -r 34ebed37aecc micropython/__init__.py
--- a/micropython/__init__.py	Sun Jul 11 02:05:41 2010 +0200
+++ b/micropython/__init__.py	Sat Aug 07 02:06:15 2010 +0200
@@ -84,6 +84,23 @@
 
     # Access to finalised program information.
 
+    def finalise(self):
+
+        "Finalise the program."
+
+        # Need the tables to finalise.
+
+        objtable = self.get_object_table()
+        self.get_parameter_table()
+
+        self.importer.vacuum(objtable)
+        self.importer.finalise()
+
+        # Now remove unneeded things from the tables.
+
+        self.get_object_table(reset=1)
+        self.get_parameter_table(reset=1)
+
     def get_image(self, with_builtins=0):
 
         """
@@ -94,17 +111,9 @@
         if self.code is not None:
             return self.code
 
-        objtable = self.get_object_table()
-        paramtable = self.get_parameter_table()
-
         # Optimise and regenerate the object table.
 
-        self.importer.vacuum(objtable)
-        self.importer.finalise()
-
-        objtable = self.get_object_table(reset=1)
-        paramtable = self.get_parameter_table(reset=1)
-
+        self.finalise()
         self.code = []
 
         # Append constants to the image.
diff -r 67d7ed58ac50 -r 34ebed37aecc micropython/ast.py
--- a/micropython/ast.py	Sun Jul 11 02:05:41 2010 +0200
+++ b/micropython/ast.py	Sat Aug 07 02:06:15 2010 +0200
@@ -564,7 +564,7 @@
         self.discard_temp(temp2)
 
     def visitClass(self, node):
-        if not node.unit.parent.has_key(node.unit.name):
+        if not used_by_unit(node):
             return
 
         # Store the name.
@@ -587,7 +587,7 @@
     def visitFrom(self, node): pass
 
     def visitFunction(self, node):
-        if not node.unit.parent.has_key(node.unit.name):
+        if not used_by_unit(node):
             return
 
         # Only store the name when visiting this node from outside.
diff -r 67d7ed58ac50 -r 34ebed37aecc micropython/common.py
--- a/micropython/common.py	Sun Jul 11 02:05:41 2010 +0200
+++ b/micropython/common.py	Sat Aug 07 02:06:15 2010 +0200
@@ -47,6 +47,15 @@
                     exc.unit_name = self.full_name()
             raise
 
+def used_by_unit(node):
+
+    """
+    Return whether the definition made by a 'node' is actually employed by the
+    program unit within which it is found.
+    """
+
+    return node.unit.parent.has_key(node.unit.name)
+
 # Errors.
 
 class ProcessingError(Exception):
diff -r 67d7ed58ac50 -r 34ebed37aecc micropython/report.py
--- a/micropython/report.py	Sun Jul 11 02:05:41 2010 +0200
+++ b/micropython/report.py	Sat Aug 07 02:06:15 2010 +0200
@@ -19,8 +19,9 @@
 this program.  If not, see <http://www.gnu.org/licenses/>.
 """
 
+from micropython.common import *
 from micropython.data import *
-from compiler.visitor import ASTVisitor
+from os.path import exists, extsep, join
 import sys
 import os
 import textwrap
@@ -33,7 +34,7 @@
 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
 <html xmlns="http://www.w3.org/1999/xhtml">
 <head>
-  <title>Module</title>
+  <title>Module: %(full_name)s</title>
   <style type="text/css">
     html {
       background-color: black; color: white;
@@ -45,6 +46,10 @@
       background-color: black; color: white;
     }
 
+    a {
+      text-decoration: none;
+    }
+
     .nowrap { white-space: nowrap; }
 
     .class { margin-top: 1em; margin-bottom: 1em; }
@@ -56,7 +61,31 @@
     .function-name { color: cyan; }
     .str { color: #FF00FF; }
     .doc { color: #FF00FF; margin-top: 1em; margin-bottom: 1em; }
-    .invocation a { color: white; text-decoration: none; }
+
+    .popup {
+      display: none;
+      position: absolute;
+      top: 3ex; left: 0;
+      color: white;
+      z-index: 3;
+    }
+
+    .scope { padding: 0.5em; background-color: #007700; }
+
+    .name {
+      position: relative;
+      color: white;
+    }
+
+    .name:hover {
+      background-color: #003300;
+      padding-bottom: 1ex;
+      z-index: 2;
+    }
+
+    .name:hover .popup {
+      display: block;
+    }
 
     .summary-class {
       background-color: #004400;
@@ -98,29 +127,34 @@
 
     # Methods which write to the stream.
 
+    def _span_end(self):
+        self.stream.write("</span>")
+
     def _comment(self, comment):
         self.stream.write("<span class='comment'># %s</span>\n" % comment)
 
-    def _keyword(self, kw):
-        self.stream.write("<span class='keyword'>%s</span> " % kw)
+    def _reserved(self, token, classes, leading=0, trailing=1):
+        if leading:
+            self.stream.write(" ")
+        self.stream.write("<span class='%s'>%s</span>" % (classes, self._text(token)))
+        if trailing:
+            self.stream.write(" ")
+
+    def _keyword(self, kw, leading=0, trailing=1):
+        self._reserved(kw, "keyword", leading, trailing)
+
+    def _op(self, symbol, leading=0, trailing=1):
+        self._reserved(symbol, "operator", leading, trailing)
 
     def _doc(self, node):
         if node.doc is not None:
-            self.stream.write("<pre class='doc'>\n")
+            self.stream.write("<pre class='doc'>")
             self.stream.write('"""')
             output = textwrap.dedent(node.doc.replace('"""', '\\"\\"\\"'))
             self.stream.write(self._text(output))
             self.stream.write('"""')
             self.stream.write("</pre>\n")
 
-    def _name(self, name, classes=None):
-        self.stream.write("<span class='%s'>%s</span>" % (classes or "name", name))
-
-    def _name_link(self, module_name, full_name, name, classes=None):
-        self.stream.write("<a class='%s' href='%s%sxhtml#%s'>%s</a>" % (
-            classes or "name", module_name, os.path.extsep,
-            self._attr(full_name), self._text(name)))
-
     def _summary_link(self, module_name, full_name, name, classes=None):
         self._name_link("%s-summary" % module_name, full_name, name, classes)
 
@@ -133,6 +167,29 @@
 
         self._summary_link(module.full_name(), obj.full_name(), obj.name, classes)
 
+    def _name_start(self, classes=None):
+        self.stream.write("<span class='%s'>" % (classes or "name"))
+
+    _name_end = _span_end
+
+    def _name(self, name, classes=None):
+        self._name_start(classes)
+        self.stream.write(name)
+        self._name_end()
+
+    def _name_link(self, module_name, full_name, name, classes=None):
+        self.stream.write("<a class='%s' href='%s%sxhtml#%s'>%s</a>" % (
+            classes or "name", module_name, os.path.extsep,
+            self._attr(full_name), self._text(name)))
+
+    def _popup_start(self):
+        self.stream.write("<span class='popup'>")
+
+    _popup_end = _span_end
+
+    def _scope(self, scope):
+        self.stream.write("<div class='scope'>%s</div>\n" % scope)
+
 # Summary classes.
 
 class Summary(Writer):
@@ -148,7 +205,7 @@
 
         self.stream = stream
         self._init_details()
-        self.stream.write(html_header)
+        self.stream.write(html_header % {"full_name" : self.module.full_name()})
         self._write_classes(self.module)
         self.stream.write(html_footer)
 
@@ -221,8 +278,8 @@
         "Write the annotated code to the given 'stream'."
 
         self.stream = stream
-        self.stream.write(html_header)
-        self.dispatch(self.module.module)
+        self.stream.write(html_header % {"full_name" : self.module.full_name()})
+        self.dispatch(self.module.astnode)
         self.stream.write(html_footer)
 
     def visitModule(self, node):
@@ -243,14 +300,14 @@
         self.stream.write("<div class='assign nowrap'>\n")
         for lvalue in node.nodes:
             self.dispatch(lvalue)
-            self.stream.write("= ")
+            self.stream.write(" = ")
         self.dispatch(node.expr)
         self.stream.write("</div>\n")
 
     def visitAugAssign(self, node):
         self.stream.write("<div class='augassign nowrap'>\n")
         self.dispatch(node.node)
-        self.stream.write("%s " % node.op)
+        self.stream.write(" %s " % node.op)
         self.dispatch(node.expr)
         self.stream.write("</div>\n")
 
@@ -260,6 +317,8 @@
         self.stream.write("</div>\n")
 
     def visitClass(self, node):
+        if not used_by_unit(node):
+            return
 
         # Use inspected details where possible.
 
@@ -317,7 +376,7 @@
         self.stream.write("<div>\n")
         self._keyword("for")
         self.dispatch(node.assign)
-        self._keyword("in")
+        self._keyword("in", 1)
         self.dispatch(node.list)
         self.stream.write(":\n")
         self.stream.write("</div>\n")
@@ -326,7 +385,7 @@
         self.stream.write("</div>\n")
         if node.else_ is not None:
             self.stream.write("<div>\n")
-            self._keyword("else")
+            self._keyword("else", trailing=0)
             self.stream.write(":\n")
             self.stream.write("</div>\n")
             self.stream.write("<div class='body nowrap'>\n")
@@ -338,19 +397,22 @@
         self.stream.write("<div class='from nowrap'>\n")
         self._keyword("from")
         self._name(node.modname)
-        self._keyword("import")
+        self._keyword("import", 1)
         first = 1
         for (name, alias), _name in map(None, node.names, node._names):
             if not first:
                 self.stream.write(", ")
             if alias:
                 self.stream.write(name + " ")
-                self._keyword("as")
+                self._keyword("as", 1)
             self._name(alias or name)
             first = 0
         self.stream.write("</div>\n")
 
     def visitFunction(self, node):
+        if not used_by_unit(node):
+            return
+
         if hasattr(node, "_def"):
             fn = node._def
             self.stream.write("<div class='function nowrap' id='%s'>\n" % fn.full_name())
@@ -405,7 +467,144 @@
             first = 0
         if node.else_ is not None:
             self.stream.write("<div>\n")
-            self._keyword("else")
+            self._keyword("else", trailing=0)
+            self.stream.write(":\n")
+            self.stream.write("</div>\n")
+            self.stream.write("<div class='body nowrap'>\n")
+            self.dispatch(node.else_)
+            self.stream.write("</div>\n")
+        self.stream.write("</div>\n")
+
+    def visitImport(self, node):
+        self.stream.write("<div class='import nowrap'>\n")
+        self._keyword("import")
+        first = 1
+        for name, alias in node.names:
+            if not first:
+                self.stream.write(",\n")
+            if alias:
+                self.stream.write(name + " ")
+                self._keyword("as", 1)
+            self._name(alias or name)
+            first = 0
+        self.stream.write("</div>\n")
+
+    def visitPass(self, node):
+        self.stream.write("<div class='pass nowrap'>\n")
+        self._keyword("pass")
+        self.stream.write("</div>\n")
+
+    def visitPrint(self, node):
+        self.stream.write("<div class='print nowrap'>\n")
+        self._keyword("print")
+        if node.dest is not None:
+            self.stream.write(">>\n")
+            self.dispatch(node.dest)
+        for n in node.nodes:
+            self.dispatch(n)
+            self.stream.write(",\n")
+        self.stream.write("</div>\n")
+
+    def visitPrintnl(self, node):
+        self.stream.write("<div class='printnl nowrap'>\n")
+        self._keyword("print")
+        if node.dest is not None:
+            self.stream.write(">>\n")
+            self.dispatch(node.dest)
+        first = 1
+        for n in node.nodes:
+            if not first:
+                self.stream.write(",\n")
+            self.dispatch(n)
+            first = 0
+        self.stream.write("</div>\n")
+
+    def visitRaise(self, node):
+        self.stream.write("<div class='raise nowrap'>\n")
+        self._keyword("raise")
+        self.dispatch(node.expr1)
+        if node.expr2 is not None:
+            self.stream.write(",\n")
+            self.dispatch(node.expr2)
+        if node.expr3 is not None:
+            self.stream.write(",\n")
+            self.dispatch(node.expr3)
+        self.stream.write("</div>\n")
+
+    def visitReturn(self, node):
+        self.stream.write("<div class='return nowrap'>\n")
+        self._keyword("return")
+        self.dispatch(node.value)
+        self.stream.write("</div>\n")
+
+    def visitStmt(self, node):
+        self.stream.write("<div class='stmt nowrap'>\n")
+        self.default(node)
+        self.stream.write("</div>\n")
+
+    def visitTryExcept(self, node):
+        self.stream.write("<div class='tryexcept nowrap'>\n")
+        self.stream.write("<div>\n")
+        self._keyword("try", trailing=0)
+        self.stream.write(":\n")
+        self.stream.write("</div>\n")
+        self.stream.write("<div class='body nowrap'>\n")
+        self.dispatch(node.body)
+        self.stream.write("</div>\n")
+        for spec, assign, statement in node.handlers:
+            self.stream.write("<div>\n")
+            self._keyword("except")
+            if spec is not None:
+                self.dispatch(spec)
+            if assign is not None:
+                self.stream.write(",\n")
+                self.dispatch(assign)
+            self.stream.write(":\n")
+            self.stream.write("</div>\n")
+            self.stream.write("<div class='body nowrap'>\n")
+            self.dispatch(statement)
+            self.stream.write("</div>\n")
+        if node.else_ is not None:
+            self.stream.write("<div>\n")
+            self._keyword("else", trailing=0)
+            self.stream.write(":\n")
+            self.stream.write("</div>\n")
+            self.stream.write("<div class='body nowrap'>\n")
+            self.dispatch(node.else_)
+            self.stream.write("</div>\n")
+        self.stream.write("</div>\n")
+
+    def visitTryFinally(self, node):
+        self.stream.write("<div class='tryfinally nowrap'>\n")
+        self.stream.write("<div>\n")
+        self._keyword("try", trailing=0)
+        self.stream.write(":\n")
+        self.stream.write("</div>\n")
+        self.stream.write("<div class='body nowrap'>\n")
+        self.dispatch(node.body)
+        self.stream.write("</div>\n")
+        self.stream.write("<div>\n")
+        self._keyword("finally", trailing=0)
+        self.stream.write(":\n")
+        self.stream.write("</div>\n")
+        self.stream.write("<div class='body nowrap'>\n")
+        self.dispatch(node.final)
+        self.stream.write("</div>\n")
+        self.stream.write("</div>\n")
+
+    def visitWhile(self, node):
+        self.stream.write("<div class='while nowrap'>\n")
+        self.stream.write("<div>\n")
+        self._keyword("while")
+        self.dispatch(node.test)
+        self.stream.write(":\n")
+        self.stream.write("</div>\n")
+        self.stream.write("<div class='body nowrap'>\n")
+        self.dispatch(node.body)
+        self.stream.write("</div>\n")
+        if node.else_ is not None:
+            self.stream.write("<div>\n")
+            self._keyword("else", trailing=0)
             self.stream.write(":\n")
             self.stream.write("</div>\n")
             self.stream.write("<div class='body nowrap'>\n")
@@ -413,6 +612,210 @@
             self.stream.write("</div>\n")
         self.stream.write("</div>\n")
 
+    # Expression-related helper methods.
+
+    def _visitBitBinary(self, node, name, symbol):
+        self.stream.write("<span class='%s'>" % name)
+        first = 1
+        for node in node.nodes:
+            if not first:
+                self._op(symbol, 1)
+            self.dispatch(node)
+            first = 0
+        self.stream.write("</span>")
+
+    def _visitBinary(self, node, name, symbol):
+        self.stream.write("<span class='%s'>" % name)
+        self.dispatch(node.left)
+        self._op(symbol, 1)
+        self.dispatch(node.right)
+        self.stream.write("</span>")
+
+    def _visitUnary(self, node, name, symbol):
+        self.stream.write("<span class='%s'>" % name)
+        self._op(symbol)
+        self.dispatch(node.expr)
+        self.stream.write("</span>")
+
+    # Expressions.
+
+    def visitAdd(self, node):
+        self._visitBinary(node, "add", "+")
+
+    def visitAnd(self, node):
+        self.stream.write("<span class='and'>")
+        first = 1
+        for n in node.nodes:
+            if not first:
+                self._keyword("and", 1)
+            self.dispatch(n)
+            first = 0
+        self.stream.write("</span>")
+
+    def visitAssAttr(self, node):
+        self.stream.write("<span class='assattr'>")
+        self.dispatch(node.expr)
+        self.stream.write("<span class='attr'>")
+        self.stream.write(".")
+        self._name(node.attrname)
+        self.stream.write("</span>")
+        self.stream.write("</span>")
+
+    def visitAssList(self, node):
+        self.stream.write("<span class='list'>")
+        self.stream.write("[")
+        self._sequence(node)
+        self.stream.write("]")
+        self.stream.write("</span>")
+
+    def visitAssName(self, node):
+        self._name(node.name)
+
+    def visitAssTuple(self, node):
+        self.stream.write("<span class='tuple'>")
+        self.stream.write("(")
+        self._sequence(node)
+        self.stream.write(")")
+        self.stream.write("</span>")
+
+    def visitBitand(self, node):
+        self._visitBitBinary(node, "bitand", "&")
+
+    def visitCallFunc(self, node):
+        self.stream.write("<span class='callfunc'>")
+        self.dispatch(node.node)
+        self.stream.write("<span class='call'>")
+        self.stream.write("(")
+        first = 1
+        for arg in node.args:
+            if not first:
+                self.stream.write(", ")
+            self.dispatch(arg)
+            first = 0
+        if node.star_args is not None:
+            if not first:
+                self.stream.write(", *")
+            self.dispatch(node.star_args)
+            first = 0
+        if node.dstar_args is not None:
+            if not first:
+                self.stream.write(", **")
+            self.dispatch(node.dstar_args)
+            first = 0
+        self.stream.write(")")
+        self.stream.write("</span>")
+        self.stream.write("</span>")
+
+    def visitCompare(self, node):
+        self.stream.write("<span class='compare'>")
+        self.dispatch(node.expr)
+        for op_name, expr in node.ops:
+            self._op(op_name, 1)
+            self.dispatch(expr)
+        self.stream.write("</span>")
+
+    def visitConst(self, node):
+        if isinstance(node.value, (str, unicode)):
+            self.stream.write("<span class='str'>")
+        self.stream.write(self._text(repr(node.value)))
+        if isinstance(node.value, (str, unicode)):
+            self.stream.write("</span>")
+
+    def visitDict(self, node):
+        self.stream.write("<span class='dict'>")
+        self.stream.write("{")
+        self._mapping(node)
+        self.stream.write("}")
+        self.stream.write("</span>")
+
+    def visitDiv(self, node):
+        self._visitBinary(node, "div", "/")
+
+    def visitFloorDiv(self, node):
+        self._visitBinary(node, "floordiv", "//")
+
+    def visitGetattr(self, node):
+        self.stream.write("<span class='getattr'>")
+        self.dispatch(node.expr)
+        self.stream.write("<span class='attr'>")
+        self.stream.write(".")
+        self._name(node.attrname)
+        self.stream.write("</span>")
+        self.stream.write("</span>")
+
+    def visitKeyword(self, node):
+        self.stream.write("<span class='keyword-arg'>")
+        self.stream.write(node.name)
+        self.stream.write("=")
+        self.dispatch(node.expr)
+        self.stream.write("</span>")
+
+    def visitLambda(self, node):
+        if hasattr(node, "_def"):
+            fn = node._def
+        else:
+            print "Warning: function %s not recognised!" % node.name
+            return
+
+        self.stream.write("<span class='lambda'>")
+        self._keyword("lambda")
+        self._parameters(fn)
+        self.stream.write(": <span class='code'>")
+        self.dispatch(node.code)
+        self.stream.write("</span>")
+        self.stream.write("</span>")
+
+    visitList = visitAssList
+
+    def visitListComp(self, node):
+        self.stream.write("<span class='listcomp'>")
+        self.stream.write("[")
+        self.dispatch(node.expr)
+        for qual in node.quals:
+            self.dispatch(qual)
+        self.stream.write("]")
+        self.stream.write("</span>")
+
+    def visitListCompFor(self, node):
+        self.stream.write("<span class='listcompfor'>")
+        self._keyword("for")
+        self.stream.write("<span class='iterator'>")
+        self.dispatch(node.assign)
+        self.stream.write("</span>")
+        self._keyword("in", 1)
+        self.stream.write("<span class='collection'>")
+        self.dispatch(node.list)
+        self.stream.write("</span>")
+        for if_ in node.ifs:
+            self.dispatch(if_)
+        self.stream.write("</span>")
+
+    def visitListCompIf(self, node):
+        self.stream.write("<span class='listcompif'>")
+        self.stream.write("<span class='conditional'>")
+        self._keyword("if", 1)
+        self.dispatch(node.test)
+        self.stream.write("</span>")
+        self.stream.write("</span>")
+
+    def visitMod(self, node):
+        self._visitBinary(node, "mod", "%")
+
+    def visitMul(self, node):
+        self._visitBinary(node, "mul", "*")
+
+    def visitName(self, node):
+        if hasattr(node, "_scope"):
+            scope = node._scope
+            self._name_start()
+            self.stream.write(node.name)
+            self._popup_start()
+            self._scope(scope)
+            self._popup_end()
+            self._name_end()
+        else:
+            self._name(node.name)
+
     # Output preparation methods.
 
     def _sequence(self, node):
@@ -480,4 +883,12 @@
     finally:
         stream.close()
 
+def report(importer, directory):
+    if not exists(directory):
+        os.mkdir(directory)
+
+    for module in importer.get_modules():
+        annotate(module, join(directory, "%s%sxhtml" % (module.full_name(), extsep)))
+        summarise(module, join(directory, "%s-summary%sxhtml" % (module.full_name(), extsep)))
+
 # vim: tabstop=4 expandtab shiftwidth=4
diff -r 67d7ed58ac50 -r 34ebed37aecc test.py
--- a/test.py	Sun Jul 11 02:05:41 2010 +0200
+++ b/test.py	Sat Aug 07 02:06:15 2010 +0200
@@ -1,6 +1,7 @@
 #!/usr/bin/env python
 
 import micropython.cmd
+import micropython.report
 from micropython.graph import get_graph
 import rsvp
 import sys
@@ -55,6 +56,19 @@
             print "Loading from", filename, "..."
             m = i.load_from_file(filename)
 
+        p.finalise()
+
+        # Make a report.
+
+        if "-d" in args:
+            try:
+                directory = args[args.index("-d") + 1]
+            except IndexError:
+                print "No directory specified. Not generating report."
+            else:
+                print "Generating report in", directory
+                micropython.report.report(i, directory)
+
         # Build the program.
 
         if "-m" in args or "-t" in args: