1.1 --- a/common.py Wed Dec 14 16:23:32 2016 +0100
1.2 +++ b/common.py Wed Dec 14 16:40:00 2016 +0100
1.3 @@ -24,7 +24,8 @@
1.4 from os import listdir, makedirs, remove
1.5 from os.path import exists, isdir, join, split
1.6 from results import ConstantValueRef, LiteralSequenceRef, NameRef
1.7 -import compiler
1.8 +from compiler.transformer import Transformer
1.9 +import compiler.ast
1.10
1.11 class CommonOutput:
1.12
1.13 @@ -93,6 +94,7 @@
1.14 # Inspection-related attributes.
1.15
1.16 self.astnode = None
1.17 + self.encoding = None
1.18 self.iterators = {}
1.19 self.temp = {}
1.20 self.lambdas = {}
1.21 @@ -128,7 +130,17 @@
1.22 "Parse the file with the given 'filename', initialising attributes."
1.23
1.24 self.filename = filename
1.25 - self.astnode = compiler.parseFile(filename)
1.26 +
1.27 + # Use the Transformer directly to obtain encoding information.
1.28 +
1.29 + t = Transformer()
1.30 + f = open(filename)
1.31 +
1.32 + try:
1.33 + self.astnode = t.parsesuite(f.read() + "\n")
1.34 + self.encoding = t.encoding
1.35 + finally:
1.36 + f.close()
1.37
1.38 # Module-relative naming.
1.39
1.40 @@ -221,13 +233,23 @@
1.41
1.42 # Constant and literal recording.
1.43
1.44 - def get_constant_value(self, value):
1.45 + def get_constant_value(self, value, literal=None):
1.46
1.47 - "Encode the 'value' if appropriate."
1.48 + "Encode the 'value' if appropriate, returning a value and typename."
1.49
1.50 if isinstance(value, unicode):
1.51 - value = value.encode("utf-8")
1.52 - return value
1.53 + return value.encode("utf-8"), "unicode"
1.54 +
1.55 + # Attempt to convert plain strings to text.
1.56 +
1.57 + elif isinstance(value, str) and self.encoding:
1.58 + if not literal.startswith("b"):
1.59 + try:
1.60 + return unicode(value, self.encoding).encode("utf-8"), "unicode"
1.61 + except UnicodeDecodeError:
1.62 + pass
1.63 +
1.64 + return value, value.__class__.__name__
1.65
1.66 def get_constant_reference(self, ref, value):
1.67