paul@437 | 1 | # -*- coding: utf-8 -*- |
paul@437 | 2 | import py |
paul@437 | 3 | from pyparser import pyparse |
paul@439 | 4 | from pyparser.pygram import syms |
paul@437 | 5 | from pyparser.error import SyntaxError, IndentationError |
paul@437 | 6 | from pyparser import consts |
paul@437 | 7 | |
paul@437 | 8 | |
paul@437 | 9 | class TestPythonParser: |
paul@437 | 10 | |
paul@437 | 11 | def setup_class(self): |
paul@437 | 12 | self.parser = pyparse.PythonParser() |
paul@437 | 13 | |
paul@437 | 14 | def parse(self, source, mode="exec", info=None): |
paul@437 | 15 | if info is None: |
paul@437 | 16 | info = pyparse.CompileInfo("<test>", mode) |
paul@437 | 17 | return self.parser.parse_source(source, info) |
paul@437 | 18 | |
paul@437 | 19 | def test_with_and_as(self): |
paul@437 | 20 | py.test.raises(SyntaxError, self.parse, "with = 23") |
paul@437 | 21 | py.test.raises(SyntaxError, self.parse, "as = 2") |
paul@437 | 22 | |
paul@437 | 23 | def test_dont_imply_dedent(self): |
paul@437 | 24 | info = pyparse.CompileInfo("<test>", "single", |
paul@437 | 25 | consts.PyCF_DONT_IMPLY_DEDENT) |
paul@437 | 26 | self.parse('if 1:\n x\n', info=info) |
paul@437 | 27 | self.parse('x = 5 ', info=info) |
paul@437 | 28 | |
paul@437 | 29 | def test_clear_state(self): |
paul@437 | 30 | assert self.parser.root is None |
paul@437 | 31 | tree = self.parse("name = 32") |
paul@437 | 32 | assert self.parser.root is None |
paul@437 | 33 | |
paul@437 | 34 | def test_encoding(self): |
paul@437 | 35 | info = pyparse.CompileInfo("<test>", "exec") |
paul@437 | 36 | tree = self.parse("""# coding: latin-1 |
paul@437 | 37 | stuff = "nothing" |
paul@437 | 38 | """, info=info) |
paul@439 | 39 | assert tree.type == syms["encoding_decl"] |
paul@439 | 40 | assert tree.encoding == "iso-8859-1" |
paul@437 | 41 | assert info.encoding == "iso-8859-1" |
paul@437 | 42 | sentence = u"u'Die M??nner ??rgen sich!'" |
paul@437 | 43 | input = (u"# coding: utf-7\nstuff = %s" % (sentence,)).encode("utf-7") |
paul@437 | 44 | tree = self.parse(input, info=info) |
paul@437 | 45 | assert info.encoding == "utf-7" |
paul@437 | 46 | input = "# coding: iso-8859-15\nx" |
paul@437 | 47 | self.parse(input, info=info) |
paul@437 | 48 | assert info.encoding == "iso-8859-15" |
paul@437 | 49 | input = "\xEF\xBB\xBF# coding: utf-8\nx" |
paul@437 | 50 | self.parse(input, info=info) |
paul@437 | 51 | assert info.encoding == "utf-8" |
paul@437 | 52 | input = "# coding: utf-8\nx" |
paul@437 | 53 | info.flags |= consts.PyCF_SOURCE_IS_UTF8 |
paul@437 | 54 | exc = py.test.raises(SyntaxError, self.parse, input, info=info).value |
paul@437 | 55 | info.flags &= ~consts.PyCF_SOURCE_IS_UTF8 |
paul@437 | 56 | assert exc.msg == "coding declaration in unicode string" |
paul@437 | 57 | input = "\xEF\xBB\xBF# coding: latin-1\nx" |
paul@437 | 58 | exc = py.test.raises(SyntaxError, self.parse, input).value |
paul@437 | 59 | assert exc.msg == "UTF-8 BOM with latin-1 coding cookie" |
paul@437 | 60 | input = "# coding: not-here" |
paul@437 | 61 | exc = py.test.raises(SyntaxError, self.parse, input).value |
paul@437 | 62 | assert exc.msg == "Unknown encoding: not-here" |
paul@437 | 63 | input = u"# coding: ascii\n\xe2".encode('utf-8') |
paul@437 | 64 | exc = py.test.raises(SyntaxError, self.parse, input).value |
paul@437 | 65 | assert exc.msg == ("'ascii' codec can't decode byte 0xc3 " |
paul@437 | 66 | "in position 16: ordinal not in range(128)") |
paul@437 | 67 | |
paul@437 | 68 | def test_non_unicode_codec(self): |
paul@437 | 69 | exc = py.test.raises(SyntaxError, self.parse, """\ |
paul@437 | 70 | # coding: string-escape |
paul@437 | 71 | \x70\x72\x69\x6e\x74\x20\x32\x2b\x32\x0a |
paul@437 | 72 | """).value |
paul@437 | 73 | assert exc.msg == "codec did not return a unicode object" |
paul@437 | 74 | |
paul@437 | 75 | def test_syntax_error(self): |
paul@437 | 76 | parse = self.parse |
paul@437 | 77 | exc = py.test.raises(SyntaxError, parse, "name another for").value |
paul@437 | 78 | assert exc.msg == "invalid syntax" |
paul@437 | 79 | assert exc.lineno == 1 |
paul@437 | 80 | assert exc.offset == 5 |
paul@437 | 81 | assert exc.text.startswith("name another for") |
paul@437 | 82 | exc = py.test.raises(SyntaxError, parse, "x = \"blah\n\n\n").value |
paul@437 | 83 | assert exc.msg == "EOL while scanning string literal" |
paul@437 | 84 | assert exc.lineno == 1 |
paul@437 | 85 | assert exc.offset == 5 |
paul@437 | 86 | exc = py.test.raises(SyntaxError, parse, "x = '''\n\n\n").value |
paul@437 | 87 | assert exc.msg == "EOF while scanning triple-quoted string literal" |
paul@437 | 88 | assert exc.lineno == 1 |
paul@437 | 89 | assert exc.offset == 5 |
paul@437 | 90 | assert exc.lastlineno == 3 |
paul@437 | 91 | for input in ("())", "(()", "((", "))"): |
paul@437 | 92 | py.test.raises(SyntaxError, parse, input) |
paul@437 | 93 | exc = py.test.raises(SyntaxError, parse, "x = (\n\n(),\n(),").value |
paul@437 | 94 | assert exc.msg == "parenthesis is never closed" |
paul@437 | 95 | assert exc.lineno == 1 |
paul@437 | 96 | assert exc.offset == 5 |
paul@437 | 97 | assert exc.lastlineno == 5 |
paul@437 | 98 | exc = py.test.raises(SyntaxError, parse, "abc)").value |
paul@437 | 99 | assert exc.msg == "unmatched ')'" |
paul@437 | 100 | assert exc.lineno == 1 |
paul@437 | 101 | assert exc.offset == 4 |
paul@437 | 102 | |
paul@437 | 103 | def test_is(self): |
paul@437 | 104 | self.parse("x is y") |
paul@437 | 105 | self.parse("x is not y") |
paul@437 | 106 | |
paul@437 | 107 | def test_indentation_error(self): |
paul@437 | 108 | parse = self.parse |
paul@437 | 109 | input = """ |
paul@437 | 110 | def f(): |
paul@437 | 111 | pass""" |
paul@437 | 112 | exc = py.test.raises(IndentationError, parse, input).value |
paul@437 | 113 | assert exc.msg == "expected an indented block" |
paul@437 | 114 | assert exc.lineno == 3 |
paul@437 | 115 | assert exc.text.startswith("pass") |
paul@437 | 116 | assert exc.offset == 0 |
paul@437 | 117 | input = "hi\n indented" |
paul@437 | 118 | exc = py.test.raises(IndentationError, parse, input).value |
paul@437 | 119 | assert exc.msg == "unexpected indent" |
paul@437 | 120 | input = "def f():\n pass\n next_stmt" |
paul@437 | 121 | exc = py.test.raises(IndentationError, parse, input).value |
paul@437 | 122 | assert exc.msg == "unindent does not match any outer indentation level" |
paul@437 | 123 | assert exc.lineno == 3 |
paul@437 | 124 | |
paul@437 | 125 | def test_mac_newline(self): |
paul@437 | 126 | self.parse("this_is\ra_mac\rfile") |
paul@437 | 127 | |
paul@437 | 128 | def test_mode(self): |
paul@439 | 129 | assert self.parse("x = 43*54").type == syms["file_input"] |
paul@437 | 130 | tree = self.parse("43**54", "eval") |
paul@439 | 131 | assert tree.type == syms["eval_input"] |
paul@437 | 132 | py.test.raises(SyntaxError, self.parse, "x = 54", "eval") |
paul@437 | 133 | tree = self.parse("x = 43", "single") |
paul@439 | 134 | assert tree.type == syms["single_input"] |
paul@437 | 135 | |
paul@437 | 136 | def test_multiline_string(self): |
paul@437 | 137 | self.parse("''' \n '''") |
paul@437 | 138 | self.parse("r''' \n '''") |
paul@437 | 139 | |
paul@437 | 140 | def test_bytes_literal(self): |
paul@437 | 141 | self.parse('b" "') |
paul@437 | 142 | self.parse('br" "') |
paul@437 | 143 | self.parse('b""" """') |
paul@437 | 144 | self.parse("b''' '''") |
paul@437 | 145 | self.parse("br'\\\n'") |
paul@437 | 146 | |
paul@437 | 147 | py.test.raises(SyntaxError, self.parse, "b'a\\n") |
paul@437 | 148 | |
paul@437 | 149 | def test_new_octal_literal(self): |
paul@437 | 150 | self.parse('0777') |
paul@437 | 151 | self.parse('0o777') |
paul@437 | 152 | self.parse('0o777L') |
paul@437 | 153 | py.test.raises(SyntaxError, self.parse, "0o778") |
paul@437 | 154 | |
paul@437 | 155 | def test_new_binary_literal(self): |
paul@437 | 156 | self.parse('0b1101') |
paul@437 | 157 | self.parse('0b0l') |
paul@437 | 158 | py.test.raises(SyntaxError, self.parse, "0b112") |
paul@437 | 159 | |
paul@437 | 160 | def test_universal_newlines(self): |
paul@437 | 161 | fmt = 'stuff = """hello%sworld"""' |
paul@437 | 162 | expected_tree = self.parse(fmt % '\n') |
paul@437 | 163 | for linefeed in ["\r\n","\r"]: |
paul@437 | 164 | tree = self.parse(fmt % linefeed) |
paul@437 | 165 | assert expected_tree == tree |