Lichen

Annotated pyparser/test/test_pyparse.py

858:8f8361de472a
2018-07-21 Paul Boddie Invocations in parameter lists may require temporary storage for contexts and accessors, but the storage locations must be different. Otherwise, the code will generate a sequence-point warning. Here, distinct ranges for parameters in lists are introduced, and accessors are now also stored in arrays, permitting distinct storage.
paul@437 1
# -*- coding: utf-8 -*-
paul@437 2
import py
paul@437 3
from pyparser import pyparse
paul@439 4
from pyparser.pygram import syms
paul@437 5
from pyparser.error import SyntaxError, IndentationError
paul@437 6
from pyparser import consts
paul@437 7
paul@437 8
paul@437 9
class TestPythonParser:
paul@437 10
paul@437 11
    def setup_class(self):
paul@437 12
        self.parser = pyparse.PythonParser()
paul@437 13
paul@437 14
    def parse(self, source, mode="exec", info=None):
paul@437 15
        if info is None:
paul@437 16
            info = pyparse.CompileInfo("<test>", mode)
paul@437 17
        return self.parser.parse_source(source, info)
paul@437 18
paul@437 19
    def test_with_and_as(self):
paul@437 20
        py.test.raises(SyntaxError, self.parse, "with = 23")
paul@437 21
        py.test.raises(SyntaxError, self.parse, "as = 2")
paul@437 22
paul@437 23
    def test_dont_imply_dedent(self):
paul@437 24
        info = pyparse.CompileInfo("<test>", "single",
paul@437 25
                                   consts.PyCF_DONT_IMPLY_DEDENT)
paul@437 26
        self.parse('if 1:\n  x\n', info=info)
paul@437 27
        self.parse('x = 5 ', info=info)
paul@437 28
paul@437 29
    def test_clear_state(self):
paul@437 30
        assert self.parser.root is None
paul@437 31
        tree = self.parse("name = 32")
paul@437 32
        assert self.parser.root is None
paul@437 33
paul@437 34
    def test_encoding(self):
paul@437 35
        info = pyparse.CompileInfo("<test>", "exec")
paul@437 36
        tree = self.parse("""# coding: latin-1
paul@437 37
stuff = "nothing"
paul@437 38
""", info=info)
paul@439 39
        assert tree.type == syms["encoding_decl"]
paul@439 40
        assert tree.encoding == "iso-8859-1"
paul@437 41
        assert info.encoding == "iso-8859-1"
paul@437 42
        sentence = u"u'Die M??nner ??rgen sich!'"
paul@437 43
        input = (u"# coding: utf-7\nstuff = %s" % (sentence,)).encode("utf-7")
paul@437 44
        tree = self.parse(input, info=info)
paul@437 45
        assert info.encoding == "utf-7"
paul@437 46
        input = "# coding: iso-8859-15\nx"
paul@437 47
        self.parse(input, info=info)
paul@437 48
        assert info.encoding == "iso-8859-15"
paul@437 49
        input = "\xEF\xBB\xBF# coding: utf-8\nx"
paul@437 50
        self.parse(input, info=info)
paul@437 51
        assert info.encoding == "utf-8"
paul@437 52
        input = "# coding: utf-8\nx"
paul@437 53
        info.flags |= consts.PyCF_SOURCE_IS_UTF8
paul@437 54
        exc = py.test.raises(SyntaxError, self.parse, input, info=info).value
paul@437 55
        info.flags &= ~consts.PyCF_SOURCE_IS_UTF8
paul@437 56
        assert exc.msg == "coding declaration in unicode string"
paul@437 57
        input = "\xEF\xBB\xBF# coding: latin-1\nx"
paul@437 58
        exc = py.test.raises(SyntaxError, self.parse, input).value
paul@437 59
        assert exc.msg == "UTF-8 BOM with latin-1 coding cookie"
paul@437 60
        input = "# coding: not-here"
paul@437 61
        exc = py.test.raises(SyntaxError, self.parse, input).value
paul@437 62
        assert exc.msg == "Unknown encoding: not-here"
paul@437 63
        input = u"# coding: ascii\n\xe2".encode('utf-8')
paul@437 64
        exc = py.test.raises(SyntaxError, self.parse, input).value
paul@437 65
        assert exc.msg == ("'ascii' codec can't decode byte 0xc3 "
paul@437 66
                           "in position 16: ordinal not in range(128)")
paul@437 67
paul@437 68
    def test_non_unicode_codec(self):
paul@437 69
        exc = py.test.raises(SyntaxError, self.parse, """\
paul@437 70
# coding: string-escape
paul@437 71
\x70\x72\x69\x6e\x74\x20\x32\x2b\x32\x0a
paul@437 72
""").value
paul@437 73
        assert exc.msg == "codec did not return a unicode object"
paul@437 74
paul@437 75
    def test_syntax_error(self):
paul@437 76
        parse = self.parse
paul@437 77
        exc = py.test.raises(SyntaxError, parse, "name another for").value
paul@437 78
        assert exc.msg == "invalid syntax"
paul@437 79
        assert exc.lineno == 1
paul@437 80
        assert exc.offset == 5
paul@437 81
        assert exc.text.startswith("name another for")
paul@437 82
        exc = py.test.raises(SyntaxError, parse, "x = \"blah\n\n\n").value
paul@437 83
        assert exc.msg == "EOL while scanning string literal"
paul@437 84
        assert exc.lineno == 1
paul@437 85
        assert exc.offset == 5
paul@437 86
        exc = py.test.raises(SyntaxError, parse, "x = '''\n\n\n").value
paul@437 87
        assert exc.msg == "EOF while scanning triple-quoted string literal"
paul@437 88
        assert exc.lineno == 1
paul@437 89
        assert exc.offset == 5
paul@437 90
        assert exc.lastlineno == 3
paul@437 91
        for input in ("())", "(()", "((", "))"):
paul@437 92
            py.test.raises(SyntaxError, parse, input)
paul@437 93
        exc = py.test.raises(SyntaxError, parse, "x = (\n\n(),\n(),").value
paul@437 94
        assert exc.msg == "parenthesis is never closed"
paul@437 95
        assert exc.lineno == 1
paul@437 96
        assert exc.offset == 5
paul@437 97
        assert exc.lastlineno == 5
paul@437 98
        exc = py.test.raises(SyntaxError, parse, "abc)").value
paul@437 99
        assert exc.msg == "unmatched ')'"
paul@437 100
        assert exc.lineno == 1
paul@437 101
        assert exc.offset == 4
paul@437 102
paul@437 103
    def test_is(self):
paul@437 104
        self.parse("x is y")
paul@437 105
        self.parse("x is not y")
paul@437 106
paul@437 107
    def test_indentation_error(self):
paul@437 108
        parse = self.parse
paul@437 109
        input = """
paul@437 110
def f():
paul@437 111
pass"""
paul@437 112
        exc = py.test.raises(IndentationError, parse, input).value
paul@437 113
        assert exc.msg == "expected an indented block"
paul@437 114
        assert exc.lineno == 3
paul@437 115
        assert exc.text.startswith("pass")
paul@437 116
        assert exc.offset == 0
paul@437 117
        input = "hi\n    indented"
paul@437 118
        exc = py.test.raises(IndentationError, parse, input).value
paul@437 119
        assert exc.msg == "unexpected indent"
paul@437 120
        input = "def f():\n    pass\n  next_stmt"
paul@437 121
        exc = py.test.raises(IndentationError, parse, input).value
paul@437 122
        assert exc.msg == "unindent does not match any outer indentation level"
paul@437 123
        assert exc.lineno == 3
paul@437 124
paul@437 125
    def test_mac_newline(self):
paul@437 126
        self.parse("this_is\ra_mac\rfile")
paul@437 127
paul@437 128
    def test_mode(self):
paul@439 129
        assert self.parse("x = 43*54").type == syms["file_input"]
paul@437 130
        tree = self.parse("43**54", "eval")
paul@439 131
        assert tree.type == syms["eval_input"]
paul@437 132
        py.test.raises(SyntaxError, self.parse, "x = 54", "eval")
paul@437 133
        tree = self.parse("x = 43", "single")
paul@439 134
        assert tree.type == syms["single_input"]
paul@437 135
paul@437 136
    def test_multiline_string(self):
paul@437 137
        self.parse("''' \n '''")
paul@437 138
        self.parse("r''' \n '''")
paul@437 139
paul@437 140
    def test_bytes_literal(self):
paul@437 141
        self.parse('b" "')
paul@437 142
        self.parse('br" "')
paul@437 143
        self.parse('b""" """')
paul@437 144
        self.parse("b''' '''")
paul@437 145
        self.parse("br'\\\n'")
paul@437 146
paul@437 147
        py.test.raises(SyntaxError, self.parse, "b'a\\n")
paul@437 148
paul@437 149
    def test_new_octal_literal(self):
paul@437 150
        self.parse('0777')
paul@437 151
        self.parse('0o777')
paul@437 152
        self.parse('0o777L')
paul@437 153
        py.test.raises(SyntaxError, self.parse, "0o778")
paul@437 154
paul@437 155
    def test_new_binary_literal(self):
paul@437 156
        self.parse('0b1101')
paul@437 157
        self.parse('0b0l')
paul@437 158
        py.test.raises(SyntaxError, self.parse, "0b112")
paul@437 159
paul@437 160
    def test_universal_newlines(self):
paul@437 161
        fmt = 'stuff = """hello%sworld"""'
paul@437 162
        expected_tree = self.parse(fmt % '\n')
paul@437 163
        for linefeed in ["\r\n","\r"]:
paul@437 164
            tree = self.parse(fmt % linefeed)
paul@437 165
            assert expected_tree == tree