Lichen

Annotated pyparser/test/test_metaparser.py

829:a8a16ef13733
2018-06-24 Paul Boddie Simplify the logic around target usage for invocation contexts.
paul@437 1
import py
paul@437 2
import os
paul@437 3
import glob
paul@437 4
import tokenize
paul@437 5
import token
paul@437 6
import StringIO
paul@437 7
from pyparser.metaparser import ParserGenerator, PgenError
paul@437 8
from pyparser.pygram import PythonGrammar
paul@437 9
from pyparser import parser
paul@437 10
paul@437 11
paul@437 12
class MyGrammar(parser.Grammar):
paul@437 13
    TOKENS = token.__dict__
paul@437 14
    OPERATOR_MAP = {
paul@437 15
        "+" : token.OP,
paul@437 16
        "-" : token.OP,
paul@437 17
        }
paul@437 18
    KEYWORD_TOKEN = token.NAME
paul@437 19
paul@437 20
paul@437 21
class TestParserGenerator:
paul@437 22
paul@437 23
    def gram_for(self, grammar_source):
paul@437 24
        p = ParserGenerator(grammar_source + "\n")
paul@437 25
        return p.build_grammar(MyGrammar)
paul@437 26
paul@437 27
    def test_multiple_rules(self):
paul@437 28
        g = self.gram_for("foo: NAME bar\nbar: STRING")
paul@437 29
        assert len(g.dfas) == 2
paul@437 30
        assert g.start == g.symbol_ids["foo"]
paul@437 31
paul@437 32
    def test_simple(self):
paul@437 33
        g = self.gram_for("eval: NAME\n")
paul@437 34
        assert len(g.dfas) == 1
paul@437 35
        eval_sym = g.symbol_ids["eval"]
paul@437 36
        assert g.start == eval_sym
paul@437 37
        states, first = g.dfas[eval_sym - 256]
paul@437 38
        assert states == [([(1, 1)], False), ([], True)]
paul@437 39
        assert g.labels[0] == 0
paul@437 40
paul@437 41
    def test_load_python_grammars(self):
paul@437 42
        gram_pat = os.path.join(os.path.dirname(__file__), "..", "data",
paul@437 43
                                "Grammar*")
paul@437 44
        for gram_file in glob.glob(gram_pat):
paul@437 45
            fp = open(gram_file, "r")
paul@437 46
            try:
paul@437 47
                ParserGenerator(fp.read()).build_grammar(PythonGrammar)
paul@437 48
            finally:
paul@437 49
                fp.close()
paul@437 50
paul@437 51
    def test_items(self):
paul@437 52
        g = self.gram_for("foo: NAME STRING OP '+'")
paul@437 53
        assert len(g.dfas) == 1
paul@437 54
        states = g.dfas[g.symbol_ids["foo"] - 256][0]
paul@437 55
        last = states[0][0][0][1]
paul@437 56
        for state in states[1:-1]:
paul@437 57
            assert last < state[0][0][1]
paul@437 58
            last = state[0][0][1]
paul@437 59
paul@437 60
    def test_alternatives(self):
paul@437 61
        g = self.gram_for("foo: STRING | OP")
paul@437 62
        assert len(g.dfas) == 1
paul@437 63
paul@437 64
    def test_optional(self):
paul@437 65
        g = self.gram_for("foo: [NAME]")
paul@437 66
paul@437 67
    def test_grouping(self):
paul@437 68
        g = self.gram_for("foo: (NAME | STRING) OP")
paul@437 69
paul@437 70
    def test_keyword(self):
paul@437 71
        g = self.gram_for("foo: 'some_keyword' 'for'")
paul@437 72
        assert len(g.keyword_ids) == 2
paul@437 73
        assert len(g.token_ids) == 0
paul@437 74
paul@437 75
    def test_token(self):
paul@437 76
        g = self.gram_for("foo: NAME")
paul@437 77
        assert len(g.token_ids) == 1
paul@437 78
paul@437 79
    def test_operator(self):
paul@437 80
        g = self.gram_for("add: NUMBER '+' NUMBER")
paul@437 81
        assert len(g.keyword_ids) == 0
paul@437 82
        assert len(g.token_ids) == 2
paul@437 83
paul@437 84
        exc = py.test.raises(PgenError, self.gram_for, "add: '/'").value
paul@437 85
        assert str(exc) == "no such operator: '/'"
paul@437 86
paul@437 87
    def test_symbol(self):
paul@437 88
        g = self.gram_for("foo: some_other_rule\nsome_other_rule: NAME")
paul@437 89
        assert len(g.dfas) == 2
paul@437 90
        assert len(g.labels) == 3
paul@437 91
paul@437 92
        exc = py.test.raises(PgenError, self.gram_for, "foo: no_rule").value
paul@437 93
        assert str(exc) == "no such rule: 'no_rule'"
paul@437 94
paul@437 95
    def test_repeaters(self):
paul@437 96
        g1 = self.gram_for("foo: NAME+")
paul@437 97
        g2 = self.gram_for("foo: NAME*")
paul@437 98
        assert g1.dfas != g2.dfas
paul@437 99
paul@437 100
        g = self.gram_for("foo: (NAME | STRING)*")
paul@437 101
        g = self.gram_for("foo: (NAME | STRING)+")
paul@437 102
paul@437 103
    def test_error(self):
paul@437 104
        exc = py.test.raises(PgenError, self.gram_for, "hi").value
paul@437 105
        assert str(exc) == "expected token OP but got NEWLINE"
paul@437 106
        assert exc.location == ((1, 2), (1, 3), "hi\n")
paul@437 107
        exc = py.test.raises(PgenError, self.gram_for, "hi+").value
paul@437 108
        assert str(exc) == "expected ':' but got '+'"
paul@437 109
        assert exc.location == ((1, 2), (1, 3), "hi+\n")
paul@437 110
paul@437 111
    def test_comments_and_whitespace(self):
paul@437 112
        self.gram_for("\n\n# comment\nrule: NAME # comment")