paul@437 | 1 | import py |
paul@437 | 2 | import os |
paul@437 | 3 | import glob |
paul@437 | 4 | import tokenize |
paul@437 | 5 | import token |
paul@437 | 6 | import StringIO |
paul@437 | 7 | from pyparser.metaparser import ParserGenerator, PgenError |
paul@437 | 8 | from pyparser.pygram import PythonGrammar |
paul@437 | 9 | from pyparser import parser |
paul@437 | 10 | |
paul@437 | 11 | |
paul@437 | 12 | class MyGrammar(parser.Grammar): |
paul@437 | 13 | TOKENS = token.__dict__ |
paul@437 | 14 | OPERATOR_MAP = { |
paul@437 | 15 | "+" : token.OP, |
paul@437 | 16 | "-" : token.OP, |
paul@437 | 17 | } |
paul@437 | 18 | KEYWORD_TOKEN = token.NAME |
paul@437 | 19 | |
paul@437 | 20 | |
paul@437 | 21 | class TestParserGenerator: |
paul@437 | 22 | |
paul@437 | 23 | def gram_for(self, grammar_source): |
paul@437 | 24 | p = ParserGenerator(grammar_source + "\n") |
paul@437 | 25 | return p.build_grammar(MyGrammar) |
paul@437 | 26 | |
paul@437 | 27 | def test_multiple_rules(self): |
paul@437 | 28 | g = self.gram_for("foo: NAME bar\nbar: STRING") |
paul@437 | 29 | assert len(g.dfas) == 2 |
paul@437 | 30 | assert g.start == g.symbol_ids["foo"] |
paul@437 | 31 | |
paul@437 | 32 | def test_simple(self): |
paul@437 | 33 | g = self.gram_for("eval: NAME\n") |
paul@437 | 34 | assert len(g.dfas) == 1 |
paul@437 | 35 | eval_sym = g.symbol_ids["eval"] |
paul@437 | 36 | assert g.start == eval_sym |
paul@437 | 37 | states, first = g.dfas[eval_sym - 256] |
paul@437 | 38 | assert states == [([(1, 1)], False), ([], True)] |
paul@437 | 39 | assert g.labels[0] == 0 |
paul@437 | 40 | |
paul@437 | 41 | def test_load_python_grammars(self): |
paul@437 | 42 | gram_pat = os.path.join(os.path.dirname(__file__), "..", "data", |
paul@437 | 43 | "Grammar*") |
paul@437 | 44 | for gram_file in glob.glob(gram_pat): |
paul@437 | 45 | fp = open(gram_file, "r") |
paul@437 | 46 | try: |
paul@437 | 47 | ParserGenerator(fp.read()).build_grammar(PythonGrammar) |
paul@437 | 48 | finally: |
paul@437 | 49 | fp.close() |
paul@437 | 50 | |
paul@437 | 51 | def test_items(self): |
paul@437 | 52 | g = self.gram_for("foo: NAME STRING OP '+'") |
paul@437 | 53 | assert len(g.dfas) == 1 |
paul@437 | 54 | states = g.dfas[g.symbol_ids["foo"] - 256][0] |
paul@437 | 55 | last = states[0][0][0][1] |
paul@437 | 56 | for state in states[1:-1]: |
paul@437 | 57 | assert last < state[0][0][1] |
paul@437 | 58 | last = state[0][0][1] |
paul@437 | 59 | |
paul@437 | 60 | def test_alternatives(self): |
paul@437 | 61 | g = self.gram_for("foo: STRING | OP") |
paul@437 | 62 | assert len(g.dfas) == 1 |
paul@437 | 63 | |
paul@437 | 64 | def test_optional(self): |
paul@437 | 65 | g = self.gram_for("foo: [NAME]") |
paul@437 | 66 | |
paul@437 | 67 | def test_grouping(self): |
paul@437 | 68 | g = self.gram_for("foo: (NAME | STRING) OP") |
paul@437 | 69 | |
paul@437 | 70 | def test_keyword(self): |
paul@437 | 71 | g = self.gram_for("foo: 'some_keyword' 'for'") |
paul@437 | 72 | assert len(g.keyword_ids) == 2 |
paul@437 | 73 | assert len(g.token_ids) == 0 |
paul@437 | 74 | |
paul@437 | 75 | def test_token(self): |
paul@437 | 76 | g = self.gram_for("foo: NAME") |
paul@437 | 77 | assert len(g.token_ids) == 1 |
paul@437 | 78 | |
paul@437 | 79 | def test_operator(self): |
paul@437 | 80 | g = self.gram_for("add: NUMBER '+' NUMBER") |
paul@437 | 81 | assert len(g.keyword_ids) == 0 |
paul@437 | 82 | assert len(g.token_ids) == 2 |
paul@437 | 83 | |
paul@437 | 84 | exc = py.test.raises(PgenError, self.gram_for, "add: '/'").value |
paul@437 | 85 | assert str(exc) == "no such operator: '/'" |
paul@437 | 86 | |
paul@437 | 87 | def test_symbol(self): |
paul@437 | 88 | g = self.gram_for("foo: some_other_rule\nsome_other_rule: NAME") |
paul@437 | 89 | assert len(g.dfas) == 2 |
paul@437 | 90 | assert len(g.labels) == 3 |
paul@437 | 91 | |
paul@437 | 92 | exc = py.test.raises(PgenError, self.gram_for, "foo: no_rule").value |
paul@437 | 93 | assert str(exc) == "no such rule: 'no_rule'" |
paul@437 | 94 | |
paul@437 | 95 | def test_repeaters(self): |
paul@437 | 96 | g1 = self.gram_for("foo: NAME+") |
paul@437 | 97 | g2 = self.gram_for("foo: NAME*") |
paul@437 | 98 | assert g1.dfas != g2.dfas |
paul@437 | 99 | |
paul@437 | 100 | g = self.gram_for("foo: (NAME | STRING)*") |
paul@437 | 101 | g = self.gram_for("foo: (NAME | STRING)+") |
paul@437 | 102 | |
paul@437 | 103 | def test_error(self): |
paul@437 | 104 | exc = py.test.raises(PgenError, self.gram_for, "hi").value |
paul@437 | 105 | assert str(exc) == "expected token OP but got NEWLINE" |
paul@437 | 106 | assert exc.location == ((1, 2), (1, 3), "hi\n") |
paul@437 | 107 | exc = py.test.raises(PgenError, self.gram_for, "hi+").value |
paul@437 | 108 | assert str(exc) == "expected ':' but got '+'" |
paul@437 | 109 | assert exc.location == ((1, 2), (1, 3), "hi+\n") |
paul@437 | 110 | |
paul@437 | 111 | def test_comments_and_whitespace(self): |
paul@437 | 112 | self.gram_for("\n\n# comment\nrule: NAME # comment") |