Lichen

Annotated pyparser/pytoken.py

581:7c5a7efcaf14
2017-02-13 Paul Boddie Added __str__ to the wrapper class to delegate to wrapped objects. method-wrapper-for-context
paul@437 1
"""Python token definitions."""
paul@437 2
paul@437 3
python_tokens = {}
paul@437 4
python_opmap = {}
paul@437 5
paul@437 6
def _add_tok(name, *values):
paul@437 7
    index = len(python_tokens)
paul@437 8
    assert index < 256
paul@437 9
    python_tokens[name] = index
paul@437 10
    for value in values:
paul@437 11
        python_opmap[value] = index
paul@437 12
paul@437 13
_add_tok('ENDMARKER')
paul@437 14
_add_tok('NAME')
paul@437 15
_add_tok('NUMBER')
paul@437 16
_add_tok('STRING')
paul@437 17
_add_tok('NEWLINE')
paul@437 18
_add_tok('INDENT')
paul@437 19
_add_tok('DEDENT')
paul@437 20
_add_tok('LPAR', "(")
paul@437 21
_add_tok('RPAR', ")")
paul@437 22
_add_tok('LSQB', "[")
paul@437 23
_add_tok('RSQB', "]")
paul@437 24
_add_tok('COLON', ":")
paul@437 25
_add_tok('COMMA',  "," )
paul@437 26
_add_tok('SEMI', ";" )
paul@437 27
_add_tok('PLUS', "+" )
paul@437 28
_add_tok('MINUS', "-" )
paul@437 29
_add_tok('STAR', "*" )
paul@437 30
_add_tok('SLASH', "/" )
paul@437 31
_add_tok('VBAR', "|" )
paul@437 32
_add_tok('AMPER', "&" )
paul@437 33
_add_tok('LESS', "<" )
paul@437 34
_add_tok('GREATER', ">" )
paul@437 35
_add_tok('EQUAL', "=" )
paul@437 36
_add_tok('DOT', "." )
paul@437 37
_add_tok('PERCENT', "%" )
paul@437 38
_add_tok('BACKQUOTE', "`" )
paul@437 39
_add_tok('LBRACE', "{" )
paul@437 40
_add_tok('RBRACE', "}" )
paul@437 41
_add_tok('EQEQUAL', "==" )
paul@437 42
_add_tok('NOTEQUAL', "!=", "<>" )
paul@437 43
_add_tok('LESSEQUAL', "<=" )
paul@437 44
_add_tok('GREATEREQUAL', ">=" )
paul@437 45
_add_tok('TILDE', "~" )
paul@437 46
_add_tok('CIRCUMFLEX', "^" )
paul@437 47
_add_tok('LEFTSHIFT', "<<" )
paul@437 48
_add_tok('RIGHTSHIFT', ">>" )
paul@437 49
_add_tok('DOUBLESTAR', "**" )
paul@437 50
_add_tok('PLUSEQUAL', "+=" )
paul@437 51
_add_tok('MINEQUAL', "-=" )
paul@437 52
_add_tok('STAREQUAL', "*=" )
paul@437 53
_add_tok('SLASHEQUAL', "/=" )
paul@437 54
_add_tok('PERCENTEQUAL', "%=" )
paul@437 55
_add_tok('AMPEREQUAL', "&=" )
paul@437 56
_add_tok('VBAREQUAL', "|=" )
paul@437 57
_add_tok('CIRCUMFLEXEQUAL', "^=" )
paul@437 58
_add_tok('LEFTSHIFTEQUAL', "<<=" )
paul@437 59
_add_tok('RIGHTSHIFTEQUAL', ">>=" )
paul@437 60
_add_tok('DOUBLESTAREQUAL', "**=" )
paul@437 61
_add_tok('DOUBLESLASH', "//" )
paul@437 62
_add_tok('DOUBLESLASHEQUAL',"//=" )
paul@437 63
_add_tok('AT', "@" )
paul@437 64
_add_tok('OP')
paul@437 65
_add_tok('ERRORTOKEN')
paul@437 66
paul@437 67
# extra PyPy-specific tokens
paul@437 68
_add_tok("COMMENT")
paul@437 69
_add_tok("NL")
paul@437 70
paul@437 71
del _add_tok