Lichen

pyparser/pytoken.py

934:2989aab1b4f7
6 months ago Paul Boddie Renamed the utf8string class to unicode, eliminating the unicode function. This means that the simple case of merely returning an object if it is already a Unicode object no longer occurs when using the unicode callable, but such behaviour might be better supported with more general customised instantiation functionality.
     1 """Python token definitions."""     2      3 python_tokens = {}     4 python_opmap = {}     5      6 def _add_tok(name, *values):     7     index = len(python_tokens)     8     assert index < 256     9     python_tokens[name] = index    10     for value in values:    11         python_opmap[value] = index    12     13 _add_tok('ENDMARKER')    14 _add_tok('NAME')    15 _add_tok('NUMBER')    16 _add_tok('STRING')    17 _add_tok('NEWLINE')    18 _add_tok('INDENT')    19 _add_tok('DEDENT')    20 _add_tok('LPAR', "(")    21 _add_tok('RPAR', ")")    22 _add_tok('LSQB', "[")    23 _add_tok('RSQB', "]")    24 _add_tok('COLON', ":")    25 _add_tok('COMMA',  "," )    26 _add_tok('SEMI', ";" )    27 _add_tok('PLUS', "+" )    28 _add_tok('MINUS', "-" )    29 _add_tok('STAR', "*" )    30 _add_tok('SLASH', "/" )    31 _add_tok('VBAR', "|" )    32 _add_tok('AMPER', "&" )    33 _add_tok('LESS', "<" )    34 _add_tok('GREATER', ">" )    35 _add_tok('EQUAL', "=" )    36 _add_tok('DOT', "." )    37 _add_tok('PERCENT', "%" )    38 _add_tok('BACKQUOTE', "`" )    39 _add_tok('LBRACE', "{" )    40 _add_tok('RBRACE', "}" )    41 _add_tok('EQEQUAL', "==" )    42 _add_tok('NOTEQUAL', "!=", "<>" )    43 _add_tok('LESSEQUAL', "<=" )    44 _add_tok('GREATEREQUAL', ">=" )    45 _add_tok('TILDE', "~" )    46 _add_tok('CIRCUMFLEX', "^" )    47 _add_tok('LEFTSHIFT', "<<" )    48 _add_tok('RIGHTSHIFT', ">>" )    49 _add_tok('DOUBLESTAR', "**" )    50 _add_tok('PLUSEQUAL', "+=" )    51 _add_tok('MINEQUAL', "-=" )    52 _add_tok('STAREQUAL', "*=" )    53 _add_tok('SLASHEQUAL', "/=" )    54 _add_tok('PERCENTEQUAL', "%=" )    55 _add_tok('AMPEREQUAL', "&=" )    56 _add_tok('VBAREQUAL', "|=" )    57 _add_tok('CIRCUMFLEXEQUAL', "^=" )    58 _add_tok('LEFTSHIFTEQUAL', "<<=" )    59 _add_tok('RIGHTSHIFTEQUAL', ">>=" )    60 _add_tok('DOUBLESTAREQUAL', "**=" )    61 _add_tok('DOUBLESLASH', "//" )    62 _add_tok('DOUBLESLASHEQUAL',"//=" )    63 _add_tok('AT', "@" )    64 _add_tok('OP')    65 _add_tok('ERRORTOKEN')    66     67 # extra PyPy-specific tokens    68 _add_tok("COMMENT")    69 _add_tok("NL")    70     71 del _add_tok