tokens.py

from sys import intern

WHITESPACE = intern('whitespace')
COMMENT = intern('comment')
COMMENT_BEGIN = intern('comment_begin')
COMMENT_END = intern('comment_end')
BLOCK = intern('block')
BLOCK_BEGIN = intern('block_begin')
BLOCK_END = intern('block_end')
VARIABLE_BEGIN = intern('variable_begin')
VARIABLE_END = intern('variable_end')
DATA  = intern('data')
NAME = intern('name')
INTEGER = intern('integer')
FLOAT = intern('float')
STRING = intern('string')
OPERATOR = intern('operator')
ADD = intern('add')
SUB = intern('sub')
MOD = intern('mod')
DIV = intern('div')
MUL = intern('mul')
FLOOR_DIV = intern('floor_div')
ASSIGN = intern('assign')
EQ = intern('eq')
GT = intern('gt')
LT = intern('lt')
GTEQ = intern('gteq')
LTEQ = intern('lteq')
LPAREN = intern('lparen')
RPAREN = intern('rparen')
LBRACE = intern('lbrace')
RBRACE = intern('rbrace')
LBRACKET = intern('lbracket')
RBRACKET = intern('rbracket')
COLON = intern('colon')
SEMICOLON = intern('semicolon')
DOT = intern('dot')
TILDE = intern('tilde')
COMMA = intern('comma')
PIPE = intern('pipe')
NE = intern('ne')
POW = intern('pow')
INITIAL = intern('initial')
EOF = intern('eof')