]>
Commit | Line | Data |
---|---|---|
53e6db90 DC |
1 | from __future__ import absolute_import |
2 | ||
3 | from enum import Enum | |
4 | ||
5 | ||
6 | class TokenType: | |
7 | name: str | |
8 | contains_syntax: bool | |
9 | ||
10 | def __init__(self, name: str, contains_syntax: bool = False): | |
11 | self.name = name | |
12 | self.contains_syntax = contains_syntax | |
13 | ||
14 | def __repr__(self): | |
15 | return '%s(%s)' % (self.__class__.__name__, self.name) | |
16 | ||
17 | ||
18 | class PythonTokenTypes(Enum): | |
19 | STRING = TokenType('STRING') | |
20 | NUMBER = TokenType('NUMBER') | |
21 | NAME = TokenType('NAME', contains_syntax=True) | |
22 | ERRORTOKEN = TokenType('ERRORTOKEN') | |
23 | NEWLINE = TokenType('NEWLINE') | |
24 | INDENT = TokenType('INDENT') | |
25 | DEDENT = TokenType('DEDENT') | |
26 | ERROR_DEDENT = TokenType('ERROR_DEDENT') | |
27 | FSTRING_STRING = TokenType('FSTRING_STRING') | |
28 | FSTRING_START = TokenType('FSTRING_START') | |
29 | FSTRING_END = TokenType('FSTRING_END') | |
30 | OP = TokenType('OP', contains_syntax=True) | |
31 | ENDMARKER = TokenType('ENDMARKER') |