projects
/
fur
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
Add very rudimentary line numbers to exceptions
[fur]
/
tokenization.py
diff --git
a/tokenization.py
b/tokenization.py
index
e12b8ec
..
bc8cf85
100644
(file)
--- a/
tokenization.py
+++ b/
tokenization.py
@@
-38,12
+38,15
@@
_TOKEN_MATCHERS = [
('open_parenthese', r'\('),
('close_parenthese', r'\)'),
('comma', r','),
('open_parenthese', r'\('),
('close_parenthese', r'\)'),
('comma', r','),
+ ('colon', r':'),
+ ('period', r'\.'),
('integer_literal', r'\d+'),
('symbol', r'[a-z_]+'),
('single_quoted_string_literal', r"'.*?'"),
('integer_literal', r'\d+'),
('symbol', r'[a-z_]+'),
('single_quoted_string_literal', r"'.*?'"),
+ ('double_quoted_string_literal', r'".*?"'),
('comparison_level_operator', r'(<=|>=|==|!=|<|>)'),
('assignment_operator', r'='),
('comparison_level_operator', r'(<=|>=|==|!=|<|>)'),
('assignment_operator', r'='),
- ('addition_level_operator', r'(\+|-)'),
+ ('addition_level_operator', r'(\+
\+|\+
|-)'),
('multiplication_level_operator', r'(\*|//|%)'),
('newline', r'\n'),
]
('multiplication_level_operator', r'(\*|//|%)'),
('newline', r'\n'),
]