projects
/
fur
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
|
inline
| side by side
Add the ability to assign to and retrieve variables
[fur]
/
tokenization.py
diff --git
a/tokenization.py
b/tokenization.py
index
f316e5e
..
e9f536b
100644
(file)
--- a/
tokenization.py
+++ b/
tokenization.py
@@
-36,6
+36,7
@@
_TOKEN_MATCHERS = [
('open_parenthese', r'\('),
('close_parenthese', r'\)'),
('comma', r','),
('open_parenthese', r'\('),
('close_parenthese', r'\)'),
('comma', r','),
+ ('assignment_operator', r'='),
('integer_literal', r'\d+'),
('symbol', r'[a-z]+'),
('single_quoted_string_literal', r"'.*?'"),
('integer_literal', r'\d+'),
('symbol', r'[a-z]+'),
('single_quoted_string_literal', r"'.*?'"),
@@
-185,6
+186,16
@@
if __name__ == '__main__':
),),
)
),),
)
+ def test_tokenizes_assignment_operator(self):
+ self.assertEqual(
+ tokenize('='),
+ (Token(
+ type='assignment_operator',
+ match='=',
+ index=0,
+ line=1,
+ ),),
+ )
def test_handles_trailing_newline(self):
self.assertEqual(
def test_handles_trailing_newline(self):
self.assertEqual(