projects
/
fur
/ blobdiff
commit
grep
author
committer
pickaxe
?
search:
re
summary
|
shortlog
|
log
|
commit
|
commitdiff
|
tree
raw
| inline |
side by side
Normalize symbol expressions
[fur]
/
tokenization.py
diff --git
a/tokenization.py
b/tokenization.py
index
1a2653e
..
3c4dc6f
100644
(file)
--- a/
tokenization.py
+++ b/
tokenization.py
@@
-58,6
+58,12
@@
def tokenize(source):
index += 1
continue
+ if source[index] == '#':
+ while index < len(source) and source[index] != '\n':
+ index += 1
+
+ continue
+
success = False
for matcher in _TOKEN_MATCHERS: