Implement negatives, use typedef

[fur] / tokenization.py

diff --git a/tokenization.py b/tokenization.py
index 33f4331..819a0de 100644 (file)
--- a/tokenization.py
+++ b/tokenization.py
@@ -5,12 +5,19 @@ import util
 
 Token = collections.namedtuple(
     'Token',
-    [
+    (
         'type',
         'match',
+        'metadata',
+    ),
+)
+
+NodeMetadata = collections.namedtuple(
+    'NodeMetadata',
+    (
         'index',
         'line',
-    ],
+    ),
 )
 
 def _make_token_matcher(definition):
@@ -26,21 +33,31 @@ def _make_token_matcher(definition):
         return (
             True,
             index + len(match.group()),
-            Token(type=name, match=match.group(), index=index, line=line),
+            Token(
+                type=name,
+                match=match.group(),
+                metadata=NodeMetadata(
+                    index=index,
+                    line=line,
+                ),
+            ),
         )
 
     return token_matcher
 
 _TOKEN_MATCHERS = [
-    ('keyword',                         r'(def|do|else|end|if)(?![a-z_])'),
+    ('keyword',                         r'(def|do|else|end|if|lambda)(?![a-z_])'),
     ('open_bracket',                    r'\['),
     ('close_bracket',                   r'\]'),
     ('open_parenthese',                 r'\('),
     ('close_parenthese',                r'\)'),
     ('comma',                           r','),
+    ('colon',                           r':'),
+    ('period',                          r'\.'),
     ('integer_literal',                 r'\d+'),
     ('symbol',                          r'[a-z_]+'),
     ('single_quoted_string_literal',    r"'.*?'"),
+    ('double_quoted_string_literal',    r'".*?"'),
     ('comparison_level_operator',       r'(<=|>=|==|!=|<|>)'),
     ('assignment_operator',             r'='),
     ('addition_level_operator',         r'(\+\+|\+|-)'),