X-Git-Url: https://code.kerkeslager.com/?p=fur;a=blobdiff_plain;f=tokenization.py;h=bc8cf85a8f5a12aa556095f905bb190ef292c446;hp=1a2653e3c0dc93d6c1208357d29d703b788eadac;hb=1f2895250c61aeb81c0f0ba7efed156c9386dfac;hpb=6e7ba75736ca8b463a263e7d58aa8f6bff62faaf

diff --git a/tokenization.py b/tokenization.py
index 1a2653e..bc8cf85 100644
--- a/tokenization.py
+++ b/tokenization.py
@@ -32,16 +32,21 @@ def _make_token_matcher(definition):
     return token_matcher
 
 _TOKEN_MATCHERS = [
-    ('keyword',                         r'(def|end)(?![a-z_])'),
+    ('keyword',                         r'(def|do|else|end|if)(?![a-z_])'),
+    ('open_bracket',                    r'\['),
+    ('close_bracket',                   r'\]'),
     ('open_parenthese',                 r'\('),
     ('close_parenthese',                r'\)'),
     ('comma',                           r','),
+    ('colon',                           r':'),
+    ('period',                          r'\.'),
     ('integer_literal',                 r'\d+'),
     ('symbol',                          r'[a-z_]+'),
     ('single_quoted_string_literal',    r"'.*?'"),
-    ('comparison_level_operator',         r'(<=|>=|==|!=|<|>)'),
+    ('double_quoted_string_literal',    r'".*?"'),
+    ('comparison_level_operator',       r'(<=|>=|==|!=|<|>)'),
     ('assignment_operator',             r'='),
-    ('addition_level_operator',         r'(\+|-)'),
+    ('addition_level_operator',         r'(\+\+|\+|-)'),
     ('multiplication_level_operator',   r'(\*|//|%)'),
     ('newline',                         r'\n'),
 ]
@@ -58,6 +63,12 @@ def tokenize(source):
             index += 1
             continue
 
+        if source[index] == '#':
+            while index < len(source) and source[index] != '\n':
+                index += 1
+
+            continue
+
         success = False
 
         for matcher in _TOKEN_MATCHERS: