X-Git-Url: https://code.kerkeslager.com/?a=blobdiff_plain;f=tokenization.py;h=f73ddf8d73f6fd02340252a21a07ba721a2e8f99;hb=a96e96f3f783930707122f691cd6a08a90416a74;hp=a736912603e0832efc974b07252cf03ebe6fdc38;hpb=ec8ce6f417bad0e61e82462787fbb7d7dbe25ea0;p=fur

diff --git a/tokenization.py b/tokenization.py
index a736912..f73ddf8 100644
--- a/tokenization.py
+++ b/tokenization.py
@@ -31,13 +31,13 @@ def _make_token_matcher(definition):
 
     return token_matcher
 
-
 _TOKEN_MATCHERS = [
+    ('keyword',                         r'(def|do|else|end|if)(?![a-z_])'),
     ('open_parenthese',                 r'\('),
     ('close_parenthese',                r'\)'),
     ('comma',                           r','),
     ('integer_literal',                 r'\d+'),
-    ('symbol',                          r'[a-z]+'),
+    ('symbol',                          r'[a-z_]+'),
     ('single_quoted_string_literal',    r"'.*?'"),
     ('comparison_level_operator',         r'(<=|>=|==|!=|<|>)'),
     ('assignment_operator',             r'='),
@@ -58,6 +58,12 @@ def tokenize(source):
             index += 1
             continue
 
+        if source[index] == '#':
+            while index < len(source) and source[index] != '\n':
+                index += 1
+
+            continue
+
         success = False
 
         for matcher in _TOKEN_MATCHERS: