Added if expression statements

[fur] / tokenization.py

diff --git a/tokenization.py b/tokenization.py
index 1a2653e..f73ddf8 100644 (file)
--- a/tokenization.py
+++ b/tokenization.py
@@ -32,7 +32,7 @@ def _make_token_matcher(definition):
     return token_matcher
 
 _TOKEN_MATCHERS = [
-    ('keyword',                         r'(def|end)(?![a-z_])'),
+    ('keyword',                         r'(def|do|else|end|if)(?![a-z_])'),
     ('open_parenthese',                 r'\('),
     ('close_parenthese',                r'\)'),
     ('comma',                           r','),
@@ -58,6 +58,12 @@ def tokenize(source):
             index += 1
             continue
 
+        if source[index] == '#':
+            while index < len(source) and source[index] != '\n':
+                index += 1
+
+            continue
+
         success = False
 
         for matcher in _TOKEN_MATCHERS: