X-Git-Url: https://code.kerkeslager.com/?p=fur;a=blobdiff_plain;f=tokenization.py;fp=tokenization.py;h=e12b8ec718680018e8d3f44d2324fe335eaf6558;hp=f73ddf8d73f6fd02340252a21a07ba721a2e8f99;hb=3297817843cd6bd087505a70d8e108f1baa35cff;hpb=a96e96f3f783930707122f691cd6a08a90416a74

diff --git a/tokenization.py b/tokenization.py
index f73ddf8..e12b8ec 100644
--- a/tokenization.py
+++ b/tokenization.py
@@ -33,13 +33,15 @@ def _make_token_matcher(definition):
 
 _TOKEN_MATCHERS = [
     ('keyword',                         r'(def|do|else|end|if)(?![a-z_])'),
+    ('open_bracket',                    r'\['),
+    ('close_bracket',                   r'\]'),
     ('open_parenthese',                 r'\('),
     ('close_parenthese',                r'\)'),
     ('comma',                           r','),
     ('integer_literal',                 r'\d+'),
     ('symbol',                          r'[a-z_]+'),
     ('single_quoted_string_literal',    r"'.*?'"),
-    ('comparison_level_operator',         r'(<=|>=|==|!=|<|>)'),
+    ('comparison_level_operator',       r'(<=|>=|==|!=|<|>)'),
     ('assignment_operator',             r'='),
     ('addition_level_operator',         r'(\+|-)'),
     ('multiplication_level_operator',   r'(\*|//|%)'),