Added support for integers

[fur] / tokenization.py

diff --git a/tokenization.py b/tokenization.py
index c1b30df..10ed80f 100644 (file)
--- a/tokenization.py
+++ b/tokenization.py
@@ -29,6 +29,7 @@ def _make_token_matcher(definition):
 _TOKEN_MATCHERS = [
     ('open_parenthese',                 r'\('),
     ('close_parenthese',                r'\)'),
+    ('integer_literal',                 r'-?\s*\d+'),
     ('symbol',                          r'[a-z]+'),
     ('single_quoted_string_literal',    r"'.*?'"),
 ]