Add double-quoted strings

[fur] / tokenization.py

diff --git a/tokenization.py b/tokenization.py
index 33f4331..02f9528 100644 (file)
--- a/tokenization.py
+++ b/tokenization.py
@@ -41,6 +41,7 @@ _TOKEN_MATCHERS = [
     ('integer_literal',                 r'\d+'),
     ('symbol',                          r'[a-z_]+'),
     ('single_quoted_string_literal',    r"'.*?'"),
+    ('double_quoted_string_literal',    r'".*?"'),
     ('comparison_level_operator',       r'(<=|>=|==|!=|<|>)'),
     ('assignment_operator',             r'='),
     ('addition_level_operator',         r'(\+\+|\+|-)'),