Token = collections.namedtuple(
'Token',
- [
+ (
'type',
'match',
+ 'metadata',
+ ),
+)
+
+NodeMetadata = collections.namedtuple(
+ 'NodeMetadata',
+ (
'index',
'line',
- ],
+ ),
)
def _make_token_matcher(definition):
return (
True,
index + len(match.group()),
- Token(type=name, match=match.group(), index=index, line=line),
+ Token(
+ type=name,
+ match=match.group(),
+ metadata=NodeMetadata(
+ index=index,
+ line=line,
+ ),
+ ),
)
return token_matcher
_TOKEN_MATCHERS = [
- ('keyword', r'(def|end)(?![a-z_])'),
+ ('keyword', r'(def|do|else|end|if|lambda)(?![a-z_])'),
+ ('open_bracket', r'\['),
+ ('close_bracket', r'\]'),
('open_parenthese', r'\('),
('close_parenthese', r'\)'),
('comma', r','),
+ ('colon', r':'),
+ ('period', r'\.'),
('integer_literal', r'\d+'),
('symbol', r'[a-z_]+'),
('single_quoted_string_literal', r"'.*?'"),
- ('comparison_level_operator', r'(<=|>=|==|!=|<|>)'),
+ ('double_quoted_string_literal', r'".*?"'),
+ ('comparison_level_operator', r'(<=|>=|==|!=|<|>)'),
('assignment_operator', r'='),
- ('addition_level_operator', r'(\+|-)'),
+ ('addition_level_operator', r'(\+\+|\+|-)'),
('multiplication_level_operator', r'(\*|//|%)'),
('newline', r'\n'),
]
index += 1
continue
+ if source[index] == '#':
+ while index < len(source) and source[index] != '\n':
+ index += 1
+
+ continue
+
success = False
for matcher in _TOKEN_MATCHERS: