('open_parenthese', r'\('),
('close_parenthese', r'\)'),
('comma', r','),
- ('assignment_operator', r'='),
('integer_literal', r'\d+'),
('symbol', r'[a-z]+'),
('single_quoted_string_literal', r"'.*?'"),
+ ('equality_level_operator', r'(<=|>=|==|!=|<|>)'),
('addition_level_operator', r'(\+|-)'),
- ('multiplication_level_operator', r'(\*|//|%)'),
+ ('multiplication_level_operator', r'(\*|//|%)'),
+ ('assignment_operator', r'='),
]
_TOKEN_MATCHERS = list(map(_make_token_matcher, _TOKEN_MATCHERS))
),),
)
+ def test_tokenizes_equality_operator(self):
+ self.assertEqual(
+ tokenize('=='),
+ (Token(
+ type='equality_level_operator',
+ match='==',
+ index=0,
+ line=1,
+ ),),
+ )
+
+ def test_tokenizes_greater_than_or_equal_operator(self):
+ self.assertEqual(
+ tokenize('>='),
+ (Token(
+ type='equality_level_operator',
+ match='>=',
+ index=0,
+ line=1,
+ ),),
+ )
+
+ def test_tokenizes_less_than_or_equal_operator(self):
+ self.assertEqual(
+ tokenize('<='),
+ (Token(
+ type='equality_level_operator',
+ match='<=',
+ index=0,
+ line=1,
+ ),),
+ )
+
+ def test_tokenizes_greater_than_equal_operator(self):
+ self.assertEqual(
+ tokenize('>'),
+ (Token(
+ type='equality_level_operator',
+ match='>',
+ index=0,
+ line=1,
+ ),),
+ )
+
+ def test_tokenizes_less_than_equal_operator(self):
+ self.assertEqual(
+ tokenize('<'),
+ (Token(
+ type='equality_level_operator',
+ match='<',
+ index=0,
+ line=1,
+ ),),
+ )
+
+ def test_tokenizes_not_equal_operator(self):
+ self.assertEqual(
+ tokenize('!='),
+ (Token(
+ type='equality_level_operator',
+ match='!=',
+ index=0,
+ line=1,
+ ),),
+ )
+
def test_handles_trailing_newline(self):
self.assertEqual(
tokenize('print\n'),