3 def consume_newlines(index, tokens):
4 while index < len(tokens) and tokens[index].type == 'newline':
7 return True, index, None
9 def _or_parser(*parsers):
10 def result_parser(index, tokens):
11 failure = (False, index, None)
13 for parser in parsers:
14 success, index, value = parser(index, tokens)
17 return (success, index, value)
23 def _zero_or_more_parser(formatter, parser):
24 def result_parser(index, tokens):
27 while index < len(tokens):
28 success, index, value = parser(index, tokens)
35 return (True, index, formatter(values))
39 FurIntegerLiteralExpression = collections.namedtuple(
40 'FurIntegerLiteralExpression',
46 FurStringLiteralExpression = collections.namedtuple(
47 'FurStringLiteralExpression',
53 FurSymbolExpression = collections.namedtuple(
54 'FurSymbolExpression',
60 FurNegationExpression = collections.namedtuple(
61 'FurNegationExpression',
67 FurInfixExpression = collections.namedtuple(
77 FurIfExpression = collections.namedtuple(
80 'condition_expression',
82 'else_statement_list',
86 def _integer_literal_expression_parser(index, tokens):
87 failure = (False, index, None)
89 if tokens[index].type != 'integer_literal':
91 value = int(tokens[index].match)
94 return True, index, FurIntegerLiteralExpression(integer=value)
96 def _string_literal_expression_parser(index, tokens):
97 if tokens[index].type == 'single_quoted_string_literal':
98 return (True, index + 1, FurStringLiteralExpression(string=tokens[index].match[1:-1]))
100 return (False, index, None)
102 def _symbol_expression_parser(index, tokens):
103 if tokens[index].type == 'symbol':
104 return (True, index + 1, FurSymbolExpression(symbol=tokens[index].match))
106 return (False, index, None)
108 def _parenthese_wrapped_parser(internal_parser):
109 def result_parser(index, tokens):
110 failure = (False, index, None)
112 if tokens[index].type == 'open_parenthese':
117 success, index, internal = internal_parser(index, tokens)
121 if tokens[index].type == 'close_parenthese':
124 raise Exception('Expected ")" on line {}, found "{}"'.format(
129 return True, index, internal
133 def _parenthesized_expression_parser(index, tokens):
134 return _parenthese_wrapped_parser(_expression_parser)(index, tokens)
136 def _negation_expression_parser(index, tokens):
137 failure = (False, index, None)
139 if tokens[index].match != '-':
142 success, index, value = _literal_level_expression_parser(index + 1, tokens)
147 return (True, index, FurNegationExpression(value=value))
149 def _literal_level_expression_parser(index, tokens):
151 _negation_expression_parser,
152 _function_call_expression_parser,
153 _parenthesized_expression_parser,
154 _integer_literal_expression_parser,
155 _string_literal_expression_parser,
156 _symbol_expression_parser,
159 def _left_recursive_infix_operator_parser(operator_token_matcher, operand_parser, order):
160 def result_parser(index, tokens):
161 failure = (False, index, None)
163 success, index, result = operand_parser(index, tokens)
168 while success and index < len(tokens) and operator_token_matcher(tokens[index]):
171 if index + 1 < len(tokens):
172 success, try_index, value = operand_parser(index + 1, tokens)
175 result = FurInfixExpression(
177 operator=tokens[index].match,
183 return True, index, result
187 def _multiplication_level_expression_parser(index, tokens):
188 return _left_recursive_infix_operator_parser(
189 lambda token: token.type == 'multiplication_level_operator',
190 _literal_level_expression_parser,
191 'multiplication_level',
194 def _addition_level_expression_parser(index, tokens):
195 return _left_recursive_infix_operator_parser(
196 lambda token: token.type == 'addition_level_operator',
197 _multiplication_level_expression_parser,
201 def _comparison_level_expression_parser(index, tokens):
202 return _left_recursive_infix_operator_parser(
203 lambda token: token.type == 'comparison_level_operator',
204 _addition_level_expression_parser,
208 def _and_level_expression_parser(index, tokens):
209 return _left_recursive_infix_operator_parser(
210 lambda token: token.type == 'symbol' and token.match == 'and',
211 _comparison_level_expression_parser,
215 def _or_level_expression_parser(index, tokens):
216 return _left_recursive_infix_operator_parser(
217 lambda token: token.type == 'symbol' and token.match == 'or',
218 _and_level_expression_parser,
222 def _comma_separated_list_parser(subparser):
223 def result_parser(index, tokens):
228 success, index, item = subparser(index, tokens)
233 return (True, start_index, ())
235 while success and index < len(tokens) and tokens[index].type == 'comma':
238 if index + 1 < len(tokens):
239 success, try_index, item = subparser(index + 1, tokens)
245 return True, index, tuple(items)
249 def _comma_separated_expression_list_parser(index, tokens):
250 return _comma_separated_list_parser(_expression_parser)(index, tokens)
252 FurFunctionCallExpression = collections.namedtuple(
253 'FurFunctionCallExpression',
260 FurExpressionStatement = collections.namedtuple(
261 'FurExpressionStatement',
267 FurAssignmentStatement = collections.namedtuple(
268 'FurAssignmentStatement',
275 FurFunctionDefinitionStatement = collections.namedtuple(
276 'FurFunctionDefinitionStatement',
279 'argument_name_list',
284 FurProgram = collections.namedtuple(
291 def _function_call_expression_parser(index, tokens):
292 failure = (False, index, None)
294 # We have to be careful what expressions we add here. Otherwise expressions
295 # like "a + b()" become ambiguous to the parser.
296 success, index, function = _or_parser(
297 _symbol_expression_parser,
298 _parenthesized_expression_parser,
304 success, index, arguments = _parenthese_wrapped_parser(_comma_separated_expression_list_parser)(
312 while success and index < len(tokens):
313 # "function" is actually the full function call if the next parse attempt doesn't succeed
314 # We can't give this a better name without a bunch of checks, however.
315 function = FurFunctionCallExpression(
320 success, index, arguments = _parenthese_wrapped_parser(_comma_separated_expression_list_parser)(
325 return True, index, function
327 def _if_expression_parser(index, tokens):
328 failure = (False, index, None)
330 if tokens[index].match == 'if':
335 success, index, condition_expression = _or_level_expression_parser(index, tokens)
338 raise Exception('Expected condition after "if" on line {}'.format(tokens[index].line))
340 if tokens[index].match == 'do':
343 raise Exception('Expected "do" after "if" on line {}'.format(tokens[index].line))
346 success, index, if_statement_list = _zero_or_more_parser(tuple, _statement_parser)(index, tokens)
347 _, index, _ = consume_newlines(index, tokens)
349 if tokens[index].match == 'else':
351 success, index, else_statement_list = _zero_or_more_parser(tuple, _statement_parser)(index, tokens)
352 _, index, _ = consume_newlines(index, tokens)
354 else_statement_list = ()
356 if tokens[index].match == 'end':
359 raise Exception('Expected "end" after "if" on line {}'.format(tokens[index].line))
365 condition_expression=condition_expression,
366 if_statement_list=if_statement_list,
367 else_statement_list=else_statement_list,
374 _expression_parser = _or_parser(
375 _or_level_expression_parser,
376 _if_expression_parser, # This should always be at the top level
379 def _expression_statement_parser(index, tokens):
380 failure = (False, index, None)
382 success, index, expression = _expression_parser(index, tokens)
387 return (True, index, FurExpressionStatement(expression=expression))
389 BUILTINS = {'print', 'pow'}
391 def _assignment_statement_parser(index, tokens):
392 failure = (False, index, None)
394 if tokens[index].type == 'symbol':
395 target = tokens[index].match
396 target_assignment_line = tokens[index].line
403 if tokens[index].type == 'assignment_operator':
404 if target in BUILTINS:
406 'Trying to assign to builtin "{}" on line {}'.format(target, target_assignment_line),
408 assignment_operator_index = index
412 success, index, expression = _expression_parser(index + 1, tokens)
416 'Expected expression after assignment operator on line {}'.format(
417 tokens[assignment_operator_index].line
421 return True, index, FurAssignmentStatement(target=target, expression=expression)
423 def _function_definition_statement_parser(index, tokens):
424 failure = (False, index, None)
426 if tokens[index].type == 'keyword' and tokens[index].match == 'def':
431 if tokens[index].type == 'symbol':
432 name = tokens[index].match
435 raise Exception('Expected function name, found "{}" on line {}'.format(
440 if tokens[index].type == 'open_parenthese':
443 raise Exception('Expected "(", found "{}" on line {}'.format(
448 success, index, argument_name_list = _comma_separated_list_parser(_symbol_expression_parser)(
453 if tokens[index].type == 'close_parenthese':
456 raise Exception('Expected ")", found "{}" on line {}'.format(
461 if tokens[index].match == 'do':
466 success, index, statement_list = _zero_or_more_parser(tuple, _statement_parser)(index, tokens)
468 _, index, _ = consume_newlines(index, tokens)
470 if tokens[index].type == 'keyword' and tokens[index].match == 'end':
475 return True, index, FurFunctionDefinitionStatement(
477 argument_name_list=tuple(an.symbol for an in argument_name_list),
478 statement_list=statement_list,
481 def _statement_parser(index, tokens):
482 _, index, _ = consume_newlines(index, tokens)
484 if index == len(tokens):
485 return (False, index, None)
488 _assignment_statement_parser,
489 _expression_statement_parser,
490 _function_definition_statement_parser,
493 def _program_formatter(statement_list):
494 return FurProgram(statement_list=statement_list)
496 _program_parser = _zero_or_more_parser(_program_formatter, _statement_parser)
498 def _parse(parser, tokens):
499 success, index, result = parser(0, tokens)
501 if index < len(tokens):
502 raise Exception('Unable to parse token {}'.format(tokens[index]))
507 raise Exception('Unable to parse')
510 return _parse(_program_parser, tokens)
512 if __name__ == '__main__':
517 class FurStringLiteralExpressionParserTests(unittest.TestCase):
518 def test_parses_single_quoted_string_literal(self):
520 _string_literal_expression_parser(0, tokenization.tokenize("'Hello, world'")),
524 FurStringLiteralExpression(string='Hello, world'),
528 class FurFunctionCallExpressionParserTests(unittest.TestCase):
529 def test_parses_function_with_string_literal_argument(self):
531 _function_call_expression_parser(0, tokenization.tokenize("print('Hello, world')")),
535 FurFunctionCallExpression(
537 arguments=(FurStringLiteralExpression(string='Hello, world'),),