From f60d1b48bbf73c51d214c5ae5c22ea3cdee087c1 Mon Sep 17 00:00:00 2001 From: David Kerkeslager Date: Sat, 5 Aug 2017 15:37:52 -0400 Subject: [PATCH] A pretty featureful commit: * New builtin pow() * Support for multiple arguments to functions * Support for the results of function calls being passed to functions --- examples/07_power.fur | 1 + examples/07_power.fur.output.txt | 1 + generation.py | 16 +++++++--- parsing.py | 50 +++++++++++++++++++++++------- templates/program.c | 13 ++++++++ tokenization.py | 53 ++++++++++++++++++++++++++++++-- transformation.py | 15 +++++---- 7 files changed, 124 insertions(+), 25 deletions(-) create mode 100644 examples/07_power.fur create mode 100644 examples/07_power.fur.output.txt diff --git a/examples/07_power.fur b/examples/07_power.fur new file mode 100644 index 0000000..9b64c60 --- /dev/null +++ b/examples/07_power.fur @@ -0,0 +1 @@ +print(pow(3, 4)) diff --git a/examples/07_power.fur.output.txt b/examples/07_power.fur.output.txt new file mode 100644 index 0000000..c147342 --- /dev/null +++ b/examples/07_power.fur.output.txt @@ -0,0 +1 @@ +81 \ No newline at end of file diff --git a/generation.py b/generation.py index b51650b..808b0d8 100644 --- a/generation.py +++ b/generation.py @@ -24,6 +24,9 @@ def generate_string_literal(c_string_literal): ) def generate_argument(c_argument): + if isinstance(c_argument, transformation.CFunctionCallExpression): + return generate_function_call(c_argument) + LITERAL_TYPE_MAPPING = { transformation.CIntegerLiteral: generate_integer_literal, transformation.CStringLiteral: generate_string_literal, @@ -46,18 +49,21 @@ def generate_argument(c_argument): generate_argument(c_argument.right), ) -def generate_statement(c_function_call_statement): - return '{}({});'.format( - c_function_call_statement.name, - ', '.join(generate_argument(argument) for argument in c_function_call_statement.arguments), +def generate_function_call(c_function_call): + return '{}({})'.format( + c_function_call.name, + ', '.join(generate_argument(argument) for argument in c_function_call.arguments), ) +def generate_statement(c_function_call_statement): + return '{};'.format(generate_function_call(c_function_call_statement)) + def generate(c_program): template = ENV.get_template('program.c') return template.render( builtins=list(sorted(c_program.builtins)), statements=[generate_statement(statement) for statement in c_program.statements], - standard_libraries=set(['stdio.h']), + standard_libraries=list(sorted(c_program.standard_libraries)), ) if __name__ == '__main__': diff --git a/parsing.py b/parsing.py index e8f9068..1ab6df4 100644 --- a/parsing.py +++ b/parsing.py @@ -106,6 +106,7 @@ def _string_literal_expression_parser(index, tokens): def _literal_level_expression_parser(index, tokens): return _or_parser( + _function_call_expression_parser, _integer_literal_expression_parser, _string_literal_expression_parser, )(index, tokens) @@ -157,8 +158,33 @@ def _addition_level_expression_parser(index, tokens): return True, index, result -FunctionCall = collections.namedtuple( - 'FunctionCall', +def _comma_separated_list_parser(index, tokens): + failure = (False, index, None) + + expressions = [] + + success, index, expression = _addition_level_expression_parser(index, tokens) + + if success: + expressions.append(expression) + else: + return failure + + while success and index < len(tokens) and tokens[index].type == 'comma': + success = False + + if index + 1 < len(tokens): + success, try_index, expression = _addition_level_expression_parser(index + 1, tokens) + + if success: + expressions.append(expression) + index = try_index + + return True, index, tuple(expressions) + + +FurFunctionCallExpression = collections.namedtuple( + 'FurFunctionCallExpression', [ 'name', 'arguments', @@ -172,7 +198,7 @@ FurProgram = collections.namedtuple( ], ) -def _function_call_parser(index, tokens): +def _function_call_expression_parser(index, tokens): failure = (False, index, None) if tokens[index].type != 'symbol': @@ -184,21 +210,24 @@ def _function_call_parser(index, tokens): return failure index += 1 - success, index, argument = _addition_level_expression_parser(index, tokens) + success, index, arguments = _comma_separated_list_parser(index, tokens) if not success: return failure if tokens[index].type != 'close_parenthese': - return failure + raise Exception('Expected ")", found "{}" on line {}'.format( + tokens[index].match, + tokens[index].line, + )) index += 1 - return True, index, FunctionCall(name=name, arguments=(argument,)) + return True, index, FurFunctionCallExpression(name=name, arguments=arguments) def _program_formatter(statement_list): return FurProgram(statement_list=statement_list) -_program_parser = _zero_or_more_parser(_program_formatter, _function_call_parser) +_program_parser = _zero_or_more_parser(_program_formatter, _function_call_expression_parser) def _parse(parser, tokens): success, index, result = parser(0, tokens) @@ -211,7 +240,6 @@ def _parse(parser, tokens): raise Exception('Unable to parse') - def parse(tokens): return _parse(_program_parser, tokens) @@ -231,14 +259,14 @@ if __name__ == '__main__': ), ) - class FunctionCallParserTests(unittest.TestCase): + class FurFunctionCallExpressionParserTests(unittest.TestCase): def test_parses_function_with_string_literal_argument(self): self.assertEqual( - _function_call_parser(0, tokenization.tokenize("print('Hello, world')")), + _function_call_expression_parser(0, tokenization.tokenize("print('Hello, world')")), ( True, 4, - FunctionCall( + FurFunctionCallExpression( name='print', arguments=(FurStringLiteralExpression(value='Hello, world'),), ), diff --git a/templates/program.c b/templates/program.c index 274aa33..063587d 100644 --- a/templates/program.c +++ b/templates/program.c @@ -169,6 +169,19 @@ Object builtin$modularDivide(Object left, Object right) return result; } +{% if 'pow' in builtins %} +Object builtin$pow(Object base, Object exponent) +{ + assert(base.type == INTEGER); + assert(exponent.type == INTEGER); + + Object result; + result.type = INTEGER; + result.instance.integer = pow(base.instance.integer, exponent.instance.integer); + return result; +} +{% endif %} + {% if 'print' in builtins %} void builtin$print(Object output) { diff --git a/tokenization.py b/tokenization.py index 0421b84..f316e5e 100644 --- a/tokenization.py +++ b/tokenization.py @@ -9,6 +9,7 @@ Token = collections.namedtuple( 'type', 'match', 'index', + 'line', ], ) @@ -16,7 +17,7 @@ def _make_token_matcher(definition): name, regex = definition regex_matcher = re.compile(regex) - def token_matcher(index, source): + def token_matcher(index, source, line): match = regex_matcher.match(source[index:]) if match is None: @@ -25,7 +26,7 @@ def _make_token_matcher(definition): return ( True, index + len(match.group()), - Token(type=name, match=match.group(), index=index), + Token(type=name, match=match.group(), index=index, line=line), ) return token_matcher @@ -34,6 +35,7 @@ def _make_token_matcher(definition): _TOKEN_MATCHERS = [ ('open_parenthese', r'\('), ('close_parenthese', r'\)'), + ('comma', r','), ('integer_literal', r'\d+'), ('symbol', r'[a-z]+'), ('single_quoted_string_literal', r"'.*?'"), @@ -46,6 +48,7 @@ _TOKEN_MATCHERS = list(map(_make_token_matcher, _TOKEN_MATCHERS)) @util.force_generator(tuple) def tokenize(source): index = 0 + line = 1 while index < len(source): if source[index] == ' ': @@ -55,7 +58,7 @@ def tokenize(source): success = False for matcher in _TOKEN_MATCHERS: - success, index, token = matcher(index, source) + success, index, token = matcher(index, source, line) if success: yield token @@ -65,6 +68,7 @@ def tokenize(source): raise Exception('Unexpected character "{}"'.format(source[index])) while index < len(source) and source[index] in set(['\n']): + line += 1 index += 1 if __name__ == '__main__': @@ -78,6 +82,7 @@ if __name__ == '__main__': type='open_parenthese', match='(', index=0, + line=1, ),), ) @@ -88,6 +93,7 @@ if __name__ == '__main__': type='close_parenthese', match=')', index=0, + line=1, ),), ) @@ -98,6 +104,7 @@ if __name__ == '__main__': type='symbol', match='print', index=0, + line=1, ),), ) @@ -108,6 +115,7 @@ if __name__ == '__main__': type='single_quoted_string_literal', match="'Hello, world'", index=0, + line=1, ),), ) @@ -118,6 +126,7 @@ if __name__ == '__main__': type='addition_level_operator', match='+', index=0, + line=1, ),), ) @@ -128,6 +137,7 @@ if __name__ == '__main__': type='addition_level_operator', match='-', index=0, + line=1, ),), ) @@ -138,6 +148,7 @@ if __name__ == '__main__': type='multiplication_level_operator', match='*', index=0, + line=1, ),), ) @@ -148,6 +159,7 @@ if __name__ == '__main__': type='multiplication_level_operator', match='//', index=0, + line=1, ),), ) @@ -158,9 +170,22 @@ if __name__ == '__main__': type='multiplication_level_operator', match='%', index=0, + line=1, ),), ) + def test_tokenizes_comma(self): + self.assertEqual( + tokenize(','), + (Token( + type='comma', + match=',', + index=0, + line=1, + ),), + ) + + def test_handles_trailing_newline(self): self.assertEqual( tokenize('print\n'), @@ -168,6 +193,7 @@ if __name__ == '__main__': type='symbol', match='print', index=0, + line=1, ),), ) @@ -178,7 +204,28 @@ if __name__ == '__main__': type='symbol', match='print', index=1, + line=1, ),), ) + def test_tokenizes_with_proper_line_numbers(self): + self.assertEqual( + tokenize('print\n('), + ( + Token( + type='symbol', + match='print', + index=0, + line=1, + ), + Token( + type='open_parenthese', + match='(', + index=6, + line=2, + ), + ), + ) + + unittest.main() diff --git a/transformation.py b/transformation.py index cdf8155..1fe7dc9 100644 --- a/transformation.py +++ b/transformation.py @@ -56,8 +56,8 @@ CModularDivisionExpression = collections.namedtuple( ], ) -CFunctionCallStatement = collections.namedtuple( - 'CFunctionCallStatement', +CFunctionCallExpression = collections.namedtuple( + 'CFunctionCallExpression', [ 'name', 'arguments', @@ -74,10 +74,13 @@ CProgram = collections.namedtuple( ) BUILTINS = { - 'print': ['stdio.h.'], + 'pow': ['math.h'], + 'print': ['stdio.h'], } def transform_expression(builtin_dependencies, expression): + if isinstance(expression, parsing.FurFunctionCallExpression): + return transform_function_call_expression(builtin_dependencies, expression) LITERAL_TYPE_MAPPING = { parsing.FurIntegerLiteralExpression: CIntegerLiteral, @@ -100,11 +103,11 @@ def transform_expression(builtin_dependencies, expression): right=transform_expression(builtin_dependencies, expression.right), ) -def transform_function_call_statement(builtin_dependencies, function_call): +def transform_function_call_expression(builtin_dependencies, function_call): if function_call.name in BUILTINS.keys(): builtin_dependencies.add(function_call.name) - return CFunctionCallStatement( + return CFunctionCallExpression( name='builtin$' + function_call.name, arguments=tuple(transform_expression(builtin_dependencies, arg) for arg in function_call.arguments), ) @@ -115,7 +118,7 @@ def transform(program): builtins = set() c_statements = [ - transform_function_call_statement(builtins, statement) for statement in program.statement_list + transform_function_call_expression(builtins, statement) for statement in program.statement_list ] standard_libraries = set() -- 2.20.1