code.kerkeslager.com Git - fur/blob - parsing.py

   1 import collections
   2
   3 def _or_parser(*parsers):
   4     def result_parser(index, tokens):
   5         failure = (False, index, None)
   6
   7         for parser in parsers:
   8             success, index, value = parser(index, tokens)
   9
  10             if success:
  11                 return (success, index, value)
  12
  13         return failure
  14
  15     return result_parser
  16
  17 def _zero_or_more_parser(formatter, parser):
  18     def result_parser(index, tokens):
  19         values = []
  20
  21         while index < len(tokens):
  22             success, index, value = parser(index, tokens)
  23
  24             if success:
  25                 values.append(value)
  26             else:
  27                 break
  28
  29         return (True, index, formatter(values))
  30
  31     return result_parser
  32
  33 IntegerLiteral = collections.namedtuple(
  34     'IntegerLiteral',
  35     [
  36         'value',
  37     ],
  38 )
  39
  40 StringLiteral = collections.namedtuple(
  41     'StringLiteral',
  42     [
  43         'value',
  44     ],
  45 )
  46
  47 def _integer_literal_parser(index, tokens):
  48     failure = (False, index, None)
  49
  50     if tokens[index].type != 'integer_literal':
  51         return failure
  52     value = int(tokens[index].match)
  53     index += 1
  54
  55     return True, index, IntegerLiteral(value=value)
  56
  57 def _string_literal_parser(index, tokens):
  58     failure = (False, index, None)
  59
  60     if tokens[index].type != 'single_quoted_string_literal':
  61         return failure
  62     value = tokens[index].match[1:-1]
  63     index += 1
  64
  65     return True, index, StringLiteral(value=value)
  66
  67 _argument_parser = _or_parser(_integer_literal_parser, _string_literal_parser)
  68
  69 FunctionCall = collections.namedtuple(
  70     'FunctionCall',
  71     [
  72         'name',
  73         'arguments',
  74     ],
  75 )
  76
  77 FurProgram = collections.namedtuple(
  78     'FurProgram',
  79     [
  80         'statement_list',
  81     ],
  82 )
  83
  84 def _function_call_parser(index, tokens):
  85     failure = (False, index, None)
  86
  87     if tokens[index].type != 'symbol':
  88         return failure
  89     name = tokens[index].match
  90     index += 1
  91
  92     if tokens[index].type != 'open_parenthese':
  93         return failure
  94     index += 1
  95
  96     success, index, argument = _argument_parser(index, tokens)
  97
  98     if not success:
  99         return failure
 100
 101     if tokens[index].type != 'close_parenthese':
 102         return failure
 103     index += 1
 104
 105     return True, index, FunctionCall(name=name, arguments=(argument,))
 106
 107 def _program_formatter(statement_list):
 108     return FurProgram(statement_list=statement_list)
 109
 110 _program_parser = _zero_or_more_parser(_program_formatter, _function_call_parser)
 111
 112 def _parse(parser, tokens):
 113     success, index, result = parser(0, tokens)
 114
 115     if index < len(tokens):
 116         raise Exception('Unable to parse token {}'.format(tokens[index]))
 117
 118     if success:
 119         return result
 120
 121     raise Exception('Unable to parse')
 122
 123
 124 def parse(tokens):
 125     return _parse(_program_parser, tokens)
 126
 127 if __name__ == '__main__':
 128     import unittest
 129
 130     import tokenization
 131
 132     class StringLiteralParserTests(unittest.TestCase):
 133         def test_parses_single_quoted_string_literal(self):
 134             self.assertEqual(
 135                 _string_literal_parser(0, tokenization.tokenize("'Hello, world'")),
 136                 (
 137                     True,
 138                     1,
 139                     StringLiteral(value='Hello, world'),
 140                 ),
 141             )
 142
 143     class FunctionCallParserTests(unittest.TestCase):
 144         def test_parses_function_with_string_literal_argument(self):
 145             self.assertEqual(
 146                 _function_call_parser(0, tokenization.tokenize("print('Hello, world')")),
 147                 (
 148                     True,
 149                     4,
 150                     FunctionCall(
 151                         name='print',
 152                         arguments=(StringLiteral(value='Hello, world'),),
 153                     ),
 154                 ),
 155             )
 156
 157     unittest.main()