X-Git-Url: https://code.kerkeslager.com/?p=fur;a=blobdiff_plain;f=tokenization.py;h=88576bd8ad82ef177203ddd0c5901d60a7b2dd17;hp=bc8cf85a8f5a12aa556095f905bb190ef292c446;hb=3a076cfd2c46c9f3b51815facd613445722a340b;hpb=61733d6070859e6a639ae4b34faec9aacca52a29

diff --git a/tokenization.py b/tokenization.py
index bc8cf85..88576bd 100644
--- a/tokenization.py
+++ b/tokenization.py
@@ -5,12 +5,19 @@ import util
 
 Token = collections.namedtuple(
     'Token',
-    [
+    (
         'type',
         'match',
+        'metadata',
+    ),
+)
+
+NodeMetadata = collections.namedtuple(
+    'NodeMetadata',
+    (
         'index',
         'line',
-    ],
+    ),
 )
 
 def _make_token_matcher(definition):
@@ -26,7 +33,14 @@ def _make_token_matcher(definition):
         return (
             True,
             index + len(match.group()),
-            Token(type=name, match=match.group(), index=index, line=line),
+            Token(
+                type=name,
+                match=match.group(),
+                metadata=NodeMetadata(
+                    index=index,
+                    line=line,
+                ),
+            ),
         )
 
     return token_matcher