Handle multiple statements

[fur] / tokenization.py

diff --git a/tokenization.py b/tokenization.py
index 10ed80f..7733ab7 100644 (file)
--- a/tokenization.py
+++ b/tokenization.py
@@ -36,7 +36,7 @@ _TOKEN_MATCHERS = [
 
 _TOKEN_MATCHERS = list(map(_make_token_matcher, _TOKEN_MATCHERS))
 
-@util.force_generator
+@util.force_generator(tuple)
 def tokenize(source):
     index = 0