X-Git-Url: https://code.kerkeslager.com/?p=fur;a=blobdiff_plain;f=tokenization.py;fp=tokenization.py;h=7733ab7d21cde5c5cbcc58b3312849656eb0cd94;hp=10ed80f28a5f47caa16f2c7ad03bfae631f1ecfc;hb=db1651d2c0e44a380f876b452f30c2244d3a0d06;hpb=c40954b865b48ffa4993f735d56213fb91c3e90e

diff --git a/tokenization.py b/tokenization.py
index 10ed80f..7733ab7 100644
--- a/tokenization.py
+++ b/tokenization.py
@@ -36,7 +36,7 @@ _TOKEN_MATCHERS = [
 
 _TOKEN_MATCHERS = list(map(_make_token_matcher, _TOKEN_MATCHERS))
 
-@util.force_generator
+@util.force_generator(tuple)
 def tokenize(source):
     index = 0