utokenize: Add module from pycopy v2.0.

source: https://github.com/pfalcon/pycopy-lib/tree/6a5a47b82b84deed49f6f83c22e44620286f8100/utokenize

Author: pi-anl

python-stdlib/utokenize/example_dump.py

@@ -0,0 +1,7 @@
+import sys
+import utokenize as tokenize
+
+
+f = open(sys.argv[1], "r")
+for t in tokenize.generate_tokens(f.readline):
+    print(t)

python-stdlib/utokenize/example_test_dump.py

@@ -0,0 +1,11 @@
+import sys
+import utokenize as tokenize
+
+
+f = open(sys.argv[1], "r")
+for t in tokenize.generate_tokens(f.readline):
+    # print(t)
+    print(
+        "TokenInfo(type=%d (%s), string=%r, startl=%d)"
+        % (t.type, tokenize.tok_name[t.type], t.string, t.start)
+    )

python-stdlib/utokenize/example_test_dump_cpy.py

@@ -0,0 +1,12 @@
+# Same as example_dump.py, but for CPython's tokenize module
+import sys
+import tokenize as tokenize
+
+
+f = open(sys.argv[1], "r")
+for t in tokenize.generate_tokens(f.readline):
+    # print(t)
+    print(
+        "TokenInfo(type=%d (%s), string=%r, startl=%d)"
+        % (t.type, tokenize.tok_name[t.type], t.string, t.start[0])
+    )

python-stdlib/utokenize/metadata.txt

@@ -0,0 +1,5 @@
+srctype = micropython-lib
+type = module
+version = 2.0
+author = Paul Sokolovsky
+long_desc = Simple tokenizer for Python source code.

python-stdlib/utokenize/test_corpus.py

@@ -0,0 +1,32 @@
+import sys
+import os
+import glob
+
+
+def handle_one(f):
+    rc = os.system("pycopy example_test_dump.py %s >%s.upy" % (f, f))
+    if rc != 0:
+        print("%s: FAIL" % f)
+        return False
+    rc = os.system("diff -u %s.cpy %s.upy" % (f, f))
+    if rc != 0:
+        print("%s: error" % f)
+        return False
+    print("%s: ok" % f)
+    return True
+
+
+if len(sys.argv) > 1 and sys.argv[1] == "--make-expected":
+    for f in glob.glob("testdata/*.py"):
+        rc = os.system("python3.6 example_test_dump_cpy.py %s >%s.cpy" % (f, f))
+        assert rc == 0
+    sys.exit()
+
+
+errors = False
+
+for f in sorted(glob.glob("testdata/*.py")):
+    e = handle_one(f)
+    errors = errors or not e
+
+sys.exit(int(errors))

python-stdlib/utokenize/testdata/.gitignore

@@ -0,0 +1,2 @@
+*.cpy
+*.upy

python-stdlib/utokenize/testdata/basic.py

@@ -0,0 +1,5 @@
+def foo():
+    print(1)
+    print(2)
+
+foo()

python-stdlib/utokenize/testdata/basic_empty_lines.py

@@ -0,0 +1,14 @@
+def foo():
+    print(1)
+
+    print(2)
+
+
+    print(3)
+ 
+    print(4)
+                                 
+    print(5)
+
+
+foo()

python-stdlib/utokenize/testdata/basic_multilevel.py

@@ -0,0 +1,4 @@
+def foo():
+    if 1:
+        if 2:
+            pass

python-stdlib/utokenize/testdata/comment_intra.py

@@ -0,0 +1 @@
+foo # comment

python-stdlib/utokenize/testdata/comment_line.py

@@ -0,0 +1,3 @@
+# comment
+  # comment
+   # comment           

python-stdlib/utokenize/testdata/consts.py

@@ -0,0 +1 @@
+None + True - False

python-stdlib/utokenize/testdata/decorator.py

@@ -0,0 +1,3 @@
+@foo
+def bar():
+    pass

python-stdlib/utokenize/testdata/ellipsis.py

@@ -0,0 +1 @@
+...

python-stdlib/utokenize/testdata/form_feeds.py

@@ -0,0 +1,6 @@
+a = 1
+
+
+
+# comment
+b = 2

python-stdlib/utokenize/testdata/func_ann.py

@@ -0,0 +1,2 @@
+def foo(a: int) -> int:
+    return 1

python-stdlib/utokenize/testdata/func_kwargs.py

@@ -0,0 +1,2 @@
+def foo(a, **b):
+    pass

python-stdlib/utokenize/testdata/func_varargs.py

@@ -0,0 +1,2 @@
+def foo(a, *b):
+    pass

python-stdlib/utokenize/testdata/ident_underscore.py

@@ -0,0 +1 @@
+__init__ = 1

python-stdlib/utokenize/testdata/import.py

@@ -0,0 +1 @@
+import foo.bar

python-stdlib/utokenize/testdata/import_from.py

@@ -0,0 +1,3 @@
+from . import foo
+from .bar import foo
+from .. import foo

python-stdlib/utokenize/testdata/indent.py

@@ -0,0 +1,6 @@
+def foo():
+    if 1:
+        2
+
+def bar():
+    2

python-stdlib/utokenize/testdata/line_cont.py

@@ -0,0 +1,8 @@
+foo\
+bar
+
+foo\
+    bar
+
+"foo\
+bar"

python-stdlib/utokenize/testdata/multiline.py

@@ -0,0 +1,33 @@
+(1,
+2)
+
+[
+1,
+2
+]
+
+{ 1:
+2,
+3:   
+
+4
+
+}
+
+[
+    1,
+    2,
+]
+
+foo(
+    1, 2,
+    3)
+
+{ 1:
+        2,
+3:   
+
+    4,
+        5:
+    6
+}

python-stdlib/utokenize/testdata/no_nl1.py

@@ -0,0 +1 @@
+123

python-stdlib/utokenize/testdata/no_nl2.py

@@ -0,0 +1,3 @@
+if 1:
+    if 2:
+        123

python-stdlib/utokenize/testdata/numbers.py

@@ -0,0 +1,11 @@
+1
+-1
+0x1
+0xff
+0o123
+1.79769313486231e+308
+.5
+.a
+.
+0b01010101
+3.0.__format__

python-stdlib/utokenize/testdata/numbers_complex.py

@@ -0,0 +1,5 @@
+1j
+10j
+10j+20
+10j-20
+j

python-stdlib/utokenize/testdata/numbers_undescore.py

@@ -0,0 +1,11 @@
+1_0
+-1_0
+0x_1
+0xf_f
+0o12_3
+1.797693_1348_6231e+3_08
+._5
+.a
+.
+0b0101_0101
+3.0.__format__

python-stdlib/utokenize/testdata/operators.py

@@ -0,0 +1,29 @@
+1 / 2
+1 // 2
+1 ** 2
+1 << 2
+1 >> 2
+1 += 2
+1 -= 2
+1 *= 2
+1 /= 2
+1 //= 2
+1 %= 2
+1 **= 2
+1 @= 2
+1 |= 2
+1 &= 2
+1 ^= 2
+1 <<= 2
+1 >>= 2
+1 == 2
+1 != 2
+1 >= 2
+1 <= 2
+1 or 2
+1 and 2
+not 1
+1 is 2
+1 is not 2
+1 in 2
+1 not in 2

python-stdlib/utokenize/testdata/slice.py

@@ -0,0 +1,6 @@
+v[1]
+v[1:2]
+v[1:]
+v[:2]
+v[1:2]
+v[::]

python-stdlib/utokenize/testdata/strings.py

@@ -0,0 +1,36 @@
+"fdfd"
+'fsdgfd'
+"12\"34"
+"12\""
+"\"34"
+"12\r\n56"
+b"foo"
+r"bar"
+u"123"
+br"2332"
+rb"321432"
+
+"""foo"""
+"""foo""" + 1
+
+"""
+foo
+"""
+
+"""\
+foo
+"""
+
+"""A minimal subset of the locale module used at interpreter startup
+(imported by the _io module), in order to reduce startup time.
+
+Don't import directly from third-party code; use the `locale` module instead!
+"""
+
+"""
+\"""
+"""
+
+"""\""""""
+
+bg="#d00" if clear == NORMAL else"#fca"

python-stdlib/utokenize/testdata/tabs.py

@@ -0,0 +1,4 @@
+if 1:
+	pass
+else:
+    2