diff --git a/.travis.yml b/.travis.yml index 19771df76a03..0b60e361e889 100644 --- a/.travis.yml +++ b/.travis.yml @@ -13,7 +13,8 @@ install: - pip install -r test-requirements.txt - python setup.py install -script: bash travis.sh +script: + - python runtests.py -v notifications: irc: diff --git a/README.md b/README.md index d5f216f161dc..3855816ce620 100644 --- a/README.md +++ b/README.md @@ -117,28 +117,44 @@ Running tests and linting First install any additional dependencies needed for testing: - $ pip install -r test-requirements.txt + $ pip3 install -r test-requirements.txt -To run tests, run the script `tests.py` in the mypy repository: +To run all tests, run the script `runtests.py` in the mypy repository: - $ python3 tests.py + $ ./runtests.py -You can also run tests without having to run `setup.py` first by -setting up the Python module search path suitably: +Note that some tests will be disabled for older python versions. - $ export PYTHONPATH=PREFIX/mypy:PREFIX/mypy/lib-typing/3.2 - $ python3 tests.py +You can run a subset of test suites by passing postive or negative filters: -Replace `PREFIX` with the path where you have the repository cloned. + $ ./runtests.py lex parse -x lint -x stub -You can also run the type checker for manual testing now without -installing anything by running `scripts/mypy`: +If you want to run individual unit tests, you can run myunit directly, or +pass inferior arguments via -a: - $ python3 PREFIX/mypy/scripts/mypy PROGRAM + $ scripts/myunit -m mypy.test.testlex -v '*backslash*' + $ ./runtests.py mypy.test.testlex -a -v -a '*backslash*' + +You can also run the type checker for manual testing without +installing anything by setting up the Python module search path suitably: + + $ export PYTHONPATH=$PWD:$PWD/lib-typing/3.2 + $ python -m mypy PROGRAM.py + +You can add the entry scripts to PATH for a single python3 version: + + $ export PATH=$PWD/scripts + $ mypy PROGRAM.py + +You can check a module or string instead of a file: + + $ mypy PROGRAM.py + $ mypy -m MODULE + $ mypy -c 'import MODULE' To run the linter: - $ ./lint.sh + $ ./runtests.py lint Development status diff --git a/lint.sh b/lint.sh deleted file mode 100755 index cbeb201643dc..000000000000 --- a/lint.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -# Run the flake8 linter against the implementation. -# -# Note that stubs are not checked; we'd need a separate set of -# settings for stubs, as they follow a different set of conventions. - -SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) -cd "$SCRIPT_DIR" - -flake8 *.py mypy scripts/mypy diff --git a/mypy/__main__.py b/mypy/__main__.py new file mode 100644 index 000000000000..c06dc7c3e38a --- /dev/null +++ b/mypy/__main__.py @@ -0,0 +1,5 @@ +"""Mypy type checker command line tool.""" + +from mypy.main import main + +main() diff --git a/mypy/build.py b/mypy/build.py index b204c43bd1f4..7bf888df6a95 100644 --- a/mypy/build.py +++ b/mypy/build.py @@ -28,6 +28,10 @@ from mypy import stats +# We need to know the location of this file to load data, but +# until Python 3.4, __file__ is relative. +__file__ = os.path.realpath(__file__) + debug = False @@ -39,6 +43,7 @@ # Build flags VERBOSE = 'verbose' # More verbose messages (for troubleshooting) MODULE = 'module' # Build module as a script +PROGRAM_TEXT = 'program-text' # Build command-line argument as a script TEST_BUILTINS = 'test-builtins' # Use stub builtins to speed up tests # State ids. These describe the states a source file / module can be in a @@ -83,6 +88,7 @@ def __init__(self, files: Dict[str, MypyFile], def build(program_path: str, target: int, module: str = None, + argument: str = None, program_text: Union[str, bytes] = None, alt_lib_path: str = None, bin_dir: str = None, @@ -123,7 +129,7 @@ def build(program_path: str, if TEST_BUILTINS in flags: # Use stub builtins (to speed up test cases and to make them easier to # debug). - lib_path.insert(0, os.path.join('mypy', 'test', 'data', 'lib-stub')) + lib_path.insert(0, os.path.join(os.path.dirname(__file__), 'test', 'data', 'lib-stub')) elif program_path: # Include directory of the program file in the module search path. lib_path.insert( @@ -147,9 +153,11 @@ def build(program_path: str, custom_typing_module=custom_typing_module, html_report_dir=html_report_dir) - program_path = program_path or lookup_program(module, lib_path) if program_text is None: + program_path = program_path or lookup_program(module, lib_path) program_text = read_program(program_path) + else: + program_path = program_path or '' # Construct information that describes the initial file. __main__ is the # implicit module id and the import context is empty initially ([]). @@ -164,16 +172,17 @@ def build(program_path: str, def default_data_dir(bin_dir: str) -> str: + # TODO fix this logic if not bin_dir: - # Default to current directory. - return '' + # Default to directory containing this file's parent. + return os.path.dirname(os.path.dirname(__file__)) base = os.path.basename(bin_dir) dir = os.path.dirname(bin_dir) - if (sys.platform == 'win32' and base.lower() == 'scripts' + if (sys.platform == 'win32' and base.lower() == 'mypy' and not os.path.isdir(os.path.join(dir, 'stubs'))): # Installed, on Windows. return os.path.join(dir, 'Lib', 'mypy') - elif base == 'scripts': + elif base == 'mypy': # Assume that we have a repo check out or unpacked source tarball. return os.path.dirname(bin_dir) elif base == 'bin': @@ -199,7 +208,14 @@ def default_lib_path(data_dir: str, target: int, pyversion: int, path[:0] = path_env.split(os.pathsep) # Add library stubs directory. By convention, they are stored in the - # stubs/x.y directory of the mypy installation. + # stubs/x.y directory of the mypy installation. Additionally, stubs + # for earlier versions in the same major version will be added, and + # as a last resort, third-party stubs will be added. + if pyversion == 2: + major, minor = 2, 7 + else: + # See bug #886 + major, minor = sys.version_info[0], sys.version_info[1] version_dir = '3.2' third_party_dir = 'third-party-3.2' if pyversion < 3: @@ -208,9 +224,12 @@ def default_lib_path(data_dir: str, target: int, pyversion: int, path.append(os.path.join(data_dir, 'stubs', version_dir)) path.append(os.path.join(data_dir, 'stubs', third_party_dir)) path.append(os.path.join(data_dir, 'stubs-auto', version_dir)) - if sys.version_info.major == 3: + if major == 3: # Add additional stub directories. versions = ['3.3', '3.4', '3.5', '3.6'] + if False: + # Ick, we really should figure out how to use this again. + versions = ['3.%d' % i for i in range(minor, -1, -1)] for v in versions: stubdir = os.path.join(data_dir, 'stubs', v) if os.path.isdir(stubdir): @@ -444,7 +463,7 @@ def lookup_state(self, module: str) -> 'State': for state in self.states: if state.id == module: return state - raise RuntimeError('%s not found' % str) + raise RuntimeError('%s not found' % module) def all_imported_modules_in_file(self, file: MypyFile) -> List[Tuple[str, int]]: @@ -459,7 +478,7 @@ def correct_rel_imp(imp: Union[ImportFrom, ImportAll]) -> str: rel = imp.relative if rel == 0: return imp.id - if os.path.basename(file.path) == '__init__.py': + if os.path.basename(file.path).startswith('__init__.'): rel -= 1 if rel != 0: file_id = ".".join(file_id.split(".")[:-rel]) diff --git a/mypy/checkstrformat.py b/mypy/checkstrformat.py index 27bb533d7c05..eb24bfbbd7c0 100644 --- a/mypy/checkstrformat.py +++ b/mypy/checkstrformat.py @@ -10,7 +10,9 @@ from mypy.nodes import ( Node, StrExpr, TupleExpr, DictExpr, Context ) -import mypy.checker +if False: + # break import cycle only needed for mypy + import mypy.checker from mypy import messages from mypy.messages import MessageBuilder diff --git a/mypy/codec/pytokenize.py b/mypy/codec/pytokenize.py index 395ced2cf353..8d3b992cef6c 100644 --- a/mypy/codec/pytokenize.py +++ b/mypy/codec/pytokenize.py @@ -92,7 +92,7 @@ from token import * import token -x = None +x = None # type: str __all__ = [x for x in dir(token) if not x.startswith("_")] __all__ += ["COMMENT", "tokenize", "generate_tokens", "NL", "untokenize"] del x @@ -187,7 +187,7 @@ def maybe(*choices): return group(*choices) + '?' 'r': None, 'R': None, 'u': None, 'U': None, 'b': None, 'B': None} -triple_quoted = {} +triple_quoted = {} # type: Dict[str, str] for t in ("'''", '"""', "r'''", 'r"""', "R'''", 'R"""', "u'''", 'u"""', "U'''", 'U"""', @@ -197,7 +197,7 @@ def maybe(*choices): return group(*choices) + '?' "br'''", 'br"""', "Br'''", 'Br"""', "bR'''", 'bR"""', "BR'''", 'BR"""'): triple_quoted[t] = t -single_quoted = {} +single_quoted = {} # type: Dict[str, str] for t in ("'", '"', "r'", 'r"', "R'", 'R"', "u'", 'u"', "U'", 'U"', @@ -333,6 +333,10 @@ def generate_tokens(readline): contline = None indents = [0] + if 0: + # type hints for mypy + strstart = (0, 0) + endprog = re.compile('') while 1: # loop over lines in stream try: line = readline() diff --git a/mypy/codec/test/test_function_translation.py b/mypy/codec/test/test_function_translation.py index d308ca10f90a..bd6296a1c98b 100644 --- a/mypy/codec/test/test_function_translation.py +++ b/mypy/codec/test/test_function_translation.py @@ -1,4 +1,5 @@ import codecs +import sys from unittest import TestCase @@ -8,56 +9,56 @@ # even indices are input functions, odd indices are expected output test_function_examples = [ # test a function without annotations -'''\ +b'''\ def f(x): x = {'a': x} return x['a'] ''', -'''\ +b'''\ def f(x): x = {'a': x} return x['a'] ''', # test parameter type annotations -'''\ +b'''\ def f(x: int, y: str = 'abc'): return x ''', -'''\ +b'''\ def f(x , y = 'abc'): return x ''', # test return type annotations -'''\ +b'''\ def f(x, y) -> str: return x ''', -'''\ +b'''\ def f(x, y) : return x ''', # test newlines in param list -'''\ +b'''\ def f(x: int, y: str) -> str: return x ''', -'''\ +b'''\ def f(x , y ) : return x ''', # test newlines in return type annotation -'''\ +b'''\ def f(x: int, y: str='abc') -> Tuple[int, str]: return x, y ''', -'''\ +b'''\ def f(x , y ='abc')\\ : return x, y @@ -65,11 +66,11 @@ def f(x , y ='abc')\\ # test unrelated continuations -'''\ +b'''\ x = 1 + \ 2 ''', -'''\ +b'''\ x = 1 + \ 2 ''', @@ -81,7 +82,11 @@ class TestFunctionTranslation(TestCase): def test_all_functions(self): for i in range(0, len(test_function_examples), 2): - func_translated = codecs.decode(test_function_examples[i], 'mypy') - #print repr(func_translated) - #print repr(test_function_examples[i + 1]) - self.assertEqual(func_translated, test_function_examples[i + 1]) + func_orig = test_function_examples[i] + func_py2 = test_function_examples[i + 1].decode('utf-8') + func_py3 = func_orig.decode('utf-8') + func_translated = codecs.decode(func_orig, 'mypy') + if sys.version_info[0] == 2: + self.assertEqual(func_translated, func_py2) + else: + self.assertEqual(func_translated, func_py3) diff --git a/mypy/main.py b/mypy/main.py new file mode 100644 index 000000000000..5353198dd5ec --- /dev/null +++ b/mypy/main.py @@ -0,0 +1,185 @@ +"""Mypy type checker command line tool.""" + +import os +import os.path +import shutil +import subprocess +import sys +import tempfile + +import typing +from typing import List, Tuple + +from mypy import build +from mypy.errors import CompileError + +from mypy.version import __version__ + + +class Options: + def __init__(self) -> None: + # Set default options. + self.target = build.TYPE_CHECK + self.build_flags = [] # type: List[str] + self.pyversion = 3 + self.custom_typing_module = None # type: str + self.html_report_dir = None # type: str + self.python_path = False + + +def main() -> None: + bin_dir = find_bin_directory() + path, module, program_text, options = process_options(sys.argv[1:]) + try: + if options.target == build.TYPE_CHECK: + type_check_only(path, module, program_text, bin_dir, options) + else: + raise RuntimeError('unsupported target %d' % options.target) + except CompileError as e: + for m in e.messages: + sys.stderr.write(m + '\n') + sys.exit(1) + + +def find_bin_directory() -> str: + """Find the directory that contains this script. + + This is used by build to find stubs and other data files. + """ + script = __file__ + # Follow up to 5 symbolic links (cap to avoid cycles). + for i in range(5): + if os.path.islink(script): + script = readlinkabs(script) + else: + break + return os.path.dirname(script) + + +def readlinkabs(link: str) -> str: + """Return an absolute path to symbolic link destination.""" + # Adapted from code by Greg Smith. + assert os.path.islink(link) + path = os.readlink(link) + if os.path.isabs(path): + return path + return os.path.join(os.path.dirname(link), path) + + +def type_check_only(path: str, module: str, program_text: str, + bin_dir: str, options: Options) -> None: + # Type check the program and dependencies and translate to Python. + build.build(path, + module=module, + program_text=program_text, + bin_dir=bin_dir, + target=build.TYPE_CHECK, + pyversion=options.pyversion, + custom_typing_module=options.custom_typing_module, + html_report_dir=options.html_report_dir, + flags=options.build_flags, + python_path=options.python_path) + + +def process_options(args: List[str]) -> Tuple[str, str, str, Options]: + """Process command line arguments. + + Return (mypy program path (or None), + module to run as script (or None), + parsed flags) + """ + options = Options() + help = False + ver = False + while args and args[0].startswith('-'): + if args[0] == '--verbose': + options.build_flags.append(build.VERBOSE) + args = args[1:] + elif args[0] == '--py2': + # Use Python 2 mode. + options.pyversion = 2 + args = args[1:] + elif args[0] == '-m' and args[1:]: + options.build_flags.append(build.MODULE) + return None, args[1], None, options + elif args[0] == '-c' and args[1:]: + options.build_flags.append(build.PROGRAM_TEXT) + return None, None, args[1], options + elif args[0] in ('-h', '--help'): + help = True + args = args[1:] + elif args[0] == '--stats': + options.build_flags.append('dump-type-stats') + args = args[1:] + elif args[0] == '--inferstats': + options.build_flags.append('dump-infer-stats') + args = args[1:] + elif args[0] == '--custom-typing' and args[1:]: + options.custom_typing_module = args[1] + args = args[2:] + elif args[0] == '--html-report' and args[1:]: + options.html_report_dir = args[1] + options.build_flags.append('html-report') + args = args[2:] + elif args[0] == '--use-python-path': + options.python_path = True + args = args[1:] + elif args[0] == '--version': + ver = True + args = args[1:] + else: + usage('Unknown option: {}'.format(args[0])) + + if help: + usage() + + if ver: + version() + + if not args: + usage('Missing target file or module') + + if args[1:]: + usage('Extra argument: {}'.format(args[1])) + + if options.python_path and options.pyversion == 2: + usage('--py2 specified, ' + 'but --use-python-path will search in sys.path of Python 3') + + return args[0], None, None, options + + +def usage(msg: str = None) -> None: + if msg: + sys.stderr.write('%s\n' % msg) + sys.stderr.write("""\ +usage: mypy [option ...] [-m mod | file] +Try 'mypy -h' for more information. +""") + else: + sys.stderr.write("""\ +usage: mypy [option ...] [-m mod | file] + +Optional arguments: + -h, --help print this help message and exit + --html-report dir generate a HTML report of type precision under dir/ + -m mod type check module + -c string type check string + --verbose more verbose messages + --use-python-path search for modules in sys.path of running Python + --version show the current version information + +Environment variables: + MYPYPATH additional module search path +""") + sys.exit(2) + + +def version() -> None: + sys.stdout.write("mypy {}\n".format(__version__)) + exit(0) + + +def fail(msg: str) -> None: + sys.stderr.write('%s\n' % msg) + sys.exit(1) diff --git a/mypy/myunit.py b/mypy/myunit/__init__.py similarity index 72% rename from mypy/myunit.py rename to mypy/myunit/__init__.py index 65a44f7f9325..6ffa3293929f 100644 --- a/mypy/myunit.py +++ b/mypy/myunit/__init__.py @@ -1,9 +1,12 @@ +import importlib +import os import sys import re +import tempfile import time import traceback -from typing import List, Tuple, Any, Callable, Union +from typing import List, Tuple, Any, Callable, Union, cast # TODO remove global state @@ -11,6 +14,8 @@ is_quiet = False patterns = [] # type: List[str] times = [] # type: List[Tuple[float, str]] +APPEND_TESTCASES = '' +UPDATE_TESTCASES = False class AssertionFailure(Exception): @@ -35,14 +40,26 @@ def assert_false(b: bool, msg: str = None) -> None: raise AssertionFailure(msg) +def good_repr(obj: object) -> str: + if isinstance(obj, str): + if obj.count('\n') > 1: + bits = ["'''\\"] + for line in obj.split('\n'): + # force repr to use ' not ", then cut it off + bits.append(repr('"' + line)[2:-1]) + bits[-1] += "'''" + return '\n'.join(bits) + return repr(obj) + + def assert_equal(a: object, b: object, fmt: str = '{} != {}') -> None: if a != b: - raise AssertionFailure(fmt.format(repr(a), repr(b))) + raise AssertionFailure(fmt.format(good_repr(a), good_repr(b))) def assert_not_equal(a: object, b: object, fmt: str = '{} == {}') -> None: if a == b: - raise AssertionFailure(fmt.format(repr(a), repr(b))) + raise AssertionFailure(fmt.format(good_repr(a), good_repr(b))) def assert_raises(typ: type, *rest: Any) -> None: @@ -91,18 +108,29 @@ def __init__(self, name: str, suite: 'Suite' = None, self.func = func self.name = name self.suite = suite + self.old_cwd = None # type: str + self.tmpdir = None # type: tempfile.TemporaryDirectory def run(self) -> None: if self.func: self.func() def set_up(self) -> None: + self.old_cwd = os.getcwd() + self.tmpdir = tempfile.TemporaryDirectory(prefix='mypy-test-', + dir=os.path.abspath('tmp-test-dirs')) + os.chdir(self.tmpdir.name) + os.mkdir('tmp') if self.suite: self.suite.set_up() def tear_down(self) -> None: if self.suite: self.suite.tear_down() + os.chdir(self.old_cwd) + self.tmpdir.cleanup() + self.old_cwd = None + self.tmpdir = None class Suite: @@ -139,12 +167,40 @@ def skip(self) -> None: raise SkipTestCaseException() -def run_test(t: Suite, args: List[str] = None) -> None: +def add_suites_from_module(suites: List[Suite], mod_name: str) -> None: + mod = importlib.import_module(mod_name) + got_suite = False + for suite in mod.__dict__.values(): + if isinstance(suite, type) and issubclass(suite, Suite) and suite is not Suite: + got_suite = True + suites.append(cast(Callable[[], Suite], suite)()) + if not got_suite: + # Sanity check in case e.g. it uses unittest instead of a myunit. + # The codecs tests do since they need to be python2-compatible. + sys.exit('Test module %s had no test!' % mod_name) + + +class ListSuite(Suite): + def __init__(self, suites: List[Suite]) -> None: + for suite in suites: + mod_name = type(suite).__module__.replace('.', '_') + mod_name = mod_name.replace('mypy_', '') + mod_name = mod_name.replace('test_', '') + mod_name = mod_name.strip('_').replace('__', '_') + type_name = type(suite).__name__ + name = 'test_%s_%s' % (mod_name, type_name) + setattr(self, name, suite) + super().__init__() + + +def main(args: List[str] = None) -> None: global patterns, is_verbose, is_quiet + global APPEND_TESTCASES, UPDATE_TESTCASES if not args: - args = [] + args = sys.argv[1:] is_verbose = False is_quiet = False + suites = [] # type: List[Suite] patterns = [] i = 0 while i < len(args): @@ -153,17 +209,29 @@ def run_test(t: Suite, args: List[str] = None) -> None: is_verbose = True elif a == '-q': is_quiet = True - # Used for updating tests, but breaks clean code usage. - elif a in ('-u', '-i'): - pass - elif len(a) > 0 and a[0] != '-': + elif a == '-u': + APPEND_TESTCASES = '.new' + UPDATE_TESTCASES = True + elif a == '-i': + APPEND_TESTCASES = '' + UPDATE_TESTCASES = True + elif a == '-m': + i += 1 + if i == len(args): + sys.exit('-m requires an argument') + add_suites_from_module(suites, args[i]) + elif not a.startswith('-'): patterns.append(a) else: - raise ValueError('Invalid arguments') + sys.exit('Usage: python -m mypy.myunit [-v] [-q] [-u | -i]' + + ' -m test.module [-m test.module ...] [filter ...]') i += 1 if len(patterns) == 0: patterns.append('*') + if not suites: + sys.exit('At least one -m argument is required') + t = ListSuite(suites) num_total, num_fail, num_skip = run_test_recursive(t, 0, 0, 0, '', 0) skip_msg = '' diff --git a/mypy/myunit/__main__.py b/mypy/myunit/__main__.py new file mode 100644 index 000000000000..34098d40648d --- /dev/null +++ b/mypy/myunit/__main__.py @@ -0,0 +1,9 @@ +# This is a separate module from mypy.myunit so it doesn't exist twice. +"""Myunit test runner command line tool. + +Usually used as a slave by runtests.py, but can be used directly. +""" + +from mypy.myunit import main + +main() diff --git a/mypy/nodes.py b/mypy/nodes.py index 8432ab9a8175..41d3b92ff874 100644 --- a/mypy/nodes.py +++ b/mypy/nodes.py @@ -20,7 +20,9 @@ class Context(metaclass=ABCMeta): def get_line(self) -> int: pass -import mypy.types +if False: + # break import cycle only needed for mypy + import mypy.types T = TypeVar('T') @@ -69,7 +71,7 @@ def get_line(self) -> int: pass } reverse_type_aliases = dict((name.replace('__builtins__', 'builtins'), alias) - for alias, name in type_aliases.items()) + for alias, name in type_aliases.items()) # type: Dict[str, str] class Node(Context): @@ -163,7 +165,7 @@ def accept(self, visitor: NodeVisitor[T]) -> T: def is_package_init_file(self) -> bool: return not (self.path is None) and len(self.path) != 0 \ - and os.path.basename(self.path) == '__init__.py' + and os.path.basename(self.path).startswith('__init__.') class ImportBase(Node): @@ -1043,7 +1045,7 @@ def accept(self, visitor: NodeVisitor[T]) -> T: '>': '__gt__', '<=': '__le__', 'in': '__contains__', -} +} # type: Dict[str, str] ops_with_inplace_method = { '+', '-', '*', '/', '%', '//', '**', '&', '|', '^', '<<', '>>'} diff --git a/mypy/subtypes.py b/mypy/subtypes.py index 97b5e2a33f29..6a8a4c665c3d 100644 --- a/mypy/subtypes.py +++ b/mypy/subtypes.py @@ -7,6 +7,8 @@ ) import mypy.applytype import mypy.constraints +# Circular import; done in the function instead. +# import mypy.solve from mypy import messages, sametypes from mypy.nodes import CONTRAVARIANT, COVARIANT from mypy.maptype import map_instance_to_supertype @@ -252,6 +254,7 @@ def unify_generic_callable(type: CallableType, target: CallableType) -> Callable Return unified CallableType if successful; otherwise, return None. """ + import mypy.solve constraints = [] # type: List[mypy.constraints.Constraint] for arg_type, target_arg_type in zip(type.arg_types, target.arg_types): c = mypy.constraints.infer_constraints( diff --git a/mypy/test/config.py b/mypy/test/config.py index 5a7b5767a3d7..83962d6972f8 100644 --- a/mypy/test/config.py +++ b/mypy/test/config.py @@ -13,10 +13,6 @@ 'Test data prefix ({}) not set correctly'.format(test_data_prefix) # Temp directory used for the temp files created when running test cases. -test_temp_dir = os.path.join(PREFIX, 'tmp') - -if not os.path.isdir(test_temp_dir): - os.mkdir(test_temp_dir) - -assert os.path.isdir(test_temp_dir), \ - 'Test temp dir ({}) not set correctly'.format(test_temp_dir) +# This is *within* the tempfile.TemporaryDirectory that is chroot'ed per testcase. +# It is also hard-coded in numerous places, so don't change it. +test_temp_dir = 'tmp' diff --git a/mypy/test/testcheck.py b/mypy/test/testcheck.py index cbe96c976774..4fc83b1411f0 100644 --- a/mypy/test/testcheck.py +++ b/mypy/test/testcheck.py @@ -7,7 +7,8 @@ from typing import Tuple from mypy import build -from mypy.myunit import Suite, run_test +import mypy.myunit # for mutable globals (ick!) +from mypy.myunit import Suite from mypy.test.config import test_temp_dir, test_data_prefix from mypy.test.data import parse_test_cases from mypy.test.helpers import ( @@ -17,9 +18,6 @@ from mypy.errors import CompileError -APPEND_TESTCASES = '.new' if '-u' in sys.argv else '' -UPDATE_TESTCASES = '-i' in sys.argv or '-u' in sys.argv - # List of files that contain test case descriptions. files = [ 'check-basic.test', @@ -80,8 +78,8 @@ def run_test(self, testcase): except CompileError as e: a = normalize_error_messages(e.messages) - if testcase.output != a and UPDATE_TESTCASES: - update_testcase_output(testcase, a, APPEND_TESTCASES) + if testcase.output != a and mypy.myunit.UPDATE_TESTCASES: + update_testcase_output(testcase, a, mypy.myunit.APPEND_TESTCASES) assert_string_arrays_equal( testcase.output, a, @@ -110,8 +108,3 @@ def parse_options(self, program_text: str) -> Tuple[str, str, str]: return m.group(1), path, program_text else: return '__main__', 'main', program_text - - -if __name__ == '__main__': - import sys - run_test(TypeCheckSuite(), sys.argv[1:]) diff --git a/mypy/test/testinfer.py b/mypy/test/testinfer.py index 2a8760671940..58d3b0b3a661 100644 --- a/mypy/test/testinfer.py +++ b/mypy/test/testinfer.py @@ -2,7 +2,7 @@ import typing -from mypy.myunit import Suite, assert_equal, assert_true, run_test +from mypy.myunit import Suite, assert_equal, assert_true from mypy.checkexpr import map_actuals_to_formals from mypy.nodes import ARG_POS, ARG_OPT, ARG_STAR, ARG_STAR2, ARG_NAMED from mypy.types import AnyType, TupleType @@ -205,8 +205,3 @@ def expand_callee_kinds(kinds_and_names): kinds.append(v) names.append(None) return kinds, names - - -if __name__ == '__main__': - import sys - run_test(MapActualsToFormalsSuite(), sys.argv[1:]) diff --git a/mypy/test/testopgen.py b/mypy/test/testopgen.py deleted file mode 100644 index 3387cacda924..000000000000 --- a/mypy/test/testopgen.py +++ /dev/null @@ -1,74 +0,0 @@ -"""Test Alore type operation generation (OBSOLETE).""" - -import os.path - -import typing - -from build import build -from myunit import Suite, run_test -from config import test_data_prefix, test_temp_dir -from helpers import assert_string_arrays_equal_wildcards -from data import parse_test_cases -from testoutput import remove_prefix -from testtransform import ( - remove_comment_lines, builtins_wrapper, TRANSFORM_BUILTINS -) -from transform import DyncheckTransformVisitor -from opgen import generate_runtime_support -from errors import CompileError - - -class DyncheckOpGenSuite(Suite): - test_case_files = ['dyncheck-opgen.test'] - - def cases(self): - c = [] - for f in self.test_case_files: - c += parse_test_cases( - os.path.join(test_data_prefix, f), - builtins_wrapper(test_op_gen, - os.path.join(test_data_prefix, - TRANSFORM_BUILTINS)), - test_temp_dir, True) - return c - - -def test_op_gen(testcase): - """Perform a type operation support data and code genereation test case.""" - expected = remove_comment_lines(testcase.output) - try: - src = '\n'.join(testcase.input) - # Parse and type check the input program. - trees, symtable, infos, types = build(src, 'main', False, - test_temp_dir, True) - a = [] - first = True - # Transform each file separately. - for t in trees: - # Skip the builtins module and files with '_skip.' in the path. - if not t.path.endswith('/builtins.py') and '_skip.' not in t.path: - if not first: - # Display path for files other than the first. - a.append('{}:'.format( - remove_prefix(t.path, test_temp_dir))) - - # Transform parse tree and produce the code for operations. - # Note that currently we generate this for each file - # separately; this needs to be fixed eventually. - v = DyncheckTransformVisitor(types, symtable, True) - t.accept(v) - s = generate_runtime_support(t) - if s != '': - a += s.split('\n') - first = False - except CompileError as e: - a = e.messages - assert_string_arrays_equal_wildcards( - expected, a, - 'Invalid source code output ({}, line {})'.format(testcase.file, - testcase.line)) - - -if __name__ == '__main__': - import sys - run_test(DyncheckOpGenSuite(), sys.argv[1:]) diff --git a/mypy/test/testparse.py b/mypy/test/testparse.py index 26c6cd249d34..7f7da33c6e6d 100644 --- a/mypy/test/testparse.py +++ b/mypy/test/testparse.py @@ -6,7 +6,7 @@ import typing -from mypy.myunit import Suite, AssertionFailure, run_test +from mypy.myunit import Suite, AssertionFailure from mypy.test.helpers import assert_string_arrays_equal from mypy.test.data import parse_test_cases from mypy.test import config @@ -72,15 +72,3 @@ def test_parse_error(testcase): testcase.output, e.messages, 'Invalid compiler output ({}, line {})'.format(testcase.file, testcase.line)) - - -class CombinedParserSuite(Suite): - def __init__(self): - self.test_parse = ParserSuite() - self.test_parse_errors = ParseErrorSuite() - super().__init__() - - -if __name__ == '__main__': - import sys - run_test(CombinedParserSuite(), sys.argv[1:]) diff --git a/mypy/test/testpythoneval.py b/mypy/test/testpythoneval.py index cdc864ceaa95..07a4a0dc4f47 100644 --- a/mypy/test/testpythoneval.py +++ b/mypy/test/testpythoneval.py @@ -17,7 +17,7 @@ import typing -from mypy.myunit import Suite, run_test, SkipTestCaseException +from mypy.myunit import Suite, SkipTestCaseException from mypy.test.config import test_data_prefix, test_temp_dir from mypy.test.data import parse_test_cases from mypy.test.helpers import assert_string_arrays_equal @@ -69,25 +69,25 @@ def test_python_evaluation(testcase): with open(program_path, 'w') as file: for s in testcase.input: file.write('{}\n'.format(s)) - # Set up module path. - typing_path = os.path.join(os.getcwd(), 'lib-typing', '3.2') - assert os.path.isdir(typing_path) - env = os.environ.copy() - env['PYTHONPATH'] = os.pathsep.join([typing_path, os.getcwd()]) # Type check the program. + # This uses the same PYTHONPATH as the current process. process = subprocess.Popen([python3_path, - os.path.join(os.getcwd(), 'scripts', 'mypy')] + args + [program], + os.path.join(testcase.old_cwd, 'scripts', 'mypy')] + + args + [program], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - cwd=test_temp_dir, - env=env) + cwd=test_temp_dir) outb = process.stdout.read() # Split output into lines. out = [s.rstrip('\n\r') for s in str(outb, 'utf8').splitlines()] if not process.wait(): - if py2: - typing_path = os.path.join(os.getcwd(), 'lib-typing', '2.7') - env['PYTHONPATH'] = typing_path + # Set up module path for the execution. + # This needs the typing module but *not* the mypy module. + vers_dir = '2.7' if py2 else '3.2' + typing_path = os.path.join(testcase.old_cwd, 'lib-typing', vers_dir) + assert os.path.isdir(typing_path) + env = os.environ.copy() + env['PYTHONPATH'] = typing_path process = subprocess.Popen([interpreter, program], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, @@ -114,7 +114,3 @@ def try_find_python2_interpreter(): except OSError: pass return None - - -if __name__ == '__main__': - run_test(PythonEvaluationSuite(), sys.argv[1:]) diff --git a/mypy/test/testsemanal.py b/mypy/test/testsemanal.py index 9f497f748360..3a9c14bccce3 100644 --- a/mypy/test/testsemanal.py +++ b/mypy/test/testsemanal.py @@ -5,7 +5,7 @@ from typing import Dict, List from mypy import build -from mypy.myunit import Suite, run_test +from mypy.myunit import Suite from mypy.test.helpers import assert_string_arrays_equal, testfile_pyversion from mypy.test.data import parse_test_cases from mypy.test.config import test_data_prefix, test_temp_dir @@ -213,17 +213,3 @@ def __str__(self) -> str: a.append(' {} : {}'.format(x, ti)) a[-1] += ')' return '\n'.join(a) - - -class CombinedSemAnalSuite(Suite): - def __init__(self): - self.test_semanal = SemAnalSuite() - self.test_semanal_errors = SemAnalErrorSuite() - self.test_semanal_symtable = SemAnalSymtableSuite() - self.test_semanal_typeinfos = SemAnalTypeInfoSuite() - super().__init__() - - -if __name__ == '__main__': - import sys - run_test(CombinedSemAnalSuite(), sys.argv[1:]) diff --git a/mypy/test/testsolve.py b/mypy/test/testsolve.py index 825d3440dc73..b84bb1714f50 100644 --- a/mypy/test/testsolve.py +++ b/mypy/test/testsolve.py @@ -2,7 +2,7 @@ import typing -from mypy.myunit import Suite, assert_equal, run_test +from mypy.myunit import Suite, assert_equal from mypy.constraints import SUPERTYPE_OF, SUBTYPE_OF, Constraint from mypy.solve import solve_constraints from mypy.typefixture import TypeFixture @@ -149,8 +149,3 @@ def supc(self, type_var, bound): def subc(self, type_var, bound): return Constraint(type_var.name, SUBTYPE_OF, bound) - - -if __name__ == '__main__': - import sys - run_test(SolveSuite(), sys.argv[1:]) diff --git a/mypy/test/teststubgen.py b/mypy/test/teststubgen.py index ac30a118fdad..893659d2f6cf 100644 --- a/mypy/test/teststubgen.py +++ b/mypy/test/teststubgen.py @@ -3,11 +3,13 @@ import os.path import random import shutil +import sys +import tempfile import time import typing -from mypy.myunit import Suite, AssertionFailure, run_test, assert_equal +from mypy.myunit import Suite, AssertionFailure, assert_equal from mypy.test.helpers import assert_string_arrays_equal from mypy.test.data import parse_test_cases from mypy.test import config @@ -102,9 +104,15 @@ def cases(self): def test_stubgen(testcase): + if 'stubgen-test-path' not in sys.path: + sys.path.insert(0, 'stubgen-test-path') + os.mkdir('stubgen-test-path') source = '\n'.join(testcase.input) - name = 'prog%d' % random.randrange(1000 * 1000 * 1000) - path = '%s.py' % name + handle = tempfile.NamedTemporaryFile(prefix='prog_', suffix='.py', dir='stubgen-test-path') + assert os.path.isabs(handle.name) + path = os.path.basename(handle.name) + name = path[:-3] + path = os.path.join('stubgen-test-path', path) out_dir = '_out' os.mkdir(out_dir) try: @@ -127,7 +135,7 @@ def test_stubgen(testcase): testcase.file, testcase.line)) finally: shutil.rmtree(out_dir) - os.remove(path) + handle.close() def reset_importlib_caches(): @@ -173,18 +181,3 @@ def test_infer_binary_op_sig(self): def test_infer_unary_op_sig(self): for op in ('neg', 'pos'): assert_equal(infer_method_sig('__%s__' % op), '()') - - -class StubgenSuite(Suite): - """Collect all the test classes defined in this file.""" - - def __init__(self): - self.test_python = StubgenPythonSuite() - self.test_c = StubgencSuite() - self.test_util = StubgenUtilSuite() - super().__init__() - - -if __name__ == '__main__': - import sys - run_test(StubgenSuite(), sys.argv[1:]) diff --git a/mypy/test/testsubtypes.py b/mypy/test/testsubtypes.py index 93025d07c0af..e8a4121c7682 100644 --- a/mypy/test/testsubtypes.py +++ b/mypy/test/testsubtypes.py @@ -1,4 +1,4 @@ -from mypy.myunit import Suite, assert_true, run_test +from mypy.myunit import Suite, assert_true from mypy.nodes import CONTRAVARIANT, INVARIANT, COVARIANT from mypy.subtypes import is_subtype from mypy.typefixture import TypeFixture, InterfaceTypeFixture @@ -205,8 +205,3 @@ def assert_equivalent(self, s, t): def assert_unrelated(self, s, t): self.assert_not_subtype(s, t) self.assert_not_subtype(t, s) - - -if __name__ == '__main__': - import sys - run_test(SubtypingSuite(), sys.argv[1:]) diff --git a/mypy/test/testtransform.py b/mypy/test/testtransform.py index d44ab21b8c49..eaab12cb1de8 100644 --- a/mypy/test/testtransform.py +++ b/mypy/test/testtransform.py @@ -5,7 +5,7 @@ from typing import Dict, List from mypy import build -from mypy.myunit import Suite, run_test +from mypy.myunit import Suite from mypy.test.helpers import assert_string_arrays_equal, testfile_pyversion from mypy.test.data import parse_test_cases from mypy.test.config import test_data_prefix, test_temp_dir @@ -74,8 +74,3 @@ class TestTransformVisitor(TransformVisitor): def type(self, type): assert type is not None return type - - -if __name__ == '__main__': - import sys - run_test(TransformSuite(), sys.argv[1:]) diff --git a/mypy/test/testtypegen.py b/mypy/test/testtypegen.py index 924ac558953d..dc1961544238 100644 --- a/mypy/test/testtypegen.py +++ b/mypy/test/testtypegen.py @@ -6,7 +6,7 @@ import typing from mypy import build -from mypy.myunit import Suite, run_test +from mypy.myunit import Suite from mypy.test import config from mypy.test.data import parse_test_cases from mypy.test.helpers import assert_string_arrays_equal @@ -105,8 +105,3 @@ def ignore_node(node): return True return False - - -if __name__ == '__main__': - import sys - run_test(TypeExportSuite(), sys.argv[1:]) diff --git a/mypy/test/testtypes.py b/mypy/test/testtypes.py index bd8e5f3fa993..ec0c3ef8bafb 100644 --- a/mypy/test/testtypes.py +++ b/mypy/test/testtypes.py @@ -3,7 +3,7 @@ from typing import List from mypy.myunit import ( - Suite, assert_equal, assert_true, assert_false, run_test + Suite, assert_equal, assert_true, assert_false ) from mypy.erasetype import erase_type from mypy.expandtype import expand_type @@ -729,17 +729,3 @@ def callable(self, *a): return CallableType(a[:-1], [ARG_POS] * n, [None] * n, a[-1], self.fx.function) - - -class CombinedTypesSuite(Suite): - def __init__(self): - self.test_types = TypesSuite() - self.test_type_ops = TypeOpsSuite() - self.test_join = JoinSuite() - self.test_meet = MeetSuite() - super().__init__() - - -if __name__ == '__main__': - import sys - run_test(CombinedTypesSuite(), sys.argv[1:]) diff --git a/mypy/visitor.py b/mypy/visitor.py index 960024f1af50..3927b00dc3b8 100644 --- a/mypy/visitor.py +++ b/mypy/visitor.py @@ -2,7 +2,9 @@ from typing import TypeVar, Generic -import mypy.nodes +if False: + # break import cycle only needed for mypy + import mypy.nodes T = TypeVar('T') diff --git a/mypy/waiter.py b/mypy/waiter.py new file mode 100644 index 000000000000..f4e34456e9fb --- /dev/null +++ b/mypy/waiter.py @@ -0,0 +1,193 @@ +from typing import Dict, List, Optional, Tuple + +import os +import pipes +from subprocess import Popen +import sys + + +class WaiterError(Exception): + pass + + +class LazySubprocess: + + def __init__(self, name: str, args: List[str], *, cwd: Optional[str] = None, + env: Optional[Dict[str, str]] = None) -> None: + self.name = name + self.args = args + self.cwd = cwd + self.env = env + + def __call__(self) -> Popen: + return Popen(self.args, cwd=self.cwd, env=self.env) + + +class Noter: + """Update stats about running jobs. + + Only used when verbosity == 0. + """ + def __init__(self, total: int) -> None: + self.total = total + self.running = set() # type: Set[int] + self.passes = 0 + self.fails = 0 + + def start(self, job: int) -> None: + self.running.add(job) + self.update() + + def stop(self, job: int, failed: bool) -> None: + self.running.remove(job) + if failed: + self.fails += 1 + else: + self.passes += 1 + self.update() + + def message(self, msg: str) -> None: + # Using a CR instead of NL will overwrite the line. + sys.stdout.write('%-80s\r' % msg) + sys.stdout.flush() + + def update(self) -> None: + pending = self.total - self.passes - self.fails - len(self.running) + running = ', '.join('#%d' % r for r in sorted(self.running)) + args = (self.passes, self.fails, pending, running) + msg = 'passed %d, failed %d, pending %d; running {%s}' % args + self.message(msg) + + def clear(self) -> None: + self.message('') + + +class Waiter: + """Run subprocesses in parallel and wait for them. + + Usage: + + waiter = Waiter() + waiter.add('sleep 9') + waiter.add('sleep 10') + if not waiter.run(): + print('error') + """ + def __init__(self, limit: int = 0, *, verbosity: int = 0, xfail: List[str] = []) -> None: + self.verbosity = verbosity + self.queue = [] # type: List[LazySubprocess] + self.next = 0 + self.current = {} # type: Dict[int, Tuple[int, Popen]] + if limit == 0: + try: + sched_getaffinity = os.sched_getaffinity + except AttributeError: + limit = 2 + else: + # Note: only count CPUs we are allowed to use. It is a + # major mistake to count *all* CPUs on the machine. + limit = len(sched_getaffinity(0)) + self.limit = limit + assert limit > 0 + if self.verbosity >= -1: + print('%-8s %d' % ('PARALLEL', limit)) + sys.stdout.flush() + self.xfail = set(xfail) + self._note = None # type: Noter + + def add(self, cmd: LazySubprocess) -> int: + rv = len(self.queue) + self.queue.append(cmd) + return rv + + def _start1(self) -> None: + cmd = self.queue[self.next] + name = cmd.name + proc = cmd() + num = self.next + self.current[proc.pid] = (num, proc) + if self.verbosity >= 1: + print('%-8s #%d %s' % ('START', num, name)) + if self.verbosity >= 2: + print('%-8s #%d %s' % ('CWD', num, cmd.cwd or '.')) + cmd_str = ' '.join(pipes.quote(a) for a in cmd.args) + print('%-8s #%d %s' % ('COMMAND', num, cmd_str)) + sys.stdout.flush() + elif self.verbosity >= 0: + self._note.start(num) + self.next += 1 + + def _wait1(self) -> List[str]: + pid, status = os.waitpid(-1, 0) + num, proc = self.current.pop(pid) + + # Inlined subprocess._handle_exitstatus, it's not a public API. + assert proc.returncode is None + if os.WIFSIGNALED(status): + proc.returncode = -os.WTERMSIG(status) + elif os.WIFEXITED(status): + proc.returncode = os.WEXITSTATUS(status) + else: + # Should never happen + raise RuntimeError("Unknown child exit status!") + assert proc.returncode is not None + + name = self.queue[num].name + rc = proc.wait() + if rc >= 0: + msg = 'EXIT %d' % rc + else: + msg = 'SIG %d' % -rc + if self.verbosity >= 1: + print('%-8s #%d %s' % (msg, num, name)) + sys.stdout.flush() + elif self.verbosity >= 0: + self._note.stop(num, bool(rc)) + elif self.verbosity >= -1: + sys.stdout.write('.' if rc == 0 else msg[0]) + num_complete = self.next - len(self.current) + if num_complete % 50 == 0 or num_complete == len(self.queue): + sys.stdout.write(' %d/%d\n' % (num_complete, len(self.queue))) + elif num_complete % 10 == 0: + sys.stdout.write(' ') + sys.stdout.flush() + + if rc != 0: + if name not in self.xfail: + fail_type = 'FAILURE' + else: + fail_type = 'XFAIL' + else: + if name not in self.xfail: + fail_type = None + else: + fail_type = 'UPASS' + + if fail_type is not None: + return ['%8s %s' % (fail_type, name)] + else: + return [] + + def run(self) -> None: + if self.verbosity == 0: + self._note = Noter(len(self.queue)) + print('SUMMARY %d tasks selected' % len(self.queue)) + sys.stdout.flush() + failures = [] # type: List[str] + while self.current or self.next < len(self.queue): + while len(self.current) < self.limit and self.next < len(self.queue): + self._start1() + failures += self._wait1() + if self.verbosity == 0: + self._note.clear() + if failures: + print('SUMMARY %d/%d tasks failed' % (len(failures), len(self.queue))) + for f in failures: + print(f) + print('SUMMARY %d/%d tasks failed' % (len(failures), len(self.queue))) + sys.stdout.flush() + if any('XFAIL' not in f for f in failures): + sys.exit(1) + else: + print('SUMMARY all %d tasks passed' % len(self.queue)) + sys.stdout.flush() diff --git a/runtests.py b/runtests.py new file mode 100755 index 000000000000..b71ab635da54 --- /dev/null +++ b/runtests.py @@ -0,0 +1,342 @@ +#!/usr/bin/env python3 + +if False: + import typing + +if True: + # When this is run as a script, `typing` is not available yet. + import sys + from os.path import join, isdir + + def get_versions(): # type: () -> typing.List[str] + major = sys.version_info[0] + minor = sys.version_info[1] + if major == 2: + return ['2.7'] + else: + # generates list of python versions to use. + # For Python2, this is only [2.7]. + # Otherwise, it is [3.4, 3.3, 3.2, 3.1, 3.0]. + return ['%d.%d' % (major, i) for i in range(minor, -1, -1)] + + sys.path[0:0] = [v for v in [join('lib-typing', v) for v in get_versions()] if isdir(v)] + # Now `typing` is available. + + +from typing import Dict, List, Optional, Set + +from mypy.waiter import Waiter, LazySubprocess + +import itertools +import os + + +# Ideally, all tests would be `discover`able so that they can be driven +# (and parallelized) by an external test driver. + +class Driver: + + def __init__(self, whitelist: List[str], blacklist: List[str], + arglist: List[str], verbosity: int, xfail: List[str]) -> None: + self.whitelist = whitelist + self.blacklist = blacklist + self.arglist = arglist + self.verbosity = verbosity + self.waiter = Waiter(verbosity=verbosity, xfail=xfail) + self.versions = get_versions() + self.cwd = os.getcwd() + self.mypy = os.path.join(self.cwd, 'scripts', 'mypy') + self.env = dict(os.environ) + + def prepend_path(self, name: str, paths: List[str]) -> None: + old_val = self.env.get(name) + paths = [p for p in paths if isdir(p)] + if not paths: + return + if old_val is not None: + new_val = ':'.join(itertools.chain(paths, [old_val])) + else: + new_val = ':'.join(paths) + self.env[name] = new_val + + def allow(self, name: str) -> bool: + if any(f in name for f in self.whitelist): + if not any(f in name for f in self.blacklist): + if self.verbosity >= 2: + print('SELECT #%d %s' % (len(self.waiter.queue), name)) + return True + if self.verbosity >= 3: + print('OMIT %s' % name) + return False + + def add_mypy(self, name, *args: str, cwd: Optional[str] = None) -> None: + name = 'check %s' % name + if not self.allow(name): + return + largs = list(args) + largs[0:0] = [sys.executable, self.mypy] + env = self.env + self.waiter.add(LazySubprocess(name, largs, cwd=cwd, env=env)) + + def add_python(self, name: str, *args: str, cwd: Optional[str] = None) -> None: + name = 'run %s' % name + if not self.allow(name): + return + largs = list(args) + largs[0:0] = [sys.executable] + env = self.env + self.waiter.add(LazySubprocess(name, largs, cwd=cwd, env=env)) + + def add_both(self, name: str, *args: str, cwd: Optional[str] = None) -> None: + self.add_mypy(name, *args, cwd=cwd) + self.add_python(name, *args, cwd=cwd) + + def add_mypy_mod(self, name: str, *args: str, cwd: Optional[str] = None) -> None: + name = 'check %s' % name + if not self.allow(name): + return + largs = list(args) + largs[0:0] = [sys.executable, self.mypy, '-m'] + env = self.env + self.waiter.add(LazySubprocess(name, largs, cwd=cwd, env=env)) + + def add_python_mod(self, name: str, *args: str, cwd: Optional[str] = None) -> None: + name = 'run %s' % name + if not self.allow(name): + return + largs = list(args) + largs[0:0] = [sys.executable, '-m'] + env = self.env + self.waiter.add(LazySubprocess(name, largs, cwd=cwd, env=env)) + + def add_both_mod(self, name: str, *args: str, cwd: Optional[str] = None) -> None: + self.add_mypy_mod(name, *args, cwd=cwd) + self.add_python_mod(name, *args, cwd=cwd) + + def add_mypy_string(self, name: str, *args: str, cwd: Optional[str] = None) -> None: + name = 'check %s' % name + if not self.allow(name): + return + largs = list(args) + largs[0:0] = [sys.executable, self.mypy, '-c'] + env = self.env + self.waiter.add(LazySubprocess(name, largs, cwd=cwd, env=env)) + + def add_python_string(self, name: str, *args: str, cwd: Optional[str] = None) -> None: + name = 'run %s' % name + if not self.allow(name): + return + largs = list(args) + largs[0:0] = [sys.executable, '-c'] + env = self.env + self.waiter.add(LazySubprocess(name, largs, cwd=cwd, env=env)) + + def add_both_string(self, name: str, *args: str, cwd: Optional[str] = None) -> None: + self.add_mypy_string(name, *args, cwd=cwd) + self.add_python_string(name, *args, cwd=cwd) + + def add_python2(self, name: str, *args: str, cwd: Optional[str] = None) -> None: + name = 'run2 %s' % name + if not self.allow(name): + return + largs = list(args) + largs[0:0] = ['python2'] + env = self.env + self.waiter.add(LazySubprocess(name, largs, cwd=cwd, env=env)) + + def add_flake8(self, name: str, file: str, cwd: Optional[str] = None) -> None: + name = 'lint %s' % name + if not self.allow(name): + return + largs = ['flake8', file] + env = self.env + self.waiter.add(LazySubprocess(name, largs, cwd=cwd, env=env)) + + +def add_basic(driver: Driver) -> None: + if False: + driver.add_mypy('file setup.py', 'setup.py') + driver.add_flake8('file setup.py', 'setup.py') + driver.add_mypy('file runtests.py', 'runtests.py') + driver.add_flake8('file runtests.py', 'runtests.py') + driver.add_mypy('legacy entry script', 'scripts/mypy') + driver.add_flake8('legacy entry script', 'scripts/mypy') + driver.add_mypy('legacy myunit script', 'scripts/myunit') + driver.add_flake8('legacy myunit script', 'scripts/myunit') + driver.add_mypy_mod('entry mod mypy', 'mypy') + driver.add_mypy_mod('entry mod mypy.stubgen', 'mypy.stubgen') + driver.add_mypy_mod('entry mod mypy.myunit', 'mypy.myunit') + + +def find_files(base: str, prefix: str = '', suffix: str = '') -> List[str]: + return [join(root, f) + for root, dirs, files in os.walk(base) + for f in files + if f.startswith(prefix) and f.endswith(suffix)] + + +def file_to_module(file: str) -> str: + rv = os.path.splitext(file)[0].replace(os.sep, '.') + if rv.endswith('.__init__'): + rv = rv[:-len('.__init__')] + return rv + + +def add_imports(driver: Driver) -> None: + # Make sure each module can be imported originally. + # There is currently a bug in mypy where a module can pass typecheck + # because of *implicit* imports from other modules. + for f in find_files('mypy', suffix='.py'): + mod = file_to_module(f) + if '.test.data.' in mod: + continue + driver.add_mypy_string('import %s' % mod, 'import %s' % mod) + if not mod.endswith('.__main__'): + driver.add_python_string('import %s' % mod, 'import %s' % mod) + driver.add_flake8('module %s' % mod, f) + + +def add_myunit(driver: Driver) -> None: + for f in find_files('mypy', prefix='test', suffix='.py'): + mod = file_to_module(f) + if '.codec.test.' in mod: + # myunit is Python3 only. + driver.add_python_mod('unittest %s' % mod, 'unittest', mod) + driver.add_python2('unittest %s' % mod, '-m', 'unittest', mod) + continue + driver.add_python_mod('unit-test %s' % mod, 'mypy.myunit', '-m', mod, *driver.arglist) + + +def add_stubs(driver: Driver) -> None: + # Only test each module once, for the latest Python version supported. + # The third-party stub modules will only be used if it is not in the version. + seen = set() # type: Set[str] + for version in driver.versions: + for pfx in ['', 'third-party-']: + stubdir = join('stubs', pfx + version) + for f in find_files(stubdir, suffix='.pyi'): + module = file_to_module(f[len(stubdir) + 1:]) + if module not in seen: + seen.add(module) + driver.add_mypy_string( + 'stub (%s) module %s' % (pfx + version, module), + 'import typing, %s' % module) + + +def add_libpython(driver: Driver) -> None: + seen = set() # type: Set[str] + for version in driver.versions: + libpython_dir = join(driver.cwd, 'lib-python', version) + for f in find_files(libpython_dir, prefix='test_', suffix='.py'): + module = file_to_module(f[len(libpython_dir) + 1:]) + if module not in seen: + seen.add(module) + driver.add_mypy_mod( + 'libpython (%s) module %s' % (version, module), + module, + cwd=libpython_dir) + + +def add_samples(driver: Driver) -> None: + for f in find_files('samples', suffix='.py'): + if 'codec' in f: + cwd, bf = os.path.dirname(f), os.path.basename(f) + bf = bf[:-len('.py')] + driver.add_mypy_string('codec file %s' % f, + 'import mypy.codec.register, %s' % bf, + cwd=cwd) + else: + driver.add_mypy('file %s' % f, f) + + +def usage(status: int) -> None: + print('Usage: %s [-h | -v | -q | [-x] filter | -a argument] ... [-- filter ...]' % sys.argv[0]) + print(' -h, --help show this help') + print(' -v, --verbose increase driver verbosity') + print(' -q, --quiet decrease driver verbosity') + print(' -a, --argument pass an argument to myunit tasks') + print(' -- treat all remaning arguments as positional') + print(' filter only include tasks matching filter') + print(' -x, --exclude filter exclude tasks matching filter') + sys.exit(status) + + +def sanity() -> None: + paths = os.getenv('PYTHONPATH') + if paths is None: + return + failed = False + for p in paths.split(os.pathsep): + if not os.path.isabs(p): + print('Relative PYTHONPATH entry %r' % p) + failed = True + if failed: + print('Please use absolute so that chdir() tests can work.') + print('Cowardly refusing to continue.') + sys.exit(1) + + +def main() -> None: + sanity() + + verbosity = 0 + whitelist = [] # type: List[str] + blacklist = [] # type: List[str] + arglist = [] # type: List[str] + + allow_opts = True + curlist = whitelist + for a in sys.argv[1:]: + if curlist is not arglist and allow_opts and a.startswith('-'): + if curlist is not whitelist: + break + if a == '--': + allow_opts = False + elif a == '-v' or a == '--verbose': + verbosity += 1 + elif a == '-q' or a == '--quiet': + verbosity -= 1 + elif a == '-x' or a == '--exclude': + curlist = blacklist + elif a == '-a' or a == '--argument': + curlist = arglist + elif a == '-h' or a == '--help': + usage(0) + else: + usage(1) + else: + curlist.append(a) + curlist = whitelist + if curlist is blacklist: + sys.exit('-x must be followed by a filter') + if curlist is arglist: + sys.exit('-a must be followed by an argument') + # empty string is a substring of all names + if not whitelist: + whitelist.append('') + + driver = Driver(whitelist=whitelist, blacklist=blacklist, arglist=arglist, + verbosity=verbosity, xfail=[ + 'check stub (third-party-3.2) module requests.packages.urllib3.connection', + 'check stub (third-party-3.2) module requests.packages.urllib3.packages', + 'check stub (third-party-3.2) module ' + + 'requests.packages.urllib3.packages.ssl_match_hostname', + 'check stub (third-party-3.2) module ' + + 'requests.packages.urllib3.packages.ssl_match_hostname._implementation', + ]) + driver.prepend_path('PATH', [join(driver.cwd, 'scripts')]) + driver.prepend_path('MYPYPATH', [driver.cwd]) + driver.prepend_path('PYTHONPATH', [driver.cwd]) + driver.prepend_path('PYTHONPATH', [join(driver.cwd, 'lib-typing', v) for v in driver.versions]) + + add_basic(driver) + add_myunit(driver) + add_imports(driver) + add_stubs(driver) + add_libpython(driver) + add_samples(driver) + + driver.waiter.run() + +if __name__ == '__main__': + main() diff --git a/samples/codec/example.py b/samples/codec/example.py index 2a0591e86e4e..ba3e5f714d68 100644 --- a/samples/codec/example.py +++ b/samples/codec/example.py @@ -2,8 +2,9 @@ # very simple script to test type annotations +from typing import Tuple -def f(x: int, y: str='abc') -> [int, +def f(x: int, y: str='abc') -> Tuple[int, str]: return x, y diff --git a/scripts/mypy b/scripts/mypy index 5c734269cf8d..cda289f55262 100755 --- a/scripts/mypy +++ b/scripts/mypy @@ -1,184 +1,6 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 """Mypy type checker command line tool.""" -import os -import os.path -import shutil -import subprocess -import sys -import tempfile +from mypy.main import main -import typing -from typing import List, Tuple - -from mypy import build -from mypy.errors import CompileError - -from mypy.version import __version__ - - -class Options: - def __init__(self) -> None: - # Set default options. - self.target = build.TYPE_CHECK - self.build_flags = [] # type: List[str] - self.pyversion = 3 - self.custom_typing_module = None # type: str - self.html_report_dir = None # type: str - self.python_path = False - - -def main() -> None: - bin_dir = find_bin_directory() - path, module, options = process_options(sys.argv[1:]) - try: - if options.target == build.TYPE_CHECK: - type_check_only(path, module, bin_dir, options) - else: - raise RuntimeError('unsupported target %d' % options.target) - except CompileError as e: - for m in e.messages: - sys.stderr.write(m + '\n') - sys.exit(1) - - -def find_bin_directory() -> str: - """Find the directory that contains this script. - - This is used by build to find stubs and other data files. - """ - script = __file__ - # Follow up to 5 symbolic links (cap to avoid cycles). - for i in range(5): - if os.path.islink(script): - script = readlinkabs(script) - else: - break - return os.path.dirname(script) - - -def readlinkabs(link: str) -> str: - """Return an absolute path to symbolic link destination.""" - # Adapted from code by Greg Smith. - assert os.path.islink(link) - path = os.readlink(link) - if os.path.isabs(path): - return path - return os.path.join(os.path.dirname(link), path) - - -def type_check_only(path: str, module: str, bin_dir: str, options: Options) -> None: - # Type check the program and dependencies and translate to Python. - build.build(path, - module=module, - bin_dir=bin_dir, - target=build.TYPE_CHECK, - pyversion=options.pyversion, - custom_typing_module=options.custom_typing_module, - html_report_dir=options.html_report_dir, - flags=options.build_flags, - python_path=options.python_path) - - -def process_options(args: List[str]) -> Tuple[str, str, Options]: - """Process command line arguments. - - Return (mypy program path (or None), - module to run as script (or None), - parsed flags) - """ - options = Options() - help = False - ver = False - while args and args[0].startswith('-'): - if args[0] == '--verbose': - options.build_flags.append(build.VERBOSE) - args = args[1:] - elif args[0] == '--py2': - # Use Python 2 mode. - options.pyversion = 2 - args = args[1:] - elif args[0] == '-m' and args[1:]: - options.build_flags.append(build.MODULE) - return None, args[1], options - elif args[0] in ('-h', '--help'): - help = True - args = args[1:] - elif args[0] == '--stats': - options.build_flags.append('dump-type-stats') - args = args[1:] - elif args[0] == '--inferstats': - options.build_flags.append('dump-infer-stats') - args = args[1:] - elif args[0] == '--custom-typing' and args[1:]: - options.custom_typing_module = args[1] - args = args[2:] - elif args[0] == '--html-report' and args[1:]: - options.html_report_dir = args[1] - options.build_flags.append('html-report') - args = args[2:] - elif args[0] == '--use-python-path': - options.python_path = True - args = args[1:] - elif args[0] == '--version': - ver = True - args = args[1:] - else: - usage('Unknown option: {}'.format(args[0])) - - if help: - usage() - - if ver: - version() - - if not args: - usage('Missing target file or module') - - if args[1:]: - usage('Extra argument: {}'.format(args[1])) - - if options.python_path and options.pyversion == 2: - usage('--py2 specified, ' - 'but --use-python-path will search in sys.path of Python 3') - - return args[0], None, options - - -def usage(msg: str = None) -> None: - if msg: - sys.stderr.write('%s\n' % msg) - sys.stderr.write("""\ -usage: mypy [option ...] [-m mod | file] -Try 'mypy -h' for more information. -""") - else: - sys.stderr.write("""\ -usage: mypy [option ...] [-m mod | file] - -Optional arguments: - -h, --help print this help message and exit - --html-report dir generate a HTML report of type precision under dir/ - -m mod type check module - --verbose more verbose messages - --use-python-path search for modules in sys.path of running Python - --version show the current version information - -Environment variables: - MYPYPATH additional module search path -""") - sys.exit(2) - - -def version() -> None: - sys.stdout.write("mypy {}\n".format(__version__)) - exit(0) - - -def fail(msg: str) -> None: - sys.stderr.write('%s\n' % msg) - sys.exit(1) - - -if __name__ == '__main__': - main() +main() diff --git a/scripts/myunit b/scripts/myunit new file mode 100755 index 000000000000..43fce063f5a2 --- /dev/null +++ b/scripts/myunit @@ -0,0 +1,9 @@ +#!/usr/bin/env python3 +"""Myunit test runner command line tool. + +Usually used as a slave by runtests.py, but can be used directly. +""" + +from mypy.myunit import main + +main() diff --git a/setup.py b/setup.py index 69553acb483e..5dc46feecdab 100644 --- a/setup.py +++ b/setup.py @@ -26,23 +26,31 @@ types. '''.lstrip() -stubs = [] -for py_version in ['3.4', '3.3', '3.2', '2.7']: - base = os.path.join('stubs', py_version) - if not os.path.exists(base): - os.mkdir(base) +def find_data_files(base, globs): + """Find all interesting data files, for setup(data_files=) - stub_dirs = [''] - for root, dirs, files in os.walk(base): - stub_dirs.extend(os.path.relpath(os.path.join(root, stub_dir), base) - for stub_dir in dirs - if stub_dir != '__pycache__') - for stub_dir in stub_dirs: - target = os.path.join('lib', 'mypy', 'stubs', py_version, stub_dir) - files = glob.glob(os.path.join(base, stub_dir, '*.py')) - files += glob.glob(os.path.join(base, stub_dir, '*.pyi')) - stubs.append((target, files)) + Arguments: + root: The directory to search in. + globs: A list of glob patterns to accept files. + """ + + rv_dirs = [root for root, dirs, files in os.walk(base)] + rv = [] + for rv_dir in rv_dirs: + files = [] + for pat in globs: + files += glob.glob(os.path.join(rv_dir, pat)) + if not files: + continue + target = os.path.join('lib', 'mypy', rv_dir) + rv.append((target, files)) + + return rv + +data_files = [] + +data_files += find_data_files('stubs', ['*.py', '*.pyi']) classifiers = [ 'Development Status :: 2 - Pre-Alpha', @@ -69,6 +77,6 @@ py_modules=['typing'], packages=['mypy'], scripts=['scripts/mypy'], - data_files=stubs, + data_files=data_files, classifiers=classifiers, ) diff --git a/stubs/3.2/docutils/parsers/rst/__init__.pyi b/stubs/2.7/xml/__init__.pyi similarity index 100% rename from stubs/3.2/docutils/parsers/rst/__init__.pyi rename to stubs/2.7/xml/__init__.pyi diff --git a/stubs/2.7/xml/__init__.py b/stubs/2.7/xml/sax/__init__.pyi similarity index 100% rename from stubs/2.7/xml/__init__.py rename to stubs/2.7/xml/sax/__init__.pyi diff --git a/stubs/3.2/bisect.pyi b/stubs/3.2/bisect.pyi new file mode 100644 index 000000000000..e2e12b696465 --- /dev/null +++ b/stubs/3.2/bisect.pyi @@ -0,0 +1,12 @@ +from typing import Sequence, TypeVar + +_T = TypeVar('_T') + +def insort_left(a: Sequence[_T], x: _T, lo: int = 0, hi: int = None): pass +def insort_right(a: Sequence[_T], x: _T, lo: int = 0, hi: int = None): pass + +def bisect_left(a: Sequence[_T], x: _T, lo: int = 0, hi: int = None): pass +def bisect_right(a: Sequence[_T], x: _T, lo: int = 0, hi: int = None): pass + +insort = insort_right +bisect = bisect_right diff --git a/stubs/3.2/builtins.pyi b/stubs/3.2/builtins.pyi index df082f30169c..838d7209beaa 100644 --- a/stubs/3.2/builtins.pyi +++ b/stubs/3.2/builtins.pyi @@ -3,8 +3,8 @@ from typing import ( TypeVar, Iterator, Iterable, overload, Sequence, MutableSequence, Mapping, MutableMapping, Tuple, List, Any, Dict, Callable, Generic, - Set, AbstractSet, MutableSet, Sized, Reversible, SupportsInt, SupportsFloat, SupportsAbs, - SupportsRound, IO, Union, ItemsView, KeysView, ValuesView, ByteString + Set, AbstractSet, MutableSet, Sized, Reversible, SupportsInt, SupportsFloat, SupportsBytes, + SupportsAbs, SupportsRound, IO, Union, ItemsView, KeysView, ValuesView, ByteString ) from abc import abstractmethod, ABCMeta @@ -255,6 +255,8 @@ class bytes(ByteString): def __init__(self, length: int) -> None: ... @overload def __init__(self) -> None: ... + @overload + def __init__(self, o: SupportsBytes) -> None: ... def capitalize(self) -> bytes: ... def center(self, width: int, fillchar: bytes = None) -> bytes: ... def count(self, x: bytes) -> int: ... diff --git a/stubs/3.2/cgi.pyi b/stubs/3.2/cgi.pyi index 8c622dfb7476..86d4eac1cf5a 100644 --- a/stubs/3.2/cgi.pyi +++ b/stubs/3.2/cgi.pyi @@ -1 +1 @@ -def escape(s: str) -> str: ... +def escape(s: str, quote: bool = False) -> str: ... diff --git a/stubs/3.2/codecs.pyi b/stubs/3.2/codecs.pyi index 0ad02b7e8d7a..c44713e71231 100644 --- a/stubs/3.2/codecs.pyi +++ b/stubs/3.2/codecs.pyi @@ -1,31 +1,194 @@ -from typing import Any, BinaryIO, Callable, IO +# Better codecs stubs hand-written by o11c. +# https://docs.python.org/3/library/codecs.html +from typing import ( + BinaryIO, + Callable, + Iterable, + Iterator, + List, + Tuple, + Union, +) -BOM_UTF8 = b'' +from abc import abstractmethod + + +# TODO: this only satisfies the most common interface, where +# bytes is the raw form and str is the cooked form. +# In the long run, both should become template parameters maybe? +# There *are* bytes->bytes and str->str encodings in the standard library. +# Python 3.5 supposedly might change something there. + +_decoded = str +_encoded = bytes + +# TODO: It is not possible to specify these signatures correctly, because +# they have an optional positional or keyword argument for errors=. +_encode_type = Callable[[_decoded], _encoded] # signature of Codec().encode +_decode_type = Callable[[_encoded], _decoded] # signature of Codec().decode +_stream_reader_type = Callable[[BinaryIO], 'StreamReader'] # signature of StreamReader __init__ +_stream_writer_type = Callable[[BinaryIO], 'StreamWriter'] # signature of StreamWriter __init__ +_incremental_encoder_type = Callable[[], 'IncrementalEncoder'] # signature of IncrementalEncoder __init__ +_incremental_decode_type = Callable[[], 'IncrementalDecoder'] # signature of IncrementalDecoder __init__ -class Codec: ... -class StreamWriter(Codec): ... -class CodecInfo(tuple): - def __init__(self, *args) -> None: ... +def encode(obj: _decoded, encoding: str = 'utf-8', errors: str = 'strict') -> _encoded: + ... +def decode(obj: _encoded, encoding: str = 'utf-8', errors: str = 'strict') -> _decoded: + ... + +def lookup(encoding: str) -> 'CodecInfo': + ... +class CodecInfo(Tuple[_encode_type, _decode_type, _stream_reader_type, _stream_writer_type]): + def __init__(self, encode: _encode_type, decode: _decode_type, streamreader: _stream_reader_type = None, streamwriter: _stream_writer_type = None, incrementalencoder: _incremental_encoder_type = None, incrementaldecoder: _incremental_decode_type = None, name: str = None) -> None: + self.encode = encode + self.decode = decode + self.streamreader = streamreader + self.streamwriter = streamwriter + self.incrementalencoder = incrementalencoder + self.incrementaldecoder = incrementaldecoder + self.name = name + +def getencoder(encoding: str) -> _encode_type: + ... +def getdecoder(encoding: str) -> _encode_type: + ... +def getincrementalencoder(encoding: str) -> _incremental_encoder_type: + ... +def getincrementaldecoder(encoding: str) -> _incremental_encoder_type: + ... +def getreader(encoding: str) -> _stream_reader_type: + ... +def getwriter(encoding: str) -> _stream_writer_type: + ... def register(search_function: Callable[[str], CodecInfo]) -> None: ... -def register_error(name: str, error_handler: Callable[[UnicodeError], Any]) -> None: ... +def open(filename: str, mode: str = 'r', encoding: str = None, errors: str = 'strict', buffering: int = 1) -> StreamReaderWriter: + ... -def lookup(encoding: str) -> CodecInfo: +def EncodedFile(file: BinaryIO, data_encoding: str, file_encoding: str = None, errors = 'strict') -> 'StreamRecoder': ... -# TODO This Callable is actually a StreamWriter constructor -def getwriter(encoding: str) -> Callable[[BinaryIO], StreamWriter]: ... +def iterencode(iterator: Iterable[_decoded], encoding: str, errors: str = 'strict') -> Iterator[_encoded]: + ... +def iterdecode(iterator: Iterable[_encoded], encoding: str, errors: str = 'strict') -> Iterator[_decoded]: + ... -class IncrementalDecoder: - errors = ... # type: Any - def __init__(self, errors=''): ... - def decode(self, input, final=False): ... - def reset(self): ... - def getstate(self): ... - def setstate(self, state): ... +BOM = b'' +BOM_BE = b'' +BOM_LE = b'' +BOM_UTF8 = b'' +BOM_UTF16 = b'' +BOM_UTF16_BE = b'' +BOM_UTF16_LE = b'' +BOM_UTF32 = b'' +BOM_UTF32_BE = b'' +BOM_UTF32_LE = b'' + +# It is expected that different actions be taken depending on which of the +# three subclasses of `UnicodeError` is actually ...ed. However, the Union +# is still needed for at least one of the cases. +def register_error(name: str, error_handler: Callable[[UnicodeError], Tuple[Union[str, bytes], int]]) -> None: + ... +def lookup_error(name: str) -> Callable[[UnicodeError], Tuple[Union[str, bytes], int]]: + ... -def open(filename: str, mode: str='rb', encoding: str=None, errors: str='strict', buffering: int=1) -> IO[Any]: +def strict_errors(exception: UnicodeError) -> Tuple[Union[str, bytes], int]: + ... +def replace_errors(exception: UnicodeError) -> Tuple[Union[str, bytes], int]: + ... +def ignore_errors(exception: UnicodeError) -> Tuple[Union[str, bytes], int]: + ... +def xmlcharrefreplace_errors(exception: UnicodeError) -> Tuple[Union[str, bytes], int]: + ... +def backslashreplace_errors(exception: UnicodeError) -> Tuple[Union[str, bytes], int]: ... + +class Codec: + # These are sort of @abstractmethod but sort of not. + # The StreamReader and StreamWriter subclasses only implement one. + def encode(self, input: _decoded, errors: str = 'strict') -> Tuple[_encoded, int]: + ... + def decode(self, input: _encoded, errors: str = 'strict') -> Tuple[_decoded, int]: + ... + +class IncrementalEncoder: + def __init__(self, errors: str = 'strict') -> None: + self.errors = errors + @abstractmethod + def encode(self, object: _decoded, final: bool = False) -> _encoded: + ... + def reset(self) -> None: + ... + # documentation says int but str is needed for the subclass. + def getstate(self) -> Union[int, _decoded]: + ... + def setstate(self, state: Union[int, _decoded]) -> None: + ... + +class IncrementalDecoder: + def __init__(self, errors: str = 'strict') -> None: + self.errors = errors + @abstractmethod + def decode(self, object: _encoded, final: bool = False) -> _decoded: + ... + def reset(self) -> None: + ... + def getstate(self) -> Tuple[_encoded, int]: + ... + def setstate(self, state: Tuple[_encoded, int]) -> None: + ... + +# These are not documented but used in encodings/*.py implementations. +class BufferedIncrementalEncoder(IncrementalEncoder): + def __init__(self, errors: str = 'strict') -> None: + IncrementalEncoder.__init__(self, errors) + self.buffer = '' + @abstractmethod + def _buffer_encode(self, input: _decoded, errors: str, final: bool) -> _encoded: + ... + def encode(self, input: _decoded, final: bool = False) -> _encoded: + ... +class BufferedIncrementalDecoder(IncrementalDecoder): + def __init__(self, errors: str = 'strict') -> None: + IncrementalDecoder.__init__(self, errors) + self.buffer = b'' + @abstractmethod + def _buffer_decode(self, input: _encoded, errors: str, final: bool) -> Tuple[_decoded, int]: + ... + def decode(self, object: _encoded, final: bool = False) -> _decoded: + ... + +# TODO: it is not possible to specify the requirement that all other +# attributes and methods are passed-through from the stream. +class StreamWriter(Codec): + def __init__(self, stream: BinaryIO, errors: str = 'strict') -> None: + self.errors = errors + def write(self, obj: _decoded) -> None: + ... + def writelines(self, list: List[str]) -> None: + ... + def reset(self) -> None: + ... + +class StreamReader(Codec): + def __init__(self, stream: BinaryIO, errors: str = 'strict') -> None: + self.errors = errors + def read(self, size: int = -1, chars: int = -1, firstline: bool = False) -> _decoded: + ... + def readline(self, size: int = -1, keepends: bool = True) -> _decoded: + ... + def readlines(self, sizehint: int = -1, keepends: bool = True) -> List[_decoded]: + ... + def reset(self) -> None: + ... + +class StreamReaderWriter: + def __init__(self, stream: BinaryIO, Reader: _stream_reader_type, Writer: _stream_writer_type, errors: str = 'strict') -> None: + ... + +class StreamRecoder(BinaryIO): + def __init__(self, stream: BinaryIO, encode: _encode_type, decode: _decode_type, Reader: _stream_reader_type, Writer: _stream_writer_type, errors: str = 'strict') -> None: + ... diff --git a/stubs/3.2/encodings.pyi b/stubs/3.2/encodings/__init__.pyi similarity index 100% rename from stubs/3.2/encodings.pyi rename to stubs/3.2/encodings/__init__.pyi diff --git a/stubs/3.2/encodings/utf_8.pyi b/stubs/3.2/encodings/utf_8.pyi new file mode 100644 index 000000000000..b10865428d50 --- /dev/null +++ b/stubs/3.2/encodings/utf_8.pyi @@ -0,0 +1,14 @@ +import codecs + +class IncrementalEncoder(codecs.IncrementalEncoder): + pass +class IncrementalDecoder(codecs.BufferedIncrementalDecoder): + pass +class StreamWriter(codecs.StreamWriter): + pass +class StreamReader(codecs.StreamReader): + pass + +def getregentry() -> codecs.CodecInfo: pass +def encode(input: str, errors: str = 'strict') -> bytes: pass +def decode(input: bytes, errors: str = 'strict') -> str: pass diff --git a/stubs/3.2/inspect.pyi b/stubs/3.2/inspect.pyi index d4fae4b71d6b..9286aa04f83a 100644 --- a/stubs/3.2/inspect.pyi +++ b/stubs/3.2/inspect.pyi @@ -1,6 +1,7 @@ # Stubs for inspect from typing import Any, Tuple, List, Callable +from types import FrameType _object = object @@ -29,3 +30,5 @@ class ArgSpec(tuple): defaults = ... # type: tuple def getargspec(func: object) -> ArgSpec: ... + +def stack() -> List[Tuple[FrameType, str, int, str, List[str], int]]: ... diff --git a/stubs/3.2/os/__init__.pyi b/stubs/3.2/os/__init__.pyi index 3ada3551f118..b6113bb34dc3 100644 --- a/stubs/3.2/os/__init__.pyi +++ b/stubs/3.2/os/__init__.pyi @@ -5,7 +5,7 @@ from typing import ( Mapping, MutableMapping, Dict, List, Any, Tuple, Iterator, overload, Union, AnyStr, - Optional, Generic + Optional, Generic, Set ) from builtins import OSError as error import os.path as path @@ -336,3 +336,11 @@ def confstr(name: str) -> str: ... # Unix only def getloadavg() -> Tuple[float, float, float]: ... # Unix only def sysconf(name: str) -> int: ... # Unix only def urandom(n: int) -> bytes: ... + +def sched_getaffinity(id: int) -> Set[int]: ... +class waitresult: + si_pid = 0 +def waitid(idtype: int, id: int, options: int) -> waitresult: ... +P_ALL = 0 +WEXITED = 0 +WNOWAIT = 0 diff --git a/stubs/3.2/pipes.pyi b/stubs/3.2/pipes.pyi new file mode 100644 index 000000000000..62163d622cca --- /dev/null +++ b/stubs/3.2/pipes.pyi @@ -0,0 +1,19 @@ +# Stubs for pipes + +# Based on http://docs.python.org/3.5/library/pipes.html + +import os + +class Template: + def __init__(self) -> None: ... + def reset(self) -> None: ... + def clone(self) -> 'Template': ... + def debug(self, flag: bool) -> None: ... + def append(self, cmd: str, kind: str) -> None: ... + def prepend(self, cmd: str, kind: str) -> None: ... + def open(self, file: str, rw: str) -> os.popen: ... + def copy(self, file: str, rw: str) -> os.popen: ... + +# Not documented, but widely used. +# Documented as shlex.quote since 3.3. +def quote(s: str) -> str: ... diff --git a/stubs/3.2/shlex.pyi b/stubs/3.2/shlex.pyi index 8d6e01c27187..431d20a8e59b 100644 --- a/stubs/3.2/shlex.pyi +++ b/stubs/3.2/shlex.pyi @@ -7,6 +7,9 @@ from typing import List, Tuple, Any, TextIO def split(s: str, comments: bool = False, posix: bool = True) -> List[str]: ... +# Added in 3.3, use (undocumented) pipes.quote in previous versions. +def quote(s: str) -> str: ... + class shlex: commenters = '' wordchars = '' diff --git a/stubs/3.2/sys.pyi b/stubs/3.2/sys.pyi index 210216bd3ebc..fe769e50b4f6 100644 --- a/stubs/3.2/sys.pyi +++ b/stubs/3.2/sys.pyi @@ -4,7 +4,7 @@ # based on http://docs.python.org/3.2/library/sys.html from typing import ( - List, Sequence, Any, Dict, Tuple, TextIO, overload, Optional + List, Sequence, Any, Dict, Tuple, TextIO, overload, Optional, Union ) from types import TracebackType @@ -115,7 +115,7 @@ def displayhook(value: Optional[int]) -> None: ... def excepthook(type_: type, value: BaseException, traceback: TracebackType) -> None: ... def exc_info() -> Tuple[type, BaseException, TracebackType]: ... -def exit(arg: int = None) -> None: ... +def exit(arg: Union[int, str] = None) -> None: ... def getcheckinterval() -> int: ... # deprecated def getdefaultencoding() -> str: ... def getdlopenflags() -> int: ... # Unix only diff --git a/stubs/3.2/textwrap.pyi b/stubs/3.2/textwrap.pyi new file mode 100644 index 000000000000..e02f020eb548 --- /dev/null +++ b/stubs/3.2/textwrap.pyi @@ -0,0 +1,118 @@ +# Better textwrap stubs hand-written by o11c. +# https://docs.python.org/3/library/textwrap.html +from typing import ( + Callable, + List, +) + +class TextWrapper: + def __init__(self, + width: int = 70, + *, + initial_indent: str = '', + subsequent_indent: str = '', + expand_tabs: bool = True, + tabsize: int = 8, + replace_whitespace: bool = True, + fix_sentence_endings: bool = False, + break_long_words: bool = True, + break_on_hyphens: bool = True, + drop_whitespace: bool = True, + max_lines: int = None, + placeholder: str = ' [...]', + ) -> None: + self.width = width + self.initial_indent = initial_indent + self.subsequent_indent = subsequent_indent + self.expand_tabs = expand_tabs + self.tabsize = tabsize + self.replace_whitespace = replace_whitespace + self.fix_sentence_endings = fix_sentence_endings + self.break_long_words = break_long_words + self.break_on_hyphens = break_on_hyphens + self.drop_whitespace = drop_whitespace + self.max_lines = max_lines + self.placeholder = placeholder + + # Private methods *are* part of the documented API for subclasses. + def _munge_whitespace(self, text: str) -> str: + ... + + def _split(self, text: str) -> List[str]: + ... + + def _fix_sentence_endings(self, chunks: List[str]) -> None: + ... + + def _handle_long_word(self, reversed_chunks: List[str], cur_line: List[str], cur_len: int, width: int) -> None: + ... + + def _wrap_chunks(self, chunks: List[str]) -> List[str]: + ... + + def _split_chunks(self, text: str) -> List[str]: + ... + + def wrap(self, text: str) -> List[str]: + ... + + def fill(self, text: str) -> str: + ... + + +def wrap( + width: int = 70, + *, + initial_indent: str = '', + subsequent_indent: str = '', + expand_tabs: bool = True, + tabsize: int = 8, + replace_whitespace: bool = True, + fix_sentence_endings: bool = False, + break_long_words: bool = True, + break_on_hyphens: bool = True, + drop_whitespace: bool = True, + max_lines: int = None, + placeholder: str = ' [...]', +) -> List[str]: + ... + +def fill( + width: int = 70, + *, + initial_indent: str = '', + subsequent_indent: str = '', + expand_tabs: bool = True, + tabsize: int = 8, + replace_whitespace: bool = True, + fix_sentence_endings: bool = False, + break_long_words: bool = True, + break_on_hyphens: bool = True, + drop_whitespace: bool = True, + max_lines: int = None, + placeholder: str = ' [...]', +): + ... + +def shorten( + width: int, + *, + initial_indent: str = '', + subsequent_indent: str = '', + expand_tabs: bool = True, + tabsize: int = 8, + replace_whitespace: bool = True, + fix_sentence_endings: bool = False, + break_long_words: bool = True, + break_on_hyphens: bool = True, + drop_whitespace: bool = True, + # Omit `max_lines: int = None`, it is forced to 1 here. + placeholder: str = ' [...]', +): + ... + +def dedent(text: str) -> str: + ... + +def indent(text: str, prefix: str, predicate: Callable[[str], bool] = None) -> str: + ... diff --git a/stubs/3.2/token.pyi b/stubs/3.2/token.pyi new file mode 100644 index 000000000000..76a746f1753c --- /dev/null +++ b/stubs/3.2/token.pyi @@ -0,0 +1,63 @@ +from typing import Dict + +ENDMARKER = 0 +NAME = 0 +NUMBER = 0 +STRING = 0 +NEWLINE = 0 +INDENT = 0 +DEDENT = 0 +LPAR = 0 +RPAR = 0 +LSQB = 0 +RSQB = 0 +COLON = 0 +COMMA = 0 +SEMI = 0 +PLUS = 0 +MINUS = 0 +STAR = 0 +SLASH = 0 +VBAR = 0 +AMPER = 0 +LESS = 0 +GREATER = 0 +EQUAL = 0 +DOT = 0 +PERCENT = 0 +LBRACE = 0 +RBRACE = 0 +EQEQUAL = 0 +NOTEQUAL = 0 +LESSEQUAL = 0 +GREATEREQUAL = 0 +TILDE = 0 +CIRCUMFLEX = 0 +LEFTSHIFT = 0 +RIGHTSHIFT = 0 +DOUBLESTAR = 0 +PLUSEQUAL = 0 +MINEQUAL = 0 +STAREQUAL = 0 +SLASHEQUAL = 0 +PERCENTEQUAL = 0 +AMPEREQUAL = 0 +VBAREQUAL = 0 +CIRCUMFLEXEQUAL = 0 +LEFTSHIFTEQUAL = 0 +RIGHTSHIFTEQUAL = 0 +DOUBLESTAREQUAL = 0 +DOUBLESLASH = 0 +DOUBLESLASHEQUAL = 0 +AT = 0 +RARROW = 0 +ELLIPSIS = 0 +OP = 0 +ERRORTOKEN = 0 +N_TOKENS = 0 +NT_OFFSET = 0 +tok_name = {} # type: Dict[int, str] + +def ISTERMINAL(x: int) -> bool: pass +def ISNONTERMINAL(x: int) -> bool: pass +def ISEOF(x: int) -> bool: pass diff --git a/stubs/3.2/types.pyi b/stubs/3.2/types.pyi index 866f05d44970..80f4ac02c386 100644 --- a/stubs/3.2/types.pyi +++ b/stubs/3.2/types.pyi @@ -2,7 +2,7 @@ # TODO this is work in progress -from typing import Any +from typing import Any, Callable, Dict, Sequence class ModuleType: __name__ = ... # type: str @@ -12,8 +12,55 @@ class ModuleType: class MethodType: ... class BuiltinMethodType: ... +class CodeType: + """Create a code object. Not for the faint of heart.""" + def __init__(self, + argcount: int, + kwonlyargcount: int, + nlocals: int, + stacksize: int, + flags: int, + codestring: bytes, + constants: Sequence[Any], + names: Sequence[str], + varnames: Sequence[str], + filename: str, + name: str, + firstlineno: int, + lnotab: bytes, + freevars: Sequence[str] = (), + cellvars: Sequence[str] = (), + ) -> None: + self.co_argcount = argcount + self.co_kwonlyargcount = kwonlyargcount + self.co_nlocals = nlocals + self.co_stacksize = stacksize + self.co_flags = flags + self.co_code = codestring + self.co_consts = constants + self.co_names = names + self.co_varnames = varnames + self.co_filename = filename + self.co_name = name + self.co_firstlineno = firstlineno + self.co_lnotab = lnotab + self.co_freevars = freevars + self.co_cellvars = cellvars + +class FrameType: + f_back = ... # type: FrameType + f_builtins = ... # type: Dict[str, Any] + f_code = ... # type: CodeType + f_globals = ... # type: Dict[str, Any] + f_lasti = ... # type: int + f_lineno = ... # type: int + f_locals = ... # type: Dict[str, Any] + f_trace = ... # type: Callable[[], None] + + def clear(self) -> None: pass + class TracebackType: - tb_frame = ... # type: Any + tb_frame = ... # type: FrameType tb_lasti = ... # type: int tb_lineno = ... # type: int - tb_next = ... # type: Any + tb_next = ... # type: TracebackType diff --git a/stubs/3.2/typing.pyi b/stubs/3.2/typing.pyi index 89fac9284aa0..c6d50b79bdc4 100644 --- a/stubs/3.2/typing.pyi +++ b/stubs/3.2/typing.pyi @@ -56,6 +56,14 @@ class SupportsFloat(metaclass=ABCMeta): @abstractmethod def __float__(self) -> float: ... +class SupportsComplex(metaclass=ABCMeta): + @abstractmethod + def __complex__(self) -> complex: pass + +class SupportsBytes(metaclass=ABCMeta): + @abstractmethod + def __bytes__(self) -> bytes: pass + class SupportsAbs(Generic[_T]): @abstractmethod def __abs__(self) -> _T: ... diff --git a/stubs/3.3/ipaddress.pyi b/stubs/3.3/ipaddress.pyi index 21696493150c..19ba2c20e765 100644 --- a/stubs/3.3/ipaddress.pyi +++ b/stubs/3.3/ipaddress.pyi @@ -194,7 +194,6 @@ class IPv6Interface(IPv6Address): class IPv6Network(_BaseV6, _BaseNetwork): network_address = ... # type: Any netmask = ... # type: Any - hosts = ... # type: Any def __init__(self, address, strict=True): ... def hosts(self): ... @property diff --git a/stubs/2.7/xml/sax/__init__.py b/stubs/third-party-2.7/Crypto/__init__.pyi similarity index 100% rename from stubs/2.7/xml/sax/__init__.py rename to stubs/third-party-2.7/Crypto/__init__.pyi diff --git a/stubs/3.2/docutils/__init__.pyi b/stubs/third-party-3.2/docutils/__init__.pyi similarity index 100% rename from stubs/3.2/docutils/__init__.pyi rename to stubs/third-party-3.2/docutils/__init__.pyi diff --git a/stubs/3.2/docutils/examples.pyi b/stubs/third-party-3.2/docutils/examples.pyi similarity index 100% rename from stubs/3.2/docutils/examples.pyi rename to stubs/third-party-3.2/docutils/examples.pyi diff --git a/stubs/3.2/docutils/nodes.pyi b/stubs/third-party-3.2/docutils/nodes.pyi similarity index 100% rename from stubs/3.2/docutils/nodes.pyi rename to stubs/third-party-3.2/docutils/nodes.pyi diff --git a/stubs/3.2/docutils/parsers/__init__.pyi b/stubs/third-party-3.2/docutils/parsers/__init__.pyi similarity index 100% rename from stubs/3.2/docutils/parsers/__init__.pyi rename to stubs/third-party-3.2/docutils/parsers/__init__.pyi diff --git a/stubs/third-party-2.7/Crypto/__init__.py b/stubs/third-party-3.2/docutils/parsers/rst/__init__.pyi similarity index 100% rename from stubs/third-party-2.7/Crypto/__init__.py rename to stubs/third-party-3.2/docutils/parsers/rst/__init__.pyi diff --git a/stubs/3.2/docutils/parsers/rst/nodes.pyi b/stubs/third-party-3.2/docutils/parsers/rst/nodes.pyi similarity index 100% rename from stubs/3.2/docutils/parsers/rst/nodes.pyi rename to stubs/third-party-3.2/docutils/parsers/rst/nodes.pyi diff --git a/stubs/3.2/docutils/parsers/rst/roles.pyi b/stubs/third-party-3.2/docutils/parsers/rst/roles.pyi similarity index 100% rename from stubs/3.2/docutils/parsers/rst/roles.pyi rename to stubs/third-party-3.2/docutils/parsers/rst/roles.pyi diff --git a/stubs/3.2/docutils/parsers/rst/states.pyi b/stubs/third-party-3.2/docutils/parsers/rst/states.pyi similarity index 100% rename from stubs/3.2/docutils/parsers/rst/states.pyi rename to stubs/third-party-3.2/docutils/parsers/rst/states.pyi diff --git a/tests.py b/tests.py deleted file mode 100644 index 63d124e4fa04..000000000000 --- a/tests.py +++ /dev/null @@ -1,43 +0,0 @@ -import sys - -from mypy.myunit import Suite, run_test -from mypy.test import testtypes -from mypy.test import testsubtypes -from mypy.test import testsolve -from mypy.test import testinfer -from mypy.test import testlex -from mypy.test import testparse -from mypy.test import testsemanal -from mypy.test import testtransform -from mypy.test import testcheck -from mypy.test import testtypegen -from mypy.test import teststubgen -from mypy.test import testdocstring - - -class AllSuite(Suite): - def __init__(self): - self.test_types = testtypes.TypesSuite() - self.test_typeops = testtypes.TypeOpsSuite() - self.test_join = testtypes.JoinSuite() - self.test_meet = testtypes.MeetSuite() - self.test_subtypes = testsubtypes.SubtypingSuite() - self.test_solve = testsolve.SolveSuite() - self.test_infer = testinfer.MapActualsToFormalsSuite() - self.test_lex = testlex.LexerSuite() - self.test_parse = testparse.ParserSuite() - self.test_parse_errors = testparse.ParseErrorSuite() - self.test_semanal = testsemanal.SemAnalSuite() - self.test_semanal_errors = testsemanal.SemAnalErrorSuite() - self.test_semanal_symtable = testsemanal.SemAnalSymtableSuite() - self.test_semanal_typeinfos = testsemanal.SemAnalTypeInfoSuite() - self.test_transform = testtransform.TransformSuite() - self.test_check = testcheck.TypeCheckSuite() - self.test_typegen = testtypegen.TypeExportSuite() - self.test_stubgen = teststubgen.StubgenSuite() - self.test_docstring = testdocstring.DocstringSuite() - super().__init__() - - -if __name__ == '__main__': - run_test(AllSuite(), sys.argv[1:]) diff --git a/tmp-test-dirs/.gitignore b/tmp-test-dirs/.gitignore new file mode 100644 index 000000000000..e6579d8f495f --- /dev/null +++ b/tmp-test-dirs/.gitignore @@ -0,0 +1,4 @@ +# This directory is used to store temporary directories for the testsuite. +# If anything manages to exist here, it means python crashed instead of +# calling tempfile.TemporaryDirectory's cleanup while unwinding. +# Therefore, don't actually provide any ignore patterns. diff --git a/travis.sh b/travis.sh deleted file mode 100755 index 874c22772aa9..000000000000 --- a/travis.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/bin/bash - -# Travis CI script that does these things: -# - run tests -# - type check the implementation -# - type check stubs -# - type check example code (for regression testing) -# - run a linter to catch style issues (flake8) - -PYTHON=${PYTHON-python} - -result=0 - -fail() -{ - result=1 -} - -# Setup stuff - -DRIVER=$PWD/scripts/mypy -export PYTHONPATH=`pwd`/lib-typing/3.2:`pwd` - -# Basic tests - -echo Running tests... -echo -echo tests.py -"$PYTHON" "$DRIVER" tests.py || fail -"$PYTHON" tests.py || fail -for t in mypy.test.testpythoneval; do - echo $t - "$PYTHON" "$DRIVER" -m $t || fail - "$PYTHON" -m $t || fail -done - -# Stub checks - -STUBTEST=_test_stubs.py -echo "import typing" > $STUBTEST -cd stubs/3.2 -ls *.pyi | sed s/\\.pyi//g | sed "s/^/import /g" >> ../../$STUBTEST -for m in os os.path; do - echo "import $m" >> ../../$STUBTEST -done -cd ../.. - -NUMSTUBS=$(( `wc -l $STUBTEST | cut -d' ' -f1` - 1 )) - -echo Type checking $NUMSTUBS stubs... -echo -"$PYTHON" "$DRIVER" $STUBTEST || fail -rm $STUBTEST - -# Checks sample code - -echo Type checking lib-python... -echo -pushd lib-python/3.2 > /dev/null -for f in test/test_*.py; do - mod=test.`basename "$f" .py` - echo $mod - "$PYTHON" "$DRIVER" -m $mod || fail -done -popd > /dev/null - -echo Linting... -echo -./lint.sh || fail - -exit $result