diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index effe90371..a88fe59a7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,13 +1,13 @@ repos: - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.3 + rev: 3.8.4 hooks: - id: flake8 exclude: cibuildwheel/resources/ additional_dependencies: [flake8-bugbear] - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.2.0 + rev: v3.4.0 hooks: - id: check-case-conflict - id: check-merge-conflict @@ -18,8 +18,8 @@ repos: - id: trailing-whitespace - repo: https://github.com/pre-commit/mirrors-mypy - rev: v0.782 + rev: v0.790 hooks: - id: mypy - files: ^(cibuildwheel/|test/|bin/projects.py) + files: ^(cibuildwheel/|test/|bin/projects.py|unit_test/) pass_filenames: false diff --git a/cibuildwheel/bashlex_eval.py b/cibuildwheel/bashlex_eval.py index 2c61bc54f..26fe1f5b2 100644 --- a/cibuildwheel/bashlex_eval.py +++ b/cibuildwheel/bashlex_eval.py @@ -1,7 +1,7 @@ import subprocess from typing import Callable, Dict, List, NamedTuple, Optional, Sequence -import bashlex # type: ignore +import bashlex # a function that takes a command and the environment, and returns the result EnvironmentExecutor = Callable[[List[str], Dict[str, str]], str] @@ -50,7 +50,7 @@ def evaluate_node(node: bashlex.ast.node, context: NodeExecutionContext) -> str: def evaluate_word_node(node: bashlex.ast.node, context: NodeExecutionContext) -> str: - value = node.word + value: str = node.word for part in node.parts: part_string = context.input[part.pos[0]:part.pos[1]] @@ -95,7 +95,7 @@ def evaluate_nodes_as_compound_command(nodes: Sequence[bashlex.ast.node], contex return result -def evaluate_nodes_as_simple_command(nodes: List[bashlex.ast.node], context: NodeExecutionContext): +def evaluate_nodes_as_simple_command(nodes: List[bashlex.ast.node], context: NodeExecutionContext) -> str: command = [evaluate_node(part, context=context) for part in nodes] return context.executor(command, context.environment) diff --git a/cibuildwheel/docker_container.py b/cibuildwheel/docker_container.py index d127d01d2..fb7f8d253 100644 --- a/cibuildwheel/docker_container.py +++ b/cibuildwheel/docker_container.py @@ -5,9 +5,11 @@ import subprocess import sys import uuid -from os import PathLike from pathlib import Path, PurePath -from typing import IO, Dict, List, Optional, Sequence, Union +from typing import cast, IO, Dict, List, Optional, Sequence, Type +from types import TracebackType + +from .typing import PathOrStr, PopenBytes class DockerContainer: @@ -23,14 +25,15 @@ class DockerContainer: ''' UTILITY_PYTHON = '/opt/python/cp38-cp38/bin/python' - process: subprocess.Popen + process: PopenBytes bash_stdin: IO[bytes] bash_stdout: IO[bytes] - def __init__(self, docker_image: str, simulate_32_bit: bool = False, cwd: Optional[Union[str, PathLike]] = None): + def __init__(self, docker_image: str, simulate_32_bit: bool = False, cwd: Optional[PathOrStr] = None): self.docker_image = docker_image self.simulate_32_bit = simulate_32_bit self.cwd = cwd + self.name: Optional[str] = None def __enter__(self) -> 'DockerContainer': self.name = f'cibuildwheel-{uuid.uuid4()}' @@ -68,11 +71,18 @@ def __enter__(self) -> 'DockerContainer': return self - def __exit__(self, exc_type, exc_val, exc_tb): + def __exit__( + self, + exc_type: Optional[Type[BaseException]], + exc_val: Optional[BaseException], + exc_tb: Optional[TracebackType]) -> None: + self.bash_stdin.close() self.process.terminate() self.process.wait() + assert isinstance(self.name, str) + subprocess.run(['docker', 'rm', '--force', '-v', self.name], stdout=subprocess.DEVNULL) self.name = None @@ -117,8 +127,12 @@ def glob(self, path: PurePath, pattern: str) -> List[PurePath]: return [PurePath(p) for p in path_strs] - def call(self, args: Sequence[Union[str, PathLike]], env: Optional[Dict[str, str]] = None, - capture_output=False, cwd: Optional[Union[str, PathLike]] = None) -> str: + def call( + self, + args: Sequence[PathOrStr], + env: Optional[Dict[str, str]] = None, + capture_output: bool = False, + cwd: Optional[PathOrStr] = None) -> str: chdir = f'cd {cwd}' if cwd else '' env_assignments = ' '.join(f'{shlex.quote(k)}={shlex.quote(v)}' @@ -178,11 +192,12 @@ def call(self, args: Sequence[Union[str, PathLike]], env: Optional[Dict[str, str return output def get_environment(self) -> Dict[str, str]: - return json.loads(self.call([ + env = json.loads(self.call([ self.UTILITY_PYTHON, '-c', 'import sys, json, os; json.dump(os.environ.copy(), sys.stdout)' ], capture_output=True)) + return cast(Dict[str, str], env) def environment_executor(self, command: List[str], environment: Dict[str, str]) -> str: # used as an EnvironmentExecutor to evaluate commands and capture output diff --git a/cibuildwheel/environment.py b/cibuildwheel/environment.py index 5ecd8c68c..fd2a824c5 100644 --- a/cibuildwheel/environment.py +++ b/cibuildwheel/environment.py @@ -1,4 +1,4 @@ -import bashlex # type: ignore +import bashlex from typing import Dict, List, Mapping, Optional diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 6a553ef73..4118fd795 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -1,9 +1,8 @@ import subprocess import sys import textwrap -from os import PathLike from pathlib import Path, PurePath -from typing import List, NamedTuple, Union +from typing import List, NamedTuple from .docker_container import DockerContainer from .logger import log @@ -11,6 +10,7 @@ Architecture, BuildOptions, BuildSelector, NonPlatformWheelError, get_build_verbosity_extra_flags, prepare_command, ) +from .typing import PathOrStr class PythonConfiguration(NamedTuple): @@ -19,7 +19,7 @@ class PythonConfiguration(NamedTuple): path_str: str @property - def path(self): + def path(self) -> PurePath: return PurePath(self.path_str) @@ -125,7 +125,7 @@ def build(options: BuildOptions) -> None: for config in platform_configs: log.build_start(config.identifier) - dependency_constraint_flags: List[Union[str, PathLike]] = [] + dependency_constraint_flags: List[PathOrStr] = [] if config.identifier.startswith("pp"): # Patch PyPy to make sure headers get installed into a venv patch_version = '_27' if config.version == '2.7' else '' diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index a8289e763..68baf2980 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -3,7 +3,7 @@ import re import sys import time -from typing import Optional, Union +from typing import Optional, Union, AnyStr, IO from cibuildwheel.util import CIProvider, detect_ci_provider @@ -35,7 +35,7 @@ class Logger: step_start_time: Optional[float] = None active_fold_group_name: Optional[str] = None - def __init__(self): + def __init__(self) -> None: if sys.platform == 'win32' and hasattr(sys.stdout, 'reconfigure'): # the encoding on Windows can be a 1-byte charmap, but all CIs # support utf8, so we hardcode that @@ -65,7 +65,7 @@ def __init__(self): self.fold_mode = 'disabled' self.colors_enabled = file_supports_color(sys.stdout) - def build_start(self, identifier: str): + def build_start(self, identifier: str) -> None: self.step_end() c = self.colors description = build_description_from_identifier(identifier) @@ -77,7 +77,7 @@ def build_start(self, identifier: str): self.build_start_time = time.time() self.active_build_identifier = identifier - def build_end(self): + def build_end(self) -> None: assert self.build_start_time is not None assert self.active_build_identifier is not None self.step_end() @@ -91,12 +91,12 @@ def build_end(self): self.build_start_time = None self.active_build_identifier = None - def step(self, step_description: str): + def step(self, step_description: str) -> None: self.step_end() self.step_start_time = time.time() self._start_fold_group(step_description) - def step_end(self, success=True): + def step_end(self, success: bool = True) -> None: if self.step_start_time is not None: self._end_fold_group() c = self.colors @@ -109,7 +109,7 @@ def step_end(self, success=True): self.step_start_time = None - def error(self, error: Union[Exception, str]): + def error(self, error: Union[BaseException, str]) -> None: self.step_end(success=False) print() @@ -119,7 +119,7 @@ def error(self, error: Union[Exception, str]): c = self.colors print(f'{c.bright_red}Error{c.end} {error}') - def _start_fold_group(self, name: str): + def _start_fold_group(self, name: str) -> None: self._end_fold_group() self.active_fold_group_name = name fold_start_pattern = FOLD_PATTERNS.get(self.fold_mode, DEFAULT_FOLD_PATTERN)[0] @@ -129,7 +129,7 @@ def _start_fold_group(self, name: str): print() sys.stdout.flush() - def _end_fold_group(self): + def _end_fold_group(self) -> None: if self.active_fold_group_name: fold_start_pattern = FOLD_PATTERNS.get(self.fold_mode, DEFAULT_FOLD_PATTERN)[1] identifier = self._fold_group_identifier(self.active_fold_group_name) @@ -137,7 +137,7 @@ def _end_fold_group(self): sys.stdout.flush() self.active_fold_group_name = None - def _fold_group_identifier(self, name: str): + def _fold_group_identifier(self, name: str) -> str: ''' Travis doesn't like fold groups identifiers that have spaces in. This method converts them to ascii identifiers @@ -152,21 +152,21 @@ def _fold_group_identifier(self, name: str): return identifier.lower()[:20] @property - def colors(self): + def colors(self) -> "Colors": if self.colors_enabled: - return Colors.enabled + return Colors(enabled=True) else: - return Colors.disabled + return Colors(enabled=False) @property - def symbols(self): + def symbols(self) -> "Symbols": if self.unicode_enabled: - return Symbols.unicode + return Symbols(unicode=True) else: - return Symbols.ascii + return Symbols(unicode=False) -def build_description_from_identifier(identifier: str): +def build_description_from_identifier(identifier: str) -> str: python_identifier, _, platform_identifier = identifier.partition('-') build_description = '' @@ -192,45 +192,31 @@ def build_description_from_identifier(identifier: str): class Colors: - class Enabled: - red = '\033[31m' - green = '\033[32m' - yellow = '\033[33m' - blue = '\033[34m' - cyan = '\033[36m' - bright_red = '\033[91m' - bright_green = '\033[92m' - white = '\033[37m\033[97m' + def __init__(self, *, enabled: bool) -> None: + self.red = '\033[31m' if enabled else '' + self.green = '\033[32m' if enabled else '' + self.yellow = '\033[33m' if enabled else '' + self.blue = '\033[34m' if enabled else '' + self.cyan = '\033[36m' if enabled else '' + self.bright_red = '\033[91m' if enabled else '' + self.bright_green = '\033[92m' if enabled else '' + self.white = '\033[37m\033[97m' if enabled else '' - bg_grey = '\033[48;5;235m' + self.bg_grey = '\033[48;5;235m' if enabled else '' - bold = '\033[1m' - faint = '\033[2m' + self.bold = '\033[1m' if enabled else '' + self.faint = '\033[2m' if enabled else '' - end = '\033[0m' - - class Disabled: - def __getattr__(self, attr: str) -> str: - return '' - - enabled = Enabled() - disabled = Disabled() + self.end = '\033[0m' if enabled else '' class Symbols: - class Unicode: - done = '✓' - error = '✕' - - class Ascii: - done = 'done' - error = 'failed' - - unicode = Unicode() - ascii = Ascii() + def __init__(self, *, unicode: bool) -> None: + self.done = '✓' if unicode else 'done' + self.error = '✕' if unicode else 'failed' -def file_supports_color(file_obj): +def file_supports_color(file_obj: IO[AnyStr]) -> bool: """ Returns True if the running system's terminal supports color. """ @@ -242,11 +228,11 @@ def file_supports_color(file_obj): return (supported_platform and is_a_tty) -def file_is_a_tty(file_obj): +def file_is_a_tty(file_obj: IO[AnyStr]) -> bool: return hasattr(file_obj, 'isatty') and file_obj.isatty() -def file_supports_unicode(file_obj): +def file_supports_unicode(file_obj: IO[AnyStr]) -> bool: encoding = getattr(file_obj, 'encoding', None) if not encoding: return False diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 2db1fb4c9..0845851c1 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -5,18 +5,18 @@ import sys import tempfile import textwrap -from os import PathLike from pathlib import Path -from typing import Dict, List, NamedTuple, Optional, Sequence, Union +from typing import Dict, List, NamedTuple, Optional, Sequence from .environment import ParsedEnvironment from .logger import log from .util import (Architecture, BuildOptions, BuildSelector, NonPlatformWheelError, download, get_build_verbosity_extra_flags, get_pip_script, install_certifi_script, prepare_command) +from .typing import PathOrStr -def call(args: Union[str, Sequence[Union[str, PathLike]]], env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None, shell: bool = False) -> int: +def call(args: Sequence[PathOrStr], env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None, shell: bool = False) -> int: # print the command executing for the logs if shell: print(f'+ {args}') @@ -120,7 +120,7 @@ def install_pypy(version: str, url: str) -> Path: def setup_python(python_configuration: PythonConfiguration, - dependency_constraint_flags: Sequence[Union[str, PathLike]], + dependency_constraint_flags: Sequence[PathOrStr], environment: ParsedEnvironment) -> Dict[str, str]: implementation_id = python_configuration.identifier.split("-")[0] log.step(f'Installing Python {implementation_id}...') @@ -209,7 +209,7 @@ def build(options: BuildOptions) -> None: for config in python_configurations: log.build_start(config.identifier) - dependency_constraint_flags: Sequence[Union[str, PathLike]] = [] + dependency_constraint_flags: Sequence[PathOrStr] = [] if options.dependency_constraints: dependency_constraint_flags = [ '-c', options.dependency_constraints.get_for_python_version(config.version) diff --git a/cibuildwheel/py.typed b/cibuildwheel/py.typed new file mode 100644 index 000000000..e69de29bb diff --git a/cibuildwheel/typing.py b/cibuildwheel/typing.py new file mode 100644 index 000000000..a3ffcfbc9 --- /dev/null +++ b/cibuildwheel/typing.py @@ -0,0 +1,11 @@ +from typing import Union, TYPE_CHECKING +import os +import subprocess + + +if TYPE_CHECKING: + PopenBytes = subprocess.Popen[bytes] + PathOrStr = Union[str, os.PathLike[str]] +else: + PopenBytes = subprocess.Popen + PathOrStr = Union[str, "os.PathLike[str]"] diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 70eaa44d1..960b9896d 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -8,14 +8,15 @@ from fnmatch import fnmatch from pathlib import Path from time import sleep -from typing import Dict, List, NamedTuple, Optional, Union +from typing import Dict, List, NamedTuple, Optional import certifi from .environment import ParsedEnvironment +from .typing import PathOrStr -def prepare_command(command: str, **kwargs: Union[str, os.PathLike]) -> str: +def prepare_command(command: str, **kwargs: PathOrStr) -> str: ''' Preprocesses a command by expanding variables like {python}. @@ -119,7 +120,7 @@ def get_for_python_version(self, version: str) -> Path: else: return self.base_file_path - def __repr__(self): + def __repr__(self) -> str: return f'{self.__class__.__name__}{self.base_file_path!r})' diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 3abb61dde..3e977e4b3 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -4,9 +4,8 @@ import sys import tempfile import textwrap -from os import PathLike from pathlib import Path -from typing import Dict, List, NamedTuple, Optional, Sequence, Union +from typing import Dict, List, NamedTuple, Optional, Sequence from zipfile import ZipFile import toml @@ -16,12 +15,13 @@ from .util import (Architecture, BuildOptions, BuildSelector, NonPlatformWheelError, download, get_build_verbosity_extra_flags, get_pip_script, prepare_command) +from .typing import PathOrStr IS_RUNNING_ON_AZURE = Path('C:\\hostedtoolcache').exists() IS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows' -def call(args: Sequence[Union[str, PathLike]], env: Optional[Dict[str, str]] = None, +def call(args: Sequence[PathOrStr], env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None) -> None: print('+ ' + ' '.join(str(a) for a in args)) # we use shell=True here, even though we don't need a shell due to a bug @@ -109,7 +109,7 @@ def install_pypy(version: str, arch: str, url: str) -> Path: return installation_path -def setup_python(python_configuration: PythonConfiguration, dependency_constraint_flags: Sequence[Union[str, PathLike]], environment: ParsedEnvironment) -> Dict[str, str]: +def setup_python(python_configuration: PythonConfiguration, dependency_constraint_flags: Sequence[PathOrStr], environment: ParsedEnvironment) -> Dict[str, str]: nuget = Path('C:\\cibw\\nuget.exe') if not nuget.exists(): log.step('Downloading nuget...') @@ -225,7 +225,7 @@ def build(options: BuildOptions) -> None: for config in python_configurations: log.build_start(config.identifier) - dependency_constraint_flags: Sequence[Union[str, PathLike]] = [] + dependency_constraint_flags: Sequence[PathOrStr] = [] if options.dependency_constraints: dependency_constraint_flags = [ '-c', options.dependency_constraints.get_for_python_version(config.version) diff --git a/setup.cfg b/setup.cfg index 8202be439..7aaa9a252 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,12 +18,14 @@ junit_family=xunit2 [mypy] python_version = 3.6 -files = cibuildwheel,test - +files = cibuildwheel,test,unit_test warn_unused_configs = True warn_redundant_casts = True -[mypy-cibuildwheel] +[mypy-test.*] +check_untyped_defs = True + +[mypy-cibuildwheel.*,unit_test.*] disallow_any_generics = True disallow_subclassing_any = True disallow_untyped_calls = True @@ -37,5 +39,13 @@ warn_return_any = True no_implicit_reexport = True strict_equality = True -[mypy-pytest,setuptools] +[mypy-setuptools.*] +ignore_missing_imports = True + +# Ignored for pre-commit to speed up check +# Not ignored if manually running and pytest installed +[mypy-pytest.*] +ignore_missing_imports = True + +[mypy-bashlex.*] ignore_missing_imports = True