fix: enable and fix full type checking (#509)

* fix: enable and fix fully type checking

* chore: pre-commit autoupdate

* refactor: pull types out to file

.pre-commit-config.yaml

@@ -1,13 +1,13 @@
 repos:
 - repo: https://gitlab.com/pycqa/flake8
-  rev: 3.8.3
+  rev: 3.8.4
   hooks:
   - id: flake8
     exclude: cibuildwheel/resources/
     additional_dependencies: [flake8-bugbear]
 
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v3.2.0
+  rev: v3.4.0
   hooks:
   - id: check-case-conflict
   - id: check-merge-conflict
@@ -18,8 +18,8 @@ repos:
   - id: trailing-whitespace
 
 - repo: https://github.com/pre-commit/mirrors-mypy
-  rev: v0.782
+  rev: v0.790
   hooks:
   - id: mypy
-    files: ^(cibuildwheel/|test/|bin/projects.py)
+    files: ^(cibuildwheel/|test/|bin/projects.py|unit_test/)
     pass_filenames: false

cibuildwheel/bashlex_eval.py

@@ -1,7 +1,7 @@
 import subprocess
 from typing import Callable, Dict, List, NamedTuple, Optional, Sequence
 
-import bashlex  # type: ignore
+import bashlex
 
 # a function that takes a command and the environment, and returns the result
 EnvironmentExecutor = Callable[[List[str], Dict[str, str]], str]
@@ -50,7 +50,7 @@ def evaluate_node(node: bashlex.ast.node, context: NodeExecutionContext) -> str:
 
 
 def evaluate_word_node(node: bashlex.ast.node, context: NodeExecutionContext) -> str:
-    value = node.word
+    value: str = node.word
 
     for part in node.parts:
         part_string = context.input[part.pos[0]:part.pos[1]]
@@ -95,7 +95,7 @@ def evaluate_nodes_as_compound_command(nodes: Sequence[bashlex.ast.node], contex
     return result
 
 
-def evaluate_nodes_as_simple_command(nodes: List[bashlex.ast.node], context: NodeExecutionContext):
+def evaluate_nodes_as_simple_command(nodes: List[bashlex.ast.node], context: NodeExecutionContext) -> str:
     command = [evaluate_node(part, context=context) for part in nodes]
     return context.executor(command, context.environment)
 

cibuildwheel/docker_container.py

@@ -5,9 +5,11 @@ import shlex
 import subprocess
 import sys
 import uuid
-from os import PathLike
 from pathlib import Path, PurePath
-from typing import IO, Dict, List, Optional, Sequence, Union
+from typing import cast, IO, Dict, List, Optional, Sequence, Type
+from types import TracebackType
+
+from .typing import PathOrStr, PopenBytes
 
 
 class DockerContainer:
@@ -23,14 +25,15 @@ class DockerContainer:
     '''
     UTILITY_PYTHON = '/opt/python/cp38-cp38/bin/python'
 
-    process: subprocess.Popen
+    process: PopenBytes
     bash_stdin: IO[bytes]
     bash_stdout: IO[bytes]
 
-    def __init__(self, docker_image: str, simulate_32_bit: bool = False, cwd: Optional[Union[str, PathLike]] = None):
+    def __init__(self, docker_image: str, simulate_32_bit: bool = False, cwd: Optional[PathOrStr] = None):
         self.docker_image = docker_image
         self.simulate_32_bit = simulate_32_bit
         self.cwd = cwd
+        self.name: Optional[str] = None
 
     def __enter__(self) -> 'DockerContainer':
         self.name = f'cibuildwheel-{uuid.uuid4()}'
@@ -68,11 +71,18 @@ class DockerContainer:
 
         return self
 
-    def __exit__(self, exc_type, exc_val, exc_tb):
+    def __exit__(
+            self,
+            exc_type: Optional[Type[BaseException]],
+            exc_val: Optional[BaseException],
+            exc_tb: Optional[TracebackType]) -> None:
+
         self.bash_stdin.close()
         self.process.terminate()
         self.process.wait()
 
+        assert isinstance(self.name, str)
+
         subprocess.run(['docker', 'rm', '--force', '-v', self.name], stdout=subprocess.DEVNULL)
         self.name = None
 
@@ -117,8 +127,12 @@ class DockerContainer:
 
         return [PurePath(p) for p in path_strs]
 
-    def call(self, args: Sequence[Union[str, PathLike]], env: Optional[Dict[str, str]] = None,
+    def call(
-             capture_output=False, cwd: Optional[Union[str, PathLike]] = None) -> str:
+            self,
+            args: Sequence[PathOrStr],
+            env: Optional[Dict[str, str]] = None,
+            capture_output: bool = False,
+            cwd: Optional[PathOrStr] = None) -> str:
 
         chdir = f'cd {cwd}' if cwd else ''
         env_assignments = ' '.join(f'{shlex.quote(k)}={shlex.quote(v)}'
@@ -178,11 +192,12 @@ class DockerContainer:
         return output
 
     def get_environment(self) -> Dict[str, str]:
-        return json.loads(self.call([
+        env = json.loads(self.call([
             self.UTILITY_PYTHON,
             '-c',
             'import sys, json, os; json.dump(os.environ.copy(), sys.stdout)'
         ], capture_output=True))
+        return cast(Dict[str, str], env)
 
     def environment_executor(self, command: List[str], environment: Dict[str, str]) -> str:
         # used as an EnvironmentExecutor to evaluate commands and capture output

cibuildwheel/environment.py

@@ -1,4 +1,4 @@
-import bashlex  # type: ignore
+import bashlex
 
 from typing import Dict, List, Mapping, Optional
 

cibuildwheel/linux.py

@@ -1,9 +1,8 @@
 import subprocess
 import sys
 import textwrap
-from os import PathLike
 from pathlib import Path, PurePath
-from typing import List, NamedTuple, Union
+from typing import List, NamedTuple
 
 from .docker_container import DockerContainer
 from .logger import log
@@ -11,6 +10,7 @@ from .util import (
     Architecture, BuildOptions, BuildSelector, NonPlatformWheelError,
     get_build_verbosity_extra_flags, prepare_command,
 )
+from .typing import PathOrStr
 
 
 class PythonConfiguration(NamedTuple):
@@ -19,7 +19,7 @@ class PythonConfiguration(NamedTuple):
     path_str: str
 
     @property
-    def path(self):
+    def path(self) -> PurePath:
         return PurePath(self.path_str)
 
 
@@ -125,7 +125,7 @@ def build(options: BuildOptions) -> None:
                 for config in platform_configs:
                     log.build_start(config.identifier)
 
-                    dependency_constraint_flags: List[Union[str, PathLike]] = []
+                    dependency_constraint_flags: List[PathOrStr] = []
                     if config.identifier.startswith("pp"):
                         # Patch PyPy to make sure headers get installed into a venv
                         patch_version = '_27' if config.version == '2.7' else ''

cibuildwheel/logger.py

@@ -3,7 +3,7 @@ import os
 import re
 import sys
 import time
-from typing import Optional, Union
+from typing import Optional, Union, AnyStr, IO
 
 from cibuildwheel.util import CIProvider, detect_ci_provider
 
@@ -35,7 +35,7 @@ class Logger:
     step_start_time: Optional[float] = None
     active_fold_group_name: Optional[str] = None
 
-    def __init__(self):
+    def __init__(self) -> None:
         if sys.platform == 'win32' and hasattr(sys.stdout, 'reconfigure'):
             # the encoding on Windows can be a 1-byte charmap, but all CIs
             # support utf8, so we hardcode that
@@ -65,7 +65,7 @@ class Logger:
             self.fold_mode = 'disabled'
             self.colors_enabled = file_supports_color(sys.stdout)
 
-    def build_start(self, identifier: str):
+    def build_start(self, identifier: str) -> None:
         self.step_end()
         c = self.colors
         description = build_description_from_identifier(identifier)
@@ -77,7 +77,7 @@ class Logger:
         self.build_start_time = time.time()
         self.active_build_identifier = identifier
 
-    def build_end(self):
+    def build_end(self) -> None:
         assert self.build_start_time is not None
         assert self.active_build_identifier is not None
         self.step_end()
@@ -91,12 +91,12 @@ class Logger:
         self.build_start_time = None
         self.active_build_identifier = None
 
-    def step(self, step_description: str):
+    def step(self, step_description: str) -> None:
         self.step_end()
         self.step_start_time = time.time()
         self._start_fold_group(step_description)
 
-    def step_end(self, success=True):
+    def step_end(self, success: bool = True) -> None:
         if self.step_start_time is not None:
             self._end_fold_group()
             c = self.colors
@@ -109,7 +109,7 @@ class Logger:
 
             self.step_start_time = None
 
-    def error(self, error: Union[Exception, str]):
+    def error(self, error: Union[BaseException, str]) -> None:
         self.step_end(success=False)
         print()
 
@@ -119,7 +119,7 @@ class Logger:
             c = self.colors
             print(f'{c.bright_red}Error{c.end} {error}')
 
-    def _start_fold_group(self, name: str):
+    def _start_fold_group(self, name: str) -> None:
         self._end_fold_group()
         self.active_fold_group_name = name
         fold_start_pattern = FOLD_PATTERNS.get(self.fold_mode, DEFAULT_FOLD_PATTERN)[0]
@@ -129,7 +129,7 @@ class Logger:
         print()
         sys.stdout.flush()
 
-    def _end_fold_group(self):
+    def _end_fold_group(self) -> None:
         if self.active_fold_group_name:
             fold_start_pattern = FOLD_PATTERNS.get(self.fold_mode, DEFAULT_FOLD_PATTERN)[1]
             identifier = self._fold_group_identifier(self.active_fold_group_name)
@@ -137,7 +137,7 @@ class Logger:
             sys.stdout.flush()
             self.active_fold_group_name = None
 
-    def _fold_group_identifier(self, name: str):
+    def _fold_group_identifier(self, name: str) -> str:
         '''
         Travis doesn't like fold groups identifiers that have spaces in. This
         method converts them to ascii identifiers
@@ -152,21 +152,21 @@ class Logger:
         return identifier.lower()[:20]
 
     @property
-    def colors(self):
+    def colors(self) -> "Colors":
         if self.colors_enabled:
-            return Colors.enabled
+            return Colors(enabled=True)
         else:
-            return Colors.disabled
+            return Colors(enabled=False)
 
     @property
-    def symbols(self):
+    def symbols(self) -> "Symbols":
         if self.unicode_enabled:
-            return Symbols.unicode
+            return Symbols(unicode=True)
         else:
-            return Symbols.ascii
+            return Symbols(unicode=False)
 
 
-def build_description_from_identifier(identifier: str):
+def build_description_from_identifier(identifier: str) -> str:
     python_identifier, _, platform_identifier = identifier.partition('-')
 
     build_description = ''
@@ -192,45 +192,31 @@ def build_description_from_identifier(identifier: str):
 
 
 class Colors:
-    class Enabled:
+    def __init__(self, *, enabled: bool) -> None:
-        red = '\033[31m'
+        self.red = '\033[31m' if enabled else ''
-        green = '\033[32m'
+        self.green = '\033[32m' if enabled else ''
-        yellow = '\033[33m'
+        self.yellow = '\033[33m' if enabled else ''
-        blue = '\033[34m'
+        self.blue = '\033[34m' if enabled else ''
-        cyan = '\033[36m'
+        self.cyan = '\033[36m' if enabled else ''
-        bright_red = '\033[91m'
+        self.bright_red = '\033[91m' if enabled else ''
-        bright_green = '\033[92m'
+        self.bright_green = '\033[92m' if enabled else ''
-        white = '\033[37m\033[97m'
+        self.white = '\033[37m\033[97m' if enabled else ''
 
-        bg_grey = '\033[48;5;235m'
+        self.bg_grey = '\033[48;5;235m' if enabled else ''
 
-        bold = '\033[1m'
+        self.bold = '\033[1m' if enabled else ''
-        faint = '\033[2m'
+        self.faint = '\033[2m' if enabled else ''
 
-        end = '\033[0m'
+        self.end = '\033[0m' if enabled else ''
-
-    class Disabled:
-        def __getattr__(self, attr: str) -> str:
-            return ''
-
-    enabled = Enabled()
-    disabled = Disabled()
 
 
 class Symbols:
-    class Unicode:
+    def __init__(self, *, unicode: bool) -> None:
-        done = '✓'
+        self.done = '✓' if unicode else 'done'
-        error = '✕'
+        self.error = '✕' if unicode else 'failed'
-
-    class Ascii:
-        done = 'done'
-        error = 'failed'
-
-    unicode = Unicode()
-    ascii = Ascii()
 
 
-def file_supports_color(file_obj):
+def file_supports_color(file_obj: IO[AnyStr]) -> bool:
     """
     Returns True if the running system's terminal supports color.
     """
@@ -242,11 +228,11 @@ def file_supports_color(file_obj):
     return (supported_platform and is_a_tty)
 
 
-def file_is_a_tty(file_obj):
+def file_is_a_tty(file_obj: IO[AnyStr]) -> bool:
     return hasattr(file_obj, 'isatty') and file_obj.isatty()
 
 
-def file_supports_unicode(file_obj):
+def file_supports_unicode(file_obj: IO[AnyStr]) -> bool:
     encoding = getattr(file_obj, 'encoding', None)
     if not encoding:
         return False

cibuildwheel/macos.py

@@ -5,18 +5,18 @@ import subprocess
 import sys
 import tempfile
 import textwrap
-from os import PathLike
 from pathlib import Path
-from typing import Dict, List, NamedTuple, Optional, Sequence, Union
+from typing import Dict, List, NamedTuple, Optional, Sequence
 
 from .environment import ParsedEnvironment
 from .logger import log
 from .util import (Architecture, BuildOptions, BuildSelector, NonPlatformWheelError,
                    download, get_build_verbosity_extra_flags, get_pip_script,
                    install_certifi_script, prepare_command)
+from .typing import PathOrStr
 
 
-def call(args: Union[str, Sequence[Union[str, PathLike]]], env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None, shell: bool = False) -> int:
+def call(args: Sequence[PathOrStr], env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None, shell: bool = False) -> int:
     # print the command executing for the logs
     if shell:
         print(f'+ {args}')
@@ -120,7 +120,7 @@ def install_pypy(version: str, url: str) -> Path:
 
 
 def setup_python(python_configuration: PythonConfiguration,
-                 dependency_constraint_flags: Sequence[Union[str, PathLike]],
+                 dependency_constraint_flags: Sequence[PathOrStr],
                  environment: ParsedEnvironment) -> Dict[str, str]:
     implementation_id = python_configuration.identifier.split("-")[0]
     log.step(f'Installing Python {implementation_id}...')
@@ -209,7 +209,7 @@ def build(options: BuildOptions) -> None:
         for config in python_configurations:
             log.build_start(config.identifier)
 
-            dependency_constraint_flags: Sequence[Union[str, PathLike]] = []
+            dependency_constraint_flags: Sequence[PathOrStr] = []
             if options.dependency_constraints:
                 dependency_constraint_flags = [
                     '-c', options.dependency_constraints.get_for_python_version(config.version)

cibuildwheel/typing.py

@@ -0,0 +1,11 @@
+from typing import Union, TYPE_CHECKING
+import os
+import subprocess
+
+
+if TYPE_CHECKING:
+    PopenBytes = subprocess.Popen[bytes]
+    PathOrStr = Union[str, os.PathLike[str]]
+else:
+    PopenBytes = subprocess.Popen
+    PathOrStr = Union[str, "os.PathLike[str]"]

cibuildwheel/util.py

@@ -8,14 +8,15 @@ from enum import Enum
 from fnmatch import fnmatch
 from pathlib import Path
 from time import sleep
-from typing import Dict, List, NamedTuple, Optional, Union
+from typing import Dict, List, NamedTuple, Optional
 
 import certifi
 
 from .environment import ParsedEnvironment
+from .typing import PathOrStr
 
 
-def prepare_command(command: str, **kwargs: Union[str, os.PathLike]) -> str:
+def prepare_command(command: str, **kwargs: PathOrStr) -> str:
     '''
     Preprocesses a command by expanding variables like {python}.
 
@@ -119,7 +120,7 @@ class DependencyConstraints:
         else:
             return self.base_file_path
 
-    def __repr__(self):
+    def __repr__(self) -> str:
         return f'{self.__class__.__name__}{self.base_file_path!r})'
 
 

cibuildwheel/windows.py

@@ -4,9 +4,8 @@ import subprocess
 import sys
 import tempfile
 import textwrap
-from os import PathLike
 from pathlib import Path
-from typing import Dict, List, NamedTuple, Optional, Sequence, Union
+from typing import Dict, List, NamedTuple, Optional, Sequence
 from zipfile import ZipFile
 
 import toml
@@ -16,12 +15,13 @@ from .logger import log
 from .util import (Architecture, BuildOptions, BuildSelector, NonPlatformWheelError,
                    download, get_build_verbosity_extra_flags, get_pip_script,
                    prepare_command)
+from .typing import PathOrStr
 
 IS_RUNNING_ON_AZURE = Path('C:\\hostedtoolcache').exists()
 IS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows'
 
 
-def call(args: Sequence[Union[str, PathLike]], env: Optional[Dict[str, str]] = None,
+def call(args: Sequence[PathOrStr], env: Optional[Dict[str, str]] = None,
          cwd: Optional[str] = None) -> None:
     print('+ ' + ' '.join(str(a) for a in args))
     # we use shell=True here, even though we don't need a shell due to a bug
@@ -109,7 +109,7 @@ def install_pypy(version: str, arch: str, url: str) -> Path:
     return installation_path
 
 
-def setup_python(python_configuration: PythonConfiguration, dependency_constraint_flags: Sequence[Union[str, PathLike]], environment: ParsedEnvironment) -> Dict[str, str]:
+def setup_python(python_configuration: PythonConfiguration, dependency_constraint_flags: Sequence[PathOrStr], environment: ParsedEnvironment) -> Dict[str, str]:
     nuget = Path('C:\\cibw\\nuget.exe')
     if not nuget.exists():
         log.step('Downloading nuget...')
@@ -225,7 +225,7 @@ def build(options: BuildOptions) -> None:
         for config in python_configurations:
             log.build_start(config.identifier)
 
-            dependency_constraint_flags: Sequence[Union[str, PathLike]] = []
+            dependency_constraint_flags: Sequence[PathOrStr] = []
             if options.dependency_constraints:
                 dependency_constraint_flags = [
                     '-c', options.dependency_constraints.get_for_python_version(config.version)

setup.cfg

@@ -79,12 +79,14 @@ junit_family=xunit2
 
 [mypy]
 python_version = 3.6
-files = cibuildwheel,test
+files = cibuildwheel,test,unit_test
-
 warn_unused_configs = True
 warn_redundant_casts = True
 
-[mypy-cibuildwheel]
+[mypy-test.*]
+check_untyped_defs = True
+
+[mypy-cibuildwheel.*,unit_test.*]
 disallow_any_generics = True
 disallow_subclassing_any = True
 disallow_untyped_calls = True
@@ -98,5 +100,13 @@ warn_return_any = True
 no_implicit_reexport = True
 strict_equality = True
 
-[mypy-pytest,setuptools]
+[mypy-setuptools.*]
+ignore_missing_imports = True
+
+# Ignored for pre-commit to speed up check
+# Not ignored if manually running and pytest installed
+[mypy-pytest.*]
+ignore_missing_imports = True
+
+[mypy-bashlex.*]
 ignore_missing_imports = True