diff --git a/.circleci/config.yml b/.circleci/config.yml index 8820ceb1..cf0e47b2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,18 +1,21 @@ version: 2 jobs: - flake8: + flake8-mypy: docker: - image: circleci/python:3.6 steps: - checkout - run: - name: Install flake8 - command: sudo python -m pip install flake8 + name: Install flake8 & mypy + command: sudo python -m pip install flake8 mypy - run: - name: Test. - command: flake8 . + name: flake8 + command: flake8 + - run: + name: mypy + command: mypy osx-python3.6: macos: @@ -30,22 +33,6 @@ jobs: command: venv/bin/python ./bin/run_tests.py no_output_timeout: 30m - osx-python3.7: - macos: - xcode: "10.0.0" - environment: - PYTHON: python3 - steps: - - checkout - - - run: - name: Prepare the environment. - command: bash .circleci/prepare.sh - - run: - name: Test. - command: venv/bin/python ./bin/run_tests.py - no_output_timeout: 30m - linux-python3.6: docker: - image: circleci/python:3.6 @@ -67,7 +54,6 @@ workflows: version: 2 all-tests: jobs: - - flake8 + - flake8-mypy - osx-python3.6 - - osx-python3.7 - linux-python3.6 diff --git a/.gitignore b/.gitignore index 3241171a..2db46b82 100644 --- a/.gitignore +++ b/.gitignore @@ -103,5 +103,5 @@ env3?/ # VSCode project settings /.vscode -# mypy -.mypy_cache +# MyPy cache +.mypy_cache/ diff --git a/CI.md b/CI.md index 69a2b3bd..6d6a8356 100644 --- a/CI.md +++ b/CI.md @@ -1,10 +1,10 @@ This is a summary of the Python versions and platforms covered by the different CI platforms: -| | 3.5 | 3.6 | 3.7 | 3.8 | -|----------|------------------|------------------|----------------------------------------------------|------------------| -| Linux | Travis CI | CircleCI | AppVeyor² / GitHub Actions | Azure Pipelines | -| macOS | Azure Pipelines | CircleCI | AppVeyor² / Travis CI¹ / CircleCI / GitHub Actions | Azure Pipelines | -| Windows | TravisCI | Azure Pipelines | AppVeyor² / GitHub Actions | Azure Pipelines | +| | 3.6 | 3.7 | 3.8 | +|----------|------------------------------|------------------------------------------|------------------| +| Linux | Travis CI / CircleCI | AppVeyor² / GitHub Actions | Azure Pipelines | +| macOS | CircleCI | AppVeyor² / Travis CI¹ / GitHub Actions | Azure Pipelines | +| Windows | Travis CI / Azure Pipelines | AppVeyor² / GitHub Actions | Azure Pipelines | > ¹ Python version not really pinned, but dependent on the (default) version of image used. > ² AppVeyor only runs the "basic" test to reduce load. diff --git a/README.md b/README.md index 28ebbf4f..93c896ce 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,7 @@ Here are some repos that use cibuildwheel. - [PyAV](https://github.com/PyAV-Org/PyAV) - [aiortc](https://github.com/aiortc/aiortc) - [aioquic](https://github.com/aiortc/aioquic) +- [pikepdf](https://github.com/pikepdf/pikepdf) > Add your repo here! Send a PR. diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 7099bc8f..bc21fbe6 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -5,6 +5,8 @@ import textwrap import traceback from configparser import ConfigParser +from typing import Any, Dict, List, Optional, overload + import cibuildwheel import cibuildwheel.linux import cibuildwheel.macos @@ -14,14 +16,18 @@ from cibuildwheel.environment import ( parse_environment, ) from cibuildwheel.util import ( + BuildOptions, BuildSelector, DependencyConstraints, Unbuffered, - BuildOptions ) -def get_option_from_environment(option_name, platform=None, default=None): +@overload +def get_option_from_environment(option_name: str, platform: Optional[str], default: str) -> str: ... # noqa: E704 +@overload +def get_option_from_environment(option_name: str, platform: Optional[str] = None, default: None = None) -> Optional[str]: ... # noqa: E704 E302 +def get_option_from_environment(option_name: str, platform: Optional[str] = None, default: Optional[str] = None) -> Optional[str]: # noqa: E302 ''' Returns an option from the environment, optionally scoped by the platform. @@ -32,20 +38,20 @@ def get_option_from_environment(option_name, platform=None, default=None): CIBW_COLOR. ''' if platform: - option = os.environ.get('%s_%s' % (option_name, platform.upper())) + option = os.environ.get(f'{option_name}_{platform.upper()}') if option is not None: return option return os.environ.get(option_name, default) -def strtobool(val): +def strtobool(val: str) -> bool: if val.lower() in ('y', 'yes', 't', 'true', 'on', '1'): return True return False -def main(): +def main() -> None: parser = argparse.ArgumentParser( description='Build wheels for all the platforms.', epilog=''' @@ -114,7 +120,7 @@ def main(): test_extras = get_option_from_environment('CIBW_TEST_EXTRAS', platform=platform, default='') package_dir = args.package_dir before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform) - build_verbosity = get_option_from_environment('CIBW_BUILD_VERBOSITY', platform=platform, default='') + build_verbosity_str = get_option_from_environment('CIBW_BUILD_VERBOSITY', platform=platform, default='') build_config, skip_config = os.environ.get('CIBW_BUILD', '*'), os.environ.get('CIBW_SKIP', '') if platform == 'linux': repair_command_default = 'auditwheel repair -w {dest_dir} {wheel}' @@ -128,24 +134,24 @@ def main(): dependency_versions = get_option_from_environment('CIBW_DEPENDENCY_VERSIONS', platform=platform, default='pinned') if dependency_versions == 'pinned': - dependency_constraints = DependencyConstraints.with_defaults() + dependency_constraints: Optional[DependencyConstraints] = DependencyConstraints.with_defaults() elif dependency_versions == 'latest': dependency_constraints = None else: dependency_constraints = DependencyConstraints(dependency_versions) if test_extras: - test_extras = '[{0}]'.format(test_extras) + test_extras = f'[{test_extras}]' try: - build_verbosity = min(3, max(-3, int(build_verbosity))) + build_verbosity = min(3, max(-3, int(build_verbosity_str))) except ValueError: build_verbosity = 0 try: environment = parse_environment(environment_config) except (EnvironmentParseError, ValueError): - print('cibuildwheel: Malformed environment option "%s"' % environment_config, file=sys.stderr) + print(f'cibuildwheel: Malformed environment option "{environment_config}"', file=sys.stderr) traceback.print_exc(None, sys.stderr) exit(2) @@ -163,6 +169,7 @@ def main(): print_build_identifiers(platform, build_selector) exit(0) + manylinux_images: Optional[Dict[str, str]] = None if platform == 'linux': pinned_docker_images_file = os.path.join( os.path.dirname(__file__), 'resources', 'pinned_docker_images.cfg' @@ -180,7 +187,7 @@ def main(): for build_platform in ['x86_64', 'i686', 'pypy_x86_64', 'aarch64', 'ppc64le', 's390x']: pinned_images = all_pinned_docker_images[build_platform] - config_name = 'CIBW_MANYLINUX_{}_IMAGE'.format(build_platform.upper()) + config_name = f'CIBW_MANYLINUX_{build_platform.upper()}_IMAGE' config_value = os.environ.get(config_name) if config_value is None: @@ -193,9 +200,6 @@ def main(): manylinux_images[build_platform] = image - else: - manylinux_images = None - build_options = BuildOptions( package_dir=package_dir, output_dir=output_dir, @@ -213,7 +217,7 @@ def main(): ) # Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print' - sys.stdout = Unbuffered(sys.stdout) + sys.stdout = Unbuffered(sys.stdout) # type: ignore print_preamble(platform, build_options) @@ -227,21 +231,21 @@ def main(): elif platform == 'macos': cibuildwheel.macos.build(build_options) else: - print('cibuildwheel: Unsupported platform: {}'.format(platform), file=sys.stderr) + print(f'cibuildwheel: Unsupported platform: {platform}', file=sys.stderr) exit(2) -def detect_obsolete_options(): +def detect_obsolete_options() -> None: # Check the old 'MANYLINUX1_*_IMAGE' options for (deprecated, alternative) in [('CIBW_MANYLINUX1_X86_64_IMAGE', 'CIBW_MANYLINUX_X86_64_IMAGE'), ('CIBW_MANYLINUX1_I686_IMAGE', 'CIBW_MANYLINUX_I686_IMAGE')]: if deprecated in os.environ: - print("'{}' has been deprecated, and will be removed in a future release. Use the option '{}' instead.".format(deprecated, alternative)) + print(f"'{deprecated}' has been deprecated, and will be removed in a future release. Use the option '{alternative}' instead.") if alternative not in os.environ: - print("Using value of option '{}' as replacement for '{}'".format(deprecated, alternative)) + print(f"Using value of option '{deprecated}' as replacement for '{alternative}'") os.environ[alternative] = os.environ[deprecated] else: - print("Option '{}' is not empty. Please unset '{}'".format(alternative, deprecated)) + print(f"Option '{alternative}' is not empty. Please unset '{deprecated}'") exit(2) # Check for deprecated identifiers in 'CIBW_BUILD' and 'CIBW_SKIP' options @@ -250,15 +254,11 @@ def detect_obsolete_options(): ('macosx_10_6_intel', 'macosx_x86_64'), ('macosx_10_9_x86_64', 'macosx_x86_64')]: if option in os.environ and deprecated in os.environ[option]: - print("Build identifiers with '{deprecated}' have been deprecated. Replacing all occurences of '{deprecated}' with '{alternative}' in the option '{option}'".format( - deprecated=deprecated, - alternative=alternative, - option=option, - )) + print(f"Build identifiers with '{deprecated}' have been deprecated. Replacing all occurences of '{deprecated}' with '{alternative}' in the option '{option}'") os.environ[option] = os.environ[option].replace(deprecated, alternative) -def print_preamble(platform, build_options): +def print_preamble(platform: str, build_options: BuildOptions) -> None: print(textwrap.dedent(''' _ _ _ _ _ _ _ ___|_| |_ _ _|_| |_| |_ _ _| |_ ___ ___| | @@ -266,12 +266,12 @@ def print_preamble(platform, build_options): |___|_|___|___|_|_|___|_____|_|_|___|___|_| ''')) - print('cibuildwheel version %s\n' % cibuildwheel.__version__) + print(f'cibuildwheel version {cibuildwheel.__version__}\n') print('Build options:') - print(' platform: %r' % platform) + print(f' platform: {platform!r}') for option, value in sorted(build_options._asdict().items()): - print(' %s: %r' % (option, value)) + print(f' {option}: {value!r}') warnings = detect_warnings(platform, build_options) if warnings: @@ -282,21 +282,20 @@ def print_preamble(platform, build_options): print('\nHere we go!\n') -def print_build_identifiers(platform, build_selector): +def print_build_identifiers(platform: str, build_selector: BuildSelector) -> None: + python_configurations: List[Any] = [] if platform == 'linux': python_configurations = cibuildwheel.linux.get_python_configurations(build_selector) elif platform == 'windows': python_configurations = cibuildwheel.windows.get_python_configurations(build_selector) elif platform == 'macos': python_configurations = cibuildwheel.macos.get_python_configurations(build_selector) - else: - python_configurations = [] for config in python_configurations: print(config.identifier) -def detect_warnings(platform, build_options): +def detect_warnings(platform: str, build_options: BuildOptions) -> List[str]: warnings = [] # warn about deprecated {python} and {pip} diff --git a/cibuildwheel/bashlex_eval.py b/cibuildwheel/bashlex_eval.py index 8e2c0831..60ce9a81 100644 --- a/cibuildwheel/bashlex_eval.py +++ b/cibuildwheel/bashlex_eval.py @@ -1,13 +1,17 @@ import shlex import subprocess -from collections import namedtuple -import bashlex +from typing import Dict, NamedTuple -NodeExecutionContext = namedtuple('NodeExecutionContext', ['environment', 'input']) +import bashlex # type: ignore -def evaluate(value, environment): +class NodeExecutionContext(NamedTuple): + environment: Dict[str, str] + input: str + + +def evaluate(value: str, environment: Dict[str, str]) -> str: if not value: # empty string evaluates to empty string # (but trips up bashlex) @@ -16,7 +20,7 @@ def evaluate(value, environment): command_node = bashlex.parsesingle(value) if len(command_node.parts) != 1: - raise ValueError('"%s" has too many parts' % value) + raise ValueError(f'"{value}" has too many parts') value_word_node = command_node.parts[0] @@ -26,7 +30,7 @@ def evaluate(value, environment): ) -def evaluate_node(node, context): +def evaluate_node(node: bashlex.ast.node, context: NodeExecutionContext) -> str: if node.kind == 'word': return evaluate_word_node(node, context=context) elif node.kind == 'commandsubstitution': @@ -34,10 +38,10 @@ def evaluate_node(node, context): elif node.kind == 'parameter': return evaluate_parameter_node(node, context=context) else: - raise ValueError('Unsupported bash construct: "%s"' % node.word) + raise ValueError(f'Unsupported bash construct: "{node.word}"') -def evaluate_word_node(node, context): +def evaluate_word_node(node: bashlex.ast.node, context: NodeExecutionContext) -> str: word_start = node.pos[0] word_end = node.pos[1] word_string = context.input[word_start:word_end] @@ -49,22 +53,22 @@ def evaluate_word_node(node, context): # Set all the characters in the part to None for i in range(part_start, part_end): - letters[i] = None + letters[i] = '' letters[part_start] = evaluate_node(part, context=context) # remove the None letters and concat - value = ''.join(l for l in letters if l is not None) + value = ''.join(letters) # apply bash-like quotes/whitespace treatment return ' '.join(word.strip() for word in shlex.split(value)) -def evaluate_command_node(node, context): +def evaluate_command_node(node: bashlex.ast.node, context: NodeExecutionContext) -> str: words = [evaluate_node(part, context=context) for part in node.parts] command = ' '.join(words) return subprocess.check_output(shlex.split(command), env=context.environment, universal_newlines=True) -def evaluate_parameter_node(node, context): +def evaluate_parameter_node(node: bashlex.ast.node, context: NodeExecutionContext) -> str: return context.environment.get(node.value, '') diff --git a/cibuildwheel/environment.py b/cibuildwheel/environment.py index f016c780..086a253c 100644 --- a/cibuildwheel/environment.py +++ b/cibuildwheel/environment.py @@ -1,4 +1,6 @@ -import bashlex +import bashlex # type: ignore + +from typing import Dict, List from . import bashlex_eval @@ -7,13 +9,7 @@ class EnvironmentParseError(Exception): pass -def parse_environment(env_string): - env_items = split_env_items(env_string) - assignments = [EnvironmentAssignment(item) for item in env_items] - return ParsedEnvironment(assignments=assignments) - - -def split_env_items(env_string): +def split_env_items(env_string: str) -> List[str]: '''Splits space-separated variable assignments into a list of individual assignments. >>> split_env_items('VAR=abc') @@ -43,29 +39,29 @@ def split_env_items(env_string): class EnvironmentAssignment: - def __init__(self, assignment): + def __init__(self, assignment: str): name, equals, value = assignment.partition('=') if not equals: raise EnvironmentParseError(assignment) self.name = name self.value = value - def evaluated_value(self, environment): + def evaluated_value(self, environment: Dict[str, str]) -> str: '''Returns the value of this assignment, as evaluated in the environment''' return bashlex_eval.evaluate(self.value, environment=environment) - def as_shell_assignment(self): - return 'export %s=%s' % (self.name, self.value) + def as_shell_assignment(self) -> str: + return f'export {self.name}={self.value}' - def __repr__(self): - return '%s=%s' % (self.name, self.value) + def __repr__(self) -> str: + return f'{self.name}={self.value}' class ParsedEnvironment: - def __init__(self, assignments): + def __init__(self, assignments: List[EnvironmentAssignment]): self.assignments = assignments - def as_dictionary(self, prev_environment): + def as_dictionary(self, prev_environment: Dict[str, str]) -> Dict[str, str]: environment = prev_environment.copy() for assignment in self.assignments: @@ -74,8 +70,14 @@ class ParsedEnvironment: return environment - def as_shell_commands(self): + def as_shell_commands(self) -> List[str]: return [a.as_shell_assignment() for a in self.assignments] - def __repr__(self): - return 'ParsedEnvironment(%r)' % [repr(a) for a in self.assignments] + def __repr__(self) -> str: + return f'ParsedEnvironment({[repr(a) for a in self.assignments]!r})' + + +def parse_environment(env_string: str) -> ParsedEnvironment: + env_items = split_env_items(env_string) + assignments = [EnvironmentAssignment(item) for item in env_items] + return ParsedEnvironment(assignments=assignments) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index d30635c3..652d9eb3 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -5,23 +5,25 @@ import subprocess import sys import textwrap import uuid -from collections import namedtuple + +from typing import List, NamedTuple, Optional, Union from .util import ( + BuildOptions, + BuildSelector, get_build_verbosity_extra_flags, prepare_command, - BuildOptions ) -def call(args, input=None, universal_newlines=False): +def call(args: List[str], input: Optional[Union[str, bytes]] = None, universal_newlines: bool = False) -> None: print('+ ' + ' '.join(shlex.quote(a) for a in args)) subprocess.run( args, input=input, universal_newlines=universal_newlines, check=True ) -def matches_platform(identifier): +def matches_platform(identifier: str) -> bool: pm = platform.machine() if pm == "x86_64": # x86_64 machines can run i686 docker containers @@ -42,8 +44,13 @@ def matches_platform(identifier): return False -def get_python_configurations(build_selector): - PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'path']) +class PythonConfiguration(NamedTuple): + version: str + identifier: str + path: str + + +def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfiguration]: python_configurations = [ PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path='/opt/python/cp27-cp27m'), PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path='/opt/python/cp27-cp27mu'), @@ -76,7 +83,7 @@ def get_python_configurations(build_selector): return [c for c in python_configurations if matches_platform(c.identifier) and build_selector(c.identifier)] -def build(options: BuildOptions): +def build(options: BuildOptions) -> None: try: subprocess.check_call(['docker', '--version']) except Exception: @@ -86,6 +93,7 @@ def build(options: BuildOptions): file=sys.stderr) exit(2) + assert options.manylinux_images is not None python_configurations = get_python_configurations(options.build_selector) platforms = [ ('cp', 'manylinux_x86_64', options.manylinux_images['x86_64']), @@ -106,7 +114,7 @@ def build(options: BuildOptions): if not platform_configs: continue - container_name = 'cibuildwheel-{}'.format(uuid.uuid4()) + container_name = f'cibuildwheel-{uuid.uuid4()}' try: shell_cmd = ['linux32', '/bin/bash'] if platform_tag.endswith("i686") else ['/bin/bash'] @@ -274,7 +282,7 @@ def build(options: BuildOptions): call(['docker', 'rm', '--force', '-v', container_name]) -def troubleshoot(package_dir, error): +def troubleshoot(package_dir: str, error: Exception) -> None: if (isinstance(error, subprocess.CalledProcessError) and 'exec' in error.cmd): # the bash script failed print('Checking for common errors...') diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 8eb7984c..745c1557 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -4,37 +4,45 @@ import shutil import subprocess import sys import tempfile -from collections import namedtuple from glob import glob +from typing import Dict, List, Optional, NamedTuple, Union + +from .environment import ParsedEnvironment from .util import ( + BuildOptions, + BuildSelector, download, get_build_verbosity_extra_flags, - prepare_command, get_pip_script, - BuildOptions + prepare_command, ) -def call(args, env=None, cwd=None, shell=False): +def call(args: Union[str, List[str]], env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None, shell: bool = False) -> int: # print the command executing for the logs if shell: - print('+ %s' % args) + print(f'+ {args}') else: print('+ ' + ' '.join(shlex.quote(a) for a in args)) return subprocess.check_call(args, env=env, cwd=cwd, shell=shell) -def get_python_configurations(build_selector): - PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'url']) +class PythonConfiguration(NamedTuple): + version: str + identifier: str + url: str + + +def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfiguration]: python_configurations = [ # CPython PythonConfiguration(version='2.7', identifier='cp27-macosx_x86_64', url='https://www.python.org/ftp/python/2.7.18/python-2.7.18-macosx10.9.pkg'), PythonConfiguration(version='3.5', identifier='cp35-macosx_x86_64', url='https://www.python.org/ftp/python/3.5.4/python-3.5.4-macosx10.6.pkg'), PythonConfiguration(version='3.6', identifier='cp36-macosx_x86_64', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg'), PythonConfiguration(version='3.7', identifier='cp37-macosx_x86_64', url='https://www.python.org/ftp/python/3.7.7/python-3.7.7-macosx10.9.pkg'), - PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.2/python-3.8.2-macosx10.9.pkg'), + PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.3/python-3.8.3-macosx10.9.pkg'), # PyPy PythonConfiguration(version='2.7', identifier='pp27-macosx_x86_64', url='https://downloads.python.org/pypy/pypy2.7-v7.3.1-osx64.tar.bz2'), PythonConfiguration(version='3.6', identifier='pp36-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.6-v7.3.1-osx64.tar.bz2'), @@ -47,7 +55,7 @@ def get_python_configurations(build_selector): SYMLINKS_DIR = '/tmp/cibw_bin' -def make_symlinks(installation_bin_path, python_executable, pip_executable): +def make_symlinks(installation_bin_path: str, python_executable: str, pip_executable: str) -> None: assert os.path.exists(os.path.join(installation_bin_path, python_executable)) # Python bin folders on Mac don't symlink `python3` to `python`, and neither @@ -62,11 +70,11 @@ def make_symlinks(installation_bin_path, python_executable, pip_executable): os.symlink(os.path.join(installation_bin_path, pip_executable), os.path.join(SYMLINKS_DIR, 'pip')) -def install_cpython(version, url): +def install_cpython(version: str, url: str) -> str: installed_system_packages = subprocess.check_output(['pkgutil', '--pkgs'], universal_newlines=True).splitlines() # if this version of python isn't installed, get it from python.org and install - python_package_identifier = 'org.python.Python.PythonFramework-{}'.format(version) + python_package_identifier = f'org.python.Python.PythonFramework-{version}' if python_package_identifier not in installed_system_packages: # download the pkg download(url, '/tmp/Python.pkg') @@ -74,11 +82,11 @@ def install_cpython(version, url): call(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/']) # patch open ssl if version == '3.5': - open_ssl_patch_url = 'https://github.com/mayeut/patch-macos-python-openssl/releases/download/v1.0.2u/patch-macos-python-%s-openssl-v1.0.2u.tar.gz' % version + open_ssl_patch_url = f'https://github.com/mayeut/patch-macos-python-openssl/releases/download/v1.0.2u/patch-macos-python-{version}-openssl-v1.0.2u.tar.gz' download(open_ssl_patch_url, '/tmp/python-patch.tar.gz') - call(['sudo', 'tar', '-C', '/Library/Frameworks/Python.framework/Versions/{}/'.format(version), '-xmf', '/tmp/python-patch.tar.gz']) + call(['sudo', 'tar', '-C', f'/Library/Frameworks/Python.framework/Versions/{version}/', '-xmf', '/tmp/python-patch.tar.gz']) - installation_bin_path = '/Library/Frameworks/Python.framework/Versions/{}/bin'.format(version) + installation_bin_path = f'/Library/Frameworks/Python.framework/Versions/{version}/bin' python_executable = 'python3' if version[0] == '3' else 'python' pip_executable = 'pip3' if version[0] == '3' else 'pip' make_symlinks(installation_bin_path, python_executable, pip_executable) @@ -86,7 +94,7 @@ def install_cpython(version, url): return installation_bin_path -def install_pypy(version, url): +def install_pypy(version: str, url: str) -> str: pypy_tar_bz2 = url.rsplit('/', 1)[-1] assert pypy_tar_bz2.endswith(".tar.bz2") pypy_base_filename = os.path.splitext(os.path.splitext(pypy_tar_bz2)[0])[0] @@ -103,7 +111,7 @@ def install_pypy(version, url): return installation_bin_path -def setup_python(python_configuration, dependency_constraint_flags, environment): +def setup_python(python_configuration: PythonConfiguration, dependency_constraint_flags: List[str], environment: ParsedEnvironment) -> Dict[str, str]: if python_configuration.identifier.startswith('cp'): installation_bin_path = install_cpython(python_configuration.version, python_configuration.url) elif python_configuration.identifier.startswith('pp'): @@ -161,7 +169,7 @@ def setup_python(python_configuration, dependency_constraint_flags, environment) return env -def build(options: BuildOptions): +def build(options: BuildOptions) -> None: temp_dir = tempfile.mkdtemp(prefix='cibuildwheel') built_wheel_dir = os.path.join(temp_dir, 'built_wheel') repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel') diff --git a/cibuildwheel/resources/constraints-python27.txt b/cibuildwheel/resources/constraints-python27.txt index 6da3b134..07f6e111 100644 --- a/cibuildwheel/resources/constraints-python27.txt +++ b/cibuildwheel/resources/constraints-python27.txt @@ -4,7 +4,7 @@ # # bin/update_constraints.py # -appdirs==1.4.3 # via virtualenv +appdirs==1.4.4 # via virtualenv configparser==4.0.2 # via importlib-metadata contextlib2==0.6.0.post1 # via importlib-metadata, importlib-resources, virtualenv delocate==0.8.0 # via -r cibuildwheel/resources/constraints.in @@ -17,7 +17,7 @@ scandir==1.10.0 # via pathlib2 singledispatch==3.4.0.3 # via importlib-resources six==1.14.0 # via pathlib2, virtualenv typing==3.7.4.1 # via importlib-resources -virtualenv==20.0.18 # via -r cibuildwheel/resources/constraints.in +virtualenv==20.0.20 # via -r cibuildwheel/resources/constraints.in wheel==0.34.2 # via -r cibuildwheel/resources/constraints.in, delocate zipp==1.2.0 # via importlib-metadata, importlib-resources diff --git a/cibuildwheel/resources/constraints-python35.txt b/cibuildwheel/resources/constraints-python35.txt index 70591458..bc05483c 100644 --- a/cibuildwheel/resources/constraints-python35.txt +++ b/cibuildwheel/resources/constraints-python35.txt @@ -4,17 +4,17 @@ # # bin/update_constraints.py # -appdirs==1.4.3 # via virtualenv +appdirs==1.4.4 # via virtualenv delocate==0.8.0 # via -r cibuildwheel/resources/constraints.in distlib==0.3.0 # via virtualenv filelock==3.0.12 # via virtualenv importlib-metadata==1.6.0 # via importlib-resources, virtualenv importlib-resources==1.5.0 # via virtualenv six==1.14.0 # via virtualenv -virtualenv==20.0.18 # via -r cibuildwheel/resources/constraints.in +virtualenv==20.0.20 # via -r cibuildwheel/resources/constraints.in wheel==0.34.2 # via -r cibuildwheel/resources/constraints.in, delocate zipp==1.2.0 # via importlib-metadata, importlib-resources # The following packages are considered to be unsafe in a requirements file: pip==20.1 # via -r cibuildwheel/resources/constraints.in -setuptools==46.1.3 # via -r cibuildwheel/resources/constraints.in +setuptools==46.3.1 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python36.txt b/cibuildwheel/resources/constraints-python36.txt index 58776947..d86266df 100644 --- a/cibuildwheel/resources/constraints-python36.txt +++ b/cibuildwheel/resources/constraints-python36.txt @@ -4,17 +4,17 @@ # # bin/update_constraints.py # -appdirs==1.4.3 # via virtualenv +appdirs==1.4.4 # via virtualenv delocate==0.8.0 # via -r cibuildwheel/resources/constraints.in distlib==0.3.0 # via virtualenv filelock==3.0.12 # via virtualenv importlib-metadata==1.6.0 # via importlib-resources, virtualenv importlib-resources==1.5.0 # via virtualenv six==1.14.0 # via virtualenv -virtualenv==20.0.18 # via -r cibuildwheel/resources/constraints.in +virtualenv==20.0.20 # via -r cibuildwheel/resources/constraints.in wheel==0.34.2 # via -r cibuildwheel/resources/constraints.in, delocate zipp==3.1.0 # via importlib-metadata, importlib-resources # The following packages are considered to be unsafe in a requirements file: pip==20.1 # via -r cibuildwheel/resources/constraints.in -setuptools==46.1.3 # via -r cibuildwheel/resources/constraints.in +setuptools==46.3.1 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints.txt b/cibuildwheel/resources/constraints.txt index a9389c61..37591565 100644 --- a/cibuildwheel/resources/constraints.txt +++ b/cibuildwheel/resources/constraints.txt @@ -4,16 +4,16 @@ # # bin/update_constraints.py # -appdirs==1.4.3 # via virtualenv +appdirs==1.4.4 # via virtualenv delocate==0.8.0 # via -r cibuildwheel/resources/constraints.in distlib==0.3.0 # via virtualenv filelock==3.0.12 # via virtualenv importlib-metadata==1.6.0 # via virtualenv six==1.14.0 # via virtualenv -virtualenv==20.0.18 # via -r cibuildwheel/resources/constraints.in +virtualenv==20.0.20 # via -r cibuildwheel/resources/constraints.in wheel==0.34.2 # via -r cibuildwheel/resources/constraints.in, delocate zipp==3.1.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: pip==20.1 # via -r cibuildwheel/resources/constraints.in -setuptools==46.1.3 # via -r cibuildwheel/resources/constraints.in +setuptools==46.3.1 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/pinned_docker_images.cfg b/cibuildwheel/resources/pinned_docker_images.cfg index bf161d3e..d2055656 100644 --- a/cibuildwheel/resources/pinned_docker_images.cfg +++ b/cibuildwheel/resources/pinned_docker_images.cfg @@ -1,22 +1,22 @@ [x86_64] -manylinux1 = quay.io/pypa/manylinux1_x86_64:2020-04-25-37c204c -manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2020-04-29-0e1afc5 -manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2020-05-01-b37d76b +manylinux1 = quay.io/pypa/manylinux1_x86_64:2020-05-15-aad4e04 +manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2020-05-15-877bc09 +manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2020-05-15-5acda28 [i686] -manylinux1 = quay.io/pypa/manylinux1_i686:2020-04-25-37c204c -manylinux2010 = quay.io/pypa/manylinux2010_i686:2020-04-29-0e1afc5 -manylinux2014 = quay.io/pypa/manylinux2014_i686:2020-05-01-b37d76b +manylinux1 = quay.io/pypa/manylinux1_i686:2020-05-15-aad4e04 +manylinux2010 = quay.io/pypa/manylinux2010_i686:2020-05-15-877bc09 +manylinux2014 = quay.io/pypa/manylinux2014_i686:2020-05-15-5acda28 [pypy_x86_64] manylinux2010 = pypywheels/manylinux2010-pypy_x86_64:2020-04-25-eb2cdff [aarch64] -manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2020-05-01-b37d76b +manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2020-05-15-5acda28 [ppc64le] -manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2020-05-01-b37d76b +manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2020-05-15-5acda28 [s390x] -manylinux2014 = quay.io/pypa/manylinux2014_s390x:2020-05-01-b37d76b +manylinux2014 = quay.io/pypa/manylinux2014_s390x:2020-05-15-5acda28 diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 64f561a1..80042039 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -2,12 +2,13 @@ import os import urllib.request from fnmatch import fnmatch from time import sleep -from typing import NamedTuple, List, Optional, Dict + +from typing import Dict, List, NamedTuple, Optional from .environment import ParsedEnvironment -def prepare_command(command, **kwargs): +def prepare_command(command: str, **kwargs: str) -> str: ''' Preprocesses a command by expanding variables like {python}. @@ -17,7 +18,7 @@ def prepare_command(command, **kwargs): return command.format(python='python', pip='pip', **kwargs) -def get_build_verbosity_extra_flags(level): +def get_build_verbosity_extra_flags(level: int) -> List[str]: if level > 0: return ['-' + level * 'v'] elif level < 0: @@ -27,37 +28,37 @@ def get_build_verbosity_extra_flags(level): class BuildSelector: - def __init__(self, build_config, skip_config): + def __init__(self, build_config: str, skip_config: str): self.build_patterns = build_config.split() self.skip_patterns = skip_config.split() - def __call__(self, build_id): - def match_any(patterns): + def __call__(self, build_id: str) -> bool: + def match_any(patterns: List[str]) -> bool: return any(fnmatch(build_id, pattern) for pattern in patterns) return match_any(self.build_patterns) and not match_any(self.skip_patterns) - def __repr__(self): - return 'BuildSelector({!r} - {!r})'.format(' '.join(self.build_patterns), ' '.join(self.skip_patterns)) + def __repr__(self) -> str: + return f'BuildSelector({" ".join(self.build_patterns)!r} - {" ".join(self.skip_patterns)!r})' # Taken from https://stackoverflow.com/a/107717 class Unbuffered: - def __init__(self, stream): + def __init__(self, stream): # type: ignore self.stream = stream - def write(self, data): + def write(self, data): # type: ignore self.stream.write(data) self.stream.flush() - def writelines(self, datas): + def writelines(self, datas): # type: ignore self.stream.writelines(datas) self.stream.flush() - def __getattr__(self, attr): + def __getattr__(self, attr): # type: ignore return getattr(self.stream, attr) -def download(url, dest): +def download(url: str, dest: str) -> None: print('+ Download ' + url + ' to ' + dest) dest_dir = os.path.dirname(dest) if not os.path.exists(dest_dir): @@ -82,23 +83,23 @@ def download(url, dest): class DependencyConstraints: - def __init__(self, base_file_path): + def __init__(self, base_file_path: str): assert os.path.exists(base_file_path) self.base_file_path = os.path.abspath(base_file_path) - @classmethod - def with_defaults(cls): - return cls( + @staticmethod + def with_defaults() -> 'DependencyConstraints': + return DependencyConstraints( base_file_path=os.path.join(os.path.dirname(__file__), 'resources', 'constraints.txt') ) - def get_for_python_version(self, version): + def get_for_python_version(self, version: str) -> str: version_parts = version.split('.') # try to find a version-specific dependency file e.g. if # ./constraints.txt is the base, look for ./constraints-python27.txt base, ext = os.path.splitext(self.base_file_path) - specific = base + '-python{}{}'.format(version_parts[0], version_parts[1]) + specific = base + f'-python{version_parts[0]}{version_parts[1]}' specific_file_path = specific + ext if os.path.exists(specific_file_path): return specific_file_path @@ -106,26 +107,21 @@ class DependencyConstraints: return self.base_file_path -BuildOptions = NamedTuple("BuildOptions", [ - ("package_dir", str), - ("output_dir", str), - ("test_command", Optional[str]), - ("test_requires", List[str]), - ("test_extras", str), - ("before_build", Optional[str]), - ("build_verbosity", int), - ("build_selector", BuildSelector), - ("repair_command", str), - ("environment", ParsedEnvironment), - ("before_test", str), - ("dependency_constraints", Optional[DependencyConstraints]), - ("manylinux_images", Optional[Dict[str, str]]), -]) +class BuildOptions(NamedTuple): + package_dir: str + output_dir: str + test_command: Optional[str] + test_requires: List[str] + test_extras: str + before_build: Optional[str] + build_verbosity: int + build_selector: BuildSelector + repair_command: str + environment: ParsedEnvironment + before_test: str + dependency_constraints: Optional[DependencyConstraints] + manylinux_images: Optional[Dict[str, str]] -""" -Replace this definition with a class-style NamedTuple in the -PEP526 style when Python 3.5 host support is dropped -""" resources_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'resources')) get_pip_script = os.path.join(resources_dir, 'get-pip.py') diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index a389fac9..9f33ad91 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -3,16 +3,19 @@ import shutil import subprocess import sys import tempfile -from collections import namedtuple from glob import glob from zipfile import ZipFile +from typing import Dict, List, Optional, NamedTuple + +from .environment import ParsedEnvironment from .util import ( + BuildOptions, + BuildSelector, download, get_build_verbosity_extra_flags, - prepare_command, get_pip_script, - BuildOptions + prepare_command, ) @@ -20,20 +23,26 @@ IS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache') IS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows' -def shell(args, env=None, cwd=None): +def shell(args: List[str], env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None) -> int: print('+ ' + ' '.join(args)) return subprocess.check_call(' '.join(args), env=env, cwd=cwd, shell=True) -def get_nuget_args(version, arch): +def get_nuget_args(version: str, arch: str) -> List[str]: python_name = 'python' if version[0] == '3' else 'python2' if arch == '32': python_name = python_name + 'x86' return [python_name, '-Version', version, '-OutputDirectory', 'C:\\cibw\\python'] -def get_python_configurations(build_selector): - PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier', 'url']) +class PythonConfiguration(NamedTuple): + version: str + arch: str + identifier: str + url: Optional[str] + + +def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfiguration]: python_configurations = [ # CPython PythonConfiguration(version='2.7.18', arch='32', identifier='cp27-win32', url=None), @@ -44,8 +53,8 @@ def get_python_configurations(build_selector): PythonConfiguration(version='3.6.8', arch='64', identifier='cp36-win_amd64', url=None), PythonConfiguration(version='3.7.7', arch='32', identifier='cp37-win32', url=None), PythonConfiguration(version='3.7.7', arch='64', identifier='cp37-win_amd64', url=None), - PythonConfiguration(version='3.8.2', arch='32', identifier='cp38-win32', url=None), - PythonConfiguration(version='3.8.2', arch='64', identifier='cp38-win_amd64', url=None), + PythonConfiguration(version='3.8.3', arch='32', identifier='cp38-win32', url=None), + PythonConfiguration(version='3.8.3', arch='64', identifier='cp38-win_amd64', url=None), # PyPy PythonConfiguration(version='2.7', arch='32', identifier='pp27-win32', url='https://downloads.python.org/pypy/pypy2.7-v7.3.1-win32.zip'), PythonConfiguration(version='3.6', arch='32', identifier='pp36-win32', url='https://downloads.python.org/pypy/pypy3.6-v7.3.1-win32.zip'), @@ -62,19 +71,19 @@ def get_python_configurations(build_selector): return python_configurations -def extract_zip(zip_src, dest): +def extract_zip(zip_src: str, dest: str) -> None: with ZipFile(zip_src) as zip: zip.extractall(dest) -def install_cpython(version, arch, nuget): +def install_cpython(version: str, arch: str, nuget: str) -> str: nuget_args = get_nuget_args(version, arch) installation_path = os.path.join(nuget_args[-1], nuget_args[0] + '.' + version, 'tools') shell([nuget, 'install'] + nuget_args) return installation_path -def install_pypy(version, arch, url): +def install_pypy(version: str, arch: str, url: str) -> str: assert arch == '32' # Inside the PyPy zip file is a directory with the same name zip_filename = url.rsplit('/', 1)[-1] @@ -89,7 +98,7 @@ def install_pypy(version, arch, url): return installation_path -def setup_python(python_configuration, dependency_constraint_flags, environment): +def setup_python(python_configuration: PythonConfiguration, dependency_constraint_flags: List[str], environment: ParsedEnvironment) -> Dict[str, str]: nuget = 'C:\\cibw\\nuget.exe' if not os.path.exists(nuget): download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget) @@ -97,6 +106,7 @@ def setup_python(python_configuration, dependency_constraint_flags, environment) if python_configuration.identifier.startswith('cp'): installation_path = install_cpython(python_configuration.version, python_configuration.arch, nuget) elif python_configuration.identifier.startswith('pp'): + assert python_configuration.url is not None installation_path = install_pypy(python_configuration.version, python_configuration.arch, python_configuration.url) else: raise ValueError("Unknown Python implementation") @@ -141,7 +151,7 @@ def setup_python(python_configuration, dependency_constraint_flags, environment) return env -def build(options: BuildOptions): +def build(options: BuildOptions) -> None: temp_dir = tempfile.mkdtemp(prefix='cibuildwheel') built_wheel_dir = os.path.join(temp_dir, 'built_wheel') repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel') diff --git a/setup.cfg b/setup.cfg index d678c699..350db702 100644 --- a/setup.cfg +++ b/setup.cfg @@ -14,3 +14,7 @@ exclude = [tool:pytest] junit_family=xunit2 + +[mypy] +files=cibuildwheel/ +strict=True diff --git a/test/test_dependency_versions.py b/test/test_dependency_versions.py index e6645dec..611350f1 100644 --- a/test/test_dependency_versions.py +++ b/test/test_dependency_versions.py @@ -77,11 +77,11 @@ def test_pinned_versions(tmp_path, python_version): constraint_versions = get_versions_from_constraint_file(constraint_file) for package in ['pip', 'setuptools', 'wheel', 'virtualenv']: - env_name = 'EXPECTED_{}_VERSION'.format(package.upper()) + env_name = f'EXPECTED_{package.upper()}_VERSION' build_environment[env_name] = constraint_versions[package] cibw_environment_option = ' '.join( - ['{}={}'.format(k, v) for k, v in build_environment.items()] + [f'{k}={v}' for k, v in build_environment.items()] ) # build and test the wheels @@ -137,11 +137,11 @@ def test_dependency_constraints_file(tmp_path, python_version): build_environment = {} for package_name, version in tool_versions.items(): - env_name = 'EXPECTED_{}_VERSION'.format(package_name.upper()) + env_name = f'EXPECTED_{package_name.upper()}_VERSION' build_environment[env_name] = version cibw_environment_option = ' '.join( - ['{}={}'.format(k, v) for k, v in build_environment.items()] + [f'{k}={v}' for k, v in build_environment.items()] ) # build and test the wheels diff --git a/test/utils.py b/test/utils.py index 1c2733c6..15ce402a 100644 --- a/test/utils.py +++ b/test/utils.py @@ -110,7 +110,7 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, architectures.append('i686') platform_tags = [ - '{}_{}'.format(manylinux_version, architecture) + f'{manylinux_version}_{architecture}' for architecture in architectures for manylinux_version in manylinux_versions ] @@ -122,16 +122,13 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, platform_tags = ['win32'] elif platform == 'macos': - platform_tags = ['macosx_{}_x86_64'.format(macosx_deployment_target.replace('.', '_'))] + platform_tags = [f'macosx_{macosx_deployment_target.replace(".", "_")}_x86_64'] else: raise Exception('unsupported platform') for platform_tag in platform_tags: - wheels.append('{package_name}-{package_version}-{python_abi_tag}-{platform_tag}.whl'.format( - package_name=package_name, package_version=package_version, - python_abi_tag=python_abi_tag, platform_tag=platform_tag - )) + wheels.append(f'{package_name}-{package_version}-{python_abi_tag}-{platform_tag}.whl') if IS_WINDOWS_RUNNING_ON_TRAVIS: # Python 2.7 isn't supported on Travis. diff --git a/unit_test/main_tests/main_options_test.py b/unit_test/main_tests/main_options_test.py index 74cbe3c1..633e9cc5 100644 --- a/unit_test/main_tests/main_options_test.py +++ b/unit_test/main_tests/main_options_test.py @@ -123,7 +123,7 @@ def test_repair_command(repair_command, platform_specific, platform, intercepted ]) @pytest.mark.parametrize('platform_specific', [False, True]) def test_environment(environment, platform_specific, platform, intercepted_build_args, monkeypatch): - env_string = ' '.join(['{}={}'.format(k, v) for k, v in environment.items()]) + env_string = ' '.join([f'{k}={v}' for k, v in environment.items()]) if platform_specific: monkeypatch.setenv('CIBW_ENVIRONMENT_' + platform.upper(), env_string) monkeypatch.setenv('CIBW_ENVIRONMENT', 'overwritten')