From 618f1cefaeb007e2b4972f3f901dc43598f3b990 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Wed, 8 Apr 2020 00:16:25 +0200 Subject: [PATCH 01/11] Added majority of type annotations (to a by now outdated version) --- cibuildwheel/__main__.py | 35 ++++++++++++++++++++++------------- cibuildwheel/bashlex_eval.py | 26 ++++++++++++++------------ cibuildwheel/environment.py | 34 ++++++++++++++++++---------------- cibuildwheel/linux.py | 16 ++++++++++++---- cibuildwheel/macos.py | 23 +++++++++++++++-------- cibuildwheel/util.py | 25 +++++++++++++------------ cibuildwheel/windows.py | 25 ++++++++++++++++--------- 7 files changed, 110 insertions(+), 74 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 7099bc8f..09471d76 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -5,6 +5,8 @@ import textwrap import traceback from configparser import ConfigParser +from typing import Any, Dict, List, Optional, Union, overload + import cibuildwheel import cibuildwheel.linux import cibuildwheel.macos @@ -14,14 +16,20 @@ from cibuildwheel.environment import ( parse_environment, ) from cibuildwheel.util import ( + BuildOptions, BuildSelector, DependencyConstraints, Unbuffered, - BuildOptions ) -def get_option_from_environment(option_name, platform=None, default=None): +@overload +def get_option_from_environment(option_name: str, platform: Optional[str], default: str) -> str: + ... +@overload +def get_option_from_environment(option_name: str, platform: Optional[str] = None, default: None = None) -> Optional[str]: + ... +def get_option_from_environment(option_name: str, platform: Optional[str] = None, default: Optional[str] = None) -> Optional[str]: ''' Returns an option from the environment, optionally scoped by the platform. @@ -39,13 +47,13 @@ def get_option_from_environment(option_name, platform=None, default=None): return os.environ.get(option_name, default) -def strtobool(val): +def strtobool(val: str) -> bool: if val.lower() in ('y', 'yes', 't', 'true', 'on', '1'): return True return False -def main(): +def main() -> None: parser = argparse.ArgumentParser( description='Build wheels for all the platforms.', epilog=''' @@ -114,7 +122,7 @@ def main(): test_extras = get_option_from_environment('CIBW_TEST_EXTRAS', platform=platform, default='') package_dir = args.package_dir before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform) - build_verbosity = get_option_from_environment('CIBW_BUILD_VERBOSITY', platform=platform, default='') + build_verbosity_str = get_option_from_environment('CIBW_BUILD_VERBOSITY', platform=platform, default='') build_config, skip_config = os.environ.get('CIBW_BUILD', '*'), os.environ.get('CIBW_SKIP', '') if platform == 'linux': repair_command_default = 'auditwheel repair -w {dest_dir} {wheel}' @@ -138,7 +146,7 @@ def main(): test_extras = '[{0}]'.format(test_extras) try: - build_verbosity = min(3, max(-3, int(build_verbosity))) + build_verbosity = min(3, max(-3, int(build_verbosity_str))) except ValueError: build_verbosity = 0 @@ -175,7 +183,8 @@ def main(): # 'pypy_x86_64': {'manylinux2010': '...' } # ... } - manylinux_images = {} + manylinux_images = {} # type: Optional[Dict[str, str]] + assert manylinux_images is not None # Weird problem with mypy for build_platform in ['x86_64', 'i686', 'pypy_x86_64', 'aarch64', 'ppc64le', 's390x']: pinned_images = all_pinned_docker_images[build_platform] @@ -213,7 +222,7 @@ def main(): ) # Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print' - sys.stdout = Unbuffered(sys.stdout) + sys.stdout = Unbuffered(sys.stdout) # type: ignore print_preamble(platform, build_options) @@ -231,7 +240,7 @@ def main(): exit(2) -def detect_obsolete_options(): +def detect_obsolete_options() -> None: # Check the old 'MANYLINUX1_*_IMAGE' options for (deprecated, alternative) in [('CIBW_MANYLINUX1_X86_64_IMAGE', 'CIBW_MANYLINUX_X86_64_IMAGE'), ('CIBW_MANYLINUX1_I686_IMAGE', 'CIBW_MANYLINUX_I686_IMAGE')]: @@ -258,7 +267,7 @@ def detect_obsolete_options(): os.environ[option] = os.environ[option].replace(deprecated, alternative) -def print_preamble(platform, build_options): +def print_preamble(platform: str, build_options: BuildOptions) -> None: print(textwrap.dedent(''' _ _ _ _ _ _ _ ___|_| |_ _ _|_| |_| |_ _ _| |_ ___ ___| | @@ -282,9 +291,9 @@ def print_preamble(platform, build_options): print('\nHere we go!\n') -def print_build_identifiers(platform, build_selector): +def print_build_identifiers(platform: str, build_selector: BuildSelector) -> None: if platform == 'linux': - python_configurations = cibuildwheel.linux.get_python_configurations(build_selector) + python_configurations = cibuildwheel.linux.get_python_configurations(build_selector) # type: List[Any] elif platform == 'windows': python_configurations = cibuildwheel.windows.get_python_configurations(build_selector) elif platform == 'macos': @@ -296,7 +305,7 @@ def print_build_identifiers(platform, build_selector): print(config.identifier) -def detect_warnings(platform, build_options): +def detect_warnings(platform: str, build_options: BuildOptions) -> List[str]: warnings = [] # warn about deprecated {python} and {pip} diff --git a/cibuildwheel/bashlex_eval.py b/cibuildwheel/bashlex_eval.py index 8e2c0831..52f444a2 100644 --- a/cibuildwheel/bashlex_eval.py +++ b/cibuildwheel/bashlex_eval.py @@ -2,12 +2,14 @@ import shlex import subprocess from collections import namedtuple -import bashlex +from typing import Dict + +import bashlex # type: ignore NodeExecutionContext = namedtuple('NodeExecutionContext', ['environment', 'input']) -def evaluate(value, environment): +def evaluate(value: str, environment: Dict[str, str]) -> str: if not value: # empty string evaluates to empty string # (but trips up bashlex) @@ -20,24 +22,24 @@ def evaluate(value, environment): value_word_node = command_node.parts[0] - return evaluate_node( + return evaluate_node( # type: ignore value_word_node, context=NodeExecutionContext(environment=environment, input=value) ) -def evaluate_node(node, context): +def evaluate_node(node, context): # type: ignore if node.kind == 'word': - return evaluate_word_node(node, context=context) + return evaluate_word_node(node, context=context) # type: ignore elif node.kind == 'commandsubstitution': - return evaluate_command_node(node.command, context=context) + return evaluate_command_node(node.command, context=context) # type: ignore elif node.kind == 'parameter': - return evaluate_parameter_node(node, context=context) + return evaluate_parameter_node(node, context=context) # type: ignore else: raise ValueError('Unsupported bash construct: "%s"' % node.word) -def evaluate_word_node(node, context): +def evaluate_word_node(node, context): # type: ignore word_start = node.pos[0] word_end = node.pos[1] word_string = context.input[word_start:word_end] @@ -51,7 +53,7 @@ def evaluate_word_node(node, context): for i in range(part_start, part_end): letters[i] = None - letters[part_start] = evaluate_node(part, context=context) + letters[part_start] = evaluate_node(part, context=context) # type: ignore # remove the None letters and concat value = ''.join(l for l in letters if l is not None) @@ -60,11 +62,11 @@ def evaluate_word_node(node, context): return ' '.join(word.strip() for word in shlex.split(value)) -def evaluate_command_node(node, context): - words = [evaluate_node(part, context=context) for part in node.parts] +def evaluate_command_node(node, context): # type: ignore + words = [evaluate_node(part, context=context) for part in node.parts] # type: ignore command = ' '.join(words) return subprocess.check_output(shlex.split(command), env=context.environment, universal_newlines=True) -def evaluate_parameter_node(node, context): +def evaluate_parameter_node(node, context): # type: ignore return context.environment.get(node.value, '') diff --git a/cibuildwheel/environment.py b/cibuildwheel/environment.py index f016c780..837ad266 100644 --- a/cibuildwheel/environment.py +++ b/cibuildwheel/environment.py @@ -1,4 +1,6 @@ -import bashlex +import bashlex # type: ignore + +from typing import Dict, List from . import bashlex_eval @@ -7,13 +9,7 @@ class EnvironmentParseError(Exception): pass -def parse_environment(env_string): - env_items = split_env_items(env_string) - assignments = [EnvironmentAssignment(item) for item in env_items] - return ParsedEnvironment(assignments=assignments) - - -def split_env_items(env_string): +def split_env_items(env_string: str) -> List[str]: '''Splits space-separated variable assignments into a list of individual assignments. >>> split_env_items('VAR=abc') @@ -43,29 +39,29 @@ def split_env_items(env_string): class EnvironmentAssignment: - def __init__(self, assignment): + def __init__(self, assignment: str): name, equals, value = assignment.partition('=') if not equals: raise EnvironmentParseError(assignment) self.name = name self.value = value - def evaluated_value(self, environment): + def evaluated_value(self, environment: Dict[str, str]) -> str: '''Returns the value of this assignment, as evaluated in the environment''' return bashlex_eval.evaluate(self.value, environment=environment) - def as_shell_assignment(self): + def as_shell_assignment(self) -> str: return 'export %s=%s' % (self.name, self.value) - def __repr__(self): + def __repr__(self) -> str: return '%s=%s' % (self.name, self.value) class ParsedEnvironment: - def __init__(self, assignments): + def __init__(self, assignments: List[EnvironmentAssignment]): self.assignments = assignments - def as_dictionary(self, prev_environment): + def as_dictionary(self, prev_environment: Dict[str, str]) -> Dict[str, str]: environment = prev_environment.copy() for assignment in self.assignments: @@ -74,8 +70,14 @@ class ParsedEnvironment: return environment - def as_shell_commands(self): + def as_shell_commands(self) -> List[str]: return [a.as_shell_assignment() for a in self.assignments] - def __repr__(self): + def __repr__(self) -> str: return 'ParsedEnvironment(%r)' % [repr(a) for a in self.assignments] + + +def parse_environment(env_string: str) -> ParsedEnvironment: + env_items = split_env_items(env_string) + assignments = [EnvironmentAssignment(item) for item in env_items] + return ParsedEnvironment(assignments=assignments) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index d30635c3..b1a27de0 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -7,10 +7,15 @@ import textwrap import uuid from collections import namedtuple +from typing import Callable, Dict, List, Optional + +from .environment import ( + ParsedEnvironment, +) from .util import ( + BuildOptions, get_build_verbosity_extra_flags, prepare_command, - BuildOptions ) @@ -42,8 +47,10 @@ def matches_platform(identifier): return False -def get_python_configurations(build_selector): - PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'path']) +PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'path']) + + +def get_python_configurations(build_selector: Callable[[str], bool]) -> List[PythonConfiguration]: python_configurations = [ PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path='/opt/python/cp27-cp27m'), PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path='/opt/python/cp27-cp27mu'), @@ -86,6 +93,7 @@ def build(options: BuildOptions): file=sys.stderr) exit(2) + assert options.manylinux_images is not None python_configurations = get_python_configurations(options.build_selector) platforms = [ ('cp', 'manylinux_x86_64', options.manylinux_images['x86_64']), @@ -274,7 +282,7 @@ def build(options: BuildOptions): call(['docker', 'rm', '--force', '-v', container_name]) -def troubleshoot(package_dir, error): +def troubleshoot(package_dir: str, error: Exception) -> None: if (isinstance(error, subprocess.CalledProcessError) and 'exec' in error.cmd): # the bash script failed print('Checking for common errors...') diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 8eb7984c..610c30e1 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -7,16 +7,21 @@ import tempfile from collections import namedtuple from glob import glob +from typing import Callable, Dict, List, Optional, Union + +from .environment import ( + ParsedEnvironment, +) from .util import ( + BuildOptions, download, get_build_verbosity_extra_flags, - prepare_command, get_pip_script, - BuildOptions + prepare_command, ) -def call(args, env=None, cwd=None, shell=False): +def call(args: Union[str, List[str]], env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None, shell: bool = False) -> int: # print the command executing for the logs if shell: print('+ %s' % args) @@ -26,8 +31,10 @@ def call(args, env=None, cwd=None, shell=False): return subprocess.check_call(args, env=env, cwd=cwd, shell=shell) -def get_python_configurations(build_selector): - PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'url']) +PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'url']) + + +def get_python_configurations(build_selector: Callable[[str], bool]) -> List[PythonConfiguration]: python_configurations = [ # CPython PythonConfiguration(version='2.7', identifier='cp27-macosx_x86_64', url='https://www.python.org/ftp/python/2.7.18/python-2.7.18-macosx10.9.pkg'), @@ -47,7 +54,7 @@ def get_python_configurations(build_selector): SYMLINKS_DIR = '/tmp/cibw_bin' -def make_symlinks(installation_bin_path, python_executable, pip_executable): +def make_symlinks(installation_bin_path: str, python_executable: str, pip_executable: str) -> None: assert os.path.exists(os.path.join(installation_bin_path, python_executable)) # Python bin folders on Mac don't symlink `python3` to `python`, and neither @@ -62,7 +69,7 @@ def make_symlinks(installation_bin_path, python_executable, pip_executable): os.symlink(os.path.join(installation_bin_path, pip_executable), os.path.join(SYMLINKS_DIR, 'pip')) -def install_cpython(version, url): +def install_cpython(version: str, url: str) -> str: installed_system_packages = subprocess.check_output(['pkgutil', '--pkgs'], universal_newlines=True).splitlines() # if this version of python isn't installed, get it from python.org and install @@ -86,7 +93,7 @@ def install_cpython(version, url): return installation_bin_path -def install_pypy(version, url): +def install_pypy(version: str, url: str) -> str: pypy_tar_bz2 = url.rsplit('/', 1)[-1] assert pypy_tar_bz2.endswith(".tar.bz2") pypy_base_filename = os.path.splitext(os.path.splitext(pypy_tar_bz2)[0])[0] diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 64f561a1..21b76f80 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -2,12 +2,13 @@ import os import urllib.request from fnmatch import fnmatch from time import sleep -from typing import NamedTuple, List, Optional, Dict + +from typing import Dict, List, NamedTuple, Optional from .environment import ParsedEnvironment -def prepare_command(command, **kwargs): +def prepare_command(command: str, **kwargs: str) -> str: ''' Preprocesses a command by expanding variables like {python}. @@ -17,7 +18,7 @@ def prepare_command(command, **kwargs): return command.format(python='python', pip='pip', **kwargs) -def get_build_verbosity_extra_flags(level): +def get_build_verbosity_extra_flags(level: int) -> List[str]: if level > 0: return ['-' + level * 'v'] elif level < 0: @@ -27,37 +28,37 @@ def get_build_verbosity_extra_flags(level): class BuildSelector: - def __init__(self, build_config, skip_config): + def __init__(self, build_config: str, skip_config: str): self.build_patterns = build_config.split() self.skip_patterns = skip_config.split() - def __call__(self, build_id): - def match_any(patterns): + def __call__(self, build_id: str) -> bool: + def match_any(patterns: List[str]) -> bool: return any(fnmatch(build_id, pattern) for pattern in patterns) return match_any(self.build_patterns) and not match_any(self.skip_patterns) - def __repr__(self): + def __repr__(self) -> str: return 'BuildSelector({!r} - {!r})'.format(' '.join(self.build_patterns), ' '.join(self.skip_patterns)) # Taken from https://stackoverflow.com/a/107717 class Unbuffered: - def __init__(self, stream): + def __init__(self, stream): # type: ignore self.stream = stream - def write(self, data): + def write(self, data): # type: ignore self.stream.write(data) self.stream.flush() - def writelines(self, datas): + def writelines(self, datas): # type: ignore self.stream.writelines(datas) self.stream.flush() - def __getattr__(self, attr): + def __getattr__(self, attr): # type: ignore return getattr(self.stream, attr) -def download(url, dest): +def download(url: str, dest: str) -> None: print('+ Download ' + url + ' to ' + dest) dest_dir = os.path.dirname(dest) if not os.path.exists(dest_dir): diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index a389fac9..48db6733 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -7,12 +7,17 @@ from collections import namedtuple from glob import glob from zipfile import ZipFile +from typing import Callable, Dict, List, Optional + +from .environment import ( + ParsedEnvironment, +) from .util import ( + BuildOptions, download, get_build_verbosity_extra_flags, - prepare_command, get_pip_script, - BuildOptions + prepare_command, ) @@ -20,20 +25,22 @@ IS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache') IS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows' -def shell(args, env=None, cwd=None): +def shell(args: List[str], env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None) -> int: print('+ ' + ' '.join(args)) return subprocess.check_call(' '.join(args), env=env, cwd=cwd, shell=True) -def get_nuget_args(version, arch): +def get_nuget_args(version: str, arch: str) -> List[str]: python_name = 'python' if version[0] == '3' else 'python2' if arch == '32': python_name = python_name + 'x86' return [python_name, '-Version', version, '-OutputDirectory', 'C:\\cibw\\python'] -def get_python_configurations(build_selector): - PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier', 'url']) +PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier', 'url']) + + +def get_python_configurations(build_selector: Callable[[str], bool]) -> List[PythonConfiguration]: python_configurations = [ # CPython PythonConfiguration(version='2.7.18', arch='32', identifier='cp27-win32', url=None), @@ -62,19 +69,19 @@ def get_python_configurations(build_selector): return python_configurations -def extract_zip(zip_src, dest): +def extract_zip(zip_src: str, dest: str) -> None: with ZipFile(zip_src) as zip: zip.extractall(dest) -def install_cpython(version, arch, nuget): +def install_cpython(version: str, arch: str, nuget: str) -> str: nuget_args = get_nuget_args(version, arch) installation_path = os.path.join(nuget_args[-1], nuget_args[0] + '.' + version, 'tools') shell([nuget, 'install'] + nuget_args) return installation_path -def install_pypy(version, arch, url): +def install_pypy(version: str, arch: str, url: str) -> str: assert arch == '32' # Inside the PyPy zip file is a directory with the same name zip_filename = url.rsplit('/', 1)[-1] From c128018590caea9dc6c06a18e9c6cdb611ac9f8b Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Wed, 8 Apr 2020 00:51:04 +0200 Subject: [PATCH 02/11] Clean up bashlex_eval.py's types --- cibuildwheel/bashlex_eval.py | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/cibuildwheel/bashlex_eval.py b/cibuildwheel/bashlex_eval.py index 52f444a2..f8bd0800 100644 --- a/cibuildwheel/bashlex_eval.py +++ b/cibuildwheel/bashlex_eval.py @@ -2,11 +2,13 @@ import shlex import subprocess from collections import namedtuple -from typing import Dict +from typing import Dict, List, NamedTuple, Optional import bashlex # type: ignore -NodeExecutionContext = namedtuple('NodeExecutionContext', ['environment', 'input']) +NodeExecutionContext = NamedTuple('NodeExecutionContext', + [('environment', Dict[str, str]), + ('input', str)]) def evaluate(value: str, environment: Dict[str, str]) -> str: @@ -22,28 +24,28 @@ def evaluate(value: str, environment: Dict[str, str]) -> str: value_word_node = command_node.parts[0] - return evaluate_node( # type: ignore + return evaluate_node( value_word_node, context=NodeExecutionContext(environment=environment, input=value) ) -def evaluate_node(node, context): # type: ignore +def evaluate_node(node: bashlex.ast.node, context: NodeExecutionContext) -> str: if node.kind == 'word': - return evaluate_word_node(node, context=context) # type: ignore + return evaluate_word_node(node, context=context) elif node.kind == 'commandsubstitution': - return evaluate_command_node(node.command, context=context) # type: ignore + return evaluate_command_node(node.command, context=context) elif node.kind == 'parameter': - return evaluate_parameter_node(node, context=context) # type: ignore + return evaluate_parameter_node(node, context=context) else: raise ValueError('Unsupported bash construct: "%s"' % node.word) -def evaluate_word_node(node, context): # type: ignore +def evaluate_word_node(node: bashlex.ast.node, context: NodeExecutionContext) -> str: word_start = node.pos[0] word_end = node.pos[1] word_string = context.input[word_start:word_end] - letters = list(word_string) + letters = list(word_string) # type: List[Optional[str]] for part in node.parts: part_start = part.pos[0] - word_start @@ -53,7 +55,7 @@ def evaluate_word_node(node, context): # type: ignore for i in range(part_start, part_end): letters[i] = None - letters[part_start] = evaluate_node(part, context=context) # type: ignore + letters[part_start] = evaluate_node(part, context=context) # remove the None letters and concat value = ''.join(l for l in letters if l is not None) @@ -62,11 +64,11 @@ def evaluate_word_node(node, context): # type: ignore return ' '.join(word.strip() for word in shlex.split(value)) -def evaluate_command_node(node, context): # type: ignore - words = [evaluate_node(part, context=context) for part in node.parts] # type: ignore +def evaluate_command_node(node: bashlex.ast.node, context: NodeExecutionContext) -> str: + words = [evaluate_node(part, context=context) for part in node.parts] command = ' '.join(words) return subprocess.check_output(shlex.split(command), env=context.environment, universal_newlines=True) -def evaluate_parameter_node(node, context): # type: ignore +def evaluate_parameter_node(node: bashlex.ast.node, context: NodeExecutionContext) -> str: return context.environment.get(node.value, '') From 40967a6572a808c2b74d8fca9af189f48911fffd Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Wed, 8 Apr 2020 11:08:23 +0200 Subject: [PATCH 03/11] collections.namedtuple to typing.NameTuple and flake8 fixes --- cibuildwheel/__main__.py | 10 ++++------ cibuildwheel/bashlex_eval.py | 1 - cibuildwheel/linux.py | 8 ++------ cibuildwheel/macos.py | 8 ++------ cibuildwheel/windows.py | 11 ++++------- 5 files changed, 12 insertions(+), 26 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 09471d76..83e57f03 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -5,7 +5,7 @@ import textwrap import traceback from configparser import ConfigParser -from typing import Any, Dict, List, Optional, Union, overload +from typing import Any, Dict, List, Optional, overload import cibuildwheel import cibuildwheel.linux @@ -24,12 +24,10 @@ from cibuildwheel.util import ( @overload -def get_option_from_environment(option_name: str, platform: Optional[str], default: str) -> str: - ... +def get_option_from_environment(option_name: str, platform: Optional[str], default: str) -> str: ... # noqa: E704 @overload -def get_option_from_environment(option_name: str, platform: Optional[str] = None, default: None = None) -> Optional[str]: - ... -def get_option_from_environment(option_name: str, platform: Optional[str] = None, default: Optional[str] = None) -> Optional[str]: +def get_option_from_environment(option_name: str, platform: Optional[str] = None, default: None = None) -> Optional[str]: ... # noqa: E704 E302 +def get_option_from_environment(option_name: str, platform: Optional[str] = None, default: Optional[str] = None) -> Optional[str]: # noqa: E302 ''' Returns an option from the environment, optionally scoped by the platform. diff --git a/cibuildwheel/bashlex_eval.py b/cibuildwheel/bashlex_eval.py index f8bd0800..a10fe832 100644 --- a/cibuildwheel/bashlex_eval.py +++ b/cibuildwheel/bashlex_eval.py @@ -1,6 +1,5 @@ import shlex import subprocess -from collections import namedtuple from typing import Dict, List, NamedTuple, Optional diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index b1a27de0..33cd493d 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -5,13 +5,9 @@ import subprocess import sys import textwrap import uuid -from collections import namedtuple -from typing import Callable, Dict, List, Optional +from typing import Callable, List, NamedTuple -from .environment import ( - ParsedEnvironment, -) from .util import ( BuildOptions, get_build_verbosity_extra_flags, @@ -47,7 +43,7 @@ def matches_platform(identifier): return False -PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'path']) +PythonConfiguration = NamedTuple('PythonConfiguration', [('version', str), ('identifier', str), ('path', str)]) def get_python_configurations(build_selector: Callable[[str], bool]) -> List[PythonConfiguration]: diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 610c30e1..2e0a2535 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -4,14 +4,10 @@ import shutil import subprocess import sys import tempfile -from collections import namedtuple from glob import glob -from typing import Callable, Dict, List, Optional, Union +from typing import Callable, Dict, List, Optional, NamedTuple, Union -from .environment import ( - ParsedEnvironment, -) from .util import ( BuildOptions, download, @@ -31,7 +27,7 @@ def call(args: Union[str, List[str]], env: Optional[Dict[str, str]] = None, cwd: return subprocess.check_call(args, env=env, cwd=cwd, shell=shell) -PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'url']) +PythonConfiguration = NamedTuple('PythonConfiguration', [('version', str), ('identifier', str), ('url', str)]) def get_python_configurations(build_selector: Callable[[str], bool]) -> List[PythonConfiguration]: diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 48db6733..cb78fe60 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -3,15 +3,11 @@ import shutil import subprocess import sys import tempfile -from collections import namedtuple from glob import glob from zipfile import ZipFile -from typing import Callable, Dict, List, Optional +from typing import Callable, Dict, List, Optional, NamedTuple -from .environment import ( - ParsedEnvironment, -) from .util import ( BuildOptions, download, @@ -37,7 +33,7 @@ def get_nuget_args(version: str, arch: str) -> List[str]: return [python_name, '-Version', version, '-OutputDirectory', 'C:\\cibw\\python'] -PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier', 'url']) +PythonConfiguration = NamedTuple('PythonConfiguration', [('version', str), ('arch', str), ('identifier', str), ('url', Optional[str])]) def get_python_configurations(build_selector: Callable[[str], bool]) -> List[PythonConfiguration]: @@ -81,8 +77,9 @@ def install_cpython(version: str, arch: str, nuget: str) -> str: return installation_path -def install_pypy(version: str, arch: str, url: str) -> str: +def install_pypy(version: str, arch: str, url: Optional[str]) -> str: assert arch == '32' + assert url is not None # Inside the PyPy zip file is a directory with the same name zip_filename = url.rsplit('/', 1)[-1] installation_path = os.path.join('C:\\cibw', os.path.splitext(zip_filename)[0]) From 5bdb21b7054afbe66daf2d4792d3cc7f174b728b Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Fri, 10 Apr 2020 01:44:02 +0200 Subject: [PATCH 04/11] More type annotations after rebasing --- cibuildwheel/__main__.py | 9 +++------ cibuildwheel/linux.py | 8 ++++---- cibuildwheel/macos.py | 5 +++-- cibuildwheel/util.py | 11 +++++++---- cibuildwheel/windows.py | 5 +++-- 5 files changed, 20 insertions(+), 18 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 83e57f03..17fdf393 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -134,7 +134,7 @@ def main() -> None: dependency_versions = get_option_from_environment('CIBW_DEPENDENCY_VERSIONS', platform=platform, default='pinned') if dependency_versions == 'pinned': - dependency_constraints = DependencyConstraints.with_defaults() + dependency_constraints = DependencyConstraints.with_defaults() # type: Optional[DependencyConstraints] elif dependency_versions == 'latest': dependency_constraints = None else: @@ -169,6 +169,7 @@ def main() -> None: print_build_identifiers(platform, build_selector) exit(0) + manylinux_images = None # type: Optional[Dict[str, str]] if platform == 'linux': pinned_docker_images_file = os.path.join( os.path.dirname(__file__), 'resources', 'pinned_docker_images.cfg' @@ -181,8 +182,7 @@ def main() -> None: # 'pypy_x86_64': {'manylinux2010': '...' } # ... } - manylinux_images = {} # type: Optional[Dict[str, str]] - assert manylinux_images is not None # Weird problem with mypy + manylinux_images = {} for build_platform in ['x86_64', 'i686', 'pypy_x86_64', 'aarch64', 'ppc64le', 's390x']: pinned_images = all_pinned_docker_images[build_platform] @@ -200,9 +200,6 @@ def main() -> None: manylinux_images[build_platform] = image - else: - manylinux_images = None - build_options = BuildOptions( package_dir=package_dir, output_dir=output_dir, diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 33cd493d..38ecf943 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -6,7 +6,7 @@ import sys import textwrap import uuid -from typing import Callable, List, NamedTuple +from typing import Callable, List, NamedTuple, Optional, Union from .util import ( BuildOptions, @@ -15,14 +15,14 @@ from .util import ( ) -def call(args, input=None, universal_newlines=False): +def call(args: List[str], input: Optional[Union[str, bytes]] = None, universal_newlines: bool = False) -> None: print('+ ' + ' '.join(shlex.quote(a) for a in args)) subprocess.run( args, input=input, universal_newlines=universal_newlines, check=True ) -def matches_platform(identifier): +def matches_platform(identifier: str) -> bool: pm = platform.machine() if pm == "x86_64": # x86_64 machines can run i686 docker containers @@ -79,7 +79,7 @@ def get_python_configurations(build_selector: Callable[[str], bool]) -> List[Pyt return [c for c in python_configurations if matches_platform(c.identifier) and build_selector(c.identifier)] -def build(options: BuildOptions): +def build(options: BuildOptions) -> None: try: subprocess.check_call(['docker', '--version']) except Exception: diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 2e0a2535..911d9978 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -8,6 +8,7 @@ from glob import glob from typing import Callable, Dict, List, Optional, NamedTuple, Union +from .environment import ParsedEnvironment from .util import ( BuildOptions, download, @@ -106,7 +107,7 @@ def install_pypy(version: str, url: str) -> str: return installation_bin_path -def setup_python(python_configuration, dependency_constraint_flags, environment): +def setup_python(python_configuration: PythonConfiguration, dependency_constraint_flags: List[str], environment: ParsedEnvironment) -> Dict[str, str]: if python_configuration.identifier.startswith('cp'): installation_bin_path = install_cpython(python_configuration.version, python_configuration.url) elif python_configuration.identifier.startswith('pp'): @@ -164,7 +165,7 @@ def setup_python(python_configuration, dependency_constraint_flags, environment) return env -def build(options: BuildOptions): +def build(options: BuildOptions) -> None: temp_dir = tempfile.mkdtemp(prefix='cibuildwheel') built_wheel_dir = os.path.join(temp_dir, 'built_wheel') repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel') diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 21b76f80..33e45adf 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -3,7 +3,7 @@ import urllib.request from fnmatch import fnmatch from time import sleep -from typing import Dict, List, NamedTuple, Optional +from typing import Dict, List, NamedTuple, Optional, Type, TypeVar from .environment import ParsedEnvironment @@ -82,18 +82,21 @@ def download(url: str, dest: str) -> None: response.close() +DependencyConstraints_T = TypeVar('DependencyConstraints_T', bound='DependencyConstraints') + + class DependencyConstraints: - def __init__(self, base_file_path): + def __init__(self, base_file_path: str): assert os.path.exists(base_file_path) self.base_file_path = os.path.abspath(base_file_path) @classmethod - def with_defaults(cls): + def with_defaults(cls: Type[DependencyConstraints_T]) -> DependencyConstraints_T: return cls( base_file_path=os.path.join(os.path.dirname(__file__), 'resources', 'constraints.txt') ) - def get_for_python_version(self, version): + def get_for_python_version(self, version: str) -> str: version_parts = version.split('.') # try to find a version-specific dependency file e.g. if diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index cb78fe60..ffa8152a 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -8,6 +8,7 @@ from zipfile import ZipFile from typing import Callable, Dict, List, Optional, NamedTuple +from .environment import ParsedEnvironment from .util import ( BuildOptions, download, @@ -93,7 +94,7 @@ def install_pypy(version: str, arch: str, url: Optional[str]) -> str: return installation_path -def setup_python(python_configuration, dependency_constraint_flags, environment): +def setup_python(python_configuration: PythonConfiguration, dependency_constraint_flags: List[str], environment: ParsedEnvironment) -> Dict[str, str]: nuget = 'C:\\cibw\\nuget.exe' if not os.path.exists(nuget): download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget) @@ -145,7 +146,7 @@ def setup_python(python_configuration, dependency_constraint_flags, environment) return env -def build(options: BuildOptions): +def build(options: BuildOptions) -> None: temp_dir = tempfile.mkdtemp(prefix='cibuildwheel') built_wheel_dir = os.path.join(temp_dir, 'built_wheel') repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel') From 1a765ac4311c118c2029df5b13a504313d2e0d7b Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Fri, 10 Apr 2020 01:53:06 +0200 Subject: [PATCH 05/11] Use variable annotations instead of workarounds --- cibuildwheel/__main__.py | 9 ++++----- cibuildwheel/bashlex_eval.py | 9 +++++---- cibuildwheel/linux.py | 5 ++++- cibuildwheel/macos.py | 5 ++++- cibuildwheel/util.py | 33 ++++++++++++++------------------- cibuildwheel/windows.py | 6 +++++- 6 files changed, 36 insertions(+), 31 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 17fdf393..a624baa9 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -134,7 +134,7 @@ def main() -> None: dependency_versions = get_option_from_environment('CIBW_DEPENDENCY_VERSIONS', platform=platform, default='pinned') if dependency_versions == 'pinned': - dependency_constraints = DependencyConstraints.with_defaults() # type: Optional[DependencyConstraints] + dependency_constraints: Optional[DependencyConstraints] = DependencyConstraints.with_defaults() elif dependency_versions == 'latest': dependency_constraints = None else: @@ -169,7 +169,7 @@ def main() -> None: print_build_identifiers(platform, build_selector) exit(0) - manylinux_images = None # type: Optional[Dict[str, str]] + manylinux_images: Optional[Dict[str, str]] = None if platform == 'linux': pinned_docker_images_file = os.path.join( os.path.dirname(__file__), 'resources', 'pinned_docker_images.cfg' @@ -287,14 +287,13 @@ def print_preamble(platform: str, build_options: BuildOptions) -> None: def print_build_identifiers(platform: str, build_selector: BuildSelector) -> None: + python_configurations: List[Any] = [] if platform == 'linux': - python_configurations = cibuildwheel.linux.get_python_configurations(build_selector) # type: List[Any] + python_configurations = cibuildwheel.linux.get_python_configurations(build_selector) elif platform == 'windows': python_configurations = cibuildwheel.windows.get_python_configurations(build_selector) elif platform == 'macos': python_configurations = cibuildwheel.macos.get_python_configurations(build_selector) - else: - python_configurations = [] for config in python_configurations: print(config.identifier) diff --git a/cibuildwheel/bashlex_eval.py b/cibuildwheel/bashlex_eval.py index a10fe832..3101089d 100644 --- a/cibuildwheel/bashlex_eval.py +++ b/cibuildwheel/bashlex_eval.py @@ -5,9 +5,10 @@ from typing import Dict, List, NamedTuple, Optional import bashlex # type: ignore -NodeExecutionContext = NamedTuple('NodeExecutionContext', - [('environment', Dict[str, str]), - ('input', str)]) + +class NodeExecutionContext(NamedTuple): + environment: Dict[str, str] + input: str def evaluate(value: str, environment: Dict[str, str]) -> str: @@ -44,7 +45,7 @@ def evaluate_word_node(node: bashlex.ast.node, context: NodeExecutionContext) -> word_start = node.pos[0] word_end = node.pos[1] word_string = context.input[word_start:word_end] - letters = list(word_string) # type: List[Optional[str]] + letters: List[Optional[str]] = list(word_string) for part in node.parts: part_start = part.pos[0] - word_start diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 38ecf943..2ceaf8f1 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -43,7 +43,10 @@ def matches_platform(identifier: str) -> bool: return False -PythonConfiguration = NamedTuple('PythonConfiguration', [('version', str), ('identifier', str), ('path', str)]) +class PythonConfiguration(NamedTuple): + version: str + identifier: str + path: str def get_python_configurations(build_selector: Callable[[str], bool]) -> List[PythonConfiguration]: diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 911d9978..e2b18b88 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -28,7 +28,10 @@ def call(args: Union[str, List[str]], env: Optional[Dict[str, str]] = None, cwd: return subprocess.check_call(args, env=env, cwd=cwd, shell=shell) -PythonConfiguration = NamedTuple('PythonConfiguration', [('version', str), ('identifier', str), ('url', str)]) +class PythonConfiguration(NamedTuple): + version: str + identifier: str + url: str def get_python_configurations(build_selector: Callable[[str], bool]) -> List[PythonConfiguration]: diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 33e45adf..c466af3f 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -110,26 +110,21 @@ class DependencyConstraints: return self.base_file_path -BuildOptions = NamedTuple("BuildOptions", [ - ("package_dir", str), - ("output_dir", str), - ("test_command", Optional[str]), - ("test_requires", List[str]), - ("test_extras", str), - ("before_build", Optional[str]), - ("build_verbosity", int), - ("build_selector", BuildSelector), - ("repair_command", str), - ("environment", ParsedEnvironment), - ("before_test", str), - ("dependency_constraints", Optional[DependencyConstraints]), - ("manylinux_images", Optional[Dict[str, str]]), -]) +class BuildOptions(NamedTuple): + package_dir: str + output_dir: str + test_command: Optional[str] + test_requires: List[str] + test_extras: str + before_build: Optional[str] + build_verbosity: int + build_selector: BuildSelector + repair_command: str + environment: ParsedEnvironment + before_test: str + dependency_constraints: Optional[DependencyConstraints] + manylinux_images: Optional[Dict[str, str]] -""" -Replace this definition with a class-style NamedTuple in the -PEP526 style when Python 3.5 host support is dropped -""" resources_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'resources')) get_pip_script = os.path.join(resources_dir, 'get-pip.py') diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index ffa8152a..4e99094b 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -34,7 +34,11 @@ def get_nuget_args(version: str, arch: str) -> List[str]: return [python_name, '-Version', version, '-OutputDirectory', 'C:\\cibw\\python'] -PythonConfiguration = NamedTuple('PythonConfiguration', [('version', str), ('arch', str), ('identifier', str), ('url', Optional[str])]) +class PythonConfiguration(NamedTuple): + version: str + arch: str + identifier: str + url: Optional[str] def get_python_configurations(build_selector: Callable[[str], bool]) -> List[PythonConfiguration]: From 575e229514ebbbbe3c031c16e4e5baad7b4cbed0 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Fri, 10 Apr 2020 02:00:29 +0200 Subject: [PATCH 06/11] Drop 3.5 host support and adapt CI configurations --- .travis.yml | 26 +++++++++++++------------- CI.md | 12 ++++++------ azure-pipelines.yml | 10 ---------- examples/travis-ci-test-and-deploy.yml | 1 - setup.py | 2 +- 5 files changed, 20 insertions(+), 31 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4bda329c..89999e04 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,22 +6,22 @@ branches: jobs: include: - - name: Linux | x86_64 + i686 | Python 3.5 + - name: Linux | x86_64 + i686 | Python 3.6 language: python - python: 3.5 + python: 3.6 services: docker env: PYTHON=python - - name: Linux | arm64 | Python 3.5 + - name: Linux | arm64 | Python 3.6 language: python - python: 3.5 + python: 3.6 services: docker arch: arm64 env: PYTHON=python - - name: Linux | ppc64le | Python 3.5 + - name: Linux | ppc64le | Python 3.6 language: python - python: 3.5 + python: 3.6 services: docker arch: ppc64le env: PYTHON=python @@ -30,25 +30,25 @@ jobs: os: osx env: PYTHON=python3 - - name: Windows | x86_64 | Python 3.5 + - name: Windows | x86_64 | Python 3.6 os: windows language: shell before_install: - - choco install python3 --version 3.5.4 --no-progress -y + - choco install python3 --version 3.6.8 --no-progress -y env: - - PYTHON=C:\\Python35\\python + - PYTHON=C:\\Python36\\python - - &linux_s390x_35 - name: Linux | s390x | Python 3.5 + - &linux_s390x_36 + name: Linux | s390x | Python 365 language: python - python: 3.5 + python: 3.6 services: docker arch: s390x env: PYTHON=python allow_failures: # must repeat the s390x job above exactly to match - - *linux_s390x_35 + - *linux_s390x_36 install: $PYTHON -m pip install -r requirements-dev.txt diff --git a/CI.md b/CI.md index abcd6d88..eb990757 100644 --- a/CI.md +++ b/CI.md @@ -1,12 +1,12 @@ This is a summary of the Python versions and platforms covered by the different CI platforms: -| | 3.5 | 3.6 | 3.7 | 3.8 | -|----------|------------------|------------------|----------------------------------------------------|------------------| -| Linux | Travis CI | CircleCI | AppVeyor² / GitHub Actions | Azure Pipelines | -| macOS | Azure Pipelines | CircleCI | AppVeyor² / Travis CI¹ / CircleCI / GitHub Actions | Azure Pipelines | -| Windows | TravisCI | Azure Pipelines | AppVeyor² / GitHub Actions | Azure Pipelines | +| | 3.6 | 3.7 | 3.8 | +|----------|------------------------------|-----------------------------------------------------|------------------| +| Linux | Travis CI / CircleCI | AppVeyor² / GitHub Actions | Azure Pipelines | +| macOS | CircleCI | AppVeyor² / Travis CI¹ / CircleCI / GitHub Actions | Azure Pipelines | +| Windows | Travis CI / Azure Pipelines | AppVeyor² / GitHub Actions | Azure Pipelines | > ¹ Python version not really pinned, but dependent on the (default) version of image used. > ² AppVeyor only runs the "basic" test to reduce load. -Non-x86 architectures are covered on Travis CI using Python 3.5. +Non-x86 architectures are covered on Travis CI using Python 3.6. diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4b5aabd3..b2d324be 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -9,16 +9,6 @@ jobs: python -m pip install -r requirements-dev.txt python ./bin/run_tests.py -- job: macos_35 - pool: {vmImage: 'macOS-10.15'} - steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.5' - - bash: | - python -m pip install -r requirements-dev.txt - python ./bin/run_tests.py - - job: macos_38 pool: {vmImage: 'macOS-10.15'} steps: diff --git a/examples/travis-ci-test-and-deploy.yml b/examples/travis-ci-test-and-deploy.yml index d2218971..eada75c8 100644 --- a/examples/travis-ci-test-and-deploy.yml +++ b/examples/travis-ci-test-and-deploy.yml @@ -7,7 +7,6 @@ language: python python: - - 3.5 - 3.6 - 3.7 - 3.8 diff --git a/setup.py b/setup.py index f6992844..143e313c 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ setup( 'cibuildwheel': ['resources/*'], }, # Supported python versions - python_requires='>=3.5', + python_requires='>=3.6', keywords='ci wheel packaging pypi travis appveyor macos linux windows', classifiers=[ 'Intended Audience :: Developers', From 891e2dd6aab50c6e0853b6f6e58b30b1fea17637 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Fri, 10 Apr 2020 20:24:52 +0200 Subject: [PATCH 07/11] Fixing some review remarks --- .travis.yml | 2 +- cibuildwheel/bashlex_eval.py | 8 ++++---- cibuildwheel/util.py | 11 ++++------- cibuildwheel/windows.py | 4 ++-- 4 files changed, 11 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index 89999e04..855b565c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -39,7 +39,7 @@ jobs: - PYTHON=C:\\Python36\\python - &linux_s390x_36 - name: Linux | s390x | Python 365 + name: Linux | s390x | Python 3.6 language: python python: 3.6 services: docker diff --git a/cibuildwheel/bashlex_eval.py b/cibuildwheel/bashlex_eval.py index 3101089d..0128cfc7 100644 --- a/cibuildwheel/bashlex_eval.py +++ b/cibuildwheel/bashlex_eval.py @@ -1,7 +1,7 @@ import shlex import subprocess -from typing import Dict, List, NamedTuple, Optional +from typing import Dict, NamedTuple import bashlex # type: ignore @@ -45,7 +45,7 @@ def evaluate_word_node(node: bashlex.ast.node, context: NodeExecutionContext) -> word_start = node.pos[0] word_end = node.pos[1] word_string = context.input[word_start:word_end] - letters: List[Optional[str]] = list(word_string) + letters = list(word_string) for part in node.parts: part_start = part.pos[0] - word_start @@ -53,12 +53,12 @@ def evaluate_word_node(node: bashlex.ast.node, context: NodeExecutionContext) -> # Set all the characters in the part to None for i in range(part_start, part_end): - letters[i] = None + letters[i] = '' letters[part_start] = evaluate_node(part, context=context) # remove the None letters and concat - value = ''.join(l for l in letters if l is not None) + value = ''.join(letters) # apply bash-like quotes/whitespace treatment return ' '.join(word.strip() for word in shlex.split(value)) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index c466af3f..ecb596b1 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -3,7 +3,7 @@ import urllib.request from fnmatch import fnmatch from time import sleep -from typing import Dict, List, NamedTuple, Optional, Type, TypeVar +from typing import Dict, List, NamedTuple, Optional from .environment import ParsedEnvironment @@ -82,17 +82,14 @@ def download(url: str, dest: str) -> None: response.close() -DependencyConstraints_T = TypeVar('DependencyConstraints_T', bound='DependencyConstraints') - - class DependencyConstraints: def __init__(self, base_file_path: str): assert os.path.exists(base_file_path) self.base_file_path = os.path.abspath(base_file_path) - @classmethod - def with_defaults(cls: Type[DependencyConstraints_T]) -> DependencyConstraints_T: - return cls( + @staticmethod + def with_defaults() -> 'DependencyConstraints': + return DependencyConstraints( base_file_path=os.path.join(os.path.dirname(__file__), 'resources', 'constraints.txt') ) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 4e99094b..89d01a6e 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -82,9 +82,8 @@ def install_cpython(version: str, arch: str, nuget: str) -> str: return installation_path -def install_pypy(version: str, arch: str, url: Optional[str]) -> str: +def install_pypy(version: str, arch: str, url: str) -> str: assert arch == '32' - assert url is not None # Inside the PyPy zip file is a directory with the same name zip_filename = url.rsplit('/', 1)[-1] installation_path = os.path.join('C:\\cibw', os.path.splitext(zip_filename)[0]) @@ -106,6 +105,7 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain if python_configuration.identifier.startswith('cp'): installation_path = install_cpython(python_configuration.version, python_configuration.arch, nuget) elif python_configuration.identifier.startswith('pp'): + assert python_configuration.url is not None installation_path = install_pypy(python_configuration.version, python_configuration.arch, python_configuration.url) else: raise ValueError("Unknown Python implementation") From 7cc85c2e4fc709de4f206a034bb2182bb69e0375 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Thu, 7 May 2020 00:07:14 +0200 Subject: [PATCH 08/11] Replace Callable[[str], bool] by BuildSelector --- cibuildwheel/linux.py | 5 +++-- cibuildwheel/macos.py | 5 +++-- cibuildwheel/windows.py | 5 +++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 2ceaf8f1..33eacf87 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -6,10 +6,11 @@ import sys import textwrap import uuid -from typing import Callable, List, NamedTuple, Optional, Union +from typing import List, NamedTuple, Optional, Union from .util import ( BuildOptions, + BuildSelector, get_build_verbosity_extra_flags, prepare_command, ) @@ -49,7 +50,7 @@ class PythonConfiguration(NamedTuple): path: str -def get_python_configurations(build_selector: Callable[[str], bool]) -> List[PythonConfiguration]: +def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfiguration]: python_configurations = [ PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path='/opt/python/cp27-cp27m'), PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path='/opt/python/cp27-cp27mu'), diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index e2b18b88..d877a8a8 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -6,11 +6,12 @@ import sys import tempfile from glob import glob -from typing import Callable, Dict, List, Optional, NamedTuple, Union +from typing import Dict, List, Optional, NamedTuple, Union from .environment import ParsedEnvironment from .util import ( BuildOptions, + BuildSelector, download, get_build_verbosity_extra_flags, get_pip_script, @@ -34,7 +35,7 @@ class PythonConfiguration(NamedTuple): url: str -def get_python_configurations(build_selector: Callable[[str], bool]) -> List[PythonConfiguration]: +def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfiguration]: python_configurations = [ # CPython PythonConfiguration(version='2.7', identifier='cp27-macosx_x86_64', url='https://www.python.org/ftp/python/2.7.18/python-2.7.18-macosx10.9.pkg'), diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 89d01a6e..cd07bbcf 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -6,11 +6,12 @@ import tempfile from glob import glob from zipfile import ZipFile -from typing import Callable, Dict, List, Optional, NamedTuple +from typing import Dict, List, Optional, NamedTuple from .environment import ParsedEnvironment from .util import ( BuildOptions, + BuildSelector, download, get_build_verbosity_extra_flags, get_pip_script, @@ -41,7 +42,7 @@ class PythonConfiguration(NamedTuple): url: Optional[str] -def get_python_configurations(build_selector: Callable[[str], bool]) -> List[PythonConfiguration]: +def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfiguration]: python_configurations = [ # CPython PythonConfiguration(version='2.7.18', arch='32', identifier='cp27-win32', url=None), From 4683ac3d6c9d308a3ed252181458551d6b18575c Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Thu, 7 May 2020 00:12:11 +0200 Subject: [PATCH 09/11] Remove macOS Python 3.7 job on CircleCI --- .circleci/config.yml | 17 ----------------- CI.md | 10 +++++----- 2 files changed, 5 insertions(+), 22 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 8820ceb1..3db4b439 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,22 +30,6 @@ jobs: command: venv/bin/python ./bin/run_tests.py no_output_timeout: 30m - osx-python3.7: - macos: - xcode: "10.0.0" - environment: - PYTHON: python3 - steps: - - checkout - - - run: - name: Prepare the environment. - command: bash .circleci/prepare.sh - - run: - name: Test. - command: venv/bin/python ./bin/run_tests.py - no_output_timeout: 30m - linux-python3.6: docker: - image: circleci/python:3.6 @@ -69,5 +53,4 @@ workflows: jobs: - flake8 - osx-python3.6 - - osx-python3.7 - linux-python3.6 diff --git a/CI.md b/CI.md index eb990757..6d6a8356 100644 --- a/CI.md +++ b/CI.md @@ -1,10 +1,10 @@ This is a summary of the Python versions and platforms covered by the different CI platforms: -| | 3.6 | 3.7 | 3.8 | -|----------|------------------------------|-----------------------------------------------------|------------------| -| Linux | Travis CI / CircleCI | AppVeyor² / GitHub Actions | Azure Pipelines | -| macOS | CircleCI | AppVeyor² / Travis CI¹ / CircleCI / GitHub Actions | Azure Pipelines | -| Windows | Travis CI / Azure Pipelines | AppVeyor² / GitHub Actions | Azure Pipelines | +| | 3.6 | 3.7 | 3.8 | +|----------|------------------------------|------------------------------------------|------------------| +| Linux | Travis CI / CircleCI | AppVeyor² / GitHub Actions | Azure Pipelines | +| macOS | CircleCI | AppVeyor² / Travis CI¹ / GitHub Actions | Azure Pipelines | +| Windows | Travis CI / Azure Pipelines | AppVeyor² / GitHub Actions | Azure Pipelines | > ¹ Python version not really pinned, but dependent on the (default) version of image used. > ² AppVeyor only runs the "basic" test to reduce load. From 9daad5c60c1b83b495c0280451a7fa43055fe413 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Thu, 7 May 2020 00:20:37 +0200 Subject: [PATCH 10/11] Ignore .mypy_cache and add mypy configuration to setup.cfg --- .gitignore | 3 +++ setup.cfg | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/.gitignore b/.gitignore index ed7b532f..2db46b82 100644 --- a/.gitignore +++ b/.gitignore @@ -102,3 +102,6 @@ env3?/ # VSCode project settings /.vscode + +# MyPy cache +.mypy_cache/ diff --git a/setup.cfg b/setup.cfg index c5d51ac7..e01a4e25 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,3 +11,7 @@ exclude = env??/, .venv/, site/ + +[mypy] +files=cibuildwheel/ +strict=True From 0368d580650e01ca2f85682262ff275d0f14f4b7 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Thu, 7 May 2020 00:40:52 +0200 Subject: [PATCH 11/11] Run mypy in CircleCI job --- .circleci/config.yml | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3db4b439..cf0e47b2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,18 +1,21 @@ version: 2 jobs: - flake8: + flake8-mypy: docker: - image: circleci/python:3.6 steps: - checkout - run: - name: Install flake8 - command: sudo python -m pip install flake8 + name: Install flake8 & mypy + command: sudo python -m pip install flake8 mypy - run: - name: Test. - command: flake8 . + name: flake8 + command: flake8 + - run: + name: mypy + command: mypy osx-python3.6: macos: @@ -51,6 +54,6 @@ workflows: version: 2 all-tests: jobs: - - flake8 + - flake8-mypy - osx-python3.6 - linux-python3.6