diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 7099bc8f..09471d76 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -5,6 +5,8 @@ import textwrap import traceback from configparser import ConfigParser +from typing import Any, Dict, List, Optional, Union, overload + import cibuildwheel import cibuildwheel.linux import cibuildwheel.macos @@ -14,14 +16,20 @@ from cibuildwheel.environment import ( parse_environment, ) from cibuildwheel.util import ( + BuildOptions, BuildSelector, DependencyConstraints, Unbuffered, - BuildOptions ) -def get_option_from_environment(option_name, platform=None, default=None): +@overload +def get_option_from_environment(option_name: str, platform: Optional[str], default: str) -> str: + ... +@overload +def get_option_from_environment(option_name: str, platform: Optional[str] = None, default: None = None) -> Optional[str]: + ... +def get_option_from_environment(option_name: str, platform: Optional[str] = None, default: Optional[str] = None) -> Optional[str]: ''' Returns an option from the environment, optionally scoped by the platform. @@ -39,13 +47,13 @@ def get_option_from_environment(option_name, platform=None, default=None): return os.environ.get(option_name, default) -def strtobool(val): +def strtobool(val: str) -> bool: if val.lower() in ('y', 'yes', 't', 'true', 'on', '1'): return True return False -def main(): +def main() -> None: parser = argparse.ArgumentParser( description='Build wheels for all the platforms.', epilog=''' @@ -114,7 +122,7 @@ def main(): test_extras = get_option_from_environment('CIBW_TEST_EXTRAS', platform=platform, default='') package_dir = args.package_dir before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform) - build_verbosity = get_option_from_environment('CIBW_BUILD_VERBOSITY', platform=platform, default='') + build_verbosity_str = get_option_from_environment('CIBW_BUILD_VERBOSITY', platform=platform, default='') build_config, skip_config = os.environ.get('CIBW_BUILD', '*'), os.environ.get('CIBW_SKIP', '') if platform == 'linux': repair_command_default = 'auditwheel repair -w {dest_dir} {wheel}' @@ -138,7 +146,7 @@ def main(): test_extras = '[{0}]'.format(test_extras) try: - build_verbosity = min(3, max(-3, int(build_verbosity))) + build_verbosity = min(3, max(-3, int(build_verbosity_str))) except ValueError: build_verbosity = 0 @@ -175,7 +183,8 @@ def main(): # 'pypy_x86_64': {'manylinux2010': '...' } # ... } - manylinux_images = {} + manylinux_images = {} # type: Optional[Dict[str, str]] + assert manylinux_images is not None # Weird problem with mypy for build_platform in ['x86_64', 'i686', 'pypy_x86_64', 'aarch64', 'ppc64le', 's390x']: pinned_images = all_pinned_docker_images[build_platform] @@ -213,7 +222,7 @@ def main(): ) # Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print' - sys.stdout = Unbuffered(sys.stdout) + sys.stdout = Unbuffered(sys.stdout) # type: ignore print_preamble(platform, build_options) @@ -231,7 +240,7 @@ def main(): exit(2) -def detect_obsolete_options(): +def detect_obsolete_options() -> None: # Check the old 'MANYLINUX1_*_IMAGE' options for (deprecated, alternative) in [('CIBW_MANYLINUX1_X86_64_IMAGE', 'CIBW_MANYLINUX_X86_64_IMAGE'), ('CIBW_MANYLINUX1_I686_IMAGE', 'CIBW_MANYLINUX_I686_IMAGE')]: @@ -258,7 +267,7 @@ def detect_obsolete_options(): os.environ[option] = os.environ[option].replace(deprecated, alternative) -def print_preamble(platform, build_options): +def print_preamble(platform: str, build_options: BuildOptions) -> None: print(textwrap.dedent(''' _ _ _ _ _ _ _ ___|_| |_ _ _|_| |_| |_ _ _| |_ ___ ___| | @@ -282,9 +291,9 @@ def print_preamble(platform, build_options): print('\nHere we go!\n') -def print_build_identifiers(platform, build_selector): +def print_build_identifiers(platform: str, build_selector: BuildSelector) -> None: if platform == 'linux': - python_configurations = cibuildwheel.linux.get_python_configurations(build_selector) + python_configurations = cibuildwheel.linux.get_python_configurations(build_selector) # type: List[Any] elif platform == 'windows': python_configurations = cibuildwheel.windows.get_python_configurations(build_selector) elif platform == 'macos': @@ -296,7 +305,7 @@ def print_build_identifiers(platform, build_selector): print(config.identifier) -def detect_warnings(platform, build_options): +def detect_warnings(platform: str, build_options: BuildOptions) -> List[str]: warnings = [] # warn about deprecated {python} and {pip} diff --git a/cibuildwheel/bashlex_eval.py b/cibuildwheel/bashlex_eval.py index 8e2c0831..52f444a2 100644 --- a/cibuildwheel/bashlex_eval.py +++ b/cibuildwheel/bashlex_eval.py @@ -2,12 +2,14 @@ import shlex import subprocess from collections import namedtuple -import bashlex +from typing import Dict + +import bashlex # type: ignore NodeExecutionContext = namedtuple('NodeExecutionContext', ['environment', 'input']) -def evaluate(value, environment): +def evaluate(value: str, environment: Dict[str, str]) -> str: if not value: # empty string evaluates to empty string # (but trips up bashlex) @@ -20,24 +22,24 @@ def evaluate(value, environment): value_word_node = command_node.parts[0] - return evaluate_node( + return evaluate_node( # type: ignore value_word_node, context=NodeExecutionContext(environment=environment, input=value) ) -def evaluate_node(node, context): +def evaluate_node(node, context): # type: ignore if node.kind == 'word': - return evaluate_word_node(node, context=context) + return evaluate_word_node(node, context=context) # type: ignore elif node.kind == 'commandsubstitution': - return evaluate_command_node(node.command, context=context) + return evaluate_command_node(node.command, context=context) # type: ignore elif node.kind == 'parameter': - return evaluate_parameter_node(node, context=context) + return evaluate_parameter_node(node, context=context) # type: ignore else: raise ValueError('Unsupported bash construct: "%s"' % node.word) -def evaluate_word_node(node, context): +def evaluate_word_node(node, context): # type: ignore word_start = node.pos[0] word_end = node.pos[1] word_string = context.input[word_start:word_end] @@ -51,7 +53,7 @@ def evaluate_word_node(node, context): for i in range(part_start, part_end): letters[i] = None - letters[part_start] = evaluate_node(part, context=context) + letters[part_start] = evaluate_node(part, context=context) # type: ignore # remove the None letters and concat value = ''.join(l for l in letters if l is not None) @@ -60,11 +62,11 @@ def evaluate_word_node(node, context): return ' '.join(word.strip() for word in shlex.split(value)) -def evaluate_command_node(node, context): - words = [evaluate_node(part, context=context) for part in node.parts] +def evaluate_command_node(node, context): # type: ignore + words = [evaluate_node(part, context=context) for part in node.parts] # type: ignore command = ' '.join(words) return subprocess.check_output(shlex.split(command), env=context.environment, universal_newlines=True) -def evaluate_parameter_node(node, context): +def evaluate_parameter_node(node, context): # type: ignore return context.environment.get(node.value, '') diff --git a/cibuildwheel/environment.py b/cibuildwheel/environment.py index f016c780..837ad266 100644 --- a/cibuildwheel/environment.py +++ b/cibuildwheel/environment.py @@ -1,4 +1,6 @@ -import bashlex +import bashlex # type: ignore + +from typing import Dict, List from . import bashlex_eval @@ -7,13 +9,7 @@ class EnvironmentParseError(Exception): pass -def parse_environment(env_string): - env_items = split_env_items(env_string) - assignments = [EnvironmentAssignment(item) for item in env_items] - return ParsedEnvironment(assignments=assignments) - - -def split_env_items(env_string): +def split_env_items(env_string: str) -> List[str]: '''Splits space-separated variable assignments into a list of individual assignments. >>> split_env_items('VAR=abc') @@ -43,29 +39,29 @@ def split_env_items(env_string): class EnvironmentAssignment: - def __init__(self, assignment): + def __init__(self, assignment: str): name, equals, value = assignment.partition('=') if not equals: raise EnvironmentParseError(assignment) self.name = name self.value = value - def evaluated_value(self, environment): + def evaluated_value(self, environment: Dict[str, str]) -> str: '''Returns the value of this assignment, as evaluated in the environment''' return bashlex_eval.evaluate(self.value, environment=environment) - def as_shell_assignment(self): + def as_shell_assignment(self) -> str: return 'export %s=%s' % (self.name, self.value) - def __repr__(self): + def __repr__(self) -> str: return '%s=%s' % (self.name, self.value) class ParsedEnvironment: - def __init__(self, assignments): + def __init__(self, assignments: List[EnvironmentAssignment]): self.assignments = assignments - def as_dictionary(self, prev_environment): + def as_dictionary(self, prev_environment: Dict[str, str]) -> Dict[str, str]: environment = prev_environment.copy() for assignment in self.assignments: @@ -74,8 +70,14 @@ class ParsedEnvironment: return environment - def as_shell_commands(self): + def as_shell_commands(self) -> List[str]: return [a.as_shell_assignment() for a in self.assignments] - def __repr__(self): + def __repr__(self) -> str: return 'ParsedEnvironment(%r)' % [repr(a) for a in self.assignments] + + +def parse_environment(env_string: str) -> ParsedEnvironment: + env_items = split_env_items(env_string) + assignments = [EnvironmentAssignment(item) for item in env_items] + return ParsedEnvironment(assignments=assignments) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index d30635c3..b1a27de0 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -7,10 +7,15 @@ import textwrap import uuid from collections import namedtuple +from typing import Callable, Dict, List, Optional + +from .environment import ( + ParsedEnvironment, +) from .util import ( + BuildOptions, get_build_verbosity_extra_flags, prepare_command, - BuildOptions ) @@ -42,8 +47,10 @@ def matches_platform(identifier): return False -def get_python_configurations(build_selector): - PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'path']) +PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'path']) + + +def get_python_configurations(build_selector: Callable[[str], bool]) -> List[PythonConfiguration]: python_configurations = [ PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path='/opt/python/cp27-cp27m'), PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path='/opt/python/cp27-cp27mu'), @@ -86,6 +93,7 @@ def build(options: BuildOptions): file=sys.stderr) exit(2) + assert options.manylinux_images is not None python_configurations = get_python_configurations(options.build_selector) platforms = [ ('cp', 'manylinux_x86_64', options.manylinux_images['x86_64']), @@ -274,7 +282,7 @@ def build(options: BuildOptions): call(['docker', 'rm', '--force', '-v', container_name]) -def troubleshoot(package_dir, error): +def troubleshoot(package_dir: str, error: Exception) -> None: if (isinstance(error, subprocess.CalledProcessError) and 'exec' in error.cmd): # the bash script failed print('Checking for common errors...') diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 8eb7984c..610c30e1 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -7,16 +7,21 @@ import tempfile from collections import namedtuple from glob import glob +from typing import Callable, Dict, List, Optional, Union + +from .environment import ( + ParsedEnvironment, +) from .util import ( + BuildOptions, download, get_build_verbosity_extra_flags, - prepare_command, get_pip_script, - BuildOptions + prepare_command, ) -def call(args, env=None, cwd=None, shell=False): +def call(args: Union[str, List[str]], env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None, shell: bool = False) -> int: # print the command executing for the logs if shell: print('+ %s' % args) @@ -26,8 +31,10 @@ def call(args, env=None, cwd=None, shell=False): return subprocess.check_call(args, env=env, cwd=cwd, shell=shell) -def get_python_configurations(build_selector): - PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'url']) +PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'url']) + + +def get_python_configurations(build_selector: Callable[[str], bool]) -> List[PythonConfiguration]: python_configurations = [ # CPython PythonConfiguration(version='2.7', identifier='cp27-macosx_x86_64', url='https://www.python.org/ftp/python/2.7.18/python-2.7.18-macosx10.9.pkg'), @@ -47,7 +54,7 @@ def get_python_configurations(build_selector): SYMLINKS_DIR = '/tmp/cibw_bin' -def make_symlinks(installation_bin_path, python_executable, pip_executable): +def make_symlinks(installation_bin_path: str, python_executable: str, pip_executable: str) -> None: assert os.path.exists(os.path.join(installation_bin_path, python_executable)) # Python bin folders on Mac don't symlink `python3` to `python`, and neither @@ -62,7 +69,7 @@ def make_symlinks(installation_bin_path, python_executable, pip_executable): os.symlink(os.path.join(installation_bin_path, pip_executable), os.path.join(SYMLINKS_DIR, 'pip')) -def install_cpython(version, url): +def install_cpython(version: str, url: str) -> str: installed_system_packages = subprocess.check_output(['pkgutil', '--pkgs'], universal_newlines=True).splitlines() # if this version of python isn't installed, get it from python.org and install @@ -86,7 +93,7 @@ def install_cpython(version, url): return installation_bin_path -def install_pypy(version, url): +def install_pypy(version: str, url: str) -> str: pypy_tar_bz2 = url.rsplit('/', 1)[-1] assert pypy_tar_bz2.endswith(".tar.bz2") pypy_base_filename = os.path.splitext(os.path.splitext(pypy_tar_bz2)[0])[0] diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 64f561a1..21b76f80 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -2,12 +2,13 @@ import os import urllib.request from fnmatch import fnmatch from time import sleep -from typing import NamedTuple, List, Optional, Dict + +from typing import Dict, List, NamedTuple, Optional from .environment import ParsedEnvironment -def prepare_command(command, **kwargs): +def prepare_command(command: str, **kwargs: str) -> str: ''' Preprocesses a command by expanding variables like {python}. @@ -17,7 +18,7 @@ def prepare_command(command, **kwargs): return command.format(python='python', pip='pip', **kwargs) -def get_build_verbosity_extra_flags(level): +def get_build_verbosity_extra_flags(level: int) -> List[str]: if level > 0: return ['-' + level * 'v'] elif level < 0: @@ -27,37 +28,37 @@ def get_build_verbosity_extra_flags(level): class BuildSelector: - def __init__(self, build_config, skip_config): + def __init__(self, build_config: str, skip_config: str): self.build_patterns = build_config.split() self.skip_patterns = skip_config.split() - def __call__(self, build_id): - def match_any(patterns): + def __call__(self, build_id: str) -> bool: + def match_any(patterns: List[str]) -> bool: return any(fnmatch(build_id, pattern) for pattern in patterns) return match_any(self.build_patterns) and not match_any(self.skip_patterns) - def __repr__(self): + def __repr__(self) -> str: return 'BuildSelector({!r} - {!r})'.format(' '.join(self.build_patterns), ' '.join(self.skip_patterns)) # Taken from https://stackoverflow.com/a/107717 class Unbuffered: - def __init__(self, stream): + def __init__(self, stream): # type: ignore self.stream = stream - def write(self, data): + def write(self, data): # type: ignore self.stream.write(data) self.stream.flush() - def writelines(self, datas): + def writelines(self, datas): # type: ignore self.stream.writelines(datas) self.stream.flush() - def __getattr__(self, attr): + def __getattr__(self, attr): # type: ignore return getattr(self.stream, attr) -def download(url, dest): +def download(url: str, dest: str) -> None: print('+ Download ' + url + ' to ' + dest) dest_dir = os.path.dirname(dest) if not os.path.exists(dest_dir): diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index a389fac9..48db6733 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -7,12 +7,17 @@ from collections import namedtuple from glob import glob from zipfile import ZipFile +from typing import Callable, Dict, List, Optional + +from .environment import ( + ParsedEnvironment, +) from .util import ( + BuildOptions, download, get_build_verbosity_extra_flags, - prepare_command, get_pip_script, - BuildOptions + prepare_command, ) @@ -20,20 +25,22 @@ IS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache') IS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows' -def shell(args, env=None, cwd=None): +def shell(args: List[str], env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None) -> int: print('+ ' + ' '.join(args)) return subprocess.check_call(' '.join(args), env=env, cwd=cwd, shell=True) -def get_nuget_args(version, arch): +def get_nuget_args(version: str, arch: str) -> List[str]: python_name = 'python' if version[0] == '3' else 'python2' if arch == '32': python_name = python_name + 'x86' return [python_name, '-Version', version, '-OutputDirectory', 'C:\\cibw\\python'] -def get_python_configurations(build_selector): - PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier', 'url']) +PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier', 'url']) + + +def get_python_configurations(build_selector: Callable[[str], bool]) -> List[PythonConfiguration]: python_configurations = [ # CPython PythonConfiguration(version='2.7.18', arch='32', identifier='cp27-win32', url=None), @@ -62,19 +69,19 @@ def get_python_configurations(build_selector): return python_configurations -def extract_zip(zip_src, dest): +def extract_zip(zip_src: str, dest: str) -> None: with ZipFile(zip_src) as zip: zip.extractall(dest) -def install_cpython(version, arch, nuget): +def install_cpython(version: str, arch: str, nuget: str) -> str: nuget_args = get_nuget_args(version, arch) installation_path = os.path.join(nuget_args[-1], nuget_args[0] + '.' + version, 'tools') shell([nuget, 'install'] + nuget_args) return installation_path -def install_pypy(version, arch, url): +def install_pypy(version: str, arch: str, url: str) -> str: assert arch == '32' # Inside the PyPy zip file is a directory with the same name zip_filename = url.rsplit('/', 1)[-1]