From a99e0ded7609702340768e38f07438e41ed703d3 Mon Sep 17 00:00:00 2001 From: Henry Fredrick Schreiner Date: Fri, 1 Jan 2021 16:53:45 -0500 Subject: [PATCH 1/2] feat: Windows filtering and sets Co-authored-by: Matthieu Darbois --- cibuildwheel/__main__.py | 6 ++--- cibuildwheel/linux.py | 6 ++--- cibuildwheel/macos.py | 2 +- cibuildwheel/util.py | 28 ++++++++++++++-------- cibuildwheel/windows.py | 25 ++++++++++++------- unit_test/main_tests/main_platform_test.py | 12 +++++----- 6 files changed, 48 insertions(+), 31 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 5f48070f..7bde8c69 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -6,7 +6,7 @@ import traceback from configparser import ConfigParser from pathlib import Path -from typing import Any, Dict, List, Optional, overload +from typing import Any, Dict, List, Optional, Set, overload import cibuildwheel import cibuildwheel.linux @@ -304,13 +304,13 @@ def print_preamble(platform: str, build_options: BuildOptions) -> None: def print_build_identifiers( - platform: str, build_selector: BuildSelector, architectures: List[Architecture] + platform: str, build_selector: BuildSelector, architectures: Set[Architecture] ) -> None: python_configurations: List[Any] = [] if platform == 'linux': python_configurations = cibuildwheel.linux.get_python_configurations(build_selector, architectures) elif platform == 'windows': - python_configurations = cibuildwheel.windows.get_python_configurations(build_selector) + python_configurations = cibuildwheel.windows.get_python_configurations(build_selector, architectures) elif platform == 'macos': python_configurations = cibuildwheel.macos.get_python_configurations(build_selector) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 4118fd79..02cf6523 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -2,7 +2,7 @@ import subprocess import sys import textwrap from pathlib import Path, PurePath -from typing import List, NamedTuple +from typing import List, NamedTuple, Set from .docker_container import DockerContainer from .logger import log @@ -24,7 +24,7 @@ class PythonConfiguration(NamedTuple): def get_python_configurations( - build_selector: BuildSelector, architectures: List[Architecture] + build_selector: BuildSelector, architectures: Set[Architecture] ) -> List[PythonConfiguration]: python_configurations = [ PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path_str='/opt/python/cp27-cp27m'), @@ -61,7 +61,7 @@ def get_python_configurations( PythonConfiguration(version='3.9', identifier='cp39-manylinux_s390x', path_str='/opt/python/cp39-cp39'), ] - # return all configurations whose arch is in our `architectures` list, + # return all configurations whose arch is in our `architectures` set, # and match the build/skip rules return [ c for c in python_configurations diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 0845851c..82772b16 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -187,7 +187,7 @@ def setup_python(python_configuration: PythonConfiguration, def build(options: BuildOptions) -> None: - if options.architectures != [Architecture.x86_64]: + if not options.architectures <= {Architecture.x86_64}: raise ValueError(textwrap.dedent(f''' Invalid archs option {options.architectures}. macOS only supports x86_64 for the moment. If you want to set emulation architectures on Linux, use CIBW_ARCHS_LINUX instead. diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 960b9896..749bda77 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -2,13 +2,14 @@ import os import platform as platform_module import re import ssl +import functools import textwrap import urllib.request from enum import Enum from fnmatch import fnmatch from pathlib import Path from time import sleep -from typing import Dict, List, NamedTuple, Optional +from typing import Dict, List, NamedTuple, Optional, Set import certifi @@ -124,7 +125,10 @@ class DependencyConstraints: return f'{self.__class__.__name__}{self.base_file_path!r})' +@functools.total_ordering class Architecture(Enum): + value: str + # mac/linux archs x86_64 = 'x86_64' i686 = 'i686' @@ -136,25 +140,29 @@ class Architecture(Enum): x86 = 'x86' AMD64 = 'AMD64' + # Allow this to be sorted + def __lt__(self, other: "Architecture") -> bool: + return self.value < other.value + @staticmethod - def parse_config(config: str, platform: str) -> 'List[Architecture]': - result = [] + def parse_config(config: str, platform: str) -> 'Set[Architecture]': + result = set() for arch_str in re.split(r'[\s,]+', config): if arch_str == 'auto': - result += Architecture.auto_archs(platform=platform) + result |= Architecture.auto_archs(platform=platform) else: - result.append(Architecture(arch_str)) + result.add(Architecture(arch_str)) return result @staticmethod - def auto_archs(platform: str) -> 'List[Architecture]': + def auto_archs(platform: str) -> 'Set[Architecture]': native_architecture = Architecture(platform_module.machine()) - result = [native_architecture] + result = {native_architecture} if platform == 'linux' and native_architecture == Architecture.x86_64: # x86_64 machines can run i686 docker containers - result.append(Architecture.i686) + result.add(Architecture.i686) if platform == 'windows' and native_architecture == Architecture.AMD64: - result.append(Architecture.x86) + result.add(Architecture.x86) return result @@ -162,7 +170,7 @@ class BuildOptions(NamedTuple): package_dir: Path output_dir: Path build_selector: BuildSelector - architectures: List[Architecture] + architectures: Set[Architecture] environment: ParsedEnvironment before_all: str before_build: Optional[str] diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 3e977e4b..09f64e1d 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -5,7 +5,7 @@ import sys import tempfile import textwrap from pathlib import Path -from typing import Dict, List, NamedTuple, Optional, Sequence +from typing import Dict, List, NamedTuple, Optional, Sequence, Set from zipfile import ZipFile import toml @@ -48,7 +48,12 @@ class PythonConfiguration(NamedTuple): url: Optional[str] -def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfiguration]: +def get_python_configurations(build_selector: BuildSelector, architectures: Set[Architecture]) -> List[PythonConfiguration]: + map_arch = { + '32': Architecture.x86, + '64': Architecture.AMD64, + } + python_configurations = [ # CPython PythonConfiguration(version='2.7.18', arch='32', identifier='cp27-win32', url=None), @@ -75,7 +80,10 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi python_configurations = [c for c in python_configurations if not c.version.startswith('2.7')] # skip builds as required - python_configurations = [c for c in python_configurations if build_selector(c.identifier)] + python_configurations = [ + c for c in python_configurations + if build_selector(c.identifier) and map_arch[c.arch] in architectures + ] return python_configurations @@ -202,11 +210,12 @@ def pep_518_cp35_workaround(package_dir: Path, env: Dict[str, str]) -> None: def build(options: BuildOptions) -> None: - if options.architectures != [Architecture.AMD64, Architecture.x86]: + allowed_architectures = {Architecture.AMD64, Architecture.x86} + if not options.architectures <= allowed_architectures: raise ValueError(textwrap.dedent(f''' - Invalid archs option {options.architectures}. Windows only supports 'amd64,x86' for the - moment. If you want to set emulation architectures on Linux, use CIBW_ARCHS_LINUX - instead. + Invalid archs option {options.architectures}. Windows only supports + {sorted(allowed_architectures)}. If you want to set emulation + architectures on Linux, use CIBW_ARCHS_LINUX instead. ''')) temp_dir = Path(tempfile.mkdtemp(prefix='cibuildwheel')) @@ -220,7 +229,7 @@ def build(options: BuildOptions) -> None: before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) shell(before_all_prepared, env=env) - python_configurations = get_python_configurations(options.build_selector) + python_configurations = get_python_configurations(options.build_selector, options.architectures) for config in python_configurations: log.build_start(config.identifier) diff --git a/unit_test/main_tests/main_platform_test.py b/unit_test/main_tests/main_platform_test.py index 9ac84ea8..add54718 100644 --- a/unit_test/main_tests/main_platform_test.py +++ b/unit_test/main_tests/main_platform_test.py @@ -75,9 +75,9 @@ def test_archs_default(platform, intercepted_build_args, monkeypatch): build_options = intercepted_build_args.args[0] if platform == 'linux': - assert build_options.architectures == [Architecture.x86_64, Architecture.i686] + assert build_options.architectures == {Architecture.x86_64, Architecture.i686} else: - assert build_options.architectures == [Architecture.x86_64] + assert build_options.architectures == {Architecture.x86_64} @pytest.mark.parametrize('use_env_var', [False, True]) @@ -92,7 +92,7 @@ def test_archs_argument(platform, intercepted_build_args, monkeypatch, use_env_v main() build_options = intercepted_build_args.args[0] - assert build_options.architectures == [Architecture.ppc64le] + assert build_options.architectures == {Architecture.ppc64le} def test_archs_platform_specific(platform, intercepted_build_args, monkeypatch): @@ -106,8 +106,8 @@ def test_archs_platform_specific(platform, intercepted_build_args, monkeypatch): build_options = intercepted_build_args.args[0] if platform == 'linux': - assert build_options.architectures == [Architecture.ppc64le] + assert build_options.architectures == {Architecture.ppc64le} elif platform == 'windows': - assert build_options.architectures == [Architecture.x86] + assert build_options.architectures == {Architecture.x86} elif platform == 'macos': - assert build_options.architectures == [Architecture.x86_64] + assert build_options.architectures == {Architecture.x86_64} From fce7c70a631b612809ea688cc3e2e912410c6595 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 5 Jan 2021 13:23:54 -0500 Subject: [PATCH 2/2] refactor: standard arch checking in one place --- cibuildwheel/linux.py | 4 +++- cibuildwheel/macos.py | 11 +++-------- cibuildwheel/util.py | 36 ++++++++++++++++++++++++++++++++++++ cibuildwheel/windows.py | 11 ++--------- setup.cfg | 1 + 5 files changed, 45 insertions(+), 18 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 02cf6523..a8cc2e78 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -8,7 +8,7 @@ from .docker_container import DockerContainer from .logger import log from .util import ( Architecture, BuildOptions, BuildSelector, NonPlatformWheelError, - get_build_verbosity_extra_flags, prepare_command, + allowed_architectures_check, get_build_verbosity_extra_flags, prepare_command, ) from .typing import PathOrStr @@ -71,6 +71,8 @@ def get_python_configurations( def build(options: BuildOptions) -> None: + allowed_architectures_check("linux", options) + try: subprocess.check_output(['docker', '--version']) except Exception: diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 82772b16..23b37bf8 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -4,15 +4,14 @@ import shutil import subprocess import sys import tempfile -import textwrap from pathlib import Path from typing import Dict, List, NamedTuple, Optional, Sequence from .environment import ParsedEnvironment from .logger import log -from .util import (Architecture, BuildOptions, BuildSelector, NonPlatformWheelError, +from .util import (BuildOptions, BuildSelector, NonPlatformWheelError, download, get_build_verbosity_extra_flags, get_pip_script, - install_certifi_script, prepare_command) + install_certifi_script, prepare_command, allowed_architectures_check) from .typing import PathOrStr @@ -187,11 +186,7 @@ def setup_python(python_configuration: PythonConfiguration, def build(options: BuildOptions) -> None: - if not options.architectures <= {Architecture.x86_64}: - raise ValueError(textwrap.dedent(f''' - Invalid archs option {options.architectures}. macOS only supports x86_64 for the moment. - If you want to set emulation architectures on Linux, use CIBW_ARCHS_LINUX instead. - ''')) + allowed_architectures_check("macos", options) temp_dir = Path(tempfile.mkdtemp(prefix='cibuildwheel')) built_wheel_dir = temp_dir / 'built_wheel' diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 749bda77..31506a30 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -2,6 +2,7 @@ import os import platform as platform_module import re import ssl +import sys import functools import textwrap import urllib.request @@ -16,6 +17,11 @@ import certifi from .environment import ParsedEnvironment from .typing import PathOrStr +if sys.version_info < (3, 8): + from typing_extensions import Literal +else: + from typing import Literal + def prepare_command(command: str, **kwargs: PathOrStr) -> str: ''' @@ -237,3 +243,33 @@ def detect_ci_provider() -> Optional[CIProvider]: return CIProvider.other else: return None + + +PRETTY_NAMES = {'linux': 'Linux', 'macos': 'macOS', 'windows': 'Windows'} + +ALLOWED_ARCHITECTURES = { + 'linux': {Architecture.x86_64, Architecture.i686, Architecture.aarch64, Architecture.ppc64le, Architecture.s390x}, + 'macos': {Architecture.x86_64}, + 'windows': {Architecture.AMD64, Architecture.x86}, +} + + +def allowed_architectures_check( + name: Literal['linux', 'macos', 'windows'], + options: BuildOptions, +) -> None: + + allowed_architectures = ALLOWED_ARCHITECTURES[name] + + msg = f'{PRETTY_NAMES[name]} only supports {sorted(allowed_architectures)} at the moment.' + + if name != 'linux': + msg += ' If you want to set emulation architectures on Linux, use CIBW_ARCHS_LINUX instead.' + + if not options.architectures <= allowed_architectures: + msg = f'Invalid archs option {options.architectures}. ' + msg + raise ValueError(msg) + + if not options.architectures: + msg = 'Empty archs option set. ' + msg + raise ValueError(msg) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 09f64e1d..24d2722e 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -3,7 +3,6 @@ import shutil import subprocess import sys import tempfile -import textwrap from pathlib import Path from typing import Dict, List, NamedTuple, Optional, Sequence, Set from zipfile import ZipFile @@ -14,7 +13,7 @@ from .environment import ParsedEnvironment from .logger import log from .util import (Architecture, BuildOptions, BuildSelector, NonPlatformWheelError, download, get_build_verbosity_extra_flags, get_pip_script, - prepare_command) + prepare_command, allowed_architectures_check) from .typing import PathOrStr IS_RUNNING_ON_AZURE = Path('C:\\hostedtoolcache').exists() @@ -210,13 +209,7 @@ def pep_518_cp35_workaround(package_dir: Path, env: Dict[str, str]) -> None: def build(options: BuildOptions) -> None: - allowed_architectures = {Architecture.AMD64, Architecture.x86} - if not options.architectures <= allowed_architectures: - raise ValueError(textwrap.dedent(f''' - Invalid archs option {options.architectures}. Windows only supports - {sorted(allowed_architectures)}. If you want to set emulation - architectures on Linux, use CIBW_ARCHS_LINUX instead. - ''')) + allowed_architectures_check("windows", options) temp_dir = Path(tempfile.mkdtemp(prefix='cibuildwheel')) built_wheel_dir = temp_dir / 'built_wheel' diff --git a/setup.cfg b/setup.cfg index 05d0ba2e..7f60b783 100644 --- a/setup.cfg +++ b/setup.cfg @@ -31,6 +31,7 @@ install_requires = bashlex!=0.13 toml certifi + typing_extensions; python_version < '3.8' [options.package_data] cibuildwheel = resources/*