From 76fe47263850f82e82089cfd3f0d3e8063bc6db9 Mon Sep 17 00:00:00 2001 From: Pavel Savchenko Date: Sat, 12 Dec 2020 21:18:19 +0000 Subject: [PATCH 01/37] Support docker qemu emulation in any linux env - currently just prints out envs --- .github/workflows/test.yml | 7 +++++++ cibuildwheel/linux.py | 2 ++ 2 files changed, 9 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3ab6656c..ab4d6fa8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -47,6 +47,13 @@ jobs: id: pr-labels uses: joerick/pr-labels-action@v1.0.6 + - name: Set up QEMU + id: qemu + uses: docker/setup-qemu-action@v1 + with: + platforms: all + if: runner.os == "Linux" + - name: Sample build if: contains(steps.pr-labels.outputs.labels, ' ci-sample-build ') run: | diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index cfd4952c..d0f36a9d 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -13,6 +13,8 @@ from .util import (BuildOptions, BuildSelector, NonPlatformWheelError, def matches_platform(identifier: str) -> bool: + output = subprocess.check_output(['docker', '--version']) + print("We found support for these platforms:", output) pm = platform.machine() if pm == "x86_64": # x86_64 machines can run i686 docker containers From 90a14b9dd29ac948f2023801bddb19d87533ef23 Mon Sep 17 00:00:00 2001 From: Pavel Savchenko Date: Sat, 12 Dec 2020 21:23:27 +0000 Subject: [PATCH 02/37] fixup! Support docker qemu emulation in any linux env --- .github/workflows/test.yml | 5 +++-- cibuildwheel/linux.py | 39 +++++++++++++++++--------------------- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index ab4d6fa8..3c7fdb4b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,7 +24,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-18.04, windows-latest, macos-latest] + os: [ubuntu-18.04] + # os: [ubuntu-18.04, windows-latest, macos-latest] python_version: ['3.7'] timeout-minutes: 180 steps: @@ -52,7 +53,7 @@ jobs: uses: docker/setup-qemu-action@v1 with: platforms: all - if: runner.os == "Linux" + if: runner.os == 'Linux' - name: Sample build if: contains(steps.pr-labels.outputs.labels, ' ci-sample-build ') diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index d0f36a9d..57e83803 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -1,4 +1,5 @@ import platform +import re import subprocess import sys import textwrap @@ -12,27 +13,15 @@ from .util import (BuildOptions, BuildSelector, NonPlatformWheelError, get_build_verbosity_extra_flags, prepare_command) -def matches_platform(identifier: str) -> bool: - output = subprocess.check_output(['docker', '--version']) - print("We found support for these platforms:", output) - pm = platform.machine() - if pm == "x86_64": - # x86_64 machines can run i686 docker containers - if identifier.endswith('x86_64') or identifier.endswith('i686'): - return True - elif pm == "i686": - if identifier.endswith('i686'): - return True - elif pm == "aarch64": - if identifier.endswith('aarch64'): - return True - elif pm == "ppc64le": - if identifier.endswith('ppc64le'): - return True - elif pm == "s390x": - if identifier.endswith('s390x'): - return True - return False +re_pattern = re.compile(r'[cp]p\d{2}-manylinux_(\w*)') + + +def matches_platform(identifier: str, supported_platforms: List[str]) -> bool: + matched_architecture = re_pattern.search(identifier) + id_architecture = matched_architecture.group(1) if matched_architecture else '' + # x86_64 machines can run i686 docker containers + id_architecture = id_architecture if id_architecture == 'i686' else 'x86_64' + return id_architecture in supported_platforms class PythonConfiguration(NamedTuple): @@ -80,8 +69,14 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi PythonConfiguration(version='3.8', identifier='cp38-manylinux_s390x', path_str='/opt/python/cp38-cp38'), PythonConfiguration(version='3.9', identifier='cp39-manylinux_s390x', path_str='/opt/python/cp39-cp39'), ] + # skip builds as required - return [c for c in python_configurations if matches_platform(c.identifier) and build_selector(c.identifier)] + target_archs = [platform.machine()] + return [ + c for c in python_configurations + if matches_platform(c.identifier, target_archs) + and build_selector(c.identifier) + ] def build(options: BuildOptions) -> None: From ec99c01b2f4908316f5caccee7ce4481421002b5 Mon Sep 17 00:00:00 2001 From: Pavel Savchenko Date: Wed, 16 Dec 2020 23:17:51 +0000 Subject: [PATCH 03/37] Allow architecture selection --- cibuildwheel/__main__.py | 26 +++++++++++++++++++++++--- cibuildwheel/linux.py | 22 ++++++++++++++-------- cibuildwheel/util.py | 9 +++++++++ 3 files changed, 46 insertions(+), 11 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index e171ab20..4486aee8 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -17,6 +17,7 @@ from cibuildwheel.environment import ( parse_environment, ) from cibuildwheel.util import ( + Architecture, BuildOptions, BuildSelector, DependencyConstraints, @@ -67,6 +68,22 @@ def main() -> None: run in Windows, and it will build and test for all versions of Python. Default: auto. ''') + + parser.add_argument( + '-a', + '--architectures', + choices=[arch.value for arch in Architecture], + default=[], + help=''' + Comma-separated list of CPU architectures to build for. + If unspecified, builds the architectures natively supported + on this machine. Set this option to build an architecture + via emulation, for example, using binfmt_misc and qemu. + ''', + action="extend", + nargs="+", + type=Architecture, + ) parser.add_argument('--output-dir', default=os.environ.get('CIBW_OUTPUT_DIR', 'wheelhouse'), help='Destination folder for the wheels.') @@ -169,7 +186,7 @@ def main() -> None: exit(2) if args.print_build_identifiers: - print_build_identifiers(platform, build_selector) + print_build_identifiers(platform, build_selector, args.architectures) exit(0) manylinux_images: Optional[Dict[str, str]] = None @@ -202,6 +219,7 @@ def main() -> None: manylinux_images[build_platform] = image build_options = BuildOptions( + architectures=args.architectures, package_dir=package_dir, output_dir=output_dir, test_command=test_command, @@ -284,10 +302,12 @@ def print_preamble(platform: str, build_options: BuildOptions) -> None: print('\nHere we go!\n') -def print_build_identifiers(platform: str, build_selector: BuildSelector) -> None: +def print_build_identifiers( + platform: str, build_selector: BuildSelector, architectures: List[Architecture] +) -> None: python_configurations: List[Any] = [] if platform == 'linux': - python_configurations = cibuildwheel.linux.get_python_configurations(build_selector) + python_configurations = cibuildwheel.linux.get_python_configurations(build_selector, architectures) elif platform == 'windows': python_configurations = cibuildwheel.windows.get_python_configurations(build_selector) elif platform == 'macos': diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 57e83803..b27b4c65 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -9,18 +9,18 @@ from typing import List, NamedTuple, Union from .docker_container import DockerContainer from .logger import log -from .util import (BuildOptions, BuildSelector, NonPlatformWheelError, - get_build_verbosity_extra_flags, prepare_command) +from .util import ( + Architecture, BuildOptions, BuildSelector, NonPlatformWheelError, + get_build_verbosity_extra_flags, prepare_command, +) re_pattern = re.compile(r'[cp]p\d{2}-manylinux_(\w*)') -def matches_platform(identifier: str, supported_platforms: List[str]) -> bool: +def matches_platform(identifier: str, supported_platforms: List[Architecture]) -> bool: matched_architecture = re_pattern.search(identifier) id_architecture = matched_architecture.group(1) if matched_architecture else '' - # x86_64 machines can run i686 docker containers - id_architecture = id_architecture if id_architecture == 'i686' else 'x86_64' return id_architecture in supported_platforms @@ -34,7 +34,9 @@ class PythonConfiguration(NamedTuple): return PurePath(self.path_str) -def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfiguration]: +def get_python_configurations( + build_selector: BuildSelector, architectures: List[Architecture] +) -> List[PythonConfiguration]: python_configurations = [ PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path_str='/opt/python/cp27-cp27m'), PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path_str='/opt/python/cp27-cp27mu'), @@ -71,7 +73,11 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi ] # skip builds as required - target_archs = [platform.machine()] + target_archs = architectures or [Architecture(platform.machine())] + # x86_64 machines can run i686 docker containers + if Architecture.i686 not in target_archs and Architecture.x86_64 in target_archs: + target_archs.append(Architecture.i686) + print("matching platforms to architectures: ", target_archs) return [ c for c in python_configurations if matches_platform(c.identifier, target_archs) @@ -90,7 +96,7 @@ def build(options: BuildOptions) -> None: exit(2) assert options.manylinux_images is not None - python_configurations = get_python_configurations(options.build_selector) + python_configurations = get_python_configurations(options.build_selector, options.architectures) platforms = [ ('cp', 'manylinux_x86_64', options.manylinux_images['x86_64']), ('cp', 'manylinux_i686', options.manylinux_images['i686']), diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 87ce5d17..80ae3a36 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -121,7 +121,16 @@ class DependencyConstraints: return f'{self.__class__.__name__}{self.base_file_path!r})' +class Architecture(str, Enum): + x86_64 = 'x86_64' + i686 = 'i686' + aarch64 = 'aarch64' + ppc64le = 'ppc64le' + s390x = 's390x' + + class BuildOptions(NamedTuple): + architectures: List[Architecture] package_dir: Path output_dir: Path build_selector: BuildSelector From 0d29bf8daef92925d2fd6399a2570f8147427b6e Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 18 Dec 2020 12:18:06 -0500 Subject: [PATCH 04/37] refactor: --archs, CIBW_ARCHS, "auto" Using `--archs` for the flag, comma separated, also accepting CIBW_ARCHS. Supports "auto" as the default that can also expand, matching CIBW_PLATFORM. --- cibuildwheel/__main__.py | 19 +++++++++---------- cibuildwheel/linux.py | 18 ++++++++++++------ cibuildwheel/util.py | 1 + 3 files changed, 22 insertions(+), 16 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 4486aee8..a7f77a0b 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -70,19 +70,16 @@ def main() -> None: ''') parser.add_argument( - '-a', - '--architectures', - choices=[arch.value for arch in Architecture], - default=[], + '--archs', + default=os.environ.get("CIBW_ARCHS", 'auto'), help=''' Comma-separated list of CPU architectures to build for. If unspecified, builds the architectures natively supported on this machine. Set this option to build an architecture via emulation, for example, using binfmt_misc and qemu. - ''', - action="extend", - nargs="+", - type=Architecture, + Default: auto + Choices: auto, {} + '''.format(", ".join(a.name for a in Architecture)), ) parser.add_argument('--output-dir', default=os.environ.get('CIBW_OUTPUT_DIR', 'wheelhouse'), @@ -185,8 +182,10 @@ def main() -> None: print('cibuildwheel: Could not find setup.py, setup.cfg or pyproject.toml at root of package', file=sys.stderr) exit(2) + archs = [Architecture(a) for a in args.archs.split(",")] + if args.print_build_identifiers: - print_build_identifiers(platform, build_selector, args.architectures) + print_build_identifiers(platform, build_selector, archs) exit(0) manylinux_images: Optional[Dict[str, str]] = None @@ -219,7 +218,7 @@ def main() -> None: manylinux_images[build_platform] = image build_options = BuildOptions( - architectures=args.architectures, + architectures=archs, package_dir=package_dir, output_dir=output_dir, test_command=test_command, diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index b27b4c65..a9554396 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -18,10 +18,10 @@ from .util import ( re_pattern = re.compile(r'[cp]p\d{2}-manylinux_(\w*)') -def matches_platform(identifier: str, supported_platforms: List[Architecture]) -> bool: +def matches_platform(identifier: str, architectures: List[Architecture]) -> bool: matched_architecture = re_pattern.search(identifier) id_architecture = matched_architecture.group(1) if matched_architecture else '' - return id_architecture in supported_platforms + return id_architecture in architectures class PythonConfiguration(NamedTuple): @@ -73,10 +73,16 @@ def get_python_configurations( ] # skip builds as required - target_archs = architectures or [Architecture(platform.machine())] - # x86_64 machines can run i686 docker containers - if Architecture.i686 not in target_archs and Architecture.x86_64 in target_archs: - target_archs.append(Architecture.i686) + target_archs = architectures + + if Architecture.auto in architectures: + target_archs.remove(Architecture.auto) + native_architecture = Architecture(platform.machine()) + target_archs.append(native_architecture) + # x86_64 machines can run i686 docker containers + if native_architecture == Architecture.x86_64: + target_archs.append(Architecture.i686) + print("matching platforms to architectures: ", target_archs) return [ c for c in python_configurations diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 80ae3a36..5ea02f5b 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -122,6 +122,7 @@ class DependencyConstraints: class Architecture(str, Enum): + auto = 'auto' # gets expanded to the native arch, possibly 32+64 bit x86_64 = 'x86_64' i686 = 'i686' aarch64 = 'aarch64' From 26b7b72d5ec3ea16a476490ff232c11348ab009d Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 18 Dec 2020 14:39:27 -0500 Subject: [PATCH 05/37] fix: drop printout --- cibuildwheel/linux.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index a9554396..a6ae6e46 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -83,7 +83,6 @@ def get_python_configurations( if native_architecture == Architecture.x86_64: target_archs.append(Architecture.i686) - print("matching platforms to architectures: ", target_archs) return [ c for c in python_configurations if matches_platform(c.identifier, target_archs) From 0c9af0357baaa5b5e34cbaa5ed592a36391582b5 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 20 Dec 2020 19:42:28 +0000 Subject: [PATCH 06/37] Add platform-specific CIBW_ARCHS options. Remove 'auto' as a possible Architecture enum value. This is resolved at option parse time. --- cibuildwheel/__main__.py | 8 ++++++-- cibuildwheel/linux.py | 9 --------- cibuildwheel/util.py | 24 +++++++++++++++++++++++- 3 files changed, 29 insertions(+), 12 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index a7f77a0b..99e2e0dd 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -71,7 +71,7 @@ def main() -> None: parser.add_argument( '--archs', - default=os.environ.get("CIBW_ARCHS", 'auto'), + default=None, help=''' Comma-separated list of CPU architectures to build for. If unspecified, builds the architectures natively supported @@ -182,7 +182,11 @@ def main() -> None: print('cibuildwheel: Could not find setup.py, setup.cfg or pyproject.toml at root of package', file=sys.stderr) exit(2) - archs = [Architecture(a) for a in args.archs.split(",")] + if args.archs is not None: + archs_config_str = args.archs + else: + archs_config_str = get_option_from_environment('CIBW_ARCHS', platform=platform, default='auto') + archs = Architecture.parse_config(archs_config_str, platform=platform) if args.print_build_identifiers: print_build_identifiers(platform, build_selector, archs) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index a6ae6e46..15932776 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -1,4 +1,3 @@ -import platform import re import subprocess import sys @@ -75,14 +74,6 @@ def get_python_configurations( # skip builds as required target_archs = architectures - if Architecture.auto in architectures: - target_archs.remove(Architecture.auto) - native_architecture = Architecture(platform.machine()) - target_archs.append(native_architecture) - # x86_64 machines can run i686 docker containers - if native_architecture == Architecture.x86_64: - target_archs.append(Architecture.i686) - return [ c for c in python_configurations if matches_platform(c.identifier, target_archs) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 5ea02f5b..a4711dd3 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -1,4 +1,6 @@ import os +import platform as platform_module +import re import ssl import textwrap import urllib.request @@ -122,12 +124,32 @@ class DependencyConstraints: class Architecture(str, Enum): - auto = 'auto' # gets expanded to the native arch, possibly 32+64 bit x86_64 = 'x86_64' i686 = 'i686' aarch64 = 'aarch64' ppc64le = 'ppc64le' s390x = 's390x' + win32 = 'win32' + win_amd64 = 'win_amd64' + + @staticmethod + def parse_config(config: str, platform: str) -> 'List[Architecture]': + result = [] + for arch_str in re.split(r'[\s,]+', config): + if arch_str == 'auto': + result += Architecture.auto_archs(platform=platform) + else: + result.append(Architecture(arch_str)) + return result + + @staticmethod + def auto_archs(platform: str) -> 'List[Architecture]': + native_architecture = Architecture(platform_module.machine()) + result = [native_architecture] + # x86_64 machines can run i686 docker containers + if platform == 'linux' and native_architecture == Architecture.x86_64: + result.append(Architecture.i686) + return result class BuildOptions(NamedTuple): From f73b6083d500d43c88d05f4031068784a2462c90 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 21 Dec 2020 11:00:33 +0000 Subject: [PATCH 07/37] Add unit test for the option parsing --- unit_test/main_tests/conftest.py | 11 +++++++- unit_test/main_tests/main_platform_test.py | 29 ++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/unit_test/main_tests/conftest.py b/unit_test/main_tests/conftest.py index 3f684b6c..8a165537 100644 --- a/unit_test/main_tests/conftest.py +++ b/unit_test/main_tests/conftest.py @@ -70,5 +70,14 @@ def platform(request, monkeypatch): @pytest.fixture def intercepted_build_args(platform, monkeypatch): intercepted = ArgsInterceptor() - monkeypatch.setattr(globals()[platform], 'build', intercepted) + + if platform == 'linux': + monkeypatch.setattr(linux, 'build', intercepted) + elif platform == 'macos': + monkeypatch.setattr(macos, 'build', intercepted) + elif platform == 'windows': + monkeypatch.setattr(windows, 'build', intercepted) + else: + raise ValueError(f'unknown platform value: {platform}') + return intercepted diff --git a/unit_test/main_tests/main_platform_test.py b/unit_test/main_tests/main_platform_test.py index 004855e8..806b9065 100644 --- a/unit_test/main_tests/main_platform_test.py +++ b/unit_test/main_tests/main_platform_test.py @@ -1,3 +1,5 @@ +import platform as platform_module +from cibuildwheel.util import Architecture import sys import pytest @@ -64,3 +66,30 @@ def test_platform_environment(platform, intercepted_build_args, monkeypatch): main() assert intercepted_build_args.args[0].package_dir == MOCK_PACKAGE_DIR + + +def test_archs_default(platform, intercepted_build_args, monkeypatch): + monkeypatch.setattr(platform_module, 'machine', lambda: 'x86_64') + + main() + build_options = intercepted_build_args.args[0] + + if platform == 'linux': + assert build_options.architectures == [Architecture.x86_64, Architecture.i686] + else: + assert build_options.architectures == [Architecture.x86_64] + + +@pytest.mark.parametrize('use_env_var', [False, True]) +def test_archs_argument(platform, intercepted_build_args, monkeypatch, use_env_var): + monkeypatch.setattr(platform_module, 'machine', lambda: 'x86_64') + if use_env_var: + monkeypatch.setenv('CIBW_ARCHS', 'ppc64le') + else: + monkeypatch.setenv('CIBW_ARCHS', 'unused') + monkeypatch.setattr(sys, 'argv', sys.argv + ['--archs', 'ppc64le']) + + main() + build_options = intercepted_build_args.args[0] + + assert build_options.architectures == [Architecture.ppc64le] From 7c86223ff8254828f05305cf5ce8bad6df093252 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 21 Dec 2020 11:05:19 +0000 Subject: [PATCH 08/37] Add platform-specific arch option unit test --- cibuildwheel/util.py | 7 +++++-- unit_test/main_tests/main_platform_test.py | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index a4711dd3..a2a52481 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -124,13 +124,16 @@ class DependencyConstraints: class Architecture(str, Enum): + # mac/linux archs x86_64 = 'x86_64' i686 = 'i686' aarch64 = 'aarch64' ppc64le = 'ppc64le' s390x = 's390x' - win32 = 'win32' - win_amd64 = 'win_amd64' + + # windows archs + x86 = 'x86' + amd64 = 'AMD64' @staticmethod def parse_config(config: str, platform: str) -> 'List[Architecture]': diff --git a/unit_test/main_tests/main_platform_test.py b/unit_test/main_tests/main_platform_test.py index 806b9065..9ac84ea8 100644 --- a/unit_test/main_tests/main_platform_test.py +++ b/unit_test/main_tests/main_platform_test.py @@ -93,3 +93,21 @@ def test_archs_argument(platform, intercepted_build_args, monkeypatch, use_env_v build_options = intercepted_build_args.args[0] assert build_options.architectures == [Architecture.ppc64le] + + +def test_archs_platform_specific(platform, intercepted_build_args, monkeypatch): + monkeypatch.setattr(platform_module, 'machine', lambda: 'x86_64') + monkeypatch.setenv('CIBW_ARCHS', 'unused') + monkeypatch.setenv('CIBW_ARCHS_LINUX', 'ppc64le') + monkeypatch.setenv('CIBW_ARCHS_WINDOWS', 'x86') + monkeypatch.setenv('CIBW_ARCHS_MACOS', 'x86_64') + + main() + build_options = intercepted_build_args.args[0] + + if platform == 'linux': + assert build_options.architectures == [Architecture.ppc64le] + elif platform == 'windows': + assert build_options.architectures == [Architecture.x86] + elif platform == 'macos': + assert build_options.architectures == [Architecture.x86_64] From 5aa85e815d4ef1ae70c7d7416383f6e8e2441a5f Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 21 Dec 2020 11:06:25 +0000 Subject: [PATCH 09/37] Add integration test for emulation --- test/conftest.py | 21 +++++++++++++++++++++ test/test_emulation.py | 35 +++++++++++++++++++++++++++++++++++ test/utils.py | 13 ++++++++----- 3 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 test/conftest.py create mode 100644 test/test_emulation.py diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 00000000..410c0707 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,21 @@ +import pytest + + +def pytest_addoption(parser): + parser.addoption( + "--run-emulation", action="store_true", default=False, help="run emulation tests" + ) + + +def pytest_configure(config): + config.addinivalue_line("markers", "emulation: mark test requiring qemu binfmt_misc to run") + + +def pytest_collection_modifyitems(config, items): + if config.getoption("--run-emulation"): + # --run-emulation given in cli: do not skip emulation tests + return + skip_emulation = pytest.mark.skip(reason="need --run-emulation option to run") + for item in items: + if "emulation" in item.keywords: + item.add_marker(skip_emulation) diff --git a/test/test_emulation.py b/test/test_emulation.py new file mode 100644 index 00000000..2b5fe7b1 --- /dev/null +++ b/test/test_emulation.py @@ -0,0 +1,35 @@ +import pytest +from . import utils +from . import test_projects + + +project_with_a_test = test_projects.new_c_project() + +project_with_a_test.files['test/spam_test.py'] = r''' +import spam + +def test_spam(): + assert spam.system('python -c "exit(0)"') == 0 + assert spam.system('python -c "exit(1)"') != 0 +''' + + +@pytest.mark.emulation +def test(tmp_path): + project_dir = tmp_path / 'project' + project_with_a_test.generate(project_dir) + + # build and test the wheels + actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ + 'CIBW_TEST_REQUIRES': 'pytest', + 'CIBW_TEST_COMMAND': 'pytest {project}/test', + 'CIBW_ARCHS': 'aarch64 ppc64le s390x', + }) + + # also check that we got the right wheels + expected_wheels = ( + utils.expected_wheels('spam', '0.1.0', machine_arch='aarch64') + + utils.expected_wheels('spam', '0.1.0', machine_arch='ppc64le') + + utils.expected_wheels('spam', '0.1.0', machine_arch='s390x') + ) + assert set(actual_wheels) == set(expected_wheels) diff --git a/test/utils.py b/test/utils.py index 04f3e147..bae3853b 100644 --- a/test/utils.py +++ b/test/utils.py @@ -76,7 +76,7 @@ def cibuildwheel_run(project_path, package_dir='.', env=None, add_env=None, outp def expected_wheels(package_name, package_version, manylinux_versions=None, - macosx_deployment_target='10.9'): + macosx_deployment_target='10.9', machine_arch=None): ''' Returns a list of expected wheels from a run of cibuildwheel. ''' @@ -85,15 +85,18 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, # {python tag} and {abi tag} are closely related to the python interpreter used to build the wheel # so we'll merge them below as python_abi_tag + if machine_arch is None: + machine_arch = pm.machine() + if manylinux_versions is None: - if pm.machine() == 'x86_64': + if machine_arch == 'x86_64': manylinux_versions = ['manylinux1', 'manylinux2010'] else: manylinux_versions = ['manylinux2014'] python_abi_tags = ['cp35-cp35m', 'cp36-cp36m', 'cp37-cp37m', 'cp38-cp38', 'cp39-cp39'] - if pm.machine() in ['x86_64', 'AMD64', 'x86']: + if machine_arch in ['x86_64', 'AMD64', 'x86']: python_abi_tags += ['cp27-cp27m', 'pp27-pypy_73', 'pp36-pypy36_pp73', 'pp37-pypy37_pp73'] if platform == 'linux': @@ -105,9 +108,9 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, platform_tags = [] if platform == 'linux': - architectures = [pm.machine()] + architectures = [machine_arch] - if pm.machine() == 'x86_64' and python_abi_tag.startswith('cp'): + if machine_arch == 'x86_64' and python_abi_tag.startswith('cp'): architectures.append('i686') platform_tags = [ From 0ca294e5390643b00cac8834f1f6f62a17173636 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 21 Dec 2020 11:11:37 +0000 Subject: [PATCH 10/37] Add separate runner for emulation tests --- .github/workflows/test.yml | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3c7fdb4b..511282a2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -48,13 +48,6 @@ jobs: id: pr-labels uses: joerick/pr-labels-action@v1.0.6 - - name: Set up QEMU - id: qemu - uses: docker/setup-qemu-action@v1 - with: - platforms: all - if: runner.os == 'Linux' - - name: Sample build if: contains(steps.pr-labels.outputs.labels, ' ci-sample-build ') run: | @@ -63,3 +56,24 @@ jobs: - name: Test cibuildwheel run: | python ./bin/run_tests.py + + test-emulated: + name: Test emulated cibuildwheel using qemu + runs-on: ubuntu-20.04 + timeout-minutes: 180 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - name: Install dependencies + run: | + python -m pip install -r requirements-dev.txt + + - name: Set up QEMU + id: qemu + uses: docker/setup-qemu-action@v1 + with: + platforms: all + + - name: Run the emulation tests + run: | + pytest --run-emulation test/test_emulation.py From 91800e4739d9b9776419fa7a82e256c9cc412ed7 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 21 Dec 2020 11:31:22 +0000 Subject: [PATCH 11/37] Update .github/workflows/test.yml Co-authored-by: Henry Schreiner --- .github/workflows/test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 511282a2..e51afde7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,8 +24,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-18.04] - # os: [ubuntu-18.04, windows-latest, macos-latest] + os: [ubuntu-latest, windows-latest, macos-latest] python_version: ['3.7'] timeout-minutes: 180 steps: From 27f1a4eafc57e3d766070100ce6e4aef7121dc99 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 21 Dec 2020 13:42:55 +0000 Subject: [PATCH 12/37] Use 'latest' for CI --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index e51afde7..a9d6f7ec 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -58,7 +58,7 @@ jobs: test-emulated: name: Test emulated cibuildwheel using qemu - runs-on: ubuntu-20.04 + runs-on: ubuntu-latest timeout-minutes: 180 steps: - uses: actions/checkout@v2 From 8a1a0a4c37440d2f776f48b63577cc582e0088e4 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 21 Dec 2020 13:44:01 +0000 Subject: [PATCH 13/37] Add asserts on macOS/Windows so that they can't be anything but default ...for now? --- cibuildwheel/macos.py | 9 ++++++++- cibuildwheel/windows.py | 10 +++++++++- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 9ffca89d..56863550 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -4,13 +4,14 @@ import shutil import subprocess import sys import tempfile +import textwrap from os import PathLike from pathlib import Path from typing import Dict, List, NamedTuple, Optional, Sequence, Union from .environment import ParsedEnvironment from .logger import log -from .util import (BuildOptions, BuildSelector, NonPlatformWheelError, +from .util import (Architecture, BuildOptions, BuildSelector, NonPlatformWheelError, download, get_build_verbosity_extra_flags, get_pip_script, install_certifi_script, prepare_command) @@ -182,6 +183,12 @@ def setup_python(python_configuration: PythonConfiguration, def build(options: BuildOptions) -> None: + if options.architectures != [Architecture.x86_64]: + raise ValueError(textwrap.dedent(f''' + Invalid archs option {options.architectures}. macOS only supports x86_64 for the moment. + If you want to set emulation architectures on Linux, use CIBW_ARCHS_LINUX instead. + ''')) + temp_dir = Path(tempfile.mkdtemp(prefix='cibuildwheel')) built_wheel_dir = temp_dir / 'built_wheel' repaired_wheel_dir = temp_dir / 'repaired_wheel' diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 4331915a..a0899fb2 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -3,6 +3,7 @@ import shutil import subprocess import sys import tempfile +import textwrap from os import PathLike from pathlib import Path from typing import Dict, List, NamedTuple, Optional, Sequence, Union @@ -12,7 +13,7 @@ import toml from .environment import ParsedEnvironment from .logger import log -from .util import (BuildOptions, BuildSelector, NonPlatformWheelError, +from .util import (Architecture, BuildOptions, BuildSelector, NonPlatformWheelError, download, get_build_verbosity_extra_flags, get_pip_script, prepare_command) @@ -201,6 +202,13 @@ def pep_518_cp35_workaround(package_dir: Path, env: Dict[str, str]) -> None: def build(options: BuildOptions) -> None: + if options.architectures != [Architecture.amd64, Architecture.x86]: + raise ValueError(textwrap.dedent(f''' + Invalid archs option {options.architectures}. Windows only supports 'amd64,x86' for the + moment. If you want to set emulation architectures on Linux, use CIBW_ARCHS_LINUX + instead. + ''')) + temp_dir = Path(tempfile.mkdtemp(prefix='cibuildwheel')) built_wheel_dir = temp_dir / 'built_wheel' repaired_wheel_dir = temp_dir / 'repaired_wheel' From b34bdeec4083046d697d8994e18cdb2043e76741 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 21 Dec 2020 16:34:36 +0000 Subject: [PATCH 14/37] Fix native archs expansion for windows --- cibuildwheel/util.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index a2a52481..4a30aacd 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -149,9 +149,11 @@ class Architecture(str, Enum): def auto_archs(platform: str) -> 'List[Architecture]': native_architecture = Architecture(platform_module.machine()) result = [native_architecture] - # x86_64 machines can run i686 docker containers if platform == 'linux' and native_architecture == Architecture.x86_64: + # x86_64 machines can run i686 docker containers result.append(Architecture.i686) + if platform == 'windows' and native_architecture == Architecture.amd64: + result.append(Architecture.x86) return result From eb03a118e291e1ed07c978077d9f0f0e748d63d5 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 23 Dec 2020 18:34:16 +0000 Subject: [PATCH 15/37] Rename Architecture to match value amd64 -> AMD64 --- cibuildwheel/util.py | 4 ++-- cibuildwheel/windows.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 4a30aacd..3000ace8 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -133,7 +133,7 @@ class Architecture(str, Enum): # windows archs x86 = 'x86' - amd64 = 'AMD64' + AMD64 = 'AMD64' @staticmethod def parse_config(config: str, platform: str) -> 'List[Architecture]': @@ -152,7 +152,7 @@ class Architecture(str, Enum): if platform == 'linux' and native_architecture == Architecture.x86_64: # x86_64 machines can run i686 docker containers result.append(Architecture.i686) - if platform == 'windows' and native_architecture == Architecture.amd64: + if platform == 'windows' and native_architecture == Architecture.AMD64: result.append(Architecture.x86) return result diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index a0899fb2..bbd0dfc2 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -202,7 +202,7 @@ def pep_518_cp35_workaround(package_dir: Path, env: Dict[str, str]) -> None: def build(options: BuildOptions) -> None: - if options.architectures != [Architecture.amd64, Architecture.x86]: + if options.architectures != [Architecture.AMD64, Architecture.x86]: raise ValueError(textwrap.dedent(f''' Invalid archs option {options.architectures}. Windows only supports 'amd64,x86' for the moment. If you want to set emulation architectures on Linux, use CIBW_ARCHS_LINUX From 3696f151099e7325357d0e900f3dde887f926d1b Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 23 Dec 2020 18:42:22 +0000 Subject: [PATCH 16/37] Add example github config --- examples/github-deploy.yml | 2 +- examples/github-minimal.yml | 2 +- examples/github-with-qemu.yml | 46 +++++++++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+), 2 deletions(-) create mode 100644 examples/github-with-qemu.yml diff --git a/examples/github-deploy.yml b/examples/github-deploy.yml index 5e33a4b1..49bd5cc0 100644 --- a/examples/github-deploy.yml +++ b/examples/github-deploy.yml @@ -16,7 +16,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-18.04, windows-latest, macos-latest] + os: [ubuntu-18.04, windows-2019, macos-10.15] steps: - uses: actions/checkout@v2 diff --git a/examples/github-minimal.yml b/examples/github-minimal.yml index 9c3620f3..b7b4a1aa 100644 --- a/examples/github-minimal.yml +++ b/examples/github-minimal.yml @@ -8,7 +8,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-18.04, windows-latest, macos-latest] + os: [ubuntu-18.04, windows-2019, macos-10.15] steps: - uses: actions/checkout@v2 diff --git a/examples/github-with-qemu.yml b/examples/github-with-qemu.yml new file mode 100644 index 00000000..e1332148 --- /dev/null +++ b/examples/github-with-qemu.yml @@ -0,0 +1,46 @@ +name: Build + +on: [push, pull_request] + +jobs: + build_wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-18.04, windows-2019, macos-10.15] + + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-python@v2 + name: Install Python + with: + python-version: '3.7' + + - name: Install cibuildwheel + run: | + python -m pip install cibuildwheel==1.7.1 + + - name: Install Visual C++ for Python 2.7 + if: runner.os == 'Windows' + run: | + choco install vcpython27 -f -y + + - name: Set up QEMU + id: qemu + uses: docker/setup-qemu-action@v1 + with: + platforms: all + + - name: Build wheels + run: | + python -m cibuildwheel --output-dir wheelhouse + env: + # configure cibuildwheel to build native archs ('auto'), and some + # emulated ones + CIBW_ARCHS_LINUX: auto aarch64 ppc64le s390x + + - uses: actions/upload-artifact@v2 + with: + path: ./wheelhouse/*.whl From 68dc2867144e40174704989471b04c2ad6cd4ee7 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 23 Dec 2020 19:08:23 +0000 Subject: [PATCH 17/37] Add docs for ARCHS option --- docs/options.md | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/docs/options.md b/docs/options.md index 73c25b3b..f30da942 100644 --- a/docs/options.md +++ b/docs/options.md @@ -155,6 +155,28 @@ CIBW_SKIP: pp* } +### `CIBW_ARCHS_LINUX` {: #archs} +> Build non-native architectures + +A space-separated list of architectures to build. Use this in conjunction with +emulation, such as that provided by [docker/setup-qemu-action][setup-qemu-action] +or [tonistiigi/binfmt][tonistiigi/binfmt], to build architectures other than +those your machine natively supports. + +Options: `auto` `x86_64` `i686` `aarch64` `ppc64le` `s390x` + +Default: `auto`, meaning the native archs supported on the build machine. For +example, on an `x86_64` machine, `auto` expands to `x86_64` and `i686`. + +[setup-qemu-action]: https://github.com/docker/setup-qemu-action +[binfmt]: https://hub.docker.com/r/tonistiigi/binfmt + +#### Examples + +```yaml +# On an intel runner with qemu installed, build Intel and ARM wheels +CIBW_ARCHS_LINUX: "auto aarch64" +``` ## Build customization @@ -488,7 +510,8 @@ CIBW_BUILD_VERBOSITY: 1 ```text usage: cibuildwheel [-h] [--platform {auto,linux,macos,windows}] - [--output-dir OUTPUT_DIR] [--print-build-identifiers] + [--archs ARCHS] [--output-dir OUTPUT_DIR] + [--print-build-identifiers] [package_dir] Build wheels for all the platforms. @@ -510,6 +533,12 @@ optional arguments: don't run on your development machine. For "windows", you need to run in Windows, and it will build and test for all versions of Python. Default: auto. + --archs ARCHS Comma-separated list of CPU architectures to build + for. If unspecified, builds the architectures natively + supported on this machine. Set this option to build an + architecture via emulation, for example, using + binfmt_misc and qemu. Default: auto Choices: auto, + x86_64, i686, aarch64, ppc64le, s390x, x86, AMD64 --output-dir OUTPUT_DIR Destination folder for the wheels. --print-build-identifiers From 3983fb204934feb688ccad5a11a47794733b4f7c Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 23 Dec 2020 19:18:53 +0000 Subject: [PATCH 18/37] Fix broken link in docs --- docs/options.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/options.md b/docs/options.md index f30da942..e12ad186 100644 --- a/docs/options.md +++ b/docs/options.md @@ -160,8 +160,8 @@ CIBW_SKIP: pp* A space-separated list of architectures to build. Use this in conjunction with emulation, such as that provided by [docker/setup-qemu-action][setup-qemu-action] -or [tonistiigi/binfmt][tonistiigi/binfmt], to build architectures other than -those your machine natively supports. +or [tonistiigi/binfmt][binfmt], to build architectures other than those your +machine natively supports. Options: `auto` `x86_64` `i686` `aarch64` `ppc64le` `s390x` From 460eaa733017ff90cd74202f178fc7be3d6ba328 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 23 Dec 2020 19:23:58 +0000 Subject: [PATCH 19/37] Add mini-guide to the FAQ section --- docs/faq.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/faq.md b/docs/faq.md index 2ff777f6..d7b91149 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -24,6 +24,26 @@ Linux wheels are built in the [`manylinux` docker images](https://github.com/pyp - Alternative dockers images can be specified with the `CIBW_MANYLINUX_X86_64_IMAGE`, `CIBW_MANYLINUX_I686_IMAGE`, and `CIBW_MANYLINUX_PYPY_X86_64_IMAGE` options to allow for a custom, preconfigured build environment for the Linux builds. See [options](options.md#manylinux-image) for more details. +### Building non-native architectures using emulation {: #emulation} + +cibuildwheel supports building non-native architectures on Linux, via +emulation through the binfmt_misc kernel feature. The easiest way to use this +is via the [docker/setup-qemu-action][setup-qemu-action] on Github Actions or +[tonistiigi/binfmt][binfmt]. + +[setup-qemu-action]: https://github.com/docker/setup-qemu-action +[binfmt]: https://hub.docker.com/r/tonistiigi/binfmt + +Check out the following config for an example of how to set it up on Github +Actions. Once qemu is set up and registered, you just need to set the +`CIBW_ARCHS_LINUX` variable, and the other architectures are emulated +automatically. + +> .github/workflows/build.yml +```yaml +{% include "../examples/github-with-qemu.yml" %} +``` + ### Building packages with optional C extensions `cibuildwheel` defines the environment variable `CIBUILDWHEEL` to the value `1` allowing projects for which the C extension is optional to make it mandatory when building wheels. From f6ce1c931d962cf826cbfc0c7085556e1b8df3c1 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 23 Dec 2020 19:29:52 +0000 Subject: [PATCH 20/37] Update example config --- examples/github-with-qemu.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/github-with-qemu.yml b/examples/github-with-qemu.yml index e1332148..95a980f2 100644 --- a/examples/github-with-qemu.yml +++ b/examples/github-with-qemu.yml @@ -28,7 +28,7 @@ jobs: choco install vcpython27 -f -y - name: Set up QEMU - id: qemu + if: runner.os == 'Linux' uses: docker/setup-qemu-action@v1 with: platforms: all From 6b00931dcad87a95e0a2758c1179ad707f9eb5ec Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 23 Dec 2020 20:54:54 +0000 Subject: [PATCH 21/37] Allow the testing of any of the example configs in /examples --- bin/run_example_ci_configs.py | 51 ++++++++++++++++++++++++----------- 1 file changed, 36 insertions(+), 15 deletions(-) diff --git a/bin/run_example_ci_configs.py b/bin/run_example_ci_configs.py index 33f54de3..85a5e671 100755 --- a/bin/run_example_ci_configs.py +++ b/bin/run_example_ci_configs.py @@ -8,6 +8,7 @@ import sys import textwrap import time import click +from glob import glob from collections import namedtuple from urllib.parse import quote @@ -30,49 +31,67 @@ def generate_basic_project(path): project.generate(path) -CIService = namedtuple('CIService', 'name src_config_path dst_config_path badge_md') +CIService = namedtuple('CIService', 'name dst_config_path badge_md') services = [ CIService( name='appveyor', - src_config_path='examples/appveyor-minimal.yml', dst_config_path='appveyor.yml', badge_md='[![Build status](https://ci.appveyor.com/api/projects/status/wbsgxshp05tt1tif/branch/{branch}?svg=true)](https://ci.appveyor.com/project/joerick/cibuildwheel/branch/{branch})', ), CIService( name='azure-pipelines', - src_config_path='examples/azure-pipelines-minimal.yml', dst_config_path='azure-pipelines.yml', badge_md='[![Build Status](https://dev.azure.com/joerick0429/cibuildwheel/_apis/build/status/joerick.cibuildwheel?branchName={branch})](https://dev.azure.com/joerick0429/cibuildwheel/_build/latest?definitionId=2&branchName={branch})', ), CIService( - name='circle-ci', - src_config_path='examples/circleci-minimal.yml', + name='circleci', dst_config_path='.circleci/config.yml', badge_md='[![CircleCI](https://circleci.com/gh/joerick/cibuildwheel/tree/{branch_escaped}.svg?style=svg)](https://circleci.com/gh/joerick/cibuildwheel/tree/{branch})', ), CIService( name='github', - src_config_path='examples/github-minimal.yml', dst_config_path='.github/workflows/example.yml', badge_md='[![Build](https://github.com/joerick/cibuildwheel/workflows/Build/badge.svg?branch={branch})](https://github.com/joerick/cibuildwheel/actions)', ), CIService( name='travis-ci', - src_config_path='examples/travis-ci-minimal.yml', dst_config_path='.travis.yml', badge_md='[![Build Status](https://travis-ci.org/joerick/cibuildwheel.svg?branch={branch})](https://travis-ci.org/joerick/cibuildwheel)', ), CIService( name='gitlab', - src_config_path='examples/gitlab-minimal.yml', dst_config_path='.gitlab-ci.yml', badge_md='[![Gitlab](https://gitlab.com/joerick/cibuildwheel/badges/{branch}/pipeline.svg)](https://gitlab.com/joerick/cibuildwheel/-/commits/{branch})' ), ] +def ci_service_for_config_file(config_file): + for service in services: + if Path(config_file).name.startswith(service.name+'-'): + return service + raise ValueError(f'unknown ci service for config file {config_file}') + + @click.command() -def run_example_ci_configs(): +@click.argument('config_files', nargs=-1, type=click.Path()) +def run_example_ci_configs(config_files=None): + ''' + Test the example configs. If no files are specified, will test + examples/*-minimal.yml + ''' + + if len(config_files) == 0: + config_files = glob('examples/*-minimal.yml') + + # check each CI service has at most 1 config file + configs_by_service = {} + for config_file in config_files: + service = ci_service_for_config_file(config_file) + if service.name in configs_by_service: + raise Exception('You cannot specify more than one config per CI service') + configs_by_service[service.name] = config_file + if git_repo_has_changes(): print('Your git repo has uncommitted changes. Commit or stash before continuing.') exit(1) @@ -91,18 +110,19 @@ def run_example_ci_configs(): example_project = Path('example_root') generate_basic_project(example_project) - for service in services: - src_config_file = Path(service.src_config_path) + for config_file in config_files: + service = ci_service_for_config_file(config_file) + src_config_file = Path(config_file) dst_config_file = example_project / service.dst_config_path dst_config_file.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(src_config_file, dst_config_file) run(['git', 'add', example_project], check=True) - run(['git', 'commit', '-m', textwrap.dedent(f''' + run(['git', 'commit', '--no-verify', '-m', textwrap.dedent(f''' Test example minimal configs - Testing files: {[s.src_config_path for s in services]} + Testing files: {config_files} Generated from branch: {previous_branch} Time: {timestamp} ''')], check=True) @@ -116,9 +136,10 @@ def run_example_ci_configs(): print('> ') print('> | Service | Config | Status |') print('> |---|---|---|') - for service in services: + for config_file in config_files: + service = ci_service_for_config_file(config_file) badge = service.badge_md.format(branch=branch_name, branch_escaped=quote(branch_name, safe='')) - print(f'> | {service.name} | `{service.src_config_path}` | {badge} |') + print(f'> | {service.name} | `{config_file}` | {badge} |') print('> ') print('> Generated by `bin/run_example_ci_config.py`') print() From 0893731090b0a839efd8dd53ddeb4c339f6b98ff Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 28 Dec 2020 00:43:24 +0000 Subject: [PATCH 22/37] Simplify arch matching code --- cibuildwheel/linux.py | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 15932776..7e71bbae 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -1,4 +1,3 @@ -import re import subprocess import sys import textwrap @@ -14,15 +13,6 @@ from .util import ( ) -re_pattern = re.compile(r'[cp]p\d{2}-manylinux_(\w*)') - - -def matches_platform(identifier: str, architectures: List[Architecture]) -> bool: - matched_architecture = re_pattern.search(identifier) - id_architecture = matched_architecture.group(1) if matched_architecture else '' - return id_architecture in architectures - - class PythonConfiguration(NamedTuple): version: str identifier: str @@ -71,12 +61,11 @@ def get_python_configurations( PythonConfiguration(version='3.9', identifier='cp39-manylinux_s390x', path_str='/opt/python/cp39-cp39'), ] - # skip builds as required - target_archs = architectures - + # return all configurations whose arch is in our `architectures` list, + # and match the build/skip rules return [ c for c in python_configurations - if matches_platform(c.identifier, target_archs) + if any(c.identifier.endswith(arch) for arch in architectures) and build_selector(c.identifier) ] From 02f38497382ff8f03f7dc37cab35a27c2e4d3cc1 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sun, 27 Dec 2020 22:22:34 -0500 Subject: [PATCH 23/37] chore: update projects based on Discussion #485 (#498) --- README.md | 77 ++++++++++++++++------------- docs/data/projects.yml | 16 ++++++ docs/data/readme_icons/apple.svg | 2 +- docs/data/readme_icons/travisci.svg | 2 +- 4 files changed, 61 insertions(+), 36 deletions(-) diff --git a/README.md b/README.md index 1041237c..ff420e2a 100644 --- a/README.md +++ b/README.md @@ -129,6 +129,7 @@ Here are some repos that use cibuildwheel. |-----------------------------------|----|----|:------| | [scikit-learn][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | The machine learning library. A complex but clean config using many of cibuildwheel's features to build a large project with Cython and C++ extensions. | | [Matplotlib][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | The venerable Matplotlib, a Python library with C++ portions | +| [psutil][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Cross-platform lib for process and system monitoring in Python | | [twisted-iocpsupport][] | ![github icon][] | ![windows icon][] | A submodule of Twisted that hooks into native C APIs using Cython. | | [scikit-image][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Image processing library. Uses cibuildwheel to build and test a project that uses Cython with platform-native code. | | [websockets][] | ![travisci icon][] | ![apple icon][] ![linux icon][] | Library for building WebSocket servers and clients. Mostly written in Python, with a small C 'speedups' extension module. | @@ -138,12 +139,14 @@ Here are some repos that use cibuildwheel. | [coverage.py][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | The coverage tool for Python | | [River][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | 🌊 Online machine learning in Python | | [PyAV][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Pythonic bindings for FFmpeg's libraries. | -| [google neuroglancer][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | WebGL-based viewer for volumetric data | +| [Dependency Injector][] | ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Dependency injection framework for Python, uses Windows TravisCI | | [aioquic][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | QUIC and HTTP/3 implementation in Python | +| [google neuroglancer][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | WebGL-based viewer for volumetric data | | [AutoPy][] | ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Includes a Windows Travis build. | | [pikepdf][] | ![azurepipelines icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | A Python library for reading and writing PDF, powered by qpdf | | [Parselmouth][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | A Python interface to the Praat software package, using pybind11, C++17 and CMake, with the core Praat static library built only once and shared between wheels. | | [python-rapidjson][] | ![travisci icon][] ![gitlab icon][] ![appveyor icon][] | ![windows icon][] ![linux icon][] | Python wrapper around rapidjson | +| [Rtree][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Rtree: spatial index for Python GIS ΒΆ | | [KDEpy][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Kernel Density Estimation in Python | | [pybind11 python_example][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Example pybind11 module built with a Python-based build system | | [pybind11 cmake_example][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Example pybind11 module built with a CMake-based build system | @@ -163,6 +166,7 @@ Here are some repos that use cibuildwheel. [scikit-learn]: https://github.com/scikit-learn/scikit-learn [Matplotlib]: https://github.com/matplotlib/matplotlib +[psutil]: https://github.com/giampaolo/psutil [twisted-iocpsupport]: https://github.com/twisted/twisted-iocpsupport [scikit-image]: https://github.com/scikit-image/scikit-image [websockets]: https://github.com/aaugustin/websockets @@ -172,12 +176,14 @@ Here are some repos that use cibuildwheel. [coverage.py]: https://github.com/nedbat/coveragepy [River]: https://github.com/online-ml/river [PyAV]: https://github.com/PyAV-Org/PyAV -[google neuroglancer]: https://github.com/google/neuroglancer +[Dependency Injector]: https://github.com/ets-labs/python-dependency-injector [aioquic]: https://github.com/aiortc/aioquic +[google neuroglancer]: https://github.com/google/neuroglancer [AutoPy]: https://github.com/autopilot-rs/autopy [pikepdf]: https://github.com/pikepdf/pikepdf [Parselmouth]: https://github.com/YannickJadoul/Parselmouth [python-rapidjson]: https://github.com/python-rapidjson/python-rapidjson +[Rtree]: https://github.com/Toblerity/rtree [KDEpy]: https://github.com/tommyod/KDEpy [pybind11 python_example]: https://github.com/pybind/python_example [pybind11 cmake_example]: https://github.com/pybind/cmake_example @@ -205,39 +211,42 @@ Here are some repos that use cibuildwheel. [apple icon]: docs/data/readme_icons/apple.svg [linux icon]: docs/data/readme_icons/linux.svg - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - + + + + + + + + diff --git a/docs/data/projects.yml b/docs/data/projects.yml index 366e96cd..19acaf94 100644 --- a/docs/data/projects.yml +++ b/docs/data/projects.yml @@ -213,3 +213,19 @@ ci: [github, travisci] os: [windows, apple, linux] notes: Supports full range of wheels, including PyPy and alternate archs. + +- name: Rtree + gh: Toblerity/rtree + ci: [github] + os: [windows, apple, linux] + +- name: psutil + gh: giampaolo/psutil + ci: [github] + os: [windows, apple, linux] + +- name: Dependency Injector + gh: ets-labs/python-dependency-injector + ci: [travisci] + os: [windows, apple, linux] + notes: Dependency injection framework for Python, uses Windows TravisCI diff --git a/docs/data/readme_icons/apple.svg b/docs/data/readme_icons/apple.svg index f8780367..abef0307 100644 --- a/docs/data/readme_icons/apple.svg +++ b/docs/data/readme_icons/apple.svg @@ -1 +1 @@ -Apple icon \ No newline at end of file +Apple icon \ No newline at end of file diff --git a/docs/data/readme_icons/travisci.svg b/docs/data/readme_icons/travisci.svg index 3efbee2a..e2dfd05c 100644 --- a/docs/data/readme_icons/travisci.svg +++ b/docs/data/readme_icons/travisci.svg @@ -1 +1 @@ -Travis CI icon \ No newline at end of file +Travis CI icon \ No newline at end of file From 723b5e5128954599de9bf03f5353b751d0af73bd Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 21 Dec 2020 11:49:31 -0500 Subject: [PATCH 24/37] feat: add support for composite-actions --- action.yml | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 action.yml diff --git a/action.yml b/action.yml new file mode 100644 index 00000000..9d23e023 --- /dev/null +++ b/action.yml @@ -0,0 +1,26 @@ +name: cibuildwheel +description: 'Installs and runs cibuildwheel on the current runner' +inputs: + package-dir: + description: 'Input directory, defaults to "."' + required: false + default: . + output-dir: + description: 'Folder to place the outputs in, defaults to "wheelhouse"' + required: false + default: wheelhouse +branding: + icon: package + color: yellow + +runs: + using: composite + steps: + + # This needs powershell-core because github.action path is a Windows-style + # path on Windows. Powershell-core understands both types of paths. + - run: python -m pip install ${{ github.action_path }} + shell: pwsh + + - run: python -m cibuildwheel ${{ inputs.package-dir }} --output-dir ${{ inputs.output-dir }} + shell: bash From 93a095d3dea6142d06c17e452b0f2551fd4e1dc7 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 23 Dec 2020 16:11:15 -0500 Subject: [PATCH 25/37] docs: include action form of cibuildwheel --- bin/bump_version.py | 7 ++++-- docs/faq.md | 53 +++++++++++++++++++++++++++++++++++++++++++++ docs/setup.md | 2 ++ 3 files changed, 60 insertions(+), 2 deletions(-) diff --git a/bin/bump_version.py b/bin/bump_version.py index 45744d28..2330efda 100755 --- a/bin/bump_version.py +++ b/bin/bump_version.py @@ -11,10 +11,13 @@ import urllib.parse config = [ # file path, version find/replace format - ('cibuildwheel/__init__.py', "__version__ = '{}'"), - ('setup.py', "version='{}'"), ('README.md', "cibuildwheel=={}"), + ('cibuildwheel/__init__.py', "__version__ = '{}'"), + ('docs/faq.md', "cibuildwheel=={}"), + ('docs/faq.md', "cibuildwheel@v{}"), + ('docs/setup.md', "cibuildwheel@v{}"), ('examples/*', "cibuildwheel=={}"), + ('setup.py', "version='{}'"), ] diff --git a/docs/faq.md b/docs/faq.md index 2ff777f6..75abfc7a 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -81,3 +81,56 @@ To add the `/d2FH4-` flag to a standard `setup.py` using `setuptools`, the `extr ``` To investigate the dependencies of a C extension (i.e., the `.pyd` file, a DLL in disguise) on Windows, [Dependency Walker](http://www.dependencywalker.com/) is a great tool. + + +### Automatic updates {: #automatic-updates} + +Selecting a moving target (like the latest release) is generally a bad idea in CI. If something breaks, you can't tell whether it was your code or an upstream update that caused the breakage, and in a worse-case scenario, it could occur during a release. +There are two suggested methods for keeping cibuildwheel up to date that instead involve scheduled pull requests using GitHub's dependabot. + +#### Option 1: GitHub Action + +If you use GitHub Actions for builds, you can use cibuildwheel as an action: + +```yaml +uses: joerick/cibuildwheel@v1.7.1 +``` + +This is a composite step that just installs and runs cibuildwheel. You can set command-line options as `with:` parameters, and use `env:` as normal. + +Then, your `dependabot.yml` file could look like this: + +```yaml +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + ignore: + # Optional: Official actions have moving tags like v1; + # if you use those, you don't need updates. + - dependency-name: "actions/*" +``` + +#### Option 2: Requirement files + +The second option, and the only one that supports other CI systems, is using a `requirements-*.txt` file. The file should have a distinct name and have only one entry: + +```bash +# requirements-cibw.txt +cibuildwheel==1.7.1 +``` + +Then your install step would have `python -m pip install -r requirements-cibw.txt` in it. Your `dependabot.yml` file could look like this: + +```yaml +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "daily" +``` + +This will also try to update other pins in all requirement files, so be sure you want to do that. The only control you have over the files used is via the directory option. diff --git a/docs/setup.md b/docs/setup.md index c454d877..0f6b52d1 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -17,6 +17,8 @@ For more info on this file, check out the [docs](https://help.github.com/en/acti [`examples/github-deploy.yml`](https://github.com/joerick/cibuildwheel/blob/master/examples/github-deploy.yml) extends this minimal example with a demonstration of how to automatically upload the built wheels to PyPI. +You can also use cibuildwheel directly as an action with `uses: joerick/cibuildwheel@v1.7.1`; this combines the download and run steps into a single action, and command line arguments are available via `with:`. This makes it easy to manage cibuildwheel updates via normal actions update mechanisms like dependabot, see [Automatic updates](faq.md#automatic-updates). + # Azure Pipelines [linux/mac/windows] {: #azure-pipelines} To build Linux, Mac, and Windows wheels on Azure Pipelines, create a `azure-pipelines.yml` file in your repo. From 4e17c275efff28bae8577d84515e78b62e204f55 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 23 Dec 2020 16:31:42 -0500 Subject: [PATCH 26/37] fix: adding pyproject.toml --- pyproject.toml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..285367ae --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[build-system] +requires = [ + "setuptools>=42", + "wheel" +] + +build-backend = "setuptools.build_meta" From 143febd9fa39f46a1ac02bd8b34a8be4e8cf42f4 Mon Sep 17 00:00:00 2001 From: mayeut Date: Mon, 28 Dec 2020 06:10:06 +0000 Subject: [PATCH 27/37] Update dependencies --- .../resources/constraints-python27.txt | 2 +- .../resources/constraints-python36.txt | 2 +- .../resources/pinned_docker_images.cfg | 18 +++++++++--------- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/cibuildwheel/resources/constraints-python27.txt b/cibuildwheel/resources/constraints-python27.txt index eb6d018c..5b4be31c 100644 --- a/cibuildwheel/resources/constraints-python27.txt +++ b/cibuildwheel/resources/constraints-python27.txt @@ -11,7 +11,7 @@ delocate==0.8.2 # via -r cibuildwheel/resources/constraints.in distlib==0.3.1 # via virtualenv filelock==3.0.12 # via virtualenv importlib-metadata==2.1.1 # via virtualenv -importlib-resources==3.3.0 # via virtualenv +importlib-resources==3.3.1 # via virtualenv pathlib2==2.3.5 # via importlib-metadata, importlib-resources, virtualenv scandir==1.10.0 # via pathlib2 singledispatch==3.4.0.3 # via importlib-resources diff --git a/cibuildwheel/resources/constraints-python36.txt b/cibuildwheel/resources/constraints-python36.txt index 4e782508..80abcb1d 100644 --- a/cibuildwheel/resources/constraints-python36.txt +++ b/cibuildwheel/resources/constraints-python36.txt @@ -9,7 +9,7 @@ delocate==0.8.2 # via -r cibuildwheel/resources/constraints.in distlib==0.3.1 # via virtualenv filelock==3.0.12 # via virtualenv importlib-metadata==3.3.0 # via virtualenv -importlib-resources==3.3.0 # via virtualenv +importlib-resources==4.1.1 # via virtualenv six==1.15.0 # via virtualenv typing-extensions==3.7.4.3 # via importlib-metadata virtualenv==20.2.2 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/pinned_docker_images.cfg b/cibuildwheel/resources/pinned_docker_images.cfg index ec1b0fd3..35033aa1 100644 --- a/cibuildwheel/resources/pinned_docker_images.cfg +++ b/cibuildwheel/resources/pinned_docker_images.cfg @@ -1,22 +1,22 @@ [x86_64] -manylinux1 = quay.io/pypa/manylinux1_x86_64:2020-12-18-c48c073 -manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2020-12-19-8df9e2d -manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2020-12-18-cb453dc +manylinux1 = quay.io/pypa/manylinux1_x86_64:2020-12-25-25a002b +manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2020-12-25-719135c +manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2020-12-25-6634688 [i686] -manylinux1 = quay.io/pypa/manylinux1_i686:2020-12-18-c48c073 -manylinux2010 = quay.io/pypa/manylinux2010_i686:2020-12-19-8df9e2d -manylinux2014 = quay.io/pypa/manylinux2014_i686:2020-12-18-cb453dc +manylinux1 = quay.io/pypa/manylinux1_i686:2020-12-25-25a002b +manylinux2010 = quay.io/pypa/manylinux2010_i686:2020-12-25-719135c +manylinux2014 = quay.io/pypa/manylinux2014_i686:2020-12-25-6634688 [pypy_x86_64] manylinux2010 = pypywheels/manylinux2010-pypy_x86_64:2020-12-11-f1e0e80 [aarch64] -manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2020-12-18-cb453dc +manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2020-12-25-6634688 [ppc64le] -manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2020-12-18-cb453dc +manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2020-12-25-6634688 [s390x] -manylinux2014 = quay.io/pypa/manylinux2014_s390x:2020-12-18-cb453dc +manylinux2014 = quay.io/pypa/manylinux2014_s390x:2020-12-25-6634688 From b5f543711495ff55f896adc409ac2293bdcc4aaf Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 31 Dec 2020 14:56:19 +0000 Subject: [PATCH 28/37] Suggestion from review --- bin/run_example_ci_configs.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/run_example_ci_configs.py b/bin/run_example_ci_configs.py index 85a5e671..9ffe1717 100755 --- a/bin/run_example_ci_configs.py +++ b/bin/run_example_ci_configs.py @@ -67,9 +67,12 @@ services = [ def ci_service_for_config_file(config_file): + service_name = Path(config_file).name.rsplit('-', 1)[0] + for service in services: - if Path(config_file).name.startswith(service.name+'-'): + if service.name == service_name: return service + raise ValueError(f'unknown ci service for config file {config_file}') From 5ee663fe125b56b7f5bea5f0d6e189ebdaac6b45 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 31 Dec 2020 15:17:24 +0000 Subject: [PATCH 29/37] Add test for failure when setting CIBW_ARCHS on macos/windows --- test/test_emulation.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/test/test_emulation.py b/test/test_emulation.py index 2b5fe7b1..e486327c 100644 --- a/test/test_emulation.py +++ b/test/test_emulation.py @@ -1,3 +1,4 @@ +import subprocess import pytest from . import utils from . import test_projects @@ -33,3 +34,20 @@ def test(tmp_path): + utils.expected_wheels('spam', '0.1.0', machine_arch='s390x') ) assert set(actual_wheels) == set(expected_wheels) + + +def test_setting_arch_on_other_platforms(tmp_path, capfd): + if utils.platform == 'linux': + pytest.skip('this test checks the behaviour on platforms other than linux') + + project_dir = tmp_path / 'project' + project_with_a_test.generate(project_dir) + + # build and test the wheels + with pytest.raises(subprocess.CalledProcessError): + utils.cibuildwheel_run(project_dir, add_env={ + 'CIBW_ARCHS': 'aarch64', + }) + + captured = capfd.readouterr() + assert "Invalid archs option" in captured.err From f43407210a84280bb26f1b348e3b4c54a6ec8f76 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 31 Dec 2020 15:22:19 +0000 Subject: [PATCH 30/37] option ordering --- cibuildwheel/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 3000ace8..a2196b09 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -158,10 +158,10 @@ class Architecture(str, Enum): class BuildOptions(NamedTuple): - architectures: List[Architecture] package_dir: Path output_dir: Path build_selector: BuildSelector + architectures: List[Architecture] environment: ParsedEnvironment before_all: str before_build: Optional[str] From d65da2ee4604b89366808934eb14d15c694af380 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 31 Dec 2020 15:44:27 +0000 Subject: [PATCH 31/37] docs edits from review --- docs/faq.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index d7b91149..5fd02f64 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -35,9 +35,9 @@ is via the [docker/setup-qemu-action][setup-qemu-action] on Github Actions or [binfmt]: https://hub.docker.com/r/tonistiigi/binfmt Check out the following config for an example of how to set it up on Github -Actions. Once qemu is set up and registered, you just need to set the -`CIBW_ARCHS_LINUX` variable, and the other architectures are emulated -automatically. +Actions. Once QEMU is set up and registered, you just need to set the +`CIBW_ARCHS_LINUX` environment variable (or use the `--archs` option on +Linux), and the other architectures are emulated automatically. > .github/workflows/build.yml ```yaml From 5b5078e5e9b5b58c0b03fc0077a2b74af3895a0e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 31 Dec 2020 15:45:02 +0000 Subject: [PATCH 32/37] Copy edit of --archs help text to make 'auto' clearer --- cibuildwheel/__main__.py | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 99e2e0dd..5f48070f 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -69,18 +69,16 @@ def main() -> None: Python. Default: auto. ''') - parser.add_argument( - '--archs', - default=None, - help=''' - Comma-separated list of CPU architectures to build for. - If unspecified, builds the architectures natively supported - on this machine. Set this option to build an architecture - via emulation, for example, using binfmt_misc and qemu. - Default: auto - Choices: auto, {} - '''.format(", ".join(a.name for a in Architecture)), - ) + parser.add_argument('--archs', + default=None, + help=''' + Comma-separated list of CPU architectures to build for. + When set to 'auto', builds the architectures natively supported + on this machine. Set this option to build an architecture + via emulation, for example, using binfmt_misc and QEMU. + Default: auto. + Choices: auto, {} + '''.format(", ".join(a.name for a in Architecture))) parser.add_argument('--output-dir', default=os.environ.get('CIBW_OUTPUT_DIR', 'wheelhouse'), help='Destination folder for the wheels.') From aa871284fcc10ee5c3a3d3e2956d867cfe718c97 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 31 Dec 2020 16:35:26 +0000 Subject: [PATCH 33/37] Make the StrEnums proper Enums --- cibuildwheel/linux.py | 2 +- cibuildwheel/util.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 7e71bbae..ce1bc057 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -65,7 +65,7 @@ def get_python_configurations( # and match the build/skip rules return [ c for c in python_configurations - if any(c.identifier.endswith(arch) for arch in architectures) + if any(c.identifier.endswith(arch.value) for arch in architectures) and build_selector(c.identifier) ] diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index a2196b09..70eaa44d 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -123,7 +123,7 @@ class DependencyConstraints: return f'{self.__class__.__name__}{self.base_file_path!r})' -class Architecture(str, Enum): +class Architecture(Enum): # mac/linux archs x86_64 = 'x86_64' i686 = 'i686' @@ -201,7 +201,7 @@ def strtobool(val: str) -> bool: return False -class CIProvider(str, Enum): +class CIProvider(Enum): travis_ci = 'travis' appveyor = 'appveyor' circle_ci = 'circle_ci' From 99203a83f9ea5bff2cdeff4c65c37f4e334086e6 Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Thu, 31 Dec 2020 17:50:19 +0100 Subject: [PATCH 34/37] fix: patching pypy venv (#502) Potentially revert / remove on release of PyPy 7.3.4. * patching linux and macos * try test * add patch files for pypy2.7 * fix installing * Combine patch files into one * Simplify further, and add test to remind ourselves to remove the PyPy patch * ignore fail of patch on linux make new test failing only on linux * fix test * add timeout * fix patch on linux * change `-N` to `--force` Co-authored-by: Yannick Jadoul --- bin/run_tests.py | 2 +- cibuildwheel/linux.py | 10 +++++++ cibuildwheel/macos.py | 4 +++ cibuildwheel/resources/pypy_venv.patch | 36 +++++++++++++++++++++++ cibuildwheel/resources/pypy_venv_27.patch | 27 +++++++++++++++++ requirements-dev.txt | 1 + test/test_before_test.py | 6 ++-- test/test_testing.py | 4 +++ 8 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 cibuildwheel/resources/pypy_venv.patch create mode 100644 cibuildwheel/resources/pypy_venv_27.patch diff --git a/bin/run_tests.py b/bin/run_tests.py index 17f06aa5..b46bfbe6 100755 --- a/bin/run_tests.py +++ b/bin/run_tests.py @@ -17,4 +17,4 @@ if __name__ == '__main__': subprocess.check_call(unit_test_args) # run the integration tests - subprocess.check_call([sys.executable, '-m', 'pytest', '-x', '--durations', '0', 'test']) + subprocess.check_call([sys.executable, '-m', 'pytest', '-x', '--durations', '0', '--timeout=1200', 'test']) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index cfd4952c..e3edf494 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -138,6 +138,16 @@ def build(options: BuildOptions) -> None: log.build_start(config.identifier) dependency_constraint_flags: List[Union[str, PathLike]] = [] + if config.identifier.startswith("pp"): + # Patch PyPy to make sure headers get installed into a venv + patch_version = '_27' if config.version == '2.7' else '' + patch_path = Path(__file__).absolute().parent / 'resources' / f'pypy_venv{patch_version}.patch' + patch_docker_path = PurePath('/pypy_venv.patch') + docker.copy_into(patch_path, patch_docker_path) + try: + docker.call(['patch', '--force', '-p1', '-d', config.path, '-i', patch_docker_path]) + except subprocess.CalledProcessError: + print("PyPy patch not applied", file=sys.stderr) if options.dependency_constraints: constraints_file = options.dependency_constraints.get_for_python_version(config.version) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index c190981a..b15cf10c 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -105,6 +105,10 @@ def install_pypy(version: str, url: str) -> Path: downloaded_tar_bz2 = Path("/tmp") / pypy_tar_bz2 download(url, downloaded_tar_bz2) call(['tar', '-C', '/tmp', '-xf', downloaded_tar_bz2]) + # Patch PyPy to make sure headers get installed into a venv + patch_version = '_27' if version == '2.7' else '' + patch_path = Path(__file__).absolute().parent / 'resources' / f'pypy_venv{patch_version}.patch' + call(['patch', '--force', '-d', installation_path, patch_path]) installation_bin_path = installation_path / 'bin' python_executable = 'pypy3' if version[0] == '3' else 'pypy' diff --git a/cibuildwheel/resources/pypy_venv.patch b/cibuildwheel/resources/pypy_venv.patch new file mode 100644 index 00000000..49207b1a --- /dev/null +++ b/cibuildwheel/resources/pypy_venv.patch @@ -0,0 +1,36 @@ +diff --git a/lib-python/3/sysconfig.py b/lib-python/3/sysconfig.py +index 0d08cde66b4150e1772c13fdc6fe34d079c2b10d..30c77d165a2505d98bc7660ff0cc3b728f56c682 100644 +--- a/lib-python/3/sysconfig.py ++++ b/lib-python/3/sysconfig.py +@@ -618,9 +618,6 @@ def get_config_vars(*args): + import _osx_support + _osx_support.customize_config_vars(_CONFIG_VARS) + +- _CONFIG_VARS['INCLUDEPY'] = os.path.join(_CONFIG_VARS['prefix'], +- 'include') +- + if args: + vals = [] + for name in args: +diff --git a/lib_pypy/_sysconfigdata.py b/lib_pypy/_sysconfigdata.py +index f1cca41a4b7553819bb1a2a63af1c1a9b8591ce9..a34216e8e12b82abd72fabdd7793b3e8bb19bbcf 100644 +--- a/lib_pypy/_sysconfigdata.py ++++ b/lib_pypy/_sysconfigdata.py +@@ -5,6 +5,7 @@ from distutils.spawn import find_executable + + so_ext = _imp.extension_suffixes()[0] + ++mybase = os.path.dirname(os.path.dirname(__file__)) + + build_time_vars = { + # SOABI is PEP 3149 compliant, but CPython3 has so_ext.split('.')[1] +@@ -25,7 +26,8 @@ build_time_vars = { + 'AR': "ar", + 'ARFLAGS': "rc", + 'EXE': "", +- 'LIBDIR': os.path.join(sys.prefix, 'bin'), ++ 'LIBDIR': os.path.join(mybase, 'lib'), ++ 'INCLUDEPY': os.path.join(mybase, 'include'), + } + + if find_executable("gcc"): diff --git a/cibuildwheel/resources/pypy_venv_27.patch b/cibuildwheel/resources/pypy_venv_27.patch new file mode 100644 index 00000000..4dd31537 --- /dev/null +++ b/cibuildwheel/resources/pypy_venv_27.patch @@ -0,0 +1,27 @@ +--- a/lib-python/2.7/sysconfig.py ++++ b/lib-python/2.7/sysconfig.py +@@ -553,9 +553,6 @@ def get_config_vars(*args): + if type_ == imp.C_EXTENSION: + _CONFIG_VARS['SOABI'] = suffix.split('.')[1] + break +- _CONFIG_VARS['INCLUDEPY'] = os.path.join(_CONFIG_VARS['prefix'], +- 'include') +- + if args: + vals = [] + for name in args: +--- a/lib_pypy/_sysconfigdata.py ++++ b/lib_pypy/_sysconfigdata.py +@@ -1,5 +1,11 @@ +-import imp ++import imp, os ++ ++mybase = os.path.dirname(os.path.dirname(__file__)) + + build_time_vars = { + "SO": [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0] + } ++ ++build_time_vars['LIBDIR'] = os.path.join(mybase, 'lib') ++build_time_vars['INCLUDEPY'] = os.path.join(mybase, 'include') ++ diff --git a/requirements-dev.txt b/requirements-dev.txt index bfe96baa..304cf212 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,6 @@ -e . pytest +pytest-timeout mkdocs==1.0.4 mkdocs-include-markdown-plugin==2.1.1 pymdown-extensions diff --git a/test/test_before_test.py b/test/test_before_test.py index d0dad59d..bb87bc59 100644 --- a/test/test_before_test.py +++ b/test/test_before_test.py @@ -37,13 +37,15 @@ class TestBeforeTest(TestCase): def test(tmp_path): project_dir = tmp_path / 'project' before_test_project.generate(project_dir) + test_project_dir = project_dir / 'dependency' + test_projects.new_c_project().generate(test_project_dir) # build the wheels actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ # write python version information to a temporary file, this is # checked in setup.py - 'CIBW_BEFORE_TEST': '''python -c "import sys; open('/tmp/pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('/tmp/pythonprefix.txt', 'w').write(sys.prefix)"''', - 'CIBW_BEFORE_TEST_WINDOWS': '''python -c "import sys; open('c:\\pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('c:\\pythonprefix.txt', 'w').write(sys.prefix)"''', + 'CIBW_BEFORE_TEST': '''python -c "import sys; open('/tmp/pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('/tmp/pythonprefix.txt', 'w').write(sys.prefix)" && python -m pip install {project}/dependency''', + 'CIBW_BEFORE_TEST_WINDOWS': '''python -c "import sys; open('c:\\pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('c:\\pythonprefix.txt', 'w').write(sys.prefix)" && python -m pip install {project}/dependency''', 'CIBW_TEST_REQUIRES': 'nose', # the 'false ||' bit is to ensure this command runs in a shell on # mac/linux. diff --git a/test/test_testing.py b/test/test_testing.py index 6136c4aa..ea5122c4 100644 --- a/test/test_testing.py +++ b/test/test_testing.py @@ -64,6 +64,10 @@ class TestSpam(TestCase): bits = struct.calcsize("P") * 8 if bits == 32: self.assertEqual(platform.machine(), "i686") + + def test_time_to_remove_the_pypy_venv_patch(self): + if sys.platform == "darwin": + assert not hasattr(sys, "pypy_version_info") or sys.pypy_version_info < (7,3,4) ''' From a015f3425c1dded925f8a78cedd869b0fe5a72c2 Mon Sep 17 00:00:00 2001 From: henryiii Date: Thu, 31 Dec 2020 16:52:34 +0000 Subject: [PATCH 35/37] Update dependencies --- .../resources/constraints-python27.txt | 65 ++++++++++++++----- .../resources/constraints-python35.txt | 40 ++++++++---- .../resources/constraints-python36.txt | 43 ++++++++---- .../resources/constraints-python37.txt | 38 +++++++---- .../resources/constraints-python38.txt | 29 ++++++--- .../resources/constraints-python39.txt | 29 ++++++--- cibuildwheel/resources/constraints.txt | 29 ++++++--- .../resources/pinned_docker_images.cfg | 18 ++--- 8 files changed, 200 insertions(+), 91 deletions(-) diff --git a/cibuildwheel/resources/constraints-python27.txt b/cibuildwheel/resources/constraints-python27.txt index 5b4be31c..d5d41c2d 100644 --- a/cibuildwheel/resources/constraints-python27.txt +++ b/cibuildwheel/resources/constraints-python27.txt @@ -4,23 +4,52 @@ # # bin/update_dependencies.py # -appdirs==1.4.4 # via virtualenv -configparser==4.0.2 # via importlib-metadata -contextlib2==0.6.0.post1 # via importlib-metadata, importlib-resources -delocate==0.8.2 # via -r cibuildwheel/resources/constraints.in -distlib==0.3.1 # via virtualenv -filelock==3.0.12 # via virtualenv -importlib-metadata==2.1.1 # via virtualenv -importlib-resources==3.3.1 # via virtualenv -pathlib2==2.3.5 # via importlib-metadata, importlib-resources, virtualenv -scandir==1.10.0 # via pathlib2 -singledispatch==3.4.0.3 # via importlib-resources -six==1.15.0 # via pathlib2, virtualenv -typing==3.7.4.3 # via importlib-resources -virtualenv==20.2.2 # via -r cibuildwheel/resources/constraints.in -wheel==0.36.2 # via -r cibuildwheel/resources/constraints.in, delocate -zipp==1.2.0 # via importlib-metadata, importlib-resources +appdirs==1.4.4 + # via virtualenv +configparser==4.0.2 + # via importlib-metadata +contextlib2==0.6.0.post1 + # via + # importlib-metadata + # importlib-resources +delocate==0.8.2 + # via -r cibuildwheel/resources/constraints.in +distlib==0.3.1 + # via virtualenv +filelock==3.0.12 + # via virtualenv +importlib-metadata==2.1.1 + # via virtualenv +importlib-resources==3.3.1 + # via virtualenv +pathlib2==2.3.5 + # via + # importlib-metadata + # importlib-resources + # virtualenv +scandir==1.10.0 + # via pathlib2 +singledispatch==3.4.0.3 + # via importlib-resources +six==1.15.0 + # via + # pathlib2 + # virtualenv +typing==3.7.4.3 + # via importlib-resources +virtualenv==20.2.2 + # via -r cibuildwheel/resources/constraints.in +wheel==0.36.2 + # via + # -r cibuildwheel/resources/constraints.in + # delocate +zipp==1.2.0 + # via + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: -pip==20.3.3 # via -r cibuildwheel/resources/constraints.in -setuptools==44.1.1 # via -r cibuildwheel/resources/constraints.in +pip==20.3.3 + # via -r cibuildwheel/resources/constraints.in +setuptools==44.1.1 + # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python35.txt b/cibuildwheel/resources/constraints-python35.txt index a561e8d3..d51a1299 100644 --- a/cibuildwheel/resources/constraints-python35.txt +++ b/cibuildwheel/resources/constraints-python35.txt @@ -4,17 +4,33 @@ # # bin/update_dependencies.py # -appdirs==1.4.4 # via virtualenv -delocate==0.8.2 # via -r cibuildwheel/resources/constraints.in -distlib==0.3.1 # via virtualenv -filelock==3.0.12 # via virtualenv -importlib-metadata==2.1.1 # via virtualenv -importlib-resources==3.2.1 # via virtualenv -six==1.15.0 # via virtualenv -virtualenv==20.2.2 # via -r cibuildwheel/resources/constraints.in -wheel==0.36.2 # via -r cibuildwheel/resources/constraints.in, delocate -zipp==1.2.0 # via importlib-metadata, importlib-resources +appdirs==1.4.4 + # via virtualenv +delocate==0.8.2 + # via -r cibuildwheel/resources/constraints.in +distlib==0.3.1 + # via virtualenv +filelock==3.0.12 + # via virtualenv +importlib-metadata==2.1.1 + # via virtualenv +importlib-resources==3.2.1 + # via virtualenv +six==1.15.0 + # via virtualenv +virtualenv==20.2.2 + # via -r cibuildwheel/resources/constraints.in +wheel==0.36.2 + # via + # -r cibuildwheel/resources/constraints.in + # delocate +zipp==1.2.0 + # via + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: -pip==20.3.3 # via -r cibuildwheel/resources/constraints.in -setuptools==50.3.2 # via -r cibuildwheel/resources/constraints.in +pip==20.3.3 + # via -r cibuildwheel/resources/constraints.in +setuptools==50.3.2 + # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python36.txt b/cibuildwheel/resources/constraints-python36.txt index 80abcb1d..2d912194 100644 --- a/cibuildwheel/resources/constraints-python36.txt +++ b/cibuildwheel/resources/constraints-python36.txt @@ -4,18 +4,35 @@ # # bin/update_dependencies.py # -appdirs==1.4.4 # via virtualenv -delocate==0.8.2 # via -r cibuildwheel/resources/constraints.in -distlib==0.3.1 # via virtualenv -filelock==3.0.12 # via virtualenv -importlib-metadata==3.3.0 # via virtualenv -importlib-resources==4.1.1 # via virtualenv -six==1.15.0 # via virtualenv -typing-extensions==3.7.4.3 # via importlib-metadata -virtualenv==20.2.2 # via -r cibuildwheel/resources/constraints.in -wheel==0.36.2 # via -r cibuildwheel/resources/constraints.in, delocate -zipp==3.4.0 # via importlib-metadata, importlib-resources +appdirs==1.4.4 + # via virtualenv +delocate==0.8.2 + # via -r cibuildwheel/resources/constraints.in +distlib==0.3.1 + # via virtualenv +filelock==3.0.12 + # via virtualenv +importlib-metadata==3.3.0 + # via virtualenv +importlib-resources==4.1.1 + # via virtualenv +six==1.15.0 + # via virtualenv +typing-extensions==3.7.4.3 + # via importlib-metadata +virtualenv==20.2.2 + # via -r cibuildwheel/resources/constraints.in +wheel==0.36.2 + # via + # -r cibuildwheel/resources/constraints.in + # delocate +zipp==3.4.0 + # via + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: -pip==20.3.3 # via -r cibuildwheel/resources/constraints.in -setuptools==51.1.0 # via -r cibuildwheel/resources/constraints.in +pip==20.3.3 + # via -r cibuildwheel/resources/constraints.in +setuptools==51.1.1 + # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python37.txt b/cibuildwheel/resources/constraints-python37.txt index 9e7b421b..907b755b 100644 --- a/cibuildwheel/resources/constraints-python37.txt +++ b/cibuildwheel/resources/constraints-python37.txt @@ -4,17 +4,31 @@ # # bin/update_dependencies.py # -appdirs==1.4.4 # via virtualenv -delocate==0.8.2 # via -r cibuildwheel/resources/constraints.in -distlib==0.3.1 # via virtualenv -filelock==3.0.12 # via virtualenv -importlib-metadata==3.3.0 # via virtualenv -six==1.15.0 # via virtualenv -typing-extensions==3.7.4.3 # via importlib-metadata -virtualenv==20.2.2 # via -r cibuildwheel/resources/constraints.in -wheel==0.36.2 # via -r cibuildwheel/resources/constraints.in, delocate -zipp==3.4.0 # via importlib-metadata +appdirs==1.4.4 + # via virtualenv +delocate==0.8.2 + # via -r cibuildwheel/resources/constraints.in +distlib==0.3.1 + # via virtualenv +filelock==3.0.12 + # via virtualenv +importlib-metadata==3.3.0 + # via virtualenv +six==1.15.0 + # via virtualenv +typing-extensions==3.7.4.3 + # via importlib-metadata +virtualenv==20.2.2 + # via -r cibuildwheel/resources/constraints.in +wheel==0.36.2 + # via + # -r cibuildwheel/resources/constraints.in + # delocate +zipp==3.4.0 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -pip==20.3.3 # via -r cibuildwheel/resources/constraints.in -setuptools==51.1.0 # via -r cibuildwheel/resources/constraints.in +pip==20.3.3 + # via -r cibuildwheel/resources/constraints.in +setuptools==51.1.1 + # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python38.txt b/cibuildwheel/resources/constraints-python38.txt index 40bcd207..261ebccd 100644 --- a/cibuildwheel/resources/constraints-python38.txt +++ b/cibuildwheel/resources/constraints-python38.txt @@ -4,14 +4,25 @@ # # bin/update_dependencies.py # -appdirs==1.4.4 # via virtualenv -delocate==0.8.2 # via -r cibuildwheel/resources/constraints.in -distlib==0.3.1 # via virtualenv -filelock==3.0.12 # via virtualenv -six==1.15.0 # via virtualenv -virtualenv==20.2.2 # via -r cibuildwheel/resources/constraints.in -wheel==0.36.2 # via -r cibuildwheel/resources/constraints.in, delocate +appdirs==1.4.4 + # via virtualenv +delocate==0.8.2 + # via -r cibuildwheel/resources/constraints.in +distlib==0.3.1 + # via virtualenv +filelock==3.0.12 + # via virtualenv +six==1.15.0 + # via virtualenv +virtualenv==20.2.2 + # via -r cibuildwheel/resources/constraints.in +wheel==0.36.2 + # via + # -r cibuildwheel/resources/constraints.in + # delocate # The following packages are considered to be unsafe in a requirements file: -pip==20.3.3 # via -r cibuildwheel/resources/constraints.in -setuptools==51.1.0 # via -r cibuildwheel/resources/constraints.in +pip==20.3.3 + # via -r cibuildwheel/resources/constraints.in +setuptools==51.1.1 + # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python39.txt b/cibuildwheel/resources/constraints-python39.txt index 40bcd207..261ebccd 100644 --- a/cibuildwheel/resources/constraints-python39.txt +++ b/cibuildwheel/resources/constraints-python39.txt @@ -4,14 +4,25 @@ # # bin/update_dependencies.py # -appdirs==1.4.4 # via virtualenv -delocate==0.8.2 # via -r cibuildwheel/resources/constraints.in -distlib==0.3.1 # via virtualenv -filelock==3.0.12 # via virtualenv -six==1.15.0 # via virtualenv -virtualenv==20.2.2 # via -r cibuildwheel/resources/constraints.in -wheel==0.36.2 # via -r cibuildwheel/resources/constraints.in, delocate +appdirs==1.4.4 + # via virtualenv +delocate==0.8.2 + # via -r cibuildwheel/resources/constraints.in +distlib==0.3.1 + # via virtualenv +filelock==3.0.12 + # via virtualenv +six==1.15.0 + # via virtualenv +virtualenv==20.2.2 + # via -r cibuildwheel/resources/constraints.in +wheel==0.36.2 + # via + # -r cibuildwheel/resources/constraints.in + # delocate # The following packages are considered to be unsafe in a requirements file: -pip==20.3.3 # via -r cibuildwheel/resources/constraints.in -setuptools==51.1.0 # via -r cibuildwheel/resources/constraints.in +pip==20.3.3 + # via -r cibuildwheel/resources/constraints.in +setuptools==51.1.1 + # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints.txt b/cibuildwheel/resources/constraints.txt index 40bcd207..261ebccd 100644 --- a/cibuildwheel/resources/constraints.txt +++ b/cibuildwheel/resources/constraints.txt @@ -4,14 +4,25 @@ # # bin/update_dependencies.py # -appdirs==1.4.4 # via virtualenv -delocate==0.8.2 # via -r cibuildwheel/resources/constraints.in -distlib==0.3.1 # via virtualenv -filelock==3.0.12 # via virtualenv -six==1.15.0 # via virtualenv -virtualenv==20.2.2 # via -r cibuildwheel/resources/constraints.in -wheel==0.36.2 # via -r cibuildwheel/resources/constraints.in, delocate +appdirs==1.4.4 + # via virtualenv +delocate==0.8.2 + # via -r cibuildwheel/resources/constraints.in +distlib==0.3.1 + # via virtualenv +filelock==3.0.12 + # via virtualenv +six==1.15.0 + # via virtualenv +virtualenv==20.2.2 + # via -r cibuildwheel/resources/constraints.in +wheel==0.36.2 + # via + # -r cibuildwheel/resources/constraints.in + # delocate # The following packages are considered to be unsafe in a requirements file: -pip==20.3.3 # via -r cibuildwheel/resources/constraints.in -setuptools==51.1.0 # via -r cibuildwheel/resources/constraints.in +pip==20.3.3 + # via -r cibuildwheel/resources/constraints.in +setuptools==51.1.1 + # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/pinned_docker_images.cfg b/cibuildwheel/resources/pinned_docker_images.cfg index 35033aa1..9f46370d 100644 --- a/cibuildwheel/resources/pinned_docker_images.cfg +++ b/cibuildwheel/resources/pinned_docker_images.cfg @@ -1,22 +1,22 @@ [x86_64] -manylinux1 = quay.io/pypa/manylinux1_x86_64:2020-12-25-25a002b -manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2020-12-25-719135c -manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2020-12-25-6634688 +manylinux1 = quay.io/pypa/manylinux1_x86_64:2020-12-30-e2b3664 +manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2020-12-30-3161d69 +manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2020-12-30-68d3257 [i686] -manylinux1 = quay.io/pypa/manylinux1_i686:2020-12-25-25a002b -manylinux2010 = quay.io/pypa/manylinux2010_i686:2020-12-25-719135c -manylinux2014 = quay.io/pypa/manylinux2014_i686:2020-12-25-6634688 +manylinux1 = quay.io/pypa/manylinux1_i686:2020-12-30-e2b3664 +manylinux2010 = quay.io/pypa/manylinux2010_i686:2020-12-30-3161d69 +manylinux2014 = quay.io/pypa/manylinux2014_i686:2020-12-30-68d3257 [pypy_x86_64] manylinux2010 = pypywheels/manylinux2010-pypy_x86_64:2020-12-11-f1e0e80 [aarch64] -manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2020-12-25-6634688 +manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2020-12-30-68d3257 [ppc64le] -manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2020-12-25-6634688 +manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2020-12-30-68d3257 [s390x] -manylinux2014 = quay.io/pypa/manylinux2014_s390x:2020-12-25-6634688 +manylinux2014 = quay.io/pypa/manylinux2014_s390x:2020-12-30-68d3257 From da91d7944d48a98cf6a043db3ed9bf824d6999a6 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 1 Jan 2021 13:04:28 +0000 Subject: [PATCH 36/37] Fix docs version number --- docs/faq.md | 4 ++-- docs/setup.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/faq.md b/docs/faq.md index 75abfc7a..f00e1b5a 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -93,7 +93,7 @@ There are two suggested methods for keeping cibuildwheel up to date that instead If you use GitHub Actions for builds, you can use cibuildwheel as an action: ```yaml -uses: joerick/cibuildwheel@v1.7.1 +uses: joerick/cibuildwheel@v1.7.2 ``` This is a composite step that just installs and runs cibuildwheel. You can set command-line options as `with:` parameters, and use `env:` as normal. @@ -119,7 +119,7 @@ The second option, and the only one that supports other CI systems, is using a ` ```bash # requirements-cibw.txt -cibuildwheel==1.7.1 +cibuildwheel==1.7.2 ``` Then your install step would have `python -m pip install -r requirements-cibw.txt` in it. Your `dependabot.yml` file could look like this: diff --git a/docs/setup.md b/docs/setup.md index 0f6b52d1..fcb3341a 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -17,7 +17,7 @@ For more info on this file, check out the [docs](https://help.github.com/en/acti [`examples/github-deploy.yml`](https://github.com/joerick/cibuildwheel/blob/master/examples/github-deploy.yml) extends this minimal example with a demonstration of how to automatically upload the built wheels to PyPI. -You can also use cibuildwheel directly as an action with `uses: joerick/cibuildwheel@v1.7.1`; this combines the download and run steps into a single action, and command line arguments are available via `with:`. This makes it easy to manage cibuildwheel updates via normal actions update mechanisms like dependabot, see [Automatic updates](faq.md#automatic-updates). +You can also use cibuildwheel directly as an action with `uses: joerick/cibuildwheel@v1.7.2`; this combines the download and run steps into a single action, and command line arguments are available via `with:`. This makes it easy to manage cibuildwheel updates via normal actions update mechanisms like dependabot, see [Automatic updates](faq.md#automatic-updates). # Azure Pipelines [linux/mac/windows] {: #azure-pipelines} From eed12cf4c95141edd8bdad300469e985c10c95ef Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 1 Jan 2021 13:21:29 +0000 Subject: [PATCH 37/37] Bump version: v1.7.3 --- README.md | 14 +++++++++++++- cibuildwheel/__init__.py | 2 +- docs/faq.md | 4 ++-- docs/setup.md | 2 +- examples/appveyor-minimal.yml | 2 +- examples/azure-pipelines-minimal.yml | 6 +++--- examples/circleci-minimal.yml | 4 ++-- examples/github-deploy.yml | 2 +- examples/github-minimal.yml | 2 +- examples/travis-ci-deploy.yml | 2 +- examples/travis-ci-minimal.yml | 2 +- examples/travis-ci-test-and-deploy.yml | 6 +++--- setup.py | 2 +- 13 files changed, 31 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index ff420e2a..2bc84a5d 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ jobs: name: Install Python - name: Install cibuildwheel - run: python -m pip install cibuildwheel==1.7.2 + run: python -m pip install cibuildwheel==1.7.3 - name: Build wheels run: python -m cibuildwheel --output-dir wheelhouse @@ -270,6 +270,18 @@ Changelog +### 1.7.3 + +_1 January 2021_ + +- πŸ›  Added a patch for Pypy to ensure header files are available for building + in a virtualenv. (#502) +- πŸ›  Some preparatory work towards using cibuildwheel as a Github Action. + Check out + [the FAQ](https://cibuildwheel.readthedocs.io/en/stable/faq/#option-1-github-action) + for information on how to use it. We'll be fully updating the docs to this + approach in a subsequent release (#494) + ### 1.7.2 _21 December 2020_ diff --git a/cibuildwheel/__init__.py b/cibuildwheel/__init__.py index 8adfee4f..6da2111e 100644 --- a/cibuildwheel/__init__.py +++ b/cibuildwheel/__init__.py @@ -1 +1 @@ -__version__ = '1.7.2' +__version__ = '1.7.3' diff --git a/docs/faq.md b/docs/faq.md index f00e1b5a..58decc59 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -93,7 +93,7 @@ There are two suggested methods for keeping cibuildwheel up to date that instead If you use GitHub Actions for builds, you can use cibuildwheel as an action: ```yaml -uses: joerick/cibuildwheel@v1.7.2 +uses: joerick/cibuildwheel@v1.7.3 ``` This is a composite step that just installs and runs cibuildwheel. You can set command-line options as `with:` parameters, and use `env:` as normal. @@ -119,7 +119,7 @@ The second option, and the only one that supports other CI systems, is using a ` ```bash # requirements-cibw.txt -cibuildwheel==1.7.2 +cibuildwheel==1.7.3 ``` Then your install step would have `python -m pip install -r requirements-cibw.txt` in it. Your `dependabot.yml` file could look like this: diff --git a/docs/setup.md b/docs/setup.md index fcb3341a..7245a9ff 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -17,7 +17,7 @@ For more info on this file, check out the [docs](https://help.github.com/en/acti [`examples/github-deploy.yml`](https://github.com/joerick/cibuildwheel/blob/master/examples/github-deploy.yml) extends this minimal example with a demonstration of how to automatically upload the built wheels to PyPI. -You can also use cibuildwheel directly as an action with `uses: joerick/cibuildwheel@v1.7.2`; this combines the download and run steps into a single action, and command line arguments are available via `with:`. This makes it easy to manage cibuildwheel updates via normal actions update mechanisms like dependabot, see [Automatic updates](faq.md#automatic-updates). +You can also use cibuildwheel directly as an action with `uses: joerick/cibuildwheel@v1.7.3`; this combines the download and run steps into a single action, and command line arguments are available via `with:`. This makes it easy to manage cibuildwheel updates via normal actions update mechanisms like dependabot, see [Automatic updates](faq.md#automatic-updates). # Azure Pipelines [linux/mac/windows] {: #azure-pipelines} diff --git a/examples/appveyor-minimal.yml b/examples/appveyor-minimal.yml index 806a9859..fd490ee2 100644 --- a/examples/appveyor-minimal.yml +++ b/examples/appveyor-minimal.yml @@ -12,7 +12,7 @@ stack: python 3.7 init: - cmd: set PATH=C:\Python37;C:\Python37\Scripts;%PATH% -install: python -m pip install cibuildwheel==1.7.2 +install: python -m pip install cibuildwheel==1.7.3 build_script: python -m cibuildwheel --output-dir wheelhouse diff --git a/examples/azure-pipelines-minimal.yml b/examples/azure-pipelines-minimal.yml index 7ce34e40..8043a861 100644 --- a/examples/azure-pipelines-minimal.yml +++ b/examples/azure-pipelines-minimal.yml @@ -6,7 +6,7 @@ jobs: - bash: | set -o errexit python3 -m pip install --upgrade pip - pip3 install cibuildwheel==1.7.2 + pip3 install cibuildwheel==1.7.3 displayName: Install dependencies - bash: cibuildwheel --output-dir wheelhouse . displayName: Build wheels @@ -20,7 +20,7 @@ jobs: - bash: | set -o errexit python3 -m pip install --upgrade pip - python3 -m pip install cibuildwheel==1.7.2 + python3 -m pip install cibuildwheel==1.7.3 displayName: Install dependencies - bash: cibuildwheel --output-dir wheelhouse . displayName: Build wheels @@ -36,7 +36,7 @@ jobs: - bash: | set -o errexit python -m pip install --upgrade pip - pip install cibuildwheel==1.7.2 + pip install cibuildwheel==1.7.3 displayName: Install dependencies - bash: cibuildwheel --output-dir wheelhouse . displayName: Build wheels diff --git a/examples/circleci-minimal.yml b/examples/circleci-minimal.yml index 116dede7..5e09945d 100644 --- a/examples/circleci-minimal.yml +++ b/examples/circleci-minimal.yml @@ -11,7 +11,7 @@ jobs: - run: name: Build the Linux wheels. command: | - pip3 install --user cibuildwheel==1.7.2 + pip3 install --user cibuildwheel==1.7.3 cibuildwheel --output-dir wheelhouse - store_artifacts: path: wheelhouse/ @@ -25,7 +25,7 @@ jobs: - run: name: Build the OS X wheels. command: | - pip3 install cibuildwheel==1.7.2 + pip3 install cibuildwheel==1.7.3 cibuildwheel --output-dir wheelhouse - store_artifacts: path: wheelhouse/ diff --git a/examples/github-deploy.yml b/examples/github-deploy.yml index 43abcd66..ce936fe8 100644 --- a/examples/github-deploy.yml +++ b/examples/github-deploy.yml @@ -28,7 +28,7 @@ jobs: - name: Install cibuildwheel run: | - python -m pip install cibuildwheel==1.7.2 + python -m pip install cibuildwheel==1.7.3 - name: Install Visual C++ for Python 2.7 if: runner.os == 'Windows' diff --git a/examples/github-minimal.yml b/examples/github-minimal.yml index 29600b35..ad18fbdc 100644 --- a/examples/github-minimal.yml +++ b/examples/github-minimal.yml @@ -20,7 +20,7 @@ jobs: - name: Install cibuildwheel run: | - python -m pip install cibuildwheel==1.7.2 + python -m pip install cibuildwheel==1.7.3 - name: Install Visual C++ for Python 2.7 if: runner.os == 'Windows' diff --git a/examples/travis-ci-deploy.yml b/examples/travis-ci-deploy.yml index 402a95e1..73299b8b 100644 --- a/examples/travis-ci-deploy.yml +++ b/examples/travis-ci-deploy.yml @@ -20,7 +20,7 @@ jobs: - ln -s /c/Python38/python.exe /c/Python38/python3.exe install: - - python3 -m pip install cibuildwheel==1.7.2 + - python3 -m pip install cibuildwheel==1.7.3 script: # build the wheels, put them into './wheelhouse' diff --git a/examples/travis-ci-minimal.yml b/examples/travis-ci-minimal.yml index d9e1b64f..080b0e5c 100644 --- a/examples/travis-ci-minimal.yml +++ b/examples/travis-ci-minimal.yml @@ -26,7 +26,7 @@ jobs: - ln -s /c/Python38/python.exe /c/Python38/python3.exe install: - - python3 -m pip install cibuildwheel==1.7.2 + - python3 -m pip install cibuildwheel==1.7.3 script: # build the wheels, put them into './wheelhouse' diff --git a/examples/travis-ci-test-and-deploy.yml b/examples/travis-ci-test-and-deploy.yml index d1cfba18..58db2b02 100644 --- a/examples/travis-ci-test-and-deploy.yml +++ b/examples/travis-ci-test-and-deploy.yml @@ -55,7 +55,7 @@ jobs: - stage: deploy name: Build and deploy Linux wheels services: docker - install: python3 -m pip install cibuildwheel==1.7.2 + install: python3 -m pip install cibuildwheel==1.7.3 script: python3 -m cibuildwheel --output-dir wheelhouse after_success: | python3 -m pip install twine @@ -65,7 +65,7 @@ jobs: name: Build and deploy macOS wheels os: osx language: shell - install: python3 -m pip install cibuildwheel==1.7.2 + install: python3 -m pip install cibuildwheel==1.7.3 script: python3 -m cibuildwheel --output-dir wheelhouse after_success: | python3 -m pip install twine @@ -75,7 +75,7 @@ jobs: name: Build and deploy Windows wheels os: windows language: shell - install: python3 -m pip install cibuildwheel==1.7.2 + install: python3 -m pip install cibuildwheel==1.7.3 script: python3 -m cibuildwheel --output-dir wheelhouse after_success: | python3 -m pip install twine diff --git a/setup.py b/setup.py index 204e7ccd..5e276555 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ long_description = (this_directory / 'README.md').read_text(encoding='utf-8') setup( name='cibuildwheel', - version='1.7.2', + version='1.7.3', install_requires=['bashlex!=0.13', 'toml', 'certifi'], description="Build Python wheels on CI with minimal configuration.", long_description=long_description,