From ce76b742515486ff2cc9bfdd4e02daaeca8f3693 Mon Sep 17 00:00:00 2001 From: Matthew Treinish Date: Thu, 20 Feb 2020 13:51:37 -0500 Subject: [PATCH 01/13] Add support for manylinux wheels on non-x86 platforms This commit adds config options and identifiers for building manylinux wheels on non-x86 platforms. Based on what travis and manylinux2014 support this includes aarch64 (ARMv8), ppce64le (power pc 64bit little endian), and the s390x platforms. Fixes #269 --- .travis.yml | 24 ++++++++++++++++++++++++ cibuildwheel/__main__.py | 12 +++++++++++- cibuildwheel/linux.py | 37 +++++++++++++++++++++++++++++++++++-- docs/options.md | 22 +++++++++++----------- 4 files changed, 81 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index d5215b04..46de3bac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,30 @@ matrix: services: docker env: PYTHON=python + # Linux Python 3 + - sudo: required + language: python + python: 3.5 + services: docker + arch: arm64 + env: PYTHON=python + + # Linux Python 3 + - sudo: required + language: python + python: 3.5 + services: docker + arch: ppc64le + env: PYTHON=python + + # Linux Python 3 + - sudo: required + language: python + python: 3.5 + services: docker + arch: s390x + env: PYTHON=python + # macOS Python 3 - os: osx env: PYTHON=python3 diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 1e5b249b..aa7f6587 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -160,6 +160,9 @@ def main(): manylinux_x86_64_image = os.environ.get('CIBW_MANYLINUX_X86_64_IMAGE', 'manylinux2010') manylinux_i686_image = os.environ.get('CIBW_MANYLINUX_I686_IMAGE', 'manylinux2010') manylinux_pypy_x86_64_image = os.environ.get('CIBW_MANYLINUX_PYPY_X86_64_IMAGE', 'manylinux2010') + manylinux_aarch64_image = os.environ.get('CIBW_MANYLINUX_AARCH64_IMAGE', 'manylinux2014') + manylinux_ppc64le_image = os.environ.get('CIBW_MANYLINUX_PPC64LE_IMAGE', 'manylinux2014') + manylinux_s390x_image = os.environ.get('CIBW_MANYLINUX_S390X_IMAGE', 'manylinux2014') default_manylinux_images_x86_64 = {'manylinux1': 'quay.io/pypa/manylinux1_x86_64', 'manylinux2010': 'quay.io/pypa/manylinux2010_x86_64', @@ -168,11 +171,18 @@ def main(): 'manylinux2010': 'quay.io/pypa/manylinux2010_i686', 'manylinux2014': 'quay.io/pypa/manylinux2014_i686'} default_manylinux_images_pypy_x86_64 = {'manylinux2010': 'pypywheels/manylinux2010-pypy_x86_64'} + default_manylinux_images_aarch64 = {'manylinux2014': 'quay.io/pypa/manylinux2014_aarch64'} + default_manylinux_images_ppc64le = {'manylinux2014': 'quay.io/pypa/manylinux2014_ppc64le'} + default_manylinux_images_s390x = {'manylinux2014': 'quay.io/pypa/manylinux2014_s390x'} build_options.update( manylinux_images={'x86_64': default_manylinux_images_x86_64.get(manylinux_x86_64_image) or manylinux_x86_64_image, 'i686': default_manylinux_images_i686.get(manylinux_i686_image) or manylinux_i686_image, - 'pypy_x86_64': default_manylinux_images_pypy_x86_64.get(manylinux_pypy_x86_64_image) or manylinux_pypy_x86_64_image}, + 'pypy_x86_64': default_manylinux_images_pypy_x86_64.get(manylinux_pypy_x86_64_image) or manylinux_pypy_x86_64_image, + 'aarch64': default_manylinux_images_aarch64.get(manylinux_aarch64_image) or manylinux_aarch64_image, + 'ppc64le': default_manylinux_images_ppc64le.get(manylinux_ppc64le_image) or manylinux_ppc64le_image, + 's390x': default_manylinux_images_s390x.get(manylinux_s390x_image) or manylinux_s390x_image, + }, ) elif platform == 'macos': pass diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index b2569692..f8801129 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -1,3 +1,4 @@ +import distutils import os import shlex import subprocess @@ -29,10 +30,39 @@ def get_python_configurations(build_selector): PythonConfiguration(identifier='cp38-manylinux_i686', path='/opt/python/cp38-cp38'), PythonConfiguration(identifier='pp27-manylinux_x86_64', path='/opt/python/pp27-pypy_73'), PythonConfiguration(identifier='pp36-manylinux_x86_64', path='/opt/python/pp36-pypy36_pp73'), + PythonConfiguration(identifier='cp35-manylinux_aarch64', path='/opt/python/cp35-cp35m'), + PythonConfiguration(identifier='cp36-manylinux_aarch64', path='/opt/python/cp36-cp36m'), + PythonConfiguration(identifier='cp37-manylinux_aarch64', path='/opt/python/cp37-cp37m'), + PythonConfiguration(identifier='cp38-manylinux_aarch64', path='/opt/python/cp38-cp38'), + PythonConfiguration(identifier='cp35-manylinux_ppc64le', path='/opt/python/cp35-cp35m'), + PythonConfiguration(identifier='cp36-manylinux_ppc64le', path='/opt/python/cp36-cp36m'), + PythonConfiguration(identifier='cp37-manylinux_ppc64le', path='/opt/python/cp37-cp37m'), + PythonConfiguration(identifier='cp38-manylinux_ppc64le', path='/opt/python/cp38-cp38'), + PythonConfiguration(identifier='cp35-manylinux_s390x', path='/opt/python/cp35-cp35m'), + PythonConfiguration(identifier='cp36-manylinux_s390x', path='/opt/python/cp36-cp36m'), + PythonConfiguration(identifier='cp37-manylinux_s390x', path='/opt/python/cp37-cp37m'), + PythonConfiguration(identifier='cp38-manylinux_s390x', path='/opt/python/cp38-cp38'), ] - # skip builds as required - return [c for c in python_configurations if build_selector(c.identifier)] + configurations = [] + for c in python_configurations: + if not build_selector(c.identifier): + continue + platform = distutils.util.get_platform() + if platform == "linux-x86_64": + if c.identifier.endswith('x86_64') or c.identifier.endswith('i686'): + configurations.append(c) + elif platform == "linux-aarch64": + if c.identifier.endswith('aarch64'): + configurations.append(c) + elif platform == "linux-ppc64le": + if c.identifier.endswith('ppc64le'): + configurations.append(c) + elif platform == "linux-s390x": + if c.identifier.endswith('s390x'): + configurations.append(c) + + return configurations def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, manylinux_images): @@ -49,6 +79,9 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef platforms = [ ('cp', 'manylinux_x86_64', manylinux_images['x86_64']), ('cp', 'manylinux_i686', manylinux_images['i686']), + ('cp', 'manylinux_aarch64', manylinux_images['aarch64']), + ('cp', 'manylinux_ppc64le', manylinux_images['ppc64le']), + ('cp', 'manylinux_s390x', manylinux_images['s390x']), ('pp', 'manylinux_x86_64', manylinux_images['pypy_x86_64']), ] diff --git a/docs/options.md b/docs/options.md index f2fab00e..77afad1f 100644 --- a/docs/options.md +++ b/docs/options.md @@ -72,15 +72,15 @@ When setting the options, you can use shell-style globbing syntax (as per [`fnma
-| | macOS 64bit | Manylinux 64bit | Manylinux 32bit | Windows 64bit | Windows 32bit | -|-----------------|---------------------|------------------------|----------------------|-----------------|----------------| -| Python 2.7 | cp27-macosx_x86_64 | cp27-manylinux_x86_64 | cp27-manylinux_i686 | cp27-win_amd64 | cp27-win32 | -| Python 3.5 | cp35-macosx_x86_64 | cp35-manylinux_x86_64 | cp35-manylinux_i686 | cp35-win_amd64 | cp35-win32 | -| Python 3.6 | cp36-macosx_x86_64 | cp36-manylinux_x86_64 | cp36-manylinux_i686 | cp36-win_amd64 | cp36-win32 | -| Python 3.7 | cp37-macosx_x86_64 | cp37-manylinux_x86_64 | cp37-manylinux_i686 | cp37-win_amd64 | cp37-win32 | -| Python 3.8 | cp38-macosx_x86_64 | cp38-manylinux_x86_64 | cp38-manylinux_i686 | cp38-win_amd64 | cp38-win32 | -| PyPy 2.7 v7.3.0 | pp27-macosx_x86_64 | pp27-manylinux_x86_64 | | | pp27-win32 | -| PyPy 3.6 v7.3.0 | pp36-macosx_x86_64 | pp36-manylinux_x86_64 | | | pp36-win32 | +| | macOS 64bit | Manylinux x86 64bit | Manylinux x86 32bit | Windows 64bit | Windows 32bit | Manylinux Armv8 64bit | Manylinux PPC64LE | Manylinux s390x | +|-----------------|---------------------|------------------------|----------------------|-----------------|----------------|------------------------|------------------------|----------------------| +| Python 2.7 | cp27-macosx_x86_64 | cp27-manylinux_x86_64 | cp27-manylinux_i686 | cp27-win_amd64 | cp27-win32 | | | | +| Python 3.5 | cp35-macosx_x86_64 | cp35-manylinux_x86_64 | cp35-manylinux_i686 | cp35-win_amd64 | cp35-win32 | cp35-manylinux_aarch64 | cp35-manylinux_ppc64le | cp35-manylinux_s390x | +| Python 3.6 | cp36-macosx_x86_64 | cp36-manylinux_x86_64 | cp36-manylinux_i686 | cp36-win_amd64 | cp36-win32 | cp36-manylinux_aarch64 | cp36-manylinux_ppc64le | cp36-manylinux_s390x | +| Python 3.7 | cp37-macosx_x86_64 | cp37-manylinux_x86_64 | cp37-manylinux_i686 | cp37-win_amd64 | cp37-win32 | cp37-manylinux_aarch64 | cp37-manylinux_ppc64le | cp37-manylinux_s390x | +| Python 3.8 | cp38-macosx_x86_64 | cp38-manylinux_x86_64 | cp38-manylinux_i686 | cp38-win_amd64 | cp38-win32 | cp38-manylinux_aarch64 | cp38-manylinux_ppc64le | cp38-manylinux_s390x | +| PyPy 2.7 v7.3.0 | pp27-macosx_x86_64 | pp27-manylinux_x86_64 | | | pp27-win32 | | | | +| PyPy 3.6 v7.3.0 | pp36-macosx_x86_64 | pp36-manylinux_x86_64 | | | pp36-win32 | | | | The list of supported and currently selected build identifiers can also be retrieved by passing the `--print-build-identifiers` flag to `cibuildwheel`. The format is `python_tag-platform_tag`, with tags similar to those in [PEP 425](https://www.python.org/dev/peps/pep-0425/#details). @@ -245,10 +245,10 @@ CIBW_REPAIR_WHEEL_COMMAND_LINUX: "auditwheel repair --lib-sdir . -w {dest_dir} { ``` -### `CIBW_MANYLINUX_X86_64_IMAGE`, `CIBW_MANYLINUX_I686_IMAGE`, `CIBW_MANYLINUX_PYPY_X86_64_IMAGE` {: #manylinux-image} +### `CIBW_MANYLINUX_X86_64_IMAGE`, `CIBW_MANYLINUX_I686_IMAGE`, `CIBW_MANYLINUX_PYPY_X86_64_IMAGE`, `CIBW_MANYLINUX_AARCH64_IMAGE`, `CIBW_MANYLINUX_PPC64LE_IMAGE`, `CIBW_MANYLINUX_S390X_IMAGE` {: #manylinux-image} > Specify alternative manylinux docker images -An alternative Docker image to be used for building [`manylinux`](https://github.com/pypa/manylinux) wheels. `cibuildwheel` will then pull these instead of the default images, [`quay.io/pypa/manylinux2010_x86_64`](https://quay.io/pypa/manylinux2010_x86_64), [`quay.io/pypa/manylinux2010_i686`](https://quay.io/pypa/manylinux2010_i686), and [`pypywheels/manylinux2010-pypy_x86_64`](https://hub.docker.com/r/pypywheels/manylinux2010-pypy_x86_64). +An alternative Docker image to be used for building [`manylinux`](https://github.com/pypa/manylinux) wheels. `cibuildwheel` will then pull these instead of the default images, [`quay.io/pypa/manylinux2010_x86_64`](https://quay.io/pypa/manylinux2010_x86_64), [`quay.io/pypa/manylinux2010_i686`](https://quay.io/pypa/manylinux2010_i686), [`quay.io/pypa/manylinux2014_aarch64`](https://quay.io/pypa/manylinux2014_aarch64), [`quay.io/pypa/manylinux2014_ppc64le`](https://quay.io/pypa/manylinux2014_ppc64le), [`quay.io/pypa/manylinux2014_s390x`](https://quay.io/pypa/manylinux2010_s390x), and [`pypywheels/manylinux2010-pypy_x86_64`](https://hub.docker.com/r/pypywheels/manylinux2010-pypy_x86_64). The value of this option can either be set to `manylinux1`, `manylinux2010` or `manylinux2014` to use the [official `manylinux` images](https://github.com/pypa/manylinux) and [PyPy `manylinux` images](https://github.com/pypy/manylinux), or any other valid Docker image name. Note that for PyPy, only the official `manylinux2010` image is currently available. From ec5670a476a735c76de7a1f7d8f8ac9e8d033a37 Mon Sep 17 00:00:00 2001 From: Matthew Treinish Date: Thu, 20 Feb 2020 15:59:43 -0500 Subject: [PATCH 02/13] Use platform instead of distutils --- cibuildwheel/linux.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index f8801129..bf1e6548 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -1,5 +1,5 @@ -import distutils import os +import platform import shlex import subprocess import sys @@ -48,17 +48,19 @@ def get_python_configurations(build_selector): for c in python_configurations: if not build_selector(c.identifier): continue - platform = distutils.util.get_platform() - if platform == "linux-x86_64": + if platform.machine == "x86_64": if c.identifier.endswith('x86_64') or c.identifier.endswith('i686'): configurations.append(c) - elif platform == "linux-aarch64": + if platform.machine == "i686": + if c.identifier.endswith('i686'): + configuration.append(c) + elif platform.machine == "aarch64": if c.identifier.endswith('aarch64'): configurations.append(c) - elif platform == "linux-ppc64le": + elif platform.machine == "ppc64le": if c.identifier.endswith('ppc64le'): configurations.append(c) - elif platform == "linux-s390x": + elif platform.machine == "s390x": if c.identifier.endswith('s390x'): configurations.append(c) From ec4ea4e011fa1d1255a0dda28e0b326d511e6e29 Mon Sep 17 00:00:00 2001 From: Matthew Treinish Date: Thu, 20 Feb 2020 16:12:54 -0500 Subject: [PATCH 03/13] Adjust tests when running on non-x86 --- test/shared/utils.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/test/shared/utils.py b/test/shared/utils.py index d0db8b1f..26512995 100644 --- a/test/shared/utils.py +++ b/test/shared/utils.py @@ -5,6 +5,7 @@ This file is added to the PYTHONPATH in the test runner at bin/run_test.py. ''' import os +import platform as pm import shutil import subprocess import sys @@ -70,7 +71,7 @@ def cibuildwheel_run(project_path, env=None, add_env=None, output_dir=None): return wheels -def expected_wheels(package_name, package_version, manylinux_versions=['manylinux1', 'manylinux2010'], +def expected_wheels(package_name, package_version, manylinux_versions=None, macosx_deployment_target=None): ''' Returns a list of expected wheels from a run of cibuildwheel. @@ -79,11 +80,22 @@ def expected_wheels(package_name, package_version, manylinux_versions=['manylinu # {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl # {python tag} and {abi tag} are closely related to the python interpreter used to build the wheel # so we'll merge them below as python_abi_tag - python_abi_tags = ['cp27-cp27m', 'cp35-cp35m', 'cp36-cp36m', 'cp37-cp37m', 'cp38-cp38', - 'pp27-pypy_73', 'pp36-pypy36_pp73'] + + + python_abi_tags = ['cp35-cp35m', 'cp36-cp36m', 'cp37-cp37m', 'cp38-cp38'] + extra_python_abi_tags = ['cp27-cp27m', 'pp27-pypy_73', 'pp36-pypy36_pp73'] + if platform == 'linux': - python_abi_tags.append('cp27-cp27mu') # python 2.7 has 2 different ABI on manylinux - architectures = {'cp': ['x86_64', 'i686'], 'pp': ['x86_64']} + if pm.machine not in ['x86_64', 'i686']: + if manylinux_versions is None: + manylinux_versions = ['manylinux2014'] + architectures = {'cp': [pm.machine]} + else: + if manylinux_versions is None: + manylinux_versions = ['manylinux1', 'manylinux2010'] + python_abi_tags += extra_python_abi_tags + python_abi_tags.append('cp27-cp27mu') # python 2.7 has 2 different ABI on manylinux + architectures = {'cp': ['x86_64', 'i686'], 'pp': ['x86_64']} platform_tags = {} for python_implemention in architectures: platform_tags[python_implemention] = [ @@ -96,12 +108,14 @@ def expected_wheels(package_name, package_version, manylinux_versions=['manylinu def get_platform_tags(python_abi_tag): return platform_tags[python_abi_tag[:2]] elif platform == 'windows': + python_api_tag += extra_python_abi_tags platform_tags = {'cp': ['win32', 'win_amd64'], 'pp': ['win32']} def get_platform_tags(python_abi_tag): return platform_tags[python_abi_tag[:2]] elif platform == 'macos': + python_api_tag += extra_python_abi_tags def get_platform_tags(python_abi_tag): default_version = '10.7' if python_abi_tag.startswith('pp') else '10.9' return ['macosx_{}_x86_64'.format((macosx_deployment_target or default_version).replace('.', '_'))] From a83ac06c8d12425f4dddcc25f6f4ced0e1ccf877 Mon Sep 17 00:00:00 2001 From: Matthew Treinish Date: Thu, 20 Feb 2020 16:17:51 -0500 Subject: [PATCH 04/13] Fix small oversights --- cibuildwheel/linux.py | 11 ++++++----- test/shared/utils.py | 10 +++++----- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index bf1e6548..c84026be 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -48,19 +48,20 @@ def get_python_configurations(build_selector): for c in python_configurations: if not build_selector(c.identifier): continue - if platform.machine == "x86_64": + pm = platform.machine() + if pm == "x86_64": if c.identifier.endswith('x86_64') or c.identifier.endswith('i686'): configurations.append(c) - if platform.machine == "i686": + if pm == "i686": if c.identifier.endswith('i686'): configuration.append(c) - elif platform.machine == "aarch64": + elif pm == "aarch64": if c.identifier.endswith('aarch64'): configurations.append(c) - elif platform.machine == "ppc64le": + elif pm == "ppc64le": if c.identifier.endswith('ppc64le'): configurations.append(c) - elif platform.machine == "s390x": + elif pm == "s390x": if c.identifier.endswith('s390x'): configurations.append(c) diff --git a/test/shared/utils.py b/test/shared/utils.py index 26512995..056fe6dc 100644 --- a/test/shared/utils.py +++ b/test/shared/utils.py @@ -81,15 +81,14 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, # {python tag} and {abi tag} are closely related to the python interpreter used to build the wheel # so we'll merge them below as python_abi_tag - python_abi_tags = ['cp35-cp35m', 'cp36-cp36m', 'cp37-cp37m', 'cp38-cp38'] extra_python_abi_tags = ['cp27-cp27m', 'pp27-pypy_73', 'pp36-pypy36_pp73'] if platform == 'linux': - if pm.machine not in ['x86_64', 'i686']: + if pm.machine() not in ['x86_64', 'i686']: if manylinux_versions is None: manylinux_versions = ['manylinux2014'] - architectures = {'cp': [pm.machine]} + architectures = {'cp': [pm.machine()]} else: if manylinux_versions is None: manylinux_versions = ['manylinux1', 'manylinux2010'] @@ -108,14 +107,15 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, def get_platform_tags(python_abi_tag): return platform_tags[python_abi_tag[:2]] elif platform == 'windows': - python_api_tag += extra_python_abi_tags + python_abi_tags += extra_python_abi_tags platform_tags = {'cp': ['win32', 'win_amd64'], 'pp': ['win32']} def get_platform_tags(python_abi_tag): return platform_tags[python_abi_tag[:2]] elif platform == 'macos': - python_api_tag += extra_python_abi_tags + python_abi_tags += extra_python_abi_tags + def get_platform_tags(python_abi_tag): default_version = '10.7' if python_abi_tag.startswith('pp') else '10.9' return ['macosx_{}_x86_64'.format((macosx_deployment_target or default_version).replace('.', '_'))] From 6ddac481c0785037d419809951e65ed12d27c822 Mon Sep 17 00:00:00 2001 From: Matthew Treinish Date: Thu, 20 Feb 2020 16:18:51 -0500 Subject: [PATCH 05/13] Fix typo --- cibuildwheel/linux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index c84026be..473c0d6e 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -54,7 +54,7 @@ def get_python_configurations(build_selector): configurations.append(c) if pm == "i686": if c.identifier.endswith('i686'): - configuration.append(c) + configurations.append(c) elif pm == "aarch64": if c.identifier.endswith('aarch64'): configurations.append(c) From 43c45b6011cf3495d524f2fbcf03a77543be649a Mon Sep 17 00:00:00 2001 From: Matthew Treinish Date: Thu, 20 Feb 2020 16:24:52 -0500 Subject: [PATCH 06/13] Update tests again --- test/01_basic/cibuildwheel_test.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/test/01_basic/cibuildwheel_test.py b/test/01_basic/cibuildwheel_test.py index 76dd4857..51875a8a 100644 --- a/test/01_basic/cibuildwheel_test.py +++ b/test/01_basic/cibuildwheel_test.py @@ -1,4 +1,5 @@ import os +import platform import utils @@ -20,7 +21,10 @@ def test_build_identifiers(): # after adding CIBW_MANYLINUX_IMAGE to support manylinux2010, there # can be multiple wheels for each wheel, though, so we need to limit # the expected wheels - expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0') - if '-manylinux' not in w or '-manylinux1' in w] + if platform.machine() in ['x86_64', 'i686']: + expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0') + if '-manylinux' not in w or '-manylinux1' in w] + else: + expected_wheels = utils.expected_wheels('spam', '0.1.0') build_identifiers = utils.cibuildwheel_get_build_identifiers(project_dir) assert len(expected_wheels) == len(build_identifiers) From 90da096b141d08cfb92acd086e663841423d4ac2 Mon Sep 17 00:00:00 2001 From: Matthew Treinish Date: Thu, 20 Feb 2020 17:29:28 -0500 Subject: [PATCH 07/13] More test updates --- test/06_docker_images/cibuildwheel_test.py | 4 +++- test/08_manylinuxXXXX_only/cibuildwheel_test.py | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/test/06_docker_images/cibuildwheel_test.py b/test/06_docker_images/cibuildwheel_test.py index 7f04b088..e08f5cf9 100644 --- a/test/06_docker_images/cibuildwheel_test.py +++ b/test/06_docker_images/cibuildwheel_test.py @@ -1,4 +1,5 @@ import os +import platform import pytest @@ -8,7 +9,8 @@ import utils def test(): project_dir = os.path.dirname(__file__) - if utils.platform != 'linux': + if utils.platform != 'linux' or platform.machine() not in ['x86_64', + 'i686']: pytest.skip('the test is only relevant to the linux build') actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ diff --git a/test/08_manylinuxXXXX_only/cibuildwheel_test.py b/test/08_manylinuxXXXX_only/cibuildwheel_test.py index 8ae314db..9abdc129 100644 --- a/test/08_manylinuxXXXX_only/cibuildwheel_test.py +++ b/test/08_manylinuxXXXX_only/cibuildwheel_test.py @@ -1,4 +1,5 @@ import os +import platform import pytest @@ -11,6 +12,9 @@ def test(manylinux_image): if utils.platform != 'linux': pytest.skip('the docker test is only relevant to the linux build') + elif platform.machine not in ['x86_64', 'i686']: + if manylinux_image in ['manylinux1', 'manylinux2010']: + pytest.skip("manylinux1 and 2010 doesn't exist for non-x86 architectures") # build the wheels # CFLAGS environment variable is necessary to fail on 'malloc_info' (on manylinux1) during compilation/linking, @@ -20,6 +24,9 @@ def test(manylinux_image): 'CIBW_MANYLINUX_X86_64_IMAGE': manylinux_image, 'CIBW_MANYLINUX_I686_IMAGE': manylinux_image, 'CIBW_MANYLINUX_PYPY_X86_64_IMAGE': manylinux_image, + 'CIBW_MANYLINUX_AARCH64_IMAGE': manylinux_image, + 'CIBW_MANYLINUX_PPC64LE_IMAGE': manylinux_image, + 'CIBW_MANYLINUX_S390X_IMAGE': manylinux_image, } if manylinux_image == 'manylinux1': # We don't have a manylinux1 image for PyPy From 7b906fb5dcef001206f27c27ddee6d9ed2e36ffe Mon Sep 17 00:00:00 2001 From: Matthew Treinish Date: Thu, 20 Feb 2020 17:43:38 -0500 Subject: [PATCH 08/13] Update README table --- README.md | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 74497bbf..ca218b68 100644 --- a/README.md +++ b/README.md @@ -15,15 +15,15 @@ Python wheels are great. Building them across **Mac, Linux, Windows**, on **mult What does it do? ---------------- -| | macOS x86_64 | manylinux i686 | manylinux x86_64 | Windows 32bit | Windows 64bit | -|---|---|---|---|---|---| -| Python 2.7 | ✅ | ✅ | ✅ | ✅¹ | ✅¹ | -| Python 3.5 | ✅ | ✅ | ✅ | ✅ | ✅ | -| Python 3.6 | ✅ | ✅ | ✅ | ✅ | ✅ | -| Python 3.7 | ✅ | ✅ | ✅ | ✅ | ✅ | -| Python 3.8 | ✅ | ✅ | ✅ | ✅ | ✅ | -| PyPy 2.7 v7.3.0 | ✅ | | ✅ | ✅ | | -| PyPy 3.6 v7.3.0 | ✅ | | ✅ | ✅ | | +| | macOS x86_64 | manylinux i686 | manylinux x86_64 | Windows 32bit | Windows 64bit | manylinux aarch64 | manylinux ppc64le | manylinux s390x | +|---|---|---|---|---|---|---|---|---| +| Python 2.7 | ✅ | ✅ | ✅ | ✅¹ | ✅¹ | | | | +| Python 3.5 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| Python 3.6 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| Python 3.7 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| Python 3.8 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| PyPy 2.7 v7.3.0 | ✅ | | ✅ | ✅ | | | | | +| PyPy 3.6 v7.3.0 | ✅ | | ✅ | ✅ | | | | | > ¹ Not supported on Travis From 5549eb31658fbc3a3ccfcb053a6cebad5f13e1a0 Mon Sep 17 00:00:00 2001 From: Matthew Treinish Date: Fri, 21 Feb 2020 06:37:45 -0500 Subject: [PATCH 09/13] Update examples and CI.md --- CI.md | 2 ++ examples/travis-ci-minimal.yml | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/CI.md b/CI.md index 3b43a2e6..009b902c 100644 --- a/CI.md +++ b/CI.md @@ -7,3 +7,5 @@ This is a summary of the Python versions and platforms covered by the different | Windows | TravisCI | Azure Pipelines | AppVeyor | Azure Pipelines | > ¹ Python version not really pinned, but dependent on the (default) version of image used. + +In addition additional non-x86 architectures are covered on Travis CI using Python 3.5. diff --git a/examples/travis-ci-minimal.yml b/examples/travis-ci-minimal.yml index e71772b7..95b4a8d7 100644 --- a/examples/travis-ci-minimal.yml +++ b/examples/travis-ci-minimal.yml @@ -4,6 +4,15 @@ jobs: include: # perform a linux build - services: docker + # perform a linux ARMv8 build + - services: docker + arch: aarch64 + # perform a linux PPC64LE build + - services: docker + arch: ppc64le + # perform a linux S390X build + - services: docker + arch: s390x # and a mac build - os: osx language: shell From 549fc8a93e524b27626cf50d3848a45943622c7a Mon Sep 17 00:00:00 2001 From: Matthew Treinish Date: Fri, 21 Feb 2020 06:42:19 -0500 Subject: [PATCH 10/13] Extract platform check to separate function --- cibuildwheel/linux.py | 43 +++++++++++++++++++++---------------------- 1 file changed, 21 insertions(+), 22 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 473c0d6e..498b0750 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -13,6 +13,26 @@ from .util import ( ) +def matches_platform(identifier): + pm = platform.machine() + if pm == "x86_64": + if identifier.endswith('x86_64') or identifier.endswith('i686'): + return True + elif pm == "i686": + if identifier.endswith('i686'): + return True + elif pm == "aarch64": + if identifier.endswith('aarch64'): + return True + elif pm == "ppc64le": + if identifier.endswith('ppc64le'): + return True + elif pm == "s390x": + if identifier.endswith('s390x'): + return True + return False + + def get_python_configurations(build_selector): PythonConfiguration = namedtuple('PythonConfiguration', ['identifier', 'path']) python_configurations = [ @@ -44,28 +64,7 @@ def get_python_configurations(build_selector): PythonConfiguration(identifier='cp38-manylinux_s390x', path='/opt/python/cp38-cp38'), ] # skip builds as required - configurations = [] - for c in python_configurations: - if not build_selector(c.identifier): - continue - pm = platform.machine() - if pm == "x86_64": - if c.identifier.endswith('x86_64') or c.identifier.endswith('i686'): - configurations.append(c) - if pm == "i686": - if c.identifier.endswith('i686'): - configurations.append(c) - elif pm == "aarch64": - if c.identifier.endswith('aarch64'): - configurations.append(c) - elif pm == "ppc64le": - if c.identifier.endswith('ppc64le'): - configurations.append(c) - elif pm == "s390x": - if c.identifier.endswith('s390x'): - configurations.append(c) - - return configurations + return [c for c in python_configurations if matches_platform(c.identifier) and build_selector(c.identifier)] def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, manylinux_images): From 36e06189929534a603f9a0794d51275008c2e8e0 Mon Sep 17 00:00:00 2001 From: Matthew Treinish Date: Fri, 21 Feb 2020 06:48:10 -0500 Subject: [PATCH 11/13] Update options.md --- docs/options.md | 5 +++-- test/shared/utils.py | 8 ++++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/docs/options.md b/docs/options.md index 77afad1f..4b4069ac 100644 --- a/docs/options.md +++ b/docs/options.md @@ -248,9 +248,10 @@ CIBW_REPAIR_WHEEL_COMMAND_LINUX: "auditwheel repair --lib-sdir . -w {dest_dir} { ### `CIBW_MANYLINUX_X86_64_IMAGE`, `CIBW_MANYLINUX_I686_IMAGE`, `CIBW_MANYLINUX_PYPY_X86_64_IMAGE`, `CIBW_MANYLINUX_AARCH64_IMAGE`, `CIBW_MANYLINUX_PPC64LE_IMAGE`, `CIBW_MANYLINUX_S390X_IMAGE` {: #manylinux-image} > Specify alternative manylinux docker images -An alternative Docker image to be used for building [`manylinux`](https://github.com/pypa/manylinux) wheels. `cibuildwheel` will then pull these instead of the default images, [`quay.io/pypa/manylinux2010_x86_64`](https://quay.io/pypa/manylinux2010_x86_64), [`quay.io/pypa/manylinux2010_i686`](https://quay.io/pypa/manylinux2010_i686), [`quay.io/pypa/manylinux2014_aarch64`](https://quay.io/pypa/manylinux2014_aarch64), [`quay.io/pypa/manylinux2014_ppc64le`](https://quay.io/pypa/manylinux2014_ppc64le), [`quay.io/pypa/manylinux2014_s390x`](https://quay.io/pypa/manylinux2010_s390x), and [`pypywheels/manylinux2010-pypy_x86_64`](https://hub.docker.com/r/pypywheels/manylinux2010-pypy_x86_64). +An alternative Docker image to be used for building [`manylinux`](https://github.com/pypa/manylinux) wheels. `cibuildwheel` will then pull these instead of the default images, [`quay.io/pypa/manylinux2010_x86_64`](https://quay.io/pypa/manylinux2010_x86_64), [`quay.io/pypa/manylinux2010_i686`](https://quay.io/pypa/manylinux2010_i686), [`pypywheels/manylinux2010-pypy_x86_64`](https://hub.docker.com/r/pypywheels/manylinux2010-pypy_x86_64), [`quay.io/pypa/manylinux2014_aarch64`](https://quay.io/pypa/manylinux2014_aarch64), [`quay.io/pypa/manylinux2014_ppc64le`](https://quay.io/pypa/manylinux2014_ppc64le), and [`quay.io/pypa/manylinux2014_s390x`](https://quay.io/pypa/manylinux2010_s390x). -The value of this option can either be set to `manylinux1`, `manylinux2010` or `manylinux2014` to use the [official `manylinux` images](https://github.com/pypa/manylinux) and [PyPy `manylinux` images](https://github.com/pypy/manylinux), or any other valid Docker image name. Note that for PyPy, only the official `manylinux2010` image is currently available. +The value of this option can either be set to `manylinux1`, `manylinux2010` or `manylinux2014` to use the [official `manylinux` images](https://github.com/pypa/manylinux) and [PyPy `manylinux` images](https://github.com/pypy/manylinux), or any other valid Docker image name. Note that for PyPy, only the official `manylinux2010` image is currently available. For architectures other +than x86 (x86\_64 and i686) manylinux2014 must be used because this is the first version of the manylinux specification that supports additional architectures. Beware to specify a valid Docker image that can be used in the same way as the official, default Docker images: all necessary Python and pip versions need to be present in `/opt/python/`, and the `auditwheel` tool needs to be present for `cibuildwheel` to work. Apart from that, the architecture and relevant shared system libraries need to be manylinux1-, manylinux2010- or manylinux2014-compatible in order to produce valid `manylinux1`/`manylinux2010`/`manylinux2014` wheels (see [pypa/manylinux on GitHub](https://github.com/pypa/manylinux), [PEP 513](https://www.python.org/dev/peps/pep-0513/), [PEP 571](https://www.python.org/dev/peps/pep-0571/) and [PEP 599](https://www.python.org/dev/peps/pep-0599/) for more details). diff --git a/test/shared/utils.py b/test/shared/utils.py index 056fe6dc..862d4466 100644 --- a/test/shared/utils.py +++ b/test/shared/utils.py @@ -82,7 +82,7 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, # so we'll merge them below as python_abi_tag python_abi_tags = ['cp35-cp35m', 'cp36-cp36m', 'cp37-cp37m', 'cp38-cp38'] - extra_python_abi_tags = ['cp27-cp27m', 'pp27-pypy_73', 'pp36-pypy36_pp73'] + extra_x86_python_abi_tags = ['cp27-cp27m', 'pp27-pypy_73', 'pp36-pypy36_pp73'] if platform == 'linux': if pm.machine() not in ['x86_64', 'i686']: @@ -92,7 +92,7 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, else: if manylinux_versions is None: manylinux_versions = ['manylinux1', 'manylinux2010'] - python_abi_tags += extra_python_abi_tags + python_abi_tags += extra_x86_python_abi_tags python_abi_tags.append('cp27-cp27mu') # python 2.7 has 2 different ABI on manylinux architectures = {'cp': ['x86_64', 'i686'], 'pp': ['x86_64']} platform_tags = {} @@ -107,14 +107,14 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, def get_platform_tags(python_abi_tag): return platform_tags[python_abi_tag[:2]] elif platform == 'windows': - python_abi_tags += extra_python_abi_tags + python_abi_tags += extra_x86_python_abi_tags platform_tags = {'cp': ['win32', 'win_amd64'], 'pp': ['win32']} def get_platform_tags(python_abi_tag): return platform_tags[python_abi_tag[:2]] elif platform == 'macos': - python_abi_tags += extra_python_abi_tags + python_abi_tags += extra_x86_python_abi_tags def get_platform_tags(python_abi_tag): default_version = '10.7' if python_abi_tag.startswith('pp') else '10.9' From f8d298216675d065374dbd5b5e88b41bc53cd356 Mon Sep 17 00:00:00 2001 From: Matthew Treinish Date: Fri, 21 Feb 2020 09:27:47 -0500 Subject: [PATCH 12/13] Update CI.md Co-Authored-By: Joe Rickerby --- CI.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CI.md b/CI.md index 009b902c..d01ca50c 100644 --- a/CI.md +++ b/CI.md @@ -8,4 +8,4 @@ This is a summary of the Python versions and platforms covered by the different > ¹ Python version not really pinned, but dependent on the (default) version of image used. -In addition additional non-x86 architectures are covered on Travis CI using Python 3.5. +Non-x86 architectures are covered on Travis CI using Python 3.5. From 05df9bbfd413122bf6fbda0e317abfe882290416 Mon Sep 17 00:00:00 2001 From: Matthew Treinish Date: Fri, 21 Feb 2020 09:32:09 -0500 Subject: [PATCH 13/13] Add comment and update test_06 skip message --- cibuildwheel/linux.py | 1 + test/06_docker_images/cibuildwheel_test.py | 5 +++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 498b0750..dc7238a6 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -16,6 +16,7 @@ from .util import ( def matches_platform(identifier): pm = platform.machine() if pm == "x86_64": + # x86_64 machines can run i686 docker containers if identifier.endswith('x86_64') or identifier.endswith('i686'): return True elif pm == "i686": diff --git a/test/06_docker_images/cibuildwheel_test.py b/test/06_docker_images/cibuildwheel_test.py index e08f5cf9..7be4a4c3 100644 --- a/test/06_docker_images/cibuildwheel_test.py +++ b/test/06_docker_images/cibuildwheel_test.py @@ -9,9 +9,10 @@ import utils def test(): project_dir = os.path.dirname(__file__) - if utils.platform != 'linux' or platform.machine() not in ['x86_64', - 'i686']: + if utils.platform != 'linux': pytest.skip('the test is only relevant to the linux build') + if platform.machine() not in ['x86_64', 'i686']: + pytest.skip('this test is currently only possible on x86_64/i686 due to availability of alternative images') actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ 'CIBW_MANYLINUX_X86_64_IMAGE': 'dockcross/manylinux2010-x64',