diff --git a/.travis.yml b/.travis.yml index d5215b04..46de3bac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,30 @@ matrix: services: docker env: PYTHON=python + # Linux Python 3 + - sudo: required + language: python + python: 3.5 + services: docker + arch: arm64 + env: PYTHON=python + + # Linux Python 3 + - sudo: required + language: python + python: 3.5 + services: docker + arch: ppc64le + env: PYTHON=python + + # Linux Python 3 + - sudo: required + language: python + python: 3.5 + services: docker + arch: s390x + env: PYTHON=python + # macOS Python 3 - os: osx env: PYTHON=python3 diff --git a/CI.md b/CI.md index 3b43a2e6..d01ca50c 100644 --- a/CI.md +++ b/CI.md @@ -7,3 +7,5 @@ This is a summary of the Python versions and platforms covered by the different | Windows | TravisCI | Azure Pipelines | AppVeyor | Azure Pipelines | > ¹ Python version not really pinned, but dependent on the (default) version of image used. + +Non-x86 architectures are covered on Travis CI using Python 3.5. diff --git a/README.md b/README.md index 74497bbf..ca218b68 100644 --- a/README.md +++ b/README.md @@ -15,15 +15,15 @@ Python wheels are great. Building them across **Mac, Linux, Windows**, on **mult What does it do? ---------------- -| | macOS x86_64 | manylinux i686 | manylinux x86_64 | Windows 32bit | Windows 64bit | -|---|---|---|---|---|---| -| Python 2.7 | ✅ | ✅ | ✅ | ✅¹ | ✅¹ | -| Python 3.5 | ✅ | ✅ | ✅ | ✅ | ✅ | -| Python 3.6 | ✅ | ✅ | ✅ | ✅ | ✅ | -| Python 3.7 | ✅ | ✅ | ✅ | ✅ | ✅ | -| Python 3.8 | ✅ | ✅ | ✅ | ✅ | ✅ | -| PyPy 2.7 v7.3.0 | ✅ | | ✅ | ✅ | | -| PyPy 3.6 v7.3.0 | ✅ | | ✅ | ✅ | | +| | macOS x86_64 | manylinux i686 | manylinux x86_64 | Windows 32bit | Windows 64bit | manylinux aarch64 | manylinux ppc64le | manylinux s390x | +|---|---|---|---|---|---|---|---|---| +| Python 2.7 | ✅ | ✅ | ✅ | ✅¹ | ✅¹ | | | | +| Python 3.5 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| Python 3.6 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| Python 3.7 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| Python 3.8 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| PyPy 2.7 v7.3.0 | ✅ | | ✅ | ✅ | | | | | +| PyPy 3.6 v7.3.0 | ✅ | | ✅ | ✅ | | | | | > ¹ Not supported on Travis diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 1e5b249b..aa7f6587 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -160,6 +160,9 @@ def main(): manylinux_x86_64_image = os.environ.get('CIBW_MANYLINUX_X86_64_IMAGE', 'manylinux2010') manylinux_i686_image = os.environ.get('CIBW_MANYLINUX_I686_IMAGE', 'manylinux2010') manylinux_pypy_x86_64_image = os.environ.get('CIBW_MANYLINUX_PYPY_X86_64_IMAGE', 'manylinux2010') + manylinux_aarch64_image = os.environ.get('CIBW_MANYLINUX_AARCH64_IMAGE', 'manylinux2014') + manylinux_ppc64le_image = os.environ.get('CIBW_MANYLINUX_PPC64LE_IMAGE', 'manylinux2014') + manylinux_s390x_image = os.environ.get('CIBW_MANYLINUX_S390X_IMAGE', 'manylinux2014') default_manylinux_images_x86_64 = {'manylinux1': 'quay.io/pypa/manylinux1_x86_64', 'manylinux2010': 'quay.io/pypa/manylinux2010_x86_64', @@ -168,11 +171,18 @@ def main(): 'manylinux2010': 'quay.io/pypa/manylinux2010_i686', 'manylinux2014': 'quay.io/pypa/manylinux2014_i686'} default_manylinux_images_pypy_x86_64 = {'manylinux2010': 'pypywheels/manylinux2010-pypy_x86_64'} + default_manylinux_images_aarch64 = {'manylinux2014': 'quay.io/pypa/manylinux2014_aarch64'} + default_manylinux_images_ppc64le = {'manylinux2014': 'quay.io/pypa/manylinux2014_ppc64le'} + default_manylinux_images_s390x = {'manylinux2014': 'quay.io/pypa/manylinux2014_s390x'} build_options.update( manylinux_images={'x86_64': default_manylinux_images_x86_64.get(manylinux_x86_64_image) or manylinux_x86_64_image, 'i686': default_manylinux_images_i686.get(manylinux_i686_image) or manylinux_i686_image, - 'pypy_x86_64': default_manylinux_images_pypy_x86_64.get(manylinux_pypy_x86_64_image) or manylinux_pypy_x86_64_image}, + 'pypy_x86_64': default_manylinux_images_pypy_x86_64.get(manylinux_pypy_x86_64_image) or manylinux_pypy_x86_64_image, + 'aarch64': default_manylinux_images_aarch64.get(manylinux_aarch64_image) or manylinux_aarch64_image, + 'ppc64le': default_manylinux_images_ppc64le.get(manylinux_ppc64le_image) or manylinux_ppc64le_image, + 's390x': default_manylinux_images_s390x.get(manylinux_s390x_image) or manylinux_s390x_image, + }, ) elif platform == 'macos': pass diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index b2569692..dc7238a6 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -1,4 +1,5 @@ import os +import platform import shlex import subprocess import sys @@ -12,6 +13,27 @@ from .util import ( ) +def matches_platform(identifier): + pm = platform.machine() + if pm == "x86_64": + # x86_64 machines can run i686 docker containers + if identifier.endswith('x86_64') or identifier.endswith('i686'): + return True + elif pm == "i686": + if identifier.endswith('i686'): + return True + elif pm == "aarch64": + if identifier.endswith('aarch64'): + return True + elif pm == "ppc64le": + if identifier.endswith('ppc64le'): + return True + elif pm == "s390x": + if identifier.endswith('s390x'): + return True + return False + + def get_python_configurations(build_selector): PythonConfiguration = namedtuple('PythonConfiguration', ['identifier', 'path']) python_configurations = [ @@ -29,10 +51,21 @@ def get_python_configurations(build_selector): PythonConfiguration(identifier='cp38-manylinux_i686', path='/opt/python/cp38-cp38'), PythonConfiguration(identifier='pp27-manylinux_x86_64', path='/opt/python/pp27-pypy_73'), PythonConfiguration(identifier='pp36-manylinux_x86_64', path='/opt/python/pp36-pypy36_pp73'), + PythonConfiguration(identifier='cp35-manylinux_aarch64', path='/opt/python/cp35-cp35m'), + PythonConfiguration(identifier='cp36-manylinux_aarch64', path='/opt/python/cp36-cp36m'), + PythonConfiguration(identifier='cp37-manylinux_aarch64', path='/opt/python/cp37-cp37m'), + PythonConfiguration(identifier='cp38-manylinux_aarch64', path='/opt/python/cp38-cp38'), + PythonConfiguration(identifier='cp35-manylinux_ppc64le', path='/opt/python/cp35-cp35m'), + PythonConfiguration(identifier='cp36-manylinux_ppc64le', path='/opt/python/cp36-cp36m'), + PythonConfiguration(identifier='cp37-manylinux_ppc64le', path='/opt/python/cp37-cp37m'), + PythonConfiguration(identifier='cp38-manylinux_ppc64le', path='/opt/python/cp38-cp38'), + PythonConfiguration(identifier='cp35-manylinux_s390x', path='/opt/python/cp35-cp35m'), + PythonConfiguration(identifier='cp36-manylinux_s390x', path='/opt/python/cp36-cp36m'), + PythonConfiguration(identifier='cp37-manylinux_s390x', path='/opt/python/cp37-cp37m'), + PythonConfiguration(identifier='cp38-manylinux_s390x', path='/opt/python/cp38-cp38'), ] - # skip builds as required - return [c for c in python_configurations if build_selector(c.identifier)] + return [c for c in python_configurations if matches_platform(c.identifier) and build_selector(c.identifier)] def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, manylinux_images): @@ -49,6 +82,9 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef platforms = [ ('cp', 'manylinux_x86_64', manylinux_images['x86_64']), ('cp', 'manylinux_i686', manylinux_images['i686']), + ('cp', 'manylinux_aarch64', manylinux_images['aarch64']), + ('cp', 'manylinux_ppc64le', manylinux_images['ppc64le']), + ('cp', 'manylinux_s390x', manylinux_images['s390x']), ('pp', 'manylinux_x86_64', manylinux_images['pypy_x86_64']), ] diff --git a/docs/options.md b/docs/options.md index f2fab00e..4b4069ac 100644 --- a/docs/options.md +++ b/docs/options.md @@ -72,15 +72,15 @@ When setting the options, you can use shell-style globbing syntax (as per [`fnma
-| | macOS 64bit | Manylinux 64bit | Manylinux 32bit | Windows 64bit | Windows 32bit | -|-----------------|---------------------|------------------------|----------------------|-----------------|----------------| -| Python 2.7 | cp27-macosx_x86_64 | cp27-manylinux_x86_64 | cp27-manylinux_i686 | cp27-win_amd64 | cp27-win32 | -| Python 3.5 | cp35-macosx_x86_64 | cp35-manylinux_x86_64 | cp35-manylinux_i686 | cp35-win_amd64 | cp35-win32 | -| Python 3.6 | cp36-macosx_x86_64 | cp36-manylinux_x86_64 | cp36-manylinux_i686 | cp36-win_amd64 | cp36-win32 | -| Python 3.7 | cp37-macosx_x86_64 | cp37-manylinux_x86_64 | cp37-manylinux_i686 | cp37-win_amd64 | cp37-win32 | -| Python 3.8 | cp38-macosx_x86_64 | cp38-manylinux_x86_64 | cp38-manylinux_i686 | cp38-win_amd64 | cp38-win32 | -| PyPy 2.7 v7.3.0 | pp27-macosx_x86_64 | pp27-manylinux_x86_64 | | | pp27-win32 | -| PyPy 3.6 v7.3.0 | pp36-macosx_x86_64 | pp36-manylinux_x86_64 | | | pp36-win32 | +| | macOS 64bit | Manylinux x86 64bit | Manylinux x86 32bit | Windows 64bit | Windows 32bit | Manylinux Armv8 64bit | Manylinux PPC64LE | Manylinux s390x | +|-----------------|---------------------|------------------------|----------------------|-----------------|----------------|------------------------|------------------------|----------------------| +| Python 2.7 | cp27-macosx_x86_64 | cp27-manylinux_x86_64 | cp27-manylinux_i686 | cp27-win_amd64 | cp27-win32 | | | | +| Python 3.5 | cp35-macosx_x86_64 | cp35-manylinux_x86_64 | cp35-manylinux_i686 | cp35-win_amd64 | cp35-win32 | cp35-manylinux_aarch64 | cp35-manylinux_ppc64le | cp35-manylinux_s390x | +| Python 3.6 | cp36-macosx_x86_64 | cp36-manylinux_x86_64 | cp36-manylinux_i686 | cp36-win_amd64 | cp36-win32 | cp36-manylinux_aarch64 | cp36-manylinux_ppc64le | cp36-manylinux_s390x | +| Python 3.7 | cp37-macosx_x86_64 | cp37-manylinux_x86_64 | cp37-manylinux_i686 | cp37-win_amd64 | cp37-win32 | cp37-manylinux_aarch64 | cp37-manylinux_ppc64le | cp37-manylinux_s390x | +| Python 3.8 | cp38-macosx_x86_64 | cp38-manylinux_x86_64 | cp38-manylinux_i686 | cp38-win_amd64 | cp38-win32 | cp38-manylinux_aarch64 | cp38-manylinux_ppc64le | cp38-manylinux_s390x | +| PyPy 2.7 v7.3.0 | pp27-macosx_x86_64 | pp27-manylinux_x86_64 | | | pp27-win32 | | | | +| PyPy 3.6 v7.3.0 | pp36-macosx_x86_64 | pp36-manylinux_x86_64 | | | pp36-win32 | | | | The list of supported and currently selected build identifiers can also be retrieved by passing the `--print-build-identifiers` flag to `cibuildwheel`. The format is `python_tag-platform_tag`, with tags similar to those in [PEP 425](https://www.python.org/dev/peps/pep-0425/#details). @@ -245,12 +245,13 @@ CIBW_REPAIR_WHEEL_COMMAND_LINUX: "auditwheel repair --lib-sdir . -w {dest_dir} { ``` -### `CIBW_MANYLINUX_X86_64_IMAGE`, `CIBW_MANYLINUX_I686_IMAGE`, `CIBW_MANYLINUX_PYPY_X86_64_IMAGE` {: #manylinux-image} +### `CIBW_MANYLINUX_X86_64_IMAGE`, `CIBW_MANYLINUX_I686_IMAGE`, `CIBW_MANYLINUX_PYPY_X86_64_IMAGE`, `CIBW_MANYLINUX_AARCH64_IMAGE`, `CIBW_MANYLINUX_PPC64LE_IMAGE`, `CIBW_MANYLINUX_S390X_IMAGE` {: #manylinux-image} > Specify alternative manylinux docker images -An alternative Docker image to be used for building [`manylinux`](https://github.com/pypa/manylinux) wheels. `cibuildwheel` will then pull these instead of the default images, [`quay.io/pypa/manylinux2010_x86_64`](https://quay.io/pypa/manylinux2010_x86_64), [`quay.io/pypa/manylinux2010_i686`](https://quay.io/pypa/manylinux2010_i686), and [`pypywheels/manylinux2010-pypy_x86_64`](https://hub.docker.com/r/pypywheels/manylinux2010-pypy_x86_64). +An alternative Docker image to be used for building [`manylinux`](https://github.com/pypa/manylinux) wheels. `cibuildwheel` will then pull these instead of the default images, [`quay.io/pypa/manylinux2010_x86_64`](https://quay.io/pypa/manylinux2010_x86_64), [`quay.io/pypa/manylinux2010_i686`](https://quay.io/pypa/manylinux2010_i686), [`pypywheels/manylinux2010-pypy_x86_64`](https://hub.docker.com/r/pypywheels/manylinux2010-pypy_x86_64), [`quay.io/pypa/manylinux2014_aarch64`](https://quay.io/pypa/manylinux2014_aarch64), [`quay.io/pypa/manylinux2014_ppc64le`](https://quay.io/pypa/manylinux2014_ppc64le), and [`quay.io/pypa/manylinux2014_s390x`](https://quay.io/pypa/manylinux2010_s390x). -The value of this option can either be set to `manylinux1`, `manylinux2010` or `manylinux2014` to use the [official `manylinux` images](https://github.com/pypa/manylinux) and [PyPy `manylinux` images](https://github.com/pypy/manylinux), or any other valid Docker image name. Note that for PyPy, only the official `manylinux2010` image is currently available. +The value of this option can either be set to `manylinux1`, `manylinux2010` or `manylinux2014` to use the [official `manylinux` images](https://github.com/pypa/manylinux) and [PyPy `manylinux` images](https://github.com/pypy/manylinux), or any other valid Docker image name. Note that for PyPy, only the official `manylinux2010` image is currently available. For architectures other +than x86 (x86\_64 and i686) manylinux2014 must be used because this is the first version of the manylinux specification that supports additional architectures. Beware to specify a valid Docker image that can be used in the same way as the official, default Docker images: all necessary Python and pip versions need to be present in `/opt/python/`, and the `auditwheel` tool needs to be present for `cibuildwheel` to work. Apart from that, the architecture and relevant shared system libraries need to be manylinux1-, manylinux2010- or manylinux2014-compatible in order to produce valid `manylinux1`/`manylinux2010`/`manylinux2014` wheels (see [pypa/manylinux on GitHub](https://github.com/pypa/manylinux), [PEP 513](https://www.python.org/dev/peps/pep-0513/), [PEP 571](https://www.python.org/dev/peps/pep-0571/) and [PEP 599](https://www.python.org/dev/peps/pep-0599/) for more details). diff --git a/examples/travis-ci-minimal.yml b/examples/travis-ci-minimal.yml index e71772b7..95b4a8d7 100644 --- a/examples/travis-ci-minimal.yml +++ b/examples/travis-ci-minimal.yml @@ -4,6 +4,15 @@ jobs: include: # perform a linux build - services: docker + # perform a linux ARMv8 build + - services: docker + arch: aarch64 + # perform a linux PPC64LE build + - services: docker + arch: ppc64le + # perform a linux S390X build + - services: docker + arch: s390x # and a mac build - os: osx language: shell diff --git a/test/01_basic/cibuildwheel_test.py b/test/01_basic/cibuildwheel_test.py index 76dd4857..51875a8a 100644 --- a/test/01_basic/cibuildwheel_test.py +++ b/test/01_basic/cibuildwheel_test.py @@ -1,4 +1,5 @@ import os +import platform import utils @@ -20,7 +21,10 @@ def test_build_identifiers(): # after adding CIBW_MANYLINUX_IMAGE to support manylinux2010, there # can be multiple wheels for each wheel, though, so we need to limit # the expected wheels - expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0') - if '-manylinux' not in w or '-manylinux1' in w] + if platform.machine() in ['x86_64', 'i686']: + expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0') + if '-manylinux' not in w or '-manylinux1' in w] + else: + expected_wheels = utils.expected_wheels('spam', '0.1.0') build_identifiers = utils.cibuildwheel_get_build_identifiers(project_dir) assert len(expected_wheels) == len(build_identifiers) diff --git a/test/06_docker_images/cibuildwheel_test.py b/test/06_docker_images/cibuildwheel_test.py index 7f04b088..7be4a4c3 100644 --- a/test/06_docker_images/cibuildwheel_test.py +++ b/test/06_docker_images/cibuildwheel_test.py @@ -1,4 +1,5 @@ import os +import platform import pytest @@ -10,6 +11,8 @@ def test(): if utils.platform != 'linux': pytest.skip('the test is only relevant to the linux build') + if platform.machine() not in ['x86_64', 'i686']: + pytest.skip('this test is currently only possible on x86_64/i686 due to availability of alternative images') actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ 'CIBW_MANYLINUX_X86_64_IMAGE': 'dockcross/manylinux2010-x64', diff --git a/test/08_manylinuxXXXX_only/cibuildwheel_test.py b/test/08_manylinuxXXXX_only/cibuildwheel_test.py index 8ae314db..9abdc129 100644 --- a/test/08_manylinuxXXXX_only/cibuildwheel_test.py +++ b/test/08_manylinuxXXXX_only/cibuildwheel_test.py @@ -1,4 +1,5 @@ import os +import platform import pytest @@ -11,6 +12,9 @@ def test(manylinux_image): if utils.platform != 'linux': pytest.skip('the docker test is only relevant to the linux build') + elif platform.machine not in ['x86_64', 'i686']: + if manylinux_image in ['manylinux1', 'manylinux2010']: + pytest.skip("manylinux1 and 2010 doesn't exist for non-x86 architectures") # build the wheels # CFLAGS environment variable is necessary to fail on 'malloc_info' (on manylinux1) during compilation/linking, @@ -20,6 +24,9 @@ def test(manylinux_image): 'CIBW_MANYLINUX_X86_64_IMAGE': manylinux_image, 'CIBW_MANYLINUX_I686_IMAGE': manylinux_image, 'CIBW_MANYLINUX_PYPY_X86_64_IMAGE': manylinux_image, + 'CIBW_MANYLINUX_AARCH64_IMAGE': manylinux_image, + 'CIBW_MANYLINUX_PPC64LE_IMAGE': manylinux_image, + 'CIBW_MANYLINUX_S390X_IMAGE': manylinux_image, } if manylinux_image == 'manylinux1': # We don't have a manylinux1 image for PyPy diff --git a/test/shared/utils.py b/test/shared/utils.py index d0db8b1f..862d4466 100644 --- a/test/shared/utils.py +++ b/test/shared/utils.py @@ -5,6 +5,7 @@ This file is added to the PYTHONPATH in the test runner at bin/run_test.py. ''' import os +import platform as pm import shutil import subprocess import sys @@ -70,7 +71,7 @@ def cibuildwheel_run(project_path, env=None, add_env=None, output_dir=None): return wheels -def expected_wheels(package_name, package_version, manylinux_versions=['manylinux1', 'manylinux2010'], +def expected_wheels(package_name, package_version, manylinux_versions=None, macosx_deployment_target=None): ''' Returns a list of expected wheels from a run of cibuildwheel. @@ -79,11 +80,21 @@ def expected_wheels(package_name, package_version, manylinux_versions=['manylinu # {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl # {python tag} and {abi tag} are closely related to the python interpreter used to build the wheel # so we'll merge them below as python_abi_tag - python_abi_tags = ['cp27-cp27m', 'cp35-cp35m', 'cp36-cp36m', 'cp37-cp37m', 'cp38-cp38', - 'pp27-pypy_73', 'pp36-pypy36_pp73'] + + python_abi_tags = ['cp35-cp35m', 'cp36-cp36m', 'cp37-cp37m', 'cp38-cp38'] + extra_x86_python_abi_tags = ['cp27-cp27m', 'pp27-pypy_73', 'pp36-pypy36_pp73'] + if platform == 'linux': - python_abi_tags.append('cp27-cp27mu') # python 2.7 has 2 different ABI on manylinux - architectures = {'cp': ['x86_64', 'i686'], 'pp': ['x86_64']} + if pm.machine() not in ['x86_64', 'i686']: + if manylinux_versions is None: + manylinux_versions = ['manylinux2014'] + architectures = {'cp': [pm.machine()]} + else: + if manylinux_versions is None: + manylinux_versions = ['manylinux1', 'manylinux2010'] + python_abi_tags += extra_x86_python_abi_tags + python_abi_tags.append('cp27-cp27mu') # python 2.7 has 2 different ABI on manylinux + architectures = {'cp': ['x86_64', 'i686'], 'pp': ['x86_64']} platform_tags = {} for python_implemention in architectures: platform_tags[python_implemention] = [ @@ -96,12 +107,15 @@ def expected_wheels(package_name, package_version, manylinux_versions=['manylinu def get_platform_tags(python_abi_tag): return platform_tags[python_abi_tag[:2]] elif platform == 'windows': + python_abi_tags += extra_x86_python_abi_tags platform_tags = {'cp': ['win32', 'win_amd64'], 'pp': ['win32']} def get_platform_tags(python_abi_tag): return platform_tags[python_abi_tag[:2]] elif platform == 'macos': + python_abi_tags += extra_x86_python_abi_tags + def get_platform_tags(python_abi_tag): default_version = '10.7' if python_abi_tag.startswith('pp') else '10.9' return ['macosx_{}_x86_64'.format((macosx_deployment_target or default_version).replace('.', '_'))]