From ce76b742515486ff2cc9bfdd4e02daaeca8f3693 Mon Sep 17 00:00:00 2001 From: Matthew Treinish Date: Thu, 20 Feb 2020 13:51:37 -0500 Subject: [PATCH] Add support for manylinux wheels on non-x86 platforms This commit adds config options and identifiers for building manylinux wheels on non-x86 platforms. Based on what travis and manylinux2014 support this includes aarch64 (ARMv8), ppce64le (power pc 64bit little endian), and the s390x platforms. Fixes #269 --- .travis.yml | 24 ++++++++++++++++++++++++ cibuildwheel/__main__.py | 12 +++++++++++- cibuildwheel/linux.py | 37 +++++++++++++++++++++++++++++++++++-- docs/options.md | 22 +++++++++++----------- 4 files changed, 81 insertions(+), 14 deletions(-) diff --git a/.travis.yml b/.travis.yml index d5215b04..46de3bac 100644 --- a/.travis.yml +++ b/.travis.yml @@ -9,6 +9,30 @@ matrix: services: docker env: PYTHON=python + # Linux Python 3 + - sudo: required + language: python + python: 3.5 + services: docker + arch: arm64 + env: PYTHON=python + + # Linux Python 3 + - sudo: required + language: python + python: 3.5 + services: docker + arch: ppc64le + env: PYTHON=python + + # Linux Python 3 + - sudo: required + language: python + python: 3.5 + services: docker + arch: s390x + env: PYTHON=python + # macOS Python 3 - os: osx env: PYTHON=python3 diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 1e5b249b..aa7f6587 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -160,6 +160,9 @@ def main(): manylinux_x86_64_image = os.environ.get('CIBW_MANYLINUX_X86_64_IMAGE', 'manylinux2010') manylinux_i686_image = os.environ.get('CIBW_MANYLINUX_I686_IMAGE', 'manylinux2010') manylinux_pypy_x86_64_image = os.environ.get('CIBW_MANYLINUX_PYPY_X86_64_IMAGE', 'manylinux2010') + manylinux_aarch64_image = os.environ.get('CIBW_MANYLINUX_AARCH64_IMAGE', 'manylinux2014') + manylinux_ppc64le_image = os.environ.get('CIBW_MANYLINUX_PPC64LE_IMAGE', 'manylinux2014') + manylinux_s390x_image = os.environ.get('CIBW_MANYLINUX_S390X_IMAGE', 'manylinux2014') default_manylinux_images_x86_64 = {'manylinux1': 'quay.io/pypa/manylinux1_x86_64', 'manylinux2010': 'quay.io/pypa/manylinux2010_x86_64', @@ -168,11 +171,18 @@ def main(): 'manylinux2010': 'quay.io/pypa/manylinux2010_i686', 'manylinux2014': 'quay.io/pypa/manylinux2014_i686'} default_manylinux_images_pypy_x86_64 = {'manylinux2010': 'pypywheels/manylinux2010-pypy_x86_64'} + default_manylinux_images_aarch64 = {'manylinux2014': 'quay.io/pypa/manylinux2014_aarch64'} + default_manylinux_images_ppc64le = {'manylinux2014': 'quay.io/pypa/manylinux2014_ppc64le'} + default_manylinux_images_s390x = {'manylinux2014': 'quay.io/pypa/manylinux2014_s390x'} build_options.update( manylinux_images={'x86_64': default_manylinux_images_x86_64.get(manylinux_x86_64_image) or manylinux_x86_64_image, 'i686': default_manylinux_images_i686.get(manylinux_i686_image) or manylinux_i686_image, - 'pypy_x86_64': default_manylinux_images_pypy_x86_64.get(manylinux_pypy_x86_64_image) or manylinux_pypy_x86_64_image}, + 'pypy_x86_64': default_manylinux_images_pypy_x86_64.get(manylinux_pypy_x86_64_image) or manylinux_pypy_x86_64_image, + 'aarch64': default_manylinux_images_aarch64.get(manylinux_aarch64_image) or manylinux_aarch64_image, + 'ppc64le': default_manylinux_images_ppc64le.get(manylinux_ppc64le_image) or manylinux_ppc64le_image, + 's390x': default_manylinux_images_s390x.get(manylinux_s390x_image) or manylinux_s390x_image, + }, ) elif platform == 'macos': pass diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index b2569692..f8801129 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -1,3 +1,4 @@ +import distutils import os import shlex import subprocess @@ -29,10 +30,39 @@ def get_python_configurations(build_selector): PythonConfiguration(identifier='cp38-manylinux_i686', path='/opt/python/cp38-cp38'), PythonConfiguration(identifier='pp27-manylinux_x86_64', path='/opt/python/pp27-pypy_73'), PythonConfiguration(identifier='pp36-manylinux_x86_64', path='/opt/python/pp36-pypy36_pp73'), + PythonConfiguration(identifier='cp35-manylinux_aarch64', path='/opt/python/cp35-cp35m'), + PythonConfiguration(identifier='cp36-manylinux_aarch64', path='/opt/python/cp36-cp36m'), + PythonConfiguration(identifier='cp37-manylinux_aarch64', path='/opt/python/cp37-cp37m'), + PythonConfiguration(identifier='cp38-manylinux_aarch64', path='/opt/python/cp38-cp38'), + PythonConfiguration(identifier='cp35-manylinux_ppc64le', path='/opt/python/cp35-cp35m'), + PythonConfiguration(identifier='cp36-manylinux_ppc64le', path='/opt/python/cp36-cp36m'), + PythonConfiguration(identifier='cp37-manylinux_ppc64le', path='/opt/python/cp37-cp37m'), + PythonConfiguration(identifier='cp38-manylinux_ppc64le', path='/opt/python/cp38-cp38'), + PythonConfiguration(identifier='cp35-manylinux_s390x', path='/opt/python/cp35-cp35m'), + PythonConfiguration(identifier='cp36-manylinux_s390x', path='/opt/python/cp36-cp36m'), + PythonConfiguration(identifier='cp37-manylinux_s390x', path='/opt/python/cp37-cp37m'), + PythonConfiguration(identifier='cp38-manylinux_s390x', path='/opt/python/cp38-cp38'), ] - # skip builds as required - return [c for c in python_configurations if build_selector(c.identifier)] + configurations = [] + for c in python_configurations: + if not build_selector(c.identifier): + continue + platform = distutils.util.get_platform() + if platform == "linux-x86_64": + if c.identifier.endswith('x86_64') or c.identifier.endswith('i686'): + configurations.append(c) + elif platform == "linux-aarch64": + if c.identifier.endswith('aarch64'): + configurations.append(c) + elif platform == "linux-ppc64le": + if c.identifier.endswith('ppc64le'): + configurations.append(c) + elif platform == "linux-s390x": + if c.identifier.endswith('s390x'): + configurations.append(c) + + return configurations def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, manylinux_images): @@ -49,6 +79,9 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef platforms = [ ('cp', 'manylinux_x86_64', manylinux_images['x86_64']), ('cp', 'manylinux_i686', manylinux_images['i686']), + ('cp', 'manylinux_aarch64', manylinux_images['aarch64']), + ('cp', 'manylinux_ppc64le', manylinux_images['ppc64le']), + ('cp', 'manylinux_s390x', manylinux_images['s390x']), ('pp', 'manylinux_x86_64', manylinux_images['pypy_x86_64']), ] diff --git a/docs/options.md b/docs/options.md index f2fab00e..77afad1f 100644 --- a/docs/options.md +++ b/docs/options.md @@ -72,15 +72,15 @@ When setting the options, you can use shell-style globbing syntax (as per [`fnma
-| | macOS 64bit | Manylinux 64bit | Manylinux 32bit | Windows 64bit | Windows 32bit | -|-----------------|---------------------|------------------------|----------------------|-----------------|----------------| -| Python 2.7 | cp27-macosx_x86_64 | cp27-manylinux_x86_64 | cp27-manylinux_i686 | cp27-win_amd64 | cp27-win32 | -| Python 3.5 | cp35-macosx_x86_64 | cp35-manylinux_x86_64 | cp35-manylinux_i686 | cp35-win_amd64 | cp35-win32 | -| Python 3.6 | cp36-macosx_x86_64 | cp36-manylinux_x86_64 | cp36-manylinux_i686 | cp36-win_amd64 | cp36-win32 | -| Python 3.7 | cp37-macosx_x86_64 | cp37-manylinux_x86_64 | cp37-manylinux_i686 | cp37-win_amd64 | cp37-win32 | -| Python 3.8 | cp38-macosx_x86_64 | cp38-manylinux_x86_64 | cp38-manylinux_i686 | cp38-win_amd64 | cp38-win32 | -| PyPy 2.7 v7.3.0 | pp27-macosx_x86_64 | pp27-manylinux_x86_64 | | | pp27-win32 | -| PyPy 3.6 v7.3.0 | pp36-macosx_x86_64 | pp36-manylinux_x86_64 | | | pp36-win32 | +| | macOS 64bit | Manylinux x86 64bit | Manylinux x86 32bit | Windows 64bit | Windows 32bit | Manylinux Armv8 64bit | Manylinux PPC64LE | Manylinux s390x | +|-----------------|---------------------|------------------------|----------------------|-----------------|----------------|------------------------|------------------------|----------------------| +| Python 2.7 | cp27-macosx_x86_64 | cp27-manylinux_x86_64 | cp27-manylinux_i686 | cp27-win_amd64 | cp27-win32 | | | | +| Python 3.5 | cp35-macosx_x86_64 | cp35-manylinux_x86_64 | cp35-manylinux_i686 | cp35-win_amd64 | cp35-win32 | cp35-manylinux_aarch64 | cp35-manylinux_ppc64le | cp35-manylinux_s390x | +| Python 3.6 | cp36-macosx_x86_64 | cp36-manylinux_x86_64 | cp36-manylinux_i686 | cp36-win_amd64 | cp36-win32 | cp36-manylinux_aarch64 | cp36-manylinux_ppc64le | cp36-manylinux_s390x | +| Python 3.7 | cp37-macosx_x86_64 | cp37-manylinux_x86_64 | cp37-manylinux_i686 | cp37-win_amd64 | cp37-win32 | cp37-manylinux_aarch64 | cp37-manylinux_ppc64le | cp37-manylinux_s390x | +| Python 3.8 | cp38-macosx_x86_64 | cp38-manylinux_x86_64 | cp38-manylinux_i686 | cp38-win_amd64 | cp38-win32 | cp38-manylinux_aarch64 | cp38-manylinux_ppc64le | cp38-manylinux_s390x | +| PyPy 2.7 v7.3.0 | pp27-macosx_x86_64 | pp27-manylinux_x86_64 | | | pp27-win32 | | | | +| PyPy 3.6 v7.3.0 | pp36-macosx_x86_64 | pp36-manylinux_x86_64 | | | pp36-win32 | | | | The list of supported and currently selected build identifiers can also be retrieved by passing the `--print-build-identifiers` flag to `cibuildwheel`. The format is `python_tag-platform_tag`, with tags similar to those in [PEP 425](https://www.python.org/dev/peps/pep-0425/#details). @@ -245,10 +245,10 @@ CIBW_REPAIR_WHEEL_COMMAND_LINUX: "auditwheel repair --lib-sdir . -w {dest_dir} { ``` -### `CIBW_MANYLINUX_X86_64_IMAGE`, `CIBW_MANYLINUX_I686_IMAGE`, `CIBW_MANYLINUX_PYPY_X86_64_IMAGE` {: #manylinux-image} +### `CIBW_MANYLINUX_X86_64_IMAGE`, `CIBW_MANYLINUX_I686_IMAGE`, `CIBW_MANYLINUX_PYPY_X86_64_IMAGE`, `CIBW_MANYLINUX_AARCH64_IMAGE`, `CIBW_MANYLINUX_PPC64LE_IMAGE`, `CIBW_MANYLINUX_S390X_IMAGE` {: #manylinux-image} > Specify alternative manylinux docker images -An alternative Docker image to be used for building [`manylinux`](https://github.com/pypa/manylinux) wheels. `cibuildwheel` will then pull these instead of the default images, [`quay.io/pypa/manylinux2010_x86_64`](https://quay.io/pypa/manylinux2010_x86_64), [`quay.io/pypa/manylinux2010_i686`](https://quay.io/pypa/manylinux2010_i686), and [`pypywheels/manylinux2010-pypy_x86_64`](https://hub.docker.com/r/pypywheels/manylinux2010-pypy_x86_64). +An alternative Docker image to be used for building [`manylinux`](https://github.com/pypa/manylinux) wheels. `cibuildwheel` will then pull these instead of the default images, [`quay.io/pypa/manylinux2010_x86_64`](https://quay.io/pypa/manylinux2010_x86_64), [`quay.io/pypa/manylinux2010_i686`](https://quay.io/pypa/manylinux2010_i686), [`quay.io/pypa/manylinux2014_aarch64`](https://quay.io/pypa/manylinux2014_aarch64), [`quay.io/pypa/manylinux2014_ppc64le`](https://quay.io/pypa/manylinux2014_ppc64le), [`quay.io/pypa/manylinux2014_s390x`](https://quay.io/pypa/manylinux2010_s390x), and [`pypywheels/manylinux2010-pypy_x86_64`](https://hub.docker.com/r/pypywheels/manylinux2010-pypy_x86_64). The value of this option can either be set to `manylinux1`, `manylinux2010` or `manylinux2014` to use the [official `manylinux` images](https://github.com/pypa/manylinux) and [PyPy `manylinux` images](https://github.com/pypy/manylinux), or any other valid Docker image name. Note that for PyPy, only the official `manylinux2010` image is currently available.