Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #273 from mtreinish/non-x86-support

Browse Source 
Add support for manylinux wheels on non-x86 platforms
This commit is contained in:
Joe Rickerby
2020-02-26 20:43:37 +00:00
committed by GitHub
parent 65792bd700 05df9bbfd4
commit 6ad0aebaf8
11 changed files with 141 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.travis.yml
+24
View File
@@ -9,6 +9,30 @@ matrix:
services: docker
env: PYTHON=python
# Linux Python 3
- sudo: required
language: python
python: 3.5
services: docker
arch: arm64
env: PYTHON=python
# Linux Python 3
- sudo: required
language: python
python: 3.5
services: docker
arch: ppc64le
env: PYTHON=python
# Linux Python 3
- sudo: required
language: python
python: 3.5
services: docker
arch: s390x
env: PYTHON=python
# macOS Python 3
- os: osx
env: PYTHON=python3
CI.md
+2
View File
@@ -7,3 +7,5 @@ This is a summary of the Python versions and platforms covered by the different
| Windows | TravisCI | Azure Pipelines | AppVeyor | Azure Pipelines |
> ¹ Python version not really pinned, but dependent on the (default) version of image used.
Non-x86 architectures are covered on Travis CI using Python 3.5.
README.md
+9 -9
View File
@@ -15,15 +15,15 @@ Python wheels are great. Building them across **Mac, Linux, Windows**, on **mult
What does it do?
----------------
| | macOS x86_64 | manylinux i686 | manylinux x86_64 | Windows 32bit | Windows 64bit |
|---|---|---|---|---|---|
| Python 2.7 | ✅ | ✅ | ✅ | ✅¹ | ✅¹ |
| Python 3.5 | ✅ | ✅ | ✅ | ✅ | ✅ |
| Python 3.6 | ✅ | ✅ | ✅ | ✅ | ✅ |
| Python 3.7 | ✅ | ✅ | ✅ | ✅ | ✅ |
| Python 3.8 | ✅ | ✅ | ✅ | ✅ | ✅ |
| PyPy 2.7 v7.3.0 | ✅ | | ✅ | ✅ | |
| PyPy 3.6 v7.3.0 | ✅ | | ✅ | ✅ | |
| | macOS x86_64 | manylinux i686 | manylinux x86_64 | Windows 32bit | Windows 64bit | manylinux aarch64 | manylinux ppc64le | manylinux s390x |
|---|---|---|---|---|---|---|---|---|
| Python 2.7 | ✅ | ✅ | ✅ | ✅¹ | ✅¹ | | | |
| Python 3.5 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| Python 3.6 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| Python 3.7 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| Python 3.8 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ |
| PyPy 2.7 v7.3.0 | ✅ | | ✅ | ✅ | | | | |
| PyPy 3.6 v7.3.0 | ✅ | | ✅ | ✅ | | | | |
> ¹ Not supported on Travis
cibuildwheel/__main__.py
+11 -1
View File
@@ -160,6 +160,9 @@ def main():
manylinux_x86_64_image = os.environ.get('CIBW_MANYLINUX_X86_64_IMAGE', 'manylinux2010')
manylinux_i686_image = os.environ.get('CIBW_MANYLINUX_I686_IMAGE', 'manylinux2010')
manylinux_pypy_x86_64_image = os.environ.get('CIBW_MANYLINUX_PYPY_X86_64_IMAGE', 'manylinux2010')
manylinux_aarch64_image = os.environ.get('CIBW_MANYLINUX_AARCH64_IMAGE', 'manylinux2014')
manylinux_ppc64le_image = os.environ.get('CIBW_MANYLINUX_PPC64LE_IMAGE', 'manylinux2014')
manylinux_s390x_image = os.environ.get('CIBW_MANYLINUX_S390X_IMAGE', 'manylinux2014')
default_manylinux_images_x86_64 = {'manylinux1': 'quay.io/pypa/manylinux1_x86_64',
'manylinux2010': 'quay.io/pypa/manylinux2010_x86_64',
@@ -168,11 +171,18 @@ def main():
'manylinux2010': 'quay.io/pypa/manylinux2010_i686',
'manylinux2014': 'quay.io/pypa/manylinux2014_i686'}
default_manylinux_images_pypy_x86_64 = {'manylinux2010': 'pypywheels/manylinux2010-pypy_x86_64'}
default_manylinux_images_aarch64 = {'manylinux2014': 'quay.io/pypa/manylinux2014_aarch64'}
default_manylinux_images_ppc64le = {'manylinux2014': 'quay.io/pypa/manylinux2014_ppc64le'}
default_manylinux_images_s390x = {'manylinux2014': 'quay.io/pypa/manylinux2014_s390x'}
build_options.update(
manylinux_images={'x86_64': default_manylinux_images_x86_64.get(manylinux_x86_64_image) or manylinux_x86_64_image,
'i686': default_manylinux_images_i686.get(manylinux_i686_image) or manylinux_i686_image,
'pypy_x86_64': default_manylinux_images_pypy_x86_64.get(manylinux_pypy_x86_64_image) or manylinux_pypy_x86_64_image},
'pypy_x86_64': default_manylinux_images_pypy_x86_64.get(manylinux_pypy_x86_64_image) or manylinux_pypy_x86_64_image,
'aarch64': default_manylinux_images_aarch64.get(manylinux_aarch64_image) or manylinux_aarch64_image,
'ppc64le': default_manylinux_images_ppc64le.get(manylinux_ppc64le_image) or manylinux_ppc64le_image,
's390x': default_manylinux_images_s390x.get(manylinux_s390x_image) or manylinux_s390x_image,
},
)
elif platform == 'macos':
pass
cibuildwheel/linux.py
+38 -2
View File
@@ -1,4 +1,5 @@
import os
import platform
import shlex
import subprocess
import sys
@@ -12,6 +13,27 @@ from .util import (
)
def matches_platform(identifier):
pm = platform.machine()
if pm == "x86_64":
# x86_64 machines can run i686 docker containers
if identifier.endswith('x86_64') or identifier.endswith('i686'):
return True
elif pm == "i686":
if identifier.endswith('i686'):
return True
elif pm == "aarch64":
if identifier.endswith('aarch64'):
return True
elif pm == "ppc64le":
if identifier.endswith('ppc64le'):
return True
elif pm == "s390x":
if identifier.endswith('s390x'):
return True
return False
def get_python_configurations(build_selector):
PythonConfiguration = namedtuple('PythonConfiguration', ['identifier', 'path'])
python_configurations = [
@@ -29,10 +51,21 @@ def get_python_configurations(build_selector):
PythonConfiguration(identifier='cp38-manylinux_i686', path='/opt/python/cp38-cp38'),
PythonConfiguration(identifier='pp27-manylinux_x86_64', path='/opt/python/pp27-pypy_73'),
PythonConfiguration(identifier='pp36-manylinux_x86_64', path='/opt/python/pp36-pypy36_pp73'),
PythonConfiguration(identifier='cp35-manylinux_aarch64', path='/opt/python/cp35-cp35m'),
PythonConfiguration(identifier='cp36-manylinux_aarch64', path='/opt/python/cp36-cp36m'),
PythonConfiguration(identifier='cp37-manylinux_aarch64', path='/opt/python/cp37-cp37m'),
PythonConfiguration(identifier='cp38-manylinux_aarch64', path='/opt/python/cp38-cp38'),
PythonConfiguration(identifier='cp35-manylinux_ppc64le', path='/opt/python/cp35-cp35m'),
PythonConfiguration(identifier='cp36-manylinux_ppc64le', path='/opt/python/cp36-cp36m'),
PythonConfiguration(identifier='cp37-manylinux_ppc64le', path='/opt/python/cp37-cp37m'),
PythonConfiguration(identifier='cp38-manylinux_ppc64le', path='/opt/python/cp38-cp38'),
PythonConfiguration(identifier='cp35-manylinux_s390x', path='/opt/python/cp35-cp35m'),
PythonConfiguration(identifier='cp36-manylinux_s390x', path='/opt/python/cp36-cp36m'),
PythonConfiguration(identifier='cp37-manylinux_s390x', path='/opt/python/cp37-cp37m'),
PythonConfiguration(identifier='cp38-manylinux_s390x', path='/opt/python/cp38-cp38'),
]
# skip builds as required
return [c for c in python_configurations if build_selector(c.identifier)]
return [c for c in python_configurations if matches_platform(c.identifier) and build_selector(c.identifier)]
def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, manylinux_images):
@@ -49,6 +82,9 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
platforms = [
('cp', 'manylinux_x86_64', manylinux_images['x86_64']),
('cp', 'manylinux_i686', manylinux_images['i686']),
('cp', 'manylinux_aarch64', manylinux_images['aarch64']),
('cp', 'manylinux_ppc64le', manylinux_images['ppc64le']),
('cp', 'manylinux_s390x', manylinux_images['s390x']),
('pp', 'manylinux_x86_64', manylinux_images['pypy_x86_64']),
]
docs/options.md
+13 -12
View File
@@ -72,15 +72,15 @@ When setting the options, you can use shell-style globbing syntax (as per [`fnma
<div class="build-id-table-marker"></div>
| | macOS 64bit | Manylinux 64bit | Manylinux 32bit | Windows 64bit | Windows 32bit |
|-----------------|---------------------|------------------------|----------------------|-----------------|----------------|
| Python 2.7 | cp27-macosx_x86_64 | cp27-manylinux_x86_64 | cp27-manylinux_i686 | cp27-win_amd64 | cp27-win32 |
| Python 3.5 | cp35-macosx_x86_64 | cp35-manylinux_x86_64 | cp35-manylinux_i686 | cp35-win_amd64 | cp35-win32 |
| Python 3.6 | cp36-macosx_x86_64 | cp36-manylinux_x86_64 | cp36-manylinux_i686 | cp36-win_amd64 | cp36-win32 |
| Python 3.7 | cp37-macosx_x86_64 | cp37-manylinux_x86_64 | cp37-manylinux_i686 | cp37-win_amd64 | cp37-win32 |
| Python 3.8 | cp38-macosx_x86_64 | cp38-manylinux_x86_64 | cp38-manylinux_i686 | cp38-win_amd64 | cp38-win32 |
| PyPy 2.7 v7.3.0 | pp27-macosx_x86_64 | pp27-manylinux_x86_64 | | | pp27-win32 |
| PyPy 3.6 v7.3.0 | pp36-macosx_x86_64 | pp36-manylinux_x86_64 | | | pp36-win32 |
| | macOS 64bit | Manylinux x86 64bit | Manylinux x86 32bit | Windows 64bit | Windows 32bit | Manylinux Armv8 64bit | Manylinux PPC64LE | Manylinux s390x |
|-----------------|---------------------|------------------------|----------------------|-----------------|----------------|------------------------|------------------------|----------------------|
| Python 2.7 | cp27-macosx_x86_64 | cp27-manylinux_x86_64 | cp27-manylinux_i686 | cp27-win_amd64 | cp27-win32 | | | |
| Python 3.5 | cp35-macosx_x86_64 | cp35-manylinux_x86_64 | cp35-manylinux_i686 | cp35-win_amd64 | cp35-win32 | cp35-manylinux_aarch64 | cp35-manylinux_ppc64le | cp35-manylinux_s390x |
| Python 3.6 | cp36-macosx_x86_64 | cp36-manylinux_x86_64 | cp36-manylinux_i686 | cp36-win_amd64 | cp36-win32 | cp36-manylinux_aarch64 | cp36-manylinux_ppc64le | cp36-manylinux_s390x |
| Python 3.7 | cp37-macosx_x86_64 | cp37-manylinux_x86_64 | cp37-manylinux_i686 | cp37-win_amd64 | cp37-win32 | cp37-manylinux_aarch64 | cp37-manylinux_ppc64le | cp37-manylinux_s390x |
| Python 3.8 | cp38-macosx_x86_64 | cp38-manylinux_x86_64 | cp38-manylinux_i686 | cp38-win_amd64 | cp38-win32 | cp38-manylinux_aarch64 | cp38-manylinux_ppc64le | cp38-manylinux_s390x |
| PyPy 2.7 v7.3.0 | pp27-macosx_x86_64 | pp27-manylinux_x86_64 | | | pp27-win32 | | | |
| PyPy 3.6 v7.3.0 | pp36-macosx_x86_64 | pp36-manylinux_x86_64 | | | pp36-win32 | | | |
The list of supported and currently selected build identifiers can also be retrieved by passing the `--print-build-identifiers` flag to `cibuildwheel`.
The format is `python_tag-platform_tag`, with tags similar to those in [PEP 425](https://www.python.org/dev/peps/pep-0425/#details).
@@ -245,12 +245,13 @@ CIBW_REPAIR_WHEEL_COMMAND_LINUX: "auditwheel repair --lib-sdir . -w {dest_dir} {
```
### `CIBW_MANYLINUX_X86_64_IMAGE`, `CIBW_MANYLINUX_I686_IMAGE`, `CIBW_MANYLINUX_PYPY_X86_64_IMAGE` {: #manylinux-image}
### `CIBW_MANYLINUX_X86_64_IMAGE`, `CIBW_MANYLINUX_I686_IMAGE`, `CIBW_MANYLINUX_PYPY_X86_64_IMAGE`, `CIBW_MANYLINUX_AARCH64_IMAGE`, `CIBW_MANYLINUX_PPC64LE_IMAGE`, `CIBW_MANYLINUX_S390X_IMAGE` {: #manylinux-image}
> Specify alternative manylinux docker images
An alternative Docker image to be used for building [`manylinux`](https://github.com/pypa/manylinux) wheels. `cibuildwheel` will then pull these instead of the default images, [`quay.io/pypa/manylinux2010_x86_64`](https://quay.io/pypa/manylinux2010_x86_64), [`quay.io/pypa/manylinux2010_i686`](https://quay.io/pypa/manylinux2010_i686), and [`pypywheels/manylinux2010-pypy_x86_64`](https://hub.docker.com/r/pypywheels/manylinux2010-pypy_x86_64).
An alternative Docker image to be used for building [`manylinux`](https://github.com/pypa/manylinux) wheels. `cibuildwheel` will then pull these instead of the default images, [`quay.io/pypa/manylinux2010_x86_64`](https://quay.io/pypa/manylinux2010_x86_64), [`quay.io/pypa/manylinux2010_i686`](https://quay.io/pypa/manylinux2010_i686), [`pypywheels/manylinux2010-pypy_x86_64`](https://hub.docker.com/r/pypywheels/manylinux2010-pypy_x86_64), [`quay.io/pypa/manylinux2014_aarch64`](https://quay.io/pypa/manylinux2014_aarch64), [`quay.io/pypa/manylinux2014_ppc64le`](https://quay.io/pypa/manylinux2014_ppc64le), and [`quay.io/pypa/manylinux2014_s390x`](https://quay.io/pypa/manylinux2010_s390x).
The value of this option can either be set to `manylinux1`, `manylinux2010` or `manylinux2014` to use the [official `manylinux` images](https://github.com/pypa/manylinux) and [PyPy `manylinux` images](https://github.com/pypy/manylinux), or any other valid Docker image name. Note that for PyPy, only the official `manylinux2010` image is currently available.
The value of this option can either be set to `manylinux1`, `manylinux2010` or `manylinux2014` to use the [official `manylinux` images](https://github.com/pypa/manylinux) and [PyPy `manylinux` images](https://github.com/pypy/manylinux), or any other valid Docker image name. Note that for PyPy, only the official `manylinux2010` image is currently available. For architectures other
than x86 (x86\_64 and i686) manylinux2014 must be used because this is the first version of the manylinux specification that supports additional architectures.
Beware to specify a valid Docker image that can be used in the same way as the official, default Docker images: all necessary Python and pip versions need to be present in `/opt/python/`, and the `auditwheel` tool needs to be present for `cibuildwheel` to work. Apart from that, the architecture and relevant shared system libraries need to be manylinux1-, manylinux2010- or manylinux2014-compatible in order to produce valid `manylinux1`/`manylinux2010`/`manylinux2014` wheels (see [pypa/manylinux on GitHub](https://github.com/pypa/manylinux), [PEP 513](https://www.python.org/dev/peps/pep-0513/), [PEP 571](https://www.python.org/dev/peps/pep-0571/) and [PEP 599](https://www.python.org/dev/peps/pep-0599/) for more details).
examples/travis-ci-minimal.yml
+9
View File
@@ -4,6 +4,15 @@ jobs:
include:
# perform a linux build
- services: docker
# perform a linux ARMv8 build
- services: docker
arch: aarch64
# perform a linux PPC64LE build
- services: docker
arch: ppc64le
# perform a linux S390X build
- services: docker
arch: s390x
# and a mac build
- os: osx
language: shell
test/01_basic/cibuildwheel_test.py
+6 -2
View File
@@ -1,4 +1,5 @@
import os
import platform
import utils
@@ -20,7 +21,10 @@ def test_build_identifiers():
# after adding CIBW_MANYLINUX_IMAGE to support manylinux2010, there
# can be multiple wheels for each wheel, though, so we need to limit
# the expected wheels
expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0')
if '-manylinux' not in w or '-manylinux1' in w]
if platform.machine() in ['x86_64', 'i686']:
expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0')
if '-manylinux' not in w or '-manylinux1' in w]
else:
expected_wheels = utils.expected_wheels('spam', '0.1.0')
build_identifiers = utils.cibuildwheel_get_build_identifiers(project_dir)
assert len(expected_wheels) == len(build_identifiers)
test/06_docker_images/cibuildwheel_test.py
+3
View File
@@ -1,4 +1,5 @@
import os
import platform
import pytest
@@ -10,6 +11,8 @@ def test():
if utils.platform != 'linux':
pytest.skip('the test is only relevant to the linux build')
if platform.machine() not in ['x86_64', 'i686']:
pytest.skip('this test is currently only possible on x86_64/i686 due to availability of alternative images')
actual_wheels = utils.cibuildwheel_run(project_dir, add_env={
'CIBW_MANYLINUX_X86_64_IMAGE': 'dockcross/manylinux2010-x64',
test/08_manylinuxXXXX_only/cibuildwheel_test.py
+7
View File
@@ -1,4 +1,5 @@
import os
import platform
import pytest
@@ -11,6 +12,9 @@ def test(manylinux_image):
if utils.platform != 'linux':
pytest.skip('the docker test is only relevant to the linux build')
elif platform.machine not in ['x86_64', 'i686']:
if manylinux_image in ['manylinux1', 'manylinux2010']:
pytest.skip("manylinux1 and 2010 doesn't exist for non-x86 architectures")
# build the wheels
# CFLAGS environment variable is necessary to fail on 'malloc_info' (on manylinux1) during compilation/linking,
@@ -20,6 +24,9 @@ def test(manylinux_image):
'CIBW_MANYLINUX_X86_64_IMAGE': manylinux_image,
'CIBW_MANYLINUX_I686_IMAGE': manylinux_image,
'CIBW_MANYLINUX_PYPY_X86_64_IMAGE': manylinux_image,
'CIBW_MANYLINUX_AARCH64_IMAGE': manylinux_image,
'CIBW_MANYLINUX_PPC64LE_IMAGE': manylinux_image,
'CIBW_MANYLINUX_S390X_IMAGE': manylinux_image,
}
if manylinux_image == 'manylinux1':
# We don't have a manylinux1 image for PyPy
test/shared/utils.py
+19 -5
View File
@@ -5,6 +5,7 @@ This file is added to the PYTHONPATH in the test runner at bin/run_test.py.
'''
import os
import platform as pm
import shutil
import subprocess
import sys
@@ -70,7 +71,7 @@ def cibuildwheel_run(project_path, env=None, add_env=None, output_dir=None):
return wheels
def expected_wheels(package_name, package_version, manylinux_versions=['manylinux1', 'manylinux2010'],
def expected_wheels(package_name, package_version, manylinux_versions=None,
macosx_deployment_target=None):
'''
Returns a list of expected wheels from a run of cibuildwheel.
@@ -79,11 +80,21 @@ def expected_wheels(package_name, package_version, manylinux_versions=['manylinu
# {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl
# {python tag} and {abi tag} are closely related to the python interpreter used to build the wheel
# so we'll merge them below as python_abi_tag
python_abi_tags = ['cp27-cp27m', 'cp35-cp35m', 'cp36-cp36m', 'cp37-cp37m', 'cp38-cp38',
'pp27-pypy_73', 'pp36-pypy36_pp73']
python_abi_tags = ['cp35-cp35m', 'cp36-cp36m', 'cp37-cp37m', 'cp38-cp38']
extra_x86_python_abi_tags = ['cp27-cp27m', 'pp27-pypy_73', 'pp36-pypy36_pp73']
if platform == 'linux':
python_abi_tags.append('cp27-cp27mu') # python 2.7 has 2 different ABI on manylinux
architectures = {'cp': ['x86_64', 'i686'], 'pp': ['x86_64']}
if pm.machine() not in ['x86_64', 'i686']:
if manylinux_versions is None:
manylinux_versions = ['manylinux2014']
architectures = {'cp': [pm.machine()]}
else:
if manylinux_versions is None:
manylinux_versions = ['manylinux1', 'manylinux2010']
python_abi_tags += extra_x86_python_abi_tags
python_abi_tags.append('cp27-cp27mu') # python 2.7 has 2 different ABI on manylinux
architectures = {'cp': ['x86_64', 'i686'], 'pp': ['x86_64']}
platform_tags = {}
for python_implemention in architectures:
platform_tags[python_implemention] = [
@@ -96,12 +107,15 @@ def expected_wheels(package_name, package_version, manylinux_versions=['manylinu
def get_platform_tags(python_abi_tag):
return platform_tags[python_abi_tag[:2]]
elif platform == 'windows':
python_abi_tags += extra_x86_python_abi_tags
platform_tags = {'cp': ['win32', 'win_amd64'], 'pp': ['win32']}
def get_platform_tags(python_abi_tag):
return platform_tags[python_abi_tag[:2]]
elif platform == 'macos':
python_abi_tags += extra_x86_python_abi_tags
def get_platform_tags(python_abi_tag):
default_version = '10.7' if python_abi_tag.startswith('pp') else '10.9'
return ['macosx_{}_x86_64'.format((macosx_deployment_target or default_version).replace('.', '_'))]