Merge pull request #185 from YannickJadoul/pypy

Adding PyPy support
This commit is contained in:
Yannick Jadoul
2020-02-20 00:06:04 +01:00
committed by GitHub
13 changed files with 271 additions and 143 deletions
+11 -9
View File
@@ -15,17 +15,19 @@ Python wheels are great. Building them across **Mac, Linux, Windows**, on **mult
What does it do?
----------------
| | macOS 10.9+ x86_64 | manylinux i686 | manylinux x86_64 | Windows 32bit | Windows 64bit |
| | macOS x86_64 | manylinux i686 | manylinux x86_64 | Windows 32bit | Windows 64bit |
|---|---|---|---|---|---|
| Python 2.7 | ✅ | ✅ | ✅ | ✅¹ | ✅¹ |
| Python 3.5 | ✅ | ✅ | ✅ | ✅ | ✅ |
| Python 3.6 | ✅ | ✅ | ✅ | ✅ | ✅ |
| Python 3.7 | ✅ | ✅ | ✅ | ✅ | ✅ |
| Python 3.8 | ✅ | ✅ | ✅ | ✅ | ✅ |
| Python 2.7 | ✅ | ✅ | ✅ | ✅¹ | ✅¹ |
| Python 3.5 | ✅ | ✅ | ✅ | ✅ | ✅ |
| Python 3.6 | ✅ | ✅ | ✅ | ✅ | ✅ |
| Python 3.7 | ✅ | ✅ | ✅ | ✅ | ✅ |
| Python 3.8 | ✅ | ✅ | ✅ | ✅ | ✅ |
| PyPy 2.7 v7.3.0 | ✅ | | ✅ | ✅ | |
| PyPy 3.6 v7.3.0 | ✅ | | ✅ | ✅ | |
> ¹ Not supported on Travis
- Builds manylinux, macOS and Windows (32 and 64bit) wheels using Azure Pipelines, Travis CI, AppVeyor, and CircleCI
- Builds manylinux, macOS and Windows (32 and 64bit) wheels for CPython and PyPy using Azure Pipelines, Travis CI, AppVeyor, and CircleCI
- Bundles shared library dependencies on Linux and macOS through [auditwheel](https://github.com/pypa/auditwheel) and [delocate](https://github.com/matthew-brett/delocate)
- Runs the library test suite against the wheel-installed version of your library
@@ -48,7 +50,7 @@ Usage
Example setup
-------------
To build manylinux, macOS, and Windows wheels on Travis CI and upload them to PyPI whenever you tag a version, you could use this `.travis.yml`:
To build manylinux, macOS, and Windows wheels on Travis CI and upload them to PyPI whenever you tag a version, you could use this `.travis.yml`:
```yaml
language: python
@@ -96,7 +98,7 @@ Options
| **Build environment** | [`CIBW_ENVIRONMENT`](https://cibuildwheel.readthedocs.io/en/stable/options/#environment) | Set environment variables needed during the build |
| | [`CIBW_BEFORE_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-build) | Execute a shell command preparing each wheel's build |
| | [`CIBW_REPAIR_WHEEL_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#repair-wheel-command) | Execute a shell command to repair each (non-pure Python) built wheel |
| | [`CIBW_MANYLINUX_X86_64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) [`CIBW_MANYLINUX_I686_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) | Specify alternative manylinux docker images |
| | [`CIBW_MANYLINUX_X86_64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) [`CIBW_MANYLINUX_I686_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) [`CIBW_MANYLINUX_PYPY_X86_64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) | Specify alternative manylinux docker images |
| **Testing** | [`CIBW_TEST_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-command) | Execute a shell command to test each built wheel |
| | [`CIBW_TEST_REQUIRES`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-requires) | Install Python dependencies before running the tests |
| | [`CIBW_TEST_EXTRAS`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-extras) | Install your wheel for testing using extras_require |
+4 -1
View File
@@ -159,6 +159,7 @@ def main():
if platform == 'linux':
manylinux_x86_64_image = os.environ.get('CIBW_MANYLINUX_X86_64_IMAGE', 'manylinux2010')
manylinux_i686_image = os.environ.get('CIBW_MANYLINUX_I686_IMAGE', 'manylinux2010')
manylinux_pypy_x86_64_image = os.environ.get('CIBW_MANYLINUX_PYPY_X86_64_IMAGE', 'manylinux2010')
default_manylinux_images_x86_64 = {'manylinux1': 'quay.io/pypa/manylinux1_x86_64',
'manylinux2010': 'quay.io/pypa/manylinux2010_x86_64',
@@ -166,10 +167,12 @@ def main():
default_manylinux_images_i686 = {'manylinux1': 'quay.io/pypa/manylinux1_i686',
'manylinux2010': 'quay.io/pypa/manylinux2010_i686',
'manylinux2014': 'quay.io/pypa/manylinux2014_i686'}
default_manylinux_images_pypy_x86_64 = {'manylinux2010': 'pypywheels/manylinux2010-pypy_x86_64'}
build_options.update(
manylinux_images={'x86_64': default_manylinux_images_x86_64.get(manylinux_x86_64_image) or manylinux_x86_64_image,
'i686': default_manylinux_images_i686.get(manylinux_i686_image) or manylinux_i686_image},
'i686': default_manylinux_images_i686.get(manylinux_i686_image) or manylinux_i686_image,
'pypy_x86_64': default_manylinux_images_pypy_x86_64.get(manylinux_pypy_x86_64_image) or manylinux_pypy_x86_64_image},
)
elif platform == 'macos':
pass
+7 -4
View File
@@ -27,6 +27,8 @@ def get_python_configurations(build_selector):
PythonConfiguration(identifier='cp36-manylinux_i686', path='/opt/python/cp36-cp36m'),
PythonConfiguration(identifier='cp37-manylinux_i686', path='/opt/python/cp37-cp37m'),
PythonConfiguration(identifier='cp38-manylinux_i686', path='/opt/python/cp38-cp38'),
PythonConfiguration(identifier='pp27-manylinux_x86_64', path='/opt/python/pp27-pypy_73'),
PythonConfiguration(identifier='pp36-manylinux_x86_64', path='/opt/python/pp36-pypy36_pp73'),
]
# skip builds as required
@@ -45,12 +47,13 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
python_configurations = get_python_configurations(build_selector)
platforms = [
('manylinux_x86_64', manylinux_images['x86_64']),
('manylinux_i686', manylinux_images['i686']),
('cp', 'manylinux_x86_64', manylinux_images['x86_64']),
('cp', 'manylinux_i686', manylinux_images['i686']),
('pp', 'manylinux_x86_64', manylinux_images['pypy_x86_64']),
]
for platform_tag, docker_image in platforms:
platform_configs = [c for c in python_configurations if c.identifier.endswith(platform_tag)]
for implementation, platform_tag, docker_image in platforms:
platform_configs = [c for c in python_configurations if c.identifier.startswith(implementation) and c.identifier.endswith(platform_tag)]
if not platform_configs:
continue
+85 -40
View File
@@ -13,6 +13,16 @@ from .util import (
)
def call(args, env=None, cwd=None, shell=False):
# print the command executing for the logs
if shell:
print('+ %s' % args)
else:
print('+ ' + ' '.join(shlex.quote(a) for a in args))
return subprocess.check_call(args, env=env, cwd=cwd, shell=shell)
def get_python_configurations(build_selector):
PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'url'])
python_configurations = [
@@ -21,12 +31,79 @@ def get_python_configurations(build_selector):
PythonConfiguration(version='3.6', identifier='cp36-macosx_x86_64', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg'),
PythonConfiguration(version='3.7', identifier='cp37-macosx_x86_64', url='https://www.python.org/ftp/python/3.7.6/python-3.7.6-macosx10.9.pkg'),
PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.1/python-3.8.1-macosx10.9.pkg'),
PythonConfiguration(version='2.7-v7.3.0', identifier='pp27-macosx_x86_64', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.0-osx64.tar.bz2'),
PythonConfiguration(version='3.6-v7.3.0', identifier='pp36-macosx_x86_64', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.3.0-osx64.tar.bz2'),
]
# skip builds as required
return [c for c in python_configurations if build_selector(c.identifier)]
SYMLINKS_DIR = '/tmp/cibw_bin'
def make_symlinks(installation_bin_path, python_executable, pip_executable):
assert os.path.exists(os.path.join(installation_bin_path, python_executable))
# Python bin folders on Mac don't symlink `python3` to `python`, and neither
# does PyPy for `pypy` or `pypy3`, so we do that so `python` and `pip` always
# point to the active configuration.
if os.path.exists(SYMLINKS_DIR):
shutil.rmtree(SYMLINKS_DIR)
os.makedirs(SYMLINKS_DIR)
os.symlink(os.path.join(installation_bin_path, python_executable), os.path.join(SYMLINKS_DIR, 'python'))
os.symlink(os.path.join(installation_bin_path, python_executable + '-config'), os.path.join(SYMLINKS_DIR, 'python-config'))
os.symlink(os.path.join(installation_bin_path, pip_executable), os.path.join(SYMLINKS_DIR, 'pip'))
def install_cpython(version, url):
installed_system_packages = subprocess.check_output(['pkgutil', '--pkgs'], universal_newlines=True).splitlines()
# if this version of python isn't installed, get it from python.org and install
python_package_identifier = 'org.python.Python.PythonFramework-{}'.format(version)
if python_package_identifier not in installed_system_packages:
# download the pkg
download(url, '/tmp/Python.pkg')
# install
call(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/'])
# patch open ssl
if version == '3.5':
open_ssl_patch_url = 'https://github.com/mayeut/patch-macos-python-openssl/releases/download/v1.0.2t/patch-macos-python-%s-openssl-v1.0.2t.tar.gz' % version
download(open_ssl_patch_url, '/tmp/python-patch.tar.gz')
call(['sudo', 'tar', '-C', '/Library/Frameworks/Python.framework/Versions/{}/'.format(version), '-xmf', '/tmp/python-patch.tar.gz'])
installation_bin_path = '/Library/Frameworks/Python.framework/Versions/{}/bin'.format(version)
python_executable = 'python3' if version[0] == '3' else 'python'
pip_executable = 'pip3' if version[0] == '3' else 'pip'
make_symlinks(installation_bin_path, python_executable, pip_executable)
return installation_bin_path
def install_pypy(version, url):
pypy_tar_bz2 = url.rsplit('/', 1)[-1]
assert pypy_tar_bz2.endswith(".tar.bz2")
pypy_base_filename = os.path.splitext(os.path.splitext(pypy_tar_bz2)[0])[0]
installation_path = os.path.join('/tmp', pypy_base_filename)
if not os.path.exists(installation_path):
download(url, os.path.join("/tmp", pypy_tar_bz2))
call(['tar', '-C', '/tmp', '-xf', os.path.join("/tmp", pypy_tar_bz2)])
# fix PyPy 7.3.0 bug resulting in wrong macOS platform tag
if version.endswith("-v7.3.0") and version[0] == '3':
patch_file = os.path.abspath(os.path.join(os.path.dirname(__file__), 'resources', 'pypy3.6.patch'))
sysconfigdata_file = os.path.join(installation_path, 'lib_pypy', '_sysconfigdata.py')
call(['patch', sysconfigdata_file, patch_file, '-N']) # Always has nonzero return code
installation_bin_path = os.path.join(installation_path, 'bin')
python_executable = 'pypy3' if version[0] == '3' else 'pypy'
pip_executable = 'pip3' if version[0] == '3' else 'pip'
make_symlinks(installation_bin_path, python_executable, pip_executable)
return installation_bin_path
def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment):
abs_project_dir = os.path.abspath(project_dir)
temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
@@ -38,52 +115,20 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
get_pip_url = 'https://bootstrap.pypa.io/get-pip.py'
get_pip_script = '/tmp/get-pip.py'
pkgs_output = subprocess.check_output(['pkgutil', '--pkgs'], universal_newlines=True)
installed_system_packages = pkgs_output.splitlines()
def call(args, env=None, cwd=None, shell=False):
# print the command executing for the logs
if shell:
print('+ %s' % args)
else:
print('+ ' + ' '.join(shlex.quote(a) for a in args))
return subprocess.check_call(args, env=env, cwd=cwd, shell=shell)
# get latest pip once and for all
download(get_pip_url, get_pip_script)
for config in python_configurations:
# if this version of python isn't installed, get it from python.org and install
python_package_identifier = 'org.python.Python.PythonFramework-%s' % config.version
if python_package_identifier not in installed_system_packages:
# download the pkg
download(config.url, '/tmp/Python.pkg')
# install
call(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/'])
# patch open ssl
if config.version == '3.5':
open_ssl_patch_url = 'https://github.com/mayeut/patch-macos-python-openssl/releases/download/v1.0.2t/patch-macos-python-%s-openssl-v1.0.2t.tar.gz' % config.version
download(open_ssl_patch_url, '/tmp/python-patch.tar.gz')
call(['sudo', 'tar', '-C', '/Library/Frameworks/Python.framework/Versions/%s/' % config.version, '-xmf', '/tmp/python-patch.tar.gz'])
installation_bin_path = '/Library/Frameworks/Python.framework/Versions/{}/bin'.format(config.version)
assert os.path.exists(os.path.join(installation_bin_path, 'python3' if config.version[0] == '3' else 'python'))
# Python bin folders on Mac don't symlink python3 to python, so we do that
# so `python` and `pip` always point to the active configuration.
if os.path.exists('/tmp/cibw_bin'):
shutil.rmtree('/tmp/cibw_bin')
os.makedirs('/tmp/cibw_bin')
if config.version[0] == '3':
os.symlink(os.path.join(installation_bin_path, 'python3'), '/tmp/cibw_bin/python')
os.symlink(os.path.join(installation_bin_path, 'python3-config'), '/tmp/cibw_bin/python-config')
os.symlink(os.path.join(installation_bin_path, 'pip3'), '/tmp/cibw_bin/pip')
if config.identifier.startswith('cp'):
installation_bin_path = install_cpython(config.version, config.url)
elif config.identifier.startswith('pp'):
installation_bin_path = install_pypy(config.version, config.url)
else:
raise ValueError("Unknown Python implementation")
env = os.environ.copy()
env['PATH'] = os.pathsep.join([
'/tmp/cibw_bin',
SYMLINKS_DIR,
installation_bin_path,
env['PATH'],
])
@@ -94,7 +139,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
call(['python', '--version'], env=env)
# install pip & wheel
call(['python', get_pip_script, '--no-setuptools', '--no-wheel'], env=env, cwd="/tmp")
call(['python', get_pip_script], env=env, cwd="/tmp")
assert os.path.exists(os.path.join(installation_bin_path, 'pip'))
call(['pip', '--version'], env=env)
call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate'], env=env)
+7
View File
@@ -0,0 +1,7 @@
--- a/lib_pypy/_sysconfigdata.py Tue Jan 28 22:54:59 2020 +0200
+++ b/lib_pypy/_sysconfigdata.py Wed Jan 29 19:21:23 2020 +0200
@@ -47,4 +47,5 @@
build_time_vars['CC'] += ' -arch %s' % (arch,)
if "CXX" in build_time_vars:
build_time_vars['CXX'] += ' -arch %s' % (arch,)
+ build_time_vars['MACOSX_DEPLOYMENT_TARGET'] = '10.7'
+79 -46
View File
@@ -4,6 +4,7 @@ import subprocess
import tempfile
from collections import namedtuple
from glob import glob
from zipfile import ZipFile
from .util import (
download,
@@ -16,37 +17,54 @@ IS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache')
IS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows'
def get_python_path(config):
nuget_args = get_nuget_args(config)
return os.path.join(nuget_args[-1], nuget_args[0] + "." + config.version, "tools")
def simple_shell(args, env=None, cwd=None):
print('+ ' + ' '.join(args))
args = ['cmd', '/E:ON', '/V:ON', '/C'] + args
return subprocess.check_call(' '.join(args), env=env, cwd=cwd)
def get_nuget_args(configuration):
python_name = "python" if configuration.version[0] == '3' else "python2"
if configuration.arch == "32":
python_name = python_name + "x86"
return [python_name, "-Version", configuration.version, "-OutputDirectory", "C:/cibw/python"]
if IS_RUNNING_ON_AZURE or IS_RUNNING_ON_TRAVIS:
shell = simple_shell
else:
run_with_env = os.path.abspath(os.path.join(os.path.dirname(__file__), 'resources', 'appveyor_run_with_env.cmd'))
# run_with_env is a cmd file that sets the right environment variables
# to build on AppVeyor.
def shell(args, env=None, cwd=None):
# print the command executing for the logs
print('+ ' + ' '.join(args))
args = ['cmd', '/E:ON', '/V:ON', '/C', run_with_env] + args
return subprocess.check_call(' '.join(args), env=env, cwd=cwd)
def get_nuget_args(version, arch):
python_name = 'python' if version[0] == '3' else 'python2'
if arch == '32':
python_name = python_name + 'x86'
return [python_name, '-Version', version, '-OutputDirectory', 'C:\\cibw\\python']
def get_python_configurations(build_selector):
PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier'])
PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier', 'url'])
python_configurations = [
PythonConfiguration(version='2.7.17', arch="32", identifier='cp27-win32'),
PythonConfiguration(version='2.7.17', arch="64", identifier='cp27-win_amd64'),
PythonConfiguration(version='3.5.4', arch="32", identifier='cp35-win32'),
PythonConfiguration(version='3.5.4', arch="64", identifier='cp35-win_amd64'),
PythonConfiguration(version='3.6.8', arch="32", identifier='cp36-win32'),
PythonConfiguration(version='3.6.8', arch="64", identifier='cp36-win_amd64'),
PythonConfiguration(version='3.7.6', arch="32", identifier='cp37-win32'),
PythonConfiguration(version='3.7.6', arch="64", identifier='cp37-win_amd64'),
PythonConfiguration(version='3.8.1', arch="32", identifier='cp38-win32'),
PythonConfiguration(version='3.8.1', arch="64", identifier='cp38-win_amd64'),
PythonConfiguration(version='2.7.17', arch='32', identifier='cp27-win32', url=None),
PythonConfiguration(version='2.7.17', arch='64', identifier='cp27-win_amd64', url=None),
PythonConfiguration(version='3.5.4', arch='32', identifier='cp35-win32', url=None),
PythonConfiguration(version='3.5.4', arch='64', identifier='cp35-win_amd64', url=None),
PythonConfiguration(version='3.6.8', arch='32', identifier='cp36-win32', url=None),
PythonConfiguration(version='3.6.8', arch='64', identifier='cp36-win_amd64', url=None),
PythonConfiguration(version='3.7.6', arch='32', identifier='cp37-win32', url=None),
PythonConfiguration(version='3.7.6', arch='64', identifier='cp37-win_amd64', url=None),
PythonConfiguration(version='3.8.1', arch='32', identifier='cp38-win32', url=None),
PythonConfiguration(version='3.8.1', arch='64', identifier='cp38-win_amd64', url=None),
PythonConfiguration(version='2.7-v7.3.0', arch='32', identifier='pp27-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.0-win32.zip'),
PythonConfiguration(version='3.6-v7.3.0', arch='32', identifier='pp36-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.3.0-win32.zip'),
]
if IS_RUNNING_ON_TRAVIS:
# cannot install VCForPython27.msi which is needed for compiling C software
# try with (and similar): msiexec /i VCForPython27.msi ALLUSERS=1 ACCEPT=YES /passive
python_configurations = [c for c in python_configurations if not c.version.startswith('2.7.')]
python_configurations = [c for c in python_configurations if not c.version.startswith('2.7')]
# skip builds as required
python_configurations = [c for c in python_configurations if build_selector(c.identifier)]
@@ -54,25 +72,35 @@ def get_python_configurations(build_selector):
return python_configurations
def extract_zip(zip_src, dest):
with ZipFile(zip_src) as zip:
zip.extractall(dest)
def install_cpython(version, arch, nuget):
nuget_args = get_nuget_args(version, arch)
installation_path = os.path.join(nuget_args[-1], nuget_args[0] + '.' + version, 'tools')
simple_shell([nuget, 'install'] + nuget_args)
return installation_path
def install_pypy(version, arch, url):
assert arch == '32'
# Inside the PyPy zip file is a directory with the same name
zip_filename = url.rsplit('/', 1)[-1]
installation_path = os.path.join('C:\\cibw', os.path.splitext(zip_filename)[0])
if not os.path.exists(installation_path):
pypy_zip = os.path.join('C:\\cibw', zip_filename)
download(url, pypy_zip)
# Extract to the parent directory because the zip file still contains a directory
extract_zip(pypy_zip, os.path.dirname(installation_path))
pypy_exe = 'pypy3.exe' if version[0] == '3' else 'pypy.exe'
simple_shell(['mklink', os.path.join(installation_path, 'python.exe'), os.path.join(installation_path, pypy_exe)])
simple_shell(['mklink', '/d', os.path.join(installation_path, 'Scripts'), os.path.join(installation_path, 'bin')])
return installation_path
def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment):
def simple_shell(args, env=None, cwd=None):
print('+ ' + ' '.join(args))
args = ['cmd', '/E:ON', '/V:ON', '/C'] + args
return subprocess.check_call(' '.join(args), env=env, cwd=cwd)
if IS_RUNNING_ON_AZURE or IS_RUNNING_ON_TRAVIS:
shell = simple_shell
else:
run_with_env = os.path.abspath(os.path.join(os.path.dirname(__file__), 'resources', 'appveyor_run_with_env.cmd'))
# run_with_env is a cmd file that sets the right environment variables
# to build on AppVeyor.
def shell(args, env=None, cwd=None):
# print the command executing for the logs
print('+ ' + ' '.join(args))
args = ['cmd', '/E:ON', '/V:ON', '/C', run_with_env] + args
return subprocess.check_call(' '.join(args), env=env, cwd=cwd)
abs_project_dir = os.path.abspath(project_dir)
temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
built_wheel_dir = os.path.join(temp_dir, 'built_wheel')
@@ -88,17 +116,22 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
python_configurations = get_python_configurations(build_selector)
for config in python_configurations:
# install Python
config_python_path = get_python_path(config)
simple_shell([nuget, "install"] + get_nuget_args(config))
assert os.path.exists(os.path.join(config_python_path, 'python.exe'))
if config.identifier.startswith('cp'):
installation_path = install_cpython(config.version, config.arch, nuget)
elif config.identifier.startswith('pp'):
installation_path = install_pypy(config.version, config.arch, config.url)
else:
raise ValueError("Unknown Python implementation")
assert os.path.exists(os.path.join(installation_path, 'python.exe'))
# set up PATH and environment variables for run_with_env
env = os.environ.copy()
env['PYTHON_VERSION'] = config.version
env['PYTHON_ARCH'] = config.arch
env['PATH'] = os.pathsep.join([
config_python_path,
os.path.join(config_python_path, 'Scripts'),
installation_path,
os.path.join(installation_path, 'Scripts'),
env['PATH']
])
# update env with results from CIBW_ENVIRONMENT
@@ -107,12 +140,12 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
# for the logs - check we're running the right version of python
simple_shell(['where', 'python'], env=env)
simple_shell(['python', '--version'], env=env)
simple_shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)\"'], env=env)
simple_shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)"'], env=env)
# make sure pip is installed
if not os.path.exists(os.path.join(config_python_path, 'Scripts', 'pip.exe')):
if not os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe')):
simple_shell(['python', get_pip_script], env=env, cwd="C:\\cibw")
assert os.path.exists(os.path.join(config_python_path, 'Scripts', 'pip.exe'))
assert os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe'))
# prepare the Python environment
simple_shell(['python', '-m', 'pip', 'install', '--upgrade', 'pip'], env=env)
+1 -1
View File
@@ -22,7 +22,7 @@ Linux wheels are built in the [`manylinux` docker images](https://github.com/pyp
- The project directory is mounted in the running Docker instance as `/project`, the output directory for the wheels as `/output`. In general, this is handled transparently by `cibuildwheel`. For a more finegrained level of control however, the root of the host file system is mounted as `/host`, allowing for example to access shared files, caches, etc. on the host file system. Note that this is not available on CircleCI due to their Docker policies.
- Alternative dockers images can be specified with the `CIBW_MANYLINUX_X86_64_IMAGE` and `CIBW_MANYLINUX_I686_IMAGE` options to allow for a custom, preconfigured build environment for the Linux builds. See [options](options.md#manylinux-image) for more details.
- Alternative dockers images can be specified with the `CIBW_MANYLINUX_X86_64_IMAGE`, `CIBW_MANYLINUX_I686_IMAGE`, and `CIBW_MANYLINUX_PYPY_X86_64_IMAGE` options to allow for a custom, preconfigured build environment for the Linux builds. See [options](options.md#manylinux-image) for more details.
### Building packages with optional C extensions
+26 -14
View File
@@ -56,7 +56,7 @@ Default: `auto`
`auto` will auto-detect platform using environment variables, such as `TRAVIS_OS_NAME`/`APPVEYOR`/`CIRCLECI`.
For `linux` you need Docker running, on Mac or Linux. For `macos`, you need a Mac machine, and note that this script is going to automatically install MacPython on your system, so don't run on your development machine. For `windows`, you need to run in Windows, and it will build and test for all versions of Python at `C:\PythonXX[-x64]`.
For `linux` you need Docker running, on macOS or Linux. For `macos`, you need a Mac machine, and note that this script is going to automatically install MacPython on your system, so don't run on your development machine. For `windows`, you need to run in Windows, and `cibuildwheel` will install required versions of Python to `C:\cibw\python` using NuGet.
This option can also be set using the command-line option `--platform`.
@@ -68,21 +68,25 @@ Space-separated list of builds to build and skip. Each build has an identifier l
When both options are specified, both conditions are applied and only builds with a tag that matches `CIBW_BUILD` and does not match `CIBW_SKIP` will be built.
When setting the options, you can use shell-style globbing syntax (as per `fnmatch`). All the build identifiers supported by cibuildwheel are shown below:
When setting the options, you can use shell-style globbing syntax (as per [`fnmatch`](https://docs.python.org/3/library/fnmatch.html)). All the build identifiers supported by cibuildwheel are shown below:
<div class="build-id-table-marker"></div>
| | macOS 64bit | Manylinux 64bit | Manylinux 32bit | Windows 64bit | Windows 32bit |
|------------|--------------------|------------------------|----------------------|-----------------|----------------|
| Python 2.7 | cp27-macosx_x86_64 | cp27-manylinux_x86_64 | cp27-manylinux_i686 | cp27-win_amd64 | cp27-win32 |
| Python 3.5 | cp35-macosx_x86_64 | cp35-manylinux_x86_64 | cp35-manylinux_i686 | cp35-win_amd64 | cp35-win32 |
| Python 3.6 | cp36-macosx_x86_64 | cp36-manylinux_x86_64 | cp36-manylinux_i686 | cp36-win_amd64 | cp36-win32 |
| Python 3.7 | cp37-macosx_x86_64 | cp37-manylinux_x86_64 | cp37-manylinux_i686 | cp37-win_amd64 | cp37-win32 |
| Python 3.8 | cp38-macosx_x86_64 | cp38-manylinux_x86_64 | cp38-manylinux_i686 | cp38-win_amd64 | cp38-win32 |
| | macOS 64bit | Manylinux 64bit | Manylinux 32bit | Windows 64bit | Windows 32bit |
|-----------------|---------------------|------------------------|----------------------|-----------------|----------------|
| Python 2.7 | cp27-macosx_x86_64 | cp27-manylinux_x86_64 | cp27-manylinux_i686 | cp27-win_amd64 | cp27-win32 |
| Python 3.5 | cp35-macosx_x86_64 | cp35-manylinux_x86_64 | cp35-manylinux_i686 | cp35-win_amd64 | cp35-win32 |
| Python 3.6 | cp36-macosx_x86_64 | cp36-manylinux_x86_64 | cp36-manylinux_i686 | cp36-win_amd64 | cp36-win32 |
| Python 3.7 | cp37-macosx_x86_64 | cp37-manylinux_x86_64 | cp37-manylinux_i686 | cp37-win_amd64 | cp37-win32 |
| Python 3.8 | cp38-macosx_x86_64 | cp38-manylinux_x86_64 | cp38-manylinux_i686 | cp38-win_amd64 | cp38-win32 |
| PyPy 2.7 v7.3.0 | pp27-macosx_x86_64 | pp27-manylinux_x86_64 | | | pp27-win32 |
| PyPy 3.6 v7.3.0 | pp36-macosx_x86_64 | pp36-manylinux_x86_64 | | | pp36-win32 |
The list of supported and currently selected build identifiers can also be retrieved by passing the `--print-build-identifiers` flag to `cibuildwheel`.
The format is `python_tag-platform_tag`, with tags similar to those in [PEP 425](https://www.python.org/dev/peps/pep-0425/#details).
For CPython, the minimally supported macOS version is 10.9; for PyPy 2.7 and PyPy 3.6, respectively macOS 10.7 and 10.13 or higher is required.
#### Examples
```yaml
@@ -113,6 +117,12 @@ CIBW_SKIP: cp36-manylinux*
# Only build on Python 3 and skip 32-bit builds
CIBW_BUILD: cp3?-*
CIBW_SKIP: "*-win32 *-manylinux_i686"
# Only build PyPy and CPython 3
CIBW_BUILD: pp* cp3?-*
# Disable building PyPy wheels on all platforms
CIBW_SKIP: pp*
```
<style>
@@ -235,16 +245,16 @@ CIBW_REPAIR_WHEEL_COMMAND_LINUX: "auditwheel repair --lib-sdir . -w {dest_dir} {
```
### `CIBW_MANYLINUX_X86_64_IMAGE`, `CIBW_MANYLINUX_I686_IMAGE` {: #manylinux-image}
### `CIBW_MANYLINUX_X86_64_IMAGE`, `CIBW_MANYLINUX_I686_IMAGE`, `CIBW_MANYLINUX_PYPY_X86_64_IMAGE` {: #manylinux-image}
> Specify alternative manylinux docker images
An alternative Docker image to be used for building [`manylinux`](https://github.com/pypa/manylinux) wheels. `cibuildwheel` will then pull these instead of the default images, [`quay.io/pypa/manylinux2010_x86_64`](https://quay.io/pypa/manylinux2010_x86_64) and [`quay.io/pypa/manylinux2010_i686`](https://quay.io/pypa/manylinux2010_i686).
An alternative Docker image to be used for building [`manylinux`](https://github.com/pypa/manylinux) wheels. `cibuildwheel` will then pull these instead of the default images, [`quay.io/pypa/manylinux2010_x86_64`](https://quay.io/pypa/manylinux2010_x86_64), [`quay.io/pypa/manylinux2010_i686`](https://quay.io/pypa/manylinux2010_i686), and [`pypywheels/manylinux2010-pypy_x86_64`](https://hub.docker.com/r/pypywheels/manylinux2010-pypy_x86_64).
The value of this option can either be set to `manylinux1`, `manylinux2010` or `manylinux2014` to use the [official `manylinux` images](https://github.com/pypa/manylinux), or any other valid Docker image name.
The value of this option can either be set to `manylinux1`, `manylinux2010` or `manylinux2014` to use the [official `manylinux` images](https://github.com/pypa/manylinux) and [PyPy `manylinux` images](https://github.com/pypy/manylinux), or any other valid Docker image name. Note that for PyPy, only the official `manylinux2010` image is currently available.
Beware to specify a valid Docker image that can be used in the same way as the official, default Docker images: all necessary Python and pip versions need to be present in `/opt/python/`, and the `auditwheel` tool needs to be present for `cibuildwheel` to work. Apart from that, the architecture and relevant shared system libraries need to be manylinux1-, manylinux2010- or manylinux2014-compatible in order to produce valid `manylinux1`/`manylinux2010`/`manylinux2014` wheels (see https://github.com/pypa/manylinux, [PEP 513](https://www.python.org/dev/peps/pep-0513/), [PEP 571](https://www.python.org/dev/peps/pep-0571/ and [PEP 599](https://www.python.org/dev/peps/pep-0599/) for more details).
Beware to specify a valid Docker image that can be used in the same way as the official, default Docker images: all necessary Python and pip versions need to be present in `/opt/python/`, and the `auditwheel` tool needs to be present for `cibuildwheel` to work. Apart from that, the architecture and relevant shared system libraries need to be manylinux1-, manylinux2010- or manylinux2014-compatible in order to produce valid `manylinux1`/`manylinux2010`/`manylinux2014` wheels (see [pypa/manylinux on GitHub](https://github.com/pypa/manylinux), [PEP 513](https://www.python.org/dev/peps/pep-0513/), [PEP 571](https://www.python.org/dev/peps/pep-0571/) and [PEP 599](https://www.python.org/dev/peps/pep-0599/) for more details).
Note that `auditwheel` detects the version of the `manylinux` standard in the Docker image through the `AUDITWHEEL_PLAT` environment variable, as `cibuildwheel` has no way of detecting the correct `--plat` command line argument to pass to `auditwheel` for a custom image. If a Docker image does not correctly set this `AUDITWHEEL_PLAT` environment variable, the `CIBW_ENVIRONMENT` option can be used to do so (e.g., `CIBW_ENVIRONMENT="manylinux2010_$(uname -m)"`).
Note that `auditwheel` detects the version of the `manylinux` standard in the Docker image through the `AUDITWHEEL_PLAT` environment variable, as `cibuildwheel` has no way of detecting the correct `--plat` command line argument to pass to `auditwheel` for a custom image. If a Docker image does not correctly set this `AUDITWHEEL_PLAT` environment variable, the `CIBW_ENVIRONMENT` option can be used to do so (e.g., `CIBW_ENVIRONMENT='AUDITWHEEL_PLAT="manylinux2010_$(uname -m)"'`).
Note that `manylinux2014` doesn't support builds with Python 2.7 - when building with `manylinux2014`, skip Python 2.7 using `CIBW_SKIP` (see example below).
@@ -252,8 +262,10 @@ Note that `manylinux2014` doesn't support builds with Python 2.7 - when building
```yaml
# build using the manylinux1 image to ensure manylinux1 wheels are produced
# skip PyPy, since there is no PyPy manylinux1 image
CIBW_MANYLINUX_X86_64_IMAGE: manylinux1
CIBW_MANYLINUX_I686_IMAGE: manylinux1
CIBW_SKIP: pp*
# build using the manylinux2014 image
CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
+3 -1
View File
@@ -14,8 +14,10 @@ def test():
actual_wheels = utils.cibuildwheel_run(project_dir, add_env={
'CIBW_MANYLINUX_X86_64_IMAGE': 'dockcross/manylinux2010-x64',
'CIBW_MANYLINUX_I686_IMAGE': 'dockcross/manylinux2010-x86',
'CIBW_SKIP': 'pp*',
})
# also check that we got the right wheels built
expected_wheels = utils.expected_wheels('spam', '0.1.0')
expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0')
if '-pp' not in w]
assert set(actual_wheels) == set(expected_wheels)
@@ -19,12 +19,19 @@ def test(manylinux_image):
'CIBW_ENVIRONMENT': 'CFLAGS="$CFLAGS -Werror=implicit-function-declaration"',
'CIBW_MANYLINUX_X86_64_IMAGE': manylinux_image,
'CIBW_MANYLINUX_I686_IMAGE': manylinux_image,
'CIBW_MANYLINUX_PYPY_X86_64_IMAGE': manylinux_image,
}
if manylinux_image == 'manylinux2014':
add_env['CIBW_SKIP'] = 'cp27*' # not available on manylinux2014
if manylinux_image == 'manylinux1':
# We don't have a manylinux1 image for PyPy
add_env['CIBW_SKIP'] = 'pp*'
elif manylinux_image == 'manylinux2014':
# We don't have a manylinux2014 image for PyPy (yet?)
add_env['CIBW_SKIP'] = 'cp27* pp*' # Python 2.7 not available on manylinux2014
actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env)
expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0', manylinux_versions=[manylinux_image])]
if manylinux_image == 'manylinux2014':
expected_wheels = [w for w in expected_wheels if '-cp27' not in w]
if manylinux_image in ['manylinux1', 'manylinux2014']:
expected_wheels = [w for w in expected_wheels if '-pp' not in w]
assert set(actual_wheels) == set(expected_wheels)
+17 -11
View File
@@ -10,22 +10,23 @@ project_dir = os.path.dirname(__file__)
def test_cpp11(tmp_path):
# This test checks that the C++11 standard is supported
add_env = {'CIBW_SKIP': 'cp27-win*', 'CIBW_ENVIRONMENT': 'STANDARD=11'}
add_env = {'CIBW_SKIP': 'cp27-win* pp27-win32', 'CIBW_ENVIRONMENT': 'STANDARD=11'}
# VC++ for Python 2.7 does not support modern standards
if utils.platform == 'macos':
add_env['MACOSX_DEPLOYMENT_TARGET'] = '10.9'
actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env)
expected_wheels = [x for x in utils.expected_wheels(
expected_wheels = [w for w in utils.expected_wheels(
'spam', '0.1.0', macosx_deployment_target='10.9')
if 'cp27-cp27m-win' not in x]
if 'cp27-cp27m-win' not in w
and 'pp27-pypy_73-win32' not in w]
assert set(actual_wheels) == set(expected_wheels)
def test_cpp14():
# This test checks that the C++14 standard is supported
add_env = {'CIBW_SKIP': 'cp27-win* cp35-win*', 'CIBW_ENVIRONMENT': 'STANDARD=14'}
add_env = {'CIBW_SKIP': 'cp27-win* pp27-win32 cp35-win*', 'CIBW_ENVIRONMENT': 'STANDARD=14'}
# VC++ for Python 2.7 does not support modern standards
# The manylinux1 docker image does not have a compiler which supports C++11
# Python 3.4 and 3.5 are compiled with MSVC 10, which does not support C++14
@@ -33,27 +34,32 @@ def test_cpp14():
add_env['MACOSX_DEPLOYMENT_TARGET'] = '10.9'
actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env)
expected_wheels = [x for x in utils.expected_wheels(
expected_wheels = [w for w in utils.expected_wheels(
'spam', '0.1.0', macosx_deployment_target='10.9')
if 'cp27-cp27m-win' not in x and 'cp35-cp35m-win' not in x]
if 'cp27-cp27m-win' not in w
and 'pp27-pypy_73-win32' not in w
and 'cp35-cp35m-win' not in w]
assert set(actual_wheels) == set(expected_wheels)
def test_cpp17():
# This test checks that the C++17 standard is supported
# Python 2.7 uses the `register` keyword which is forbidden in the C++17 standard
# Python and PyPy 2.7 use the `register` keyword which is forbidden in the C++17 standard
# The manylinux1 docker image does not have a compiler which supports C++11
# Python 3.4 and 3.5 are compiled with MSVC 10, which does not support C++17
# Python 3.5 and PyPy 3.6 are compiled with MSVC 10, which does not support C++17
if os.environ.get('APPVEYOR_BUILD_WORKER_IMAGE', '') == 'Visual Studio 2015':
pytest.skip('Visual Studio 2015 does not support C++17')
add_env = {'CIBW_SKIP': 'cp27-win* cp35-win*', 'CIBW_ENVIRONMENT': 'STANDARD=17'}
add_env = {'CIBW_SKIP': 'cp27-win* pp27-win32 cp35-win* pp36-win32', 'CIBW_ENVIRONMENT': 'STANDARD=17'}
if utils.platform == 'macos':
add_env['MACOSX_DEPLOYMENT_TARGET'] = '10.13'
actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env)
expected_wheels = [x for x in utils.expected_wheels(
expected_wheels = [w for w in utils.expected_wheels(
'spam', '0.1.0', macosx_deployment_target='10.13')
if 'cp27-cp27m-win' not in x and 'cp35-cp35m-win' not in x]
if 'cp27-cp27m-win' not in w
and 'pp27-pypy_73-win32' not in w
and 'cp35-cp35m-win' not in w
and 'pp36-pypy36_pp73-win32' not in w]
assert set(actual_wheels) == set(expected_wheels)
+17 -14
View File
@@ -79,29 +79,32 @@ def expected_wheels(package_name, package_version, manylinux_versions=['manylinu
# {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl
# {python tag} and {abi tag} are closely related to the python interpreter used to build the wheel
# so we'll merge them below as python_abi_tag
python_abi_tags = ['cp27-cp27m', 'cp35-cp35m', 'cp36-cp36m', 'cp37-cp37m', 'cp38-cp38']
python_abi_tags = ['cp27-cp27m', 'cp35-cp35m', 'cp36-cp36m', 'cp37-cp37m', 'cp38-cp38',
'pp27-pypy_73', 'pp36-pypy36_pp73']
if platform == 'linux':
python_abi_tags.append('cp27-cp27mu') # python 2.7 has 2 different ABI on manylinux
platform_tags = []
for architecture in ['x86_64', 'i686']:
for manylinux_version in manylinux_versions:
platform_tags.append('{manylinux_version}_{architecture}'.format(
manylinux_version=manylinux_version, architecture=architecture
))
architectures = {'cp': ['x86_64', 'i686'], 'pp': ['x86_64']}
platform_tags = {}
for python_implemention in architectures:
platform_tags[python_implemention] = [
'{manylinux_version}_{architecture}'.format(
manylinux_version=manylinux_version, architecture=architecture)
for architecture in architectures[python_implemention]
for manylinux_version in manylinux_versions
]
def get_platform_tags(python_abi_tag):
return platform_tags
return platform_tags[python_abi_tag[:2]]
elif platform == 'windows':
platform_tags = {'cp': ['win32', 'win_amd64'], 'pp': ['win32']}
def get_platform_tags(python_abi_tag):
return ['win32', 'win_amd64']
return platform_tags[python_abi_tag[:2]]
elif platform == 'macos':
def get_platform_tags(python_abi_tag):
return ['macosx_' + (macosx_deployment_target or "10.9").replace(".", "_") + '_x86_64']
default_version = '10.7' if python_abi_tag.startswith('pp') else '10.9'
return ['macosx_{}_x86_64'.format((macosx_deployment_target or default_version).replace('.', '_'))]
else:
raise Exception('unsupported platform')
@@ -115,7 +118,7 @@ def expected_wheels(package_name, package_version, manylinux_versions=['manylinu
if IS_WINDOWS_RUNNING_ON_TRAVIS:
# Python 2.7 isn't supported on Travis.
templates = [t for t in templates if '-cp27-' not in t]
templates = [t for t in templates if '-cp27-' not in t and '-pp2' not in t]
return templates
@@ -67,6 +67,11 @@ def test_build_selector(platform, intercepted_build_args, monkeypatch):
('i686', 'manylinux2010', 'quay.io/pypa/manylinux2010_i686'),
('i686', 'manylinux2014', 'quay.io/pypa/manylinux2014_i686'),
('i686', 'custom_image', 'custom_image'),
('pypy_x86_64', None, 'pypywheels/manylinux2010-pypy_x86_64'),
('pypy_x86_64', 'manylinux1', 'manylinux1'), # Does not exist
('pypy_x86_64', 'manylinux2010', 'pypywheels/manylinux2010-pypy_x86_64'),
('pypy_x86_64', 'manylinux2014', 'manylinux2014'), # Does not exist (yet)
('pypy_x86_64', 'custom_image', 'custom_image'),
])
def test_manylinux_images(architecture, image, full_image, platform, intercepted_build_args, monkeypatch):
if image is not None: