From 100fc14411ec4b6046e0b3283859f797fc0cd465 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Wed, 30 Oct 2019 14:57:47 +0100 Subject: [PATCH 01/26] Adding PyPy builds to Linux, using the manylinux2010-pypy_x86_64 image --- cibuildwheel/__main__.py | 5 ++++- cibuildwheel/linux.py | 3 +++ test/shared/utils.py | 21 +++++++++++++-------- 3 files changed, 20 insertions(+), 9 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index b3aaacf2..1e5b249b 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -159,6 +159,7 @@ def main(): if platform == 'linux': manylinux_x86_64_image = os.environ.get('CIBW_MANYLINUX_X86_64_IMAGE', 'manylinux2010') manylinux_i686_image = os.environ.get('CIBW_MANYLINUX_I686_IMAGE', 'manylinux2010') + manylinux_pypy_x86_64_image = os.environ.get('CIBW_MANYLINUX_PYPY_X86_64_IMAGE', 'manylinux2010') default_manylinux_images_x86_64 = {'manylinux1': 'quay.io/pypa/manylinux1_x86_64', 'manylinux2010': 'quay.io/pypa/manylinux2010_x86_64', @@ -166,10 +167,12 @@ def main(): default_manylinux_images_i686 = {'manylinux1': 'quay.io/pypa/manylinux1_i686', 'manylinux2010': 'quay.io/pypa/manylinux2010_i686', 'manylinux2014': 'quay.io/pypa/manylinux2014_i686'} + default_manylinux_images_pypy_x86_64 = {'manylinux2010': 'pypywheels/manylinux2010-pypy_x86_64'} build_options.update( manylinux_images={'x86_64': default_manylinux_images_x86_64.get(manylinux_x86_64_image) or manylinux_x86_64_image, - 'i686': default_manylinux_images_i686.get(manylinux_i686_image) or manylinux_i686_image}, + 'i686': default_manylinux_images_i686.get(manylinux_i686_image) or manylinux_i686_image, + 'pypy_x86_64': default_manylinux_images_pypy_x86_64.get(manylinux_pypy_x86_64_image) or manylinux_pypy_x86_64_image}, ) elif platform == 'macos': pass diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 24a034d6..e4e60913 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -27,6 +27,8 @@ def get_python_configurations(build_selector): PythonConfiguration(identifier='cp36-manylinux_i686', path='/opt/python/cp36-cp36m'), PythonConfiguration(identifier='cp37-manylinux_i686', path='/opt/python/cp37-cp37m'), PythonConfiguration(identifier='cp38-manylinux_i686', path='/opt/python/cp38-cp38'), + PythonConfiguration(identifier='pp271-manylinux_pypy_x86_64', path='/opt/python/pp271-pypy_41'), + PythonConfiguration(identifier='pp371-manylinux_pypy_x86_64', path='/opt/python/pp371-pypy3_71'), ] # skip builds as required @@ -47,6 +49,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef platforms = [ ('manylinux_x86_64', manylinux_images['x86_64']), ('manylinux_i686', manylinux_images['i686']), + ('manylinux_pypy_x86_64', manylinux_images['pypy_x86_64']), ] for platform_tag, docker_image in platforms: diff --git a/test/shared/utils.py b/test/shared/utils.py index 4d6b96d0..beb2f357 100644 --- a/test/shared/utils.py +++ b/test/shared/utils.py @@ -82,16 +82,21 @@ def expected_wheels(package_name, package_version, manylinux_versions=['manylinu python_abi_tags = ['cp27-cp27m', 'cp35-cp35m', 'cp36-cp36m', 'cp37-cp37m', 'cp38-cp38'] if platform == 'linux': python_abi_tags.append('cp27-cp27mu') # python 2.7 has 2 different ABI on manylinux - platform_tags = [] - for architecture in ['x86_64', 'i686']: - for manylinux_version in manylinux_versions: - platform_tags.append('{manylinux_version}_{architecture}'.format( - manylinux_version=manylinux_version, architecture=architecture - )) + # Starting out by adding PyPy support on Linux + python_abi_tags.extend(['pp271-pypy_41', 'pp371-pypy3_71']) + cp_architectures = ['x86_64', 'i686'] + pp_architectures = ['x86_64'] def get_platform_tags(python_abi_tag): - return platform_tags - + if python_abi_tag.startswith('pp'): + architectures = pp_architectures + else: + architectures = cp_architectures + return ['{manylinux_version}_{architecture}'.format( + manylinux_version=manylinux_version, architecture=architecture + ) + for manylinux_version in manylinux_versions + for architecture in architectures] elif platform == 'windows': def get_platform_tags(python_abi_tag): From ef80beb433f56ab87bc9a9dfba1042c1a13f67a9 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Wed, 30 Oct 2019 17:05:17 +0100 Subject: [PATCH 02/26] Skipping PyPy for 06_docker_images test, and manylinux1 and manylinux2014 runs of 08_manylinuxXXXX_only test --- test/06_docker_images/cibuildwheel_test.py | 4 +++- test/08_manylinuxXXXX_only/cibuildwheel_test.py | 11 +++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/test/06_docker_images/cibuildwheel_test.py b/test/06_docker_images/cibuildwheel_test.py index 9f243664..4dbf40bd 100644 --- a/test/06_docker_images/cibuildwheel_test.py +++ b/test/06_docker_images/cibuildwheel_test.py @@ -14,8 +14,10 @@ def test(): actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ 'CIBW_MANYLINUX_X86_64_IMAGE': 'dockcross/manylinux2010-x64', 'CIBW_MANYLINUX_I686_IMAGE': 'dockcross/manylinux2010-x86', + 'CIBW_SKIP': 'pp*-*', }) # also check that we got the right wheels built - expected_wheels = utils.expected_wheels('spam', '0.1.0') + expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0') + if 'pypy' not in w] assert set(actual_wheels) == set(expected_wheels) diff --git a/test/08_manylinuxXXXX_only/cibuildwheel_test.py b/test/08_manylinuxXXXX_only/cibuildwheel_test.py index 439f428f..8ae314db 100644 --- a/test/08_manylinuxXXXX_only/cibuildwheel_test.py +++ b/test/08_manylinuxXXXX_only/cibuildwheel_test.py @@ -19,12 +19,19 @@ def test(manylinux_image): 'CIBW_ENVIRONMENT': 'CFLAGS="$CFLAGS -Werror=implicit-function-declaration"', 'CIBW_MANYLINUX_X86_64_IMAGE': manylinux_image, 'CIBW_MANYLINUX_I686_IMAGE': manylinux_image, + 'CIBW_MANYLINUX_PYPY_X86_64_IMAGE': manylinux_image, } - if manylinux_image == 'manylinux2014': - add_env['CIBW_SKIP'] = 'cp27*' # not available on manylinux2014 + if manylinux_image == 'manylinux1': + # We don't have a manylinux1 image for PyPy + add_env['CIBW_SKIP'] = 'pp*' + elif manylinux_image == 'manylinux2014': + # We don't have a manylinux2014 image for PyPy (yet?) + add_env['CIBW_SKIP'] = 'cp27* pp*' # Python 2.7 not available on manylinux2014 actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env) expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0', manylinux_versions=[manylinux_image])] if manylinux_image == 'manylinux2014': expected_wheels = [w for w in expected_wheels if '-cp27' not in w] + if manylinux_image in ['manylinux1', 'manylinux2014']: + expected_wheels = [w for w in expected_wheels if '-pp' not in w] assert set(actual_wheels) == set(expected_wheels) From dd26ef8ac6c5c71a32f2f41d407ce61bfce8a028 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Wed, 6 Nov 2019 19:21:51 +0100 Subject: [PATCH 03/26] Updated PyPy from 7.1.1 to 7.2.0 --- cibuildwheel/linux.py | 4 ++-- test/shared/utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index e4e60913..848a2b94 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -27,8 +27,8 @@ def get_python_configurations(build_selector): PythonConfiguration(identifier='cp36-manylinux_i686', path='/opt/python/cp36-cp36m'), PythonConfiguration(identifier='cp37-manylinux_i686', path='/opt/python/cp37-cp37m'), PythonConfiguration(identifier='cp38-manylinux_i686', path='/opt/python/cp38-cp38'), - PythonConfiguration(identifier='pp271-manylinux_pypy_x86_64', path='/opt/python/pp271-pypy_41'), - PythonConfiguration(identifier='pp371-manylinux_pypy_x86_64', path='/opt/python/pp371-pypy3_71'), + PythonConfiguration(identifier='pp272-manylinux_pypy_x86_64', path='/opt/python/pp272-pypy_41'), + PythonConfiguration(identifier='pp372-manylinux_pypy_x86_64', path='/opt/python/pp372-pypy3_72'), ] # skip builds as required diff --git a/test/shared/utils.py b/test/shared/utils.py index beb2f357..fcacceb9 100644 --- a/test/shared/utils.py +++ b/test/shared/utils.py @@ -83,7 +83,7 @@ def expected_wheels(package_name, package_version, manylinux_versions=['manylinu if platform == 'linux': python_abi_tags.append('cp27-cp27mu') # python 2.7 has 2 different ABI on manylinux # Starting out by adding PyPy support on Linux - python_abi_tags.extend(['pp271-pypy_41', 'pp371-pypy3_71']) + python_abi_tags.extend(['pp272-pypy_41', 'pp372-pypy3_72']) cp_architectures = ['x86_64', 'i686'] pp_architectures = ['x86_64'] From 129cf601da0d794bef4bc07507b79793c1eb97e5 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Thu, 7 Nov 2019 00:55:58 +0100 Subject: [PATCH 04/26] Building PyPy wheels on Windows --- cibuildwheel/windows.py | 54 +++++++++++++++++++++++++++++------------ test/shared/utils.py | 10 +++++--- 2 files changed, 45 insertions(+), 19 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index abb3a0a2..01e1a8a2 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -4,6 +4,7 @@ import subprocess import tempfile from collections import namedtuple from glob import glob +from zipfile import ZipFile from .util import ( download, @@ -16,9 +17,14 @@ IS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache') IS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows' -def get_python_path(config): - nuget_args = get_nuget_args(config) - return os.path.join(nuget_args[-1], nuget_args[0] + "." + config.version, "tools") +def get_python_path(config): + if config.identifier.startswith('cp'): + nuget_args = get_nuget_args(config) + return os.path.join(nuget_args[-1], nuget_args[0] + "." + config.version, "tools") + elif config.identifier.startswith('pp'): + # Inside the PyPy zip file is a directory with the same name + filename = config.url.rsplit('/', 1)[-1] + return os.path.join("C:\\PyPy", os.path.splitext(filename)[0]) def get_nuget_args(configuration): @@ -29,24 +35,26 @@ def get_nuget_args(configuration): def get_python_configurations(build_selector): - PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier']) + PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier', 'url']) python_configurations = [ - PythonConfiguration(version='2.7.17', arch="32", identifier='cp27-win32'), - PythonConfiguration(version='2.7.17', arch="64", identifier='cp27-win_amd64'), - PythonConfiguration(version='3.5.4', arch="32", identifier='cp35-win32'), - PythonConfiguration(version='3.5.4', arch="64", identifier='cp35-win_amd64'), - PythonConfiguration(version='3.6.8', arch="32", identifier='cp36-win32'), - PythonConfiguration(version='3.6.8', arch="64", identifier='cp36-win_amd64'), - PythonConfiguration(version='3.7.6', arch="32", identifier='cp37-win32'), - PythonConfiguration(version='3.7.6', arch="64", identifier='cp37-win_amd64'), - PythonConfiguration(version='3.8.1', arch="32", identifier='cp38-win32'), - PythonConfiguration(version='3.8.1', arch="64", identifier='cp38-win_amd64'), + PythonConfiguration(version='2.7.17', arch="32", identifier='cp27-win32', url=None), + PythonConfiguration(version='2.7.17', arch="64", identifier='cp27-win_amd64', url=None), + PythonConfiguration(version='3.5.4', arch="32", identifier='cp35-win32', url=None), + PythonConfiguration(version='3.5.4', arch="64", identifier='cp35-win_amd64', url=None), + PythonConfiguration(version='3.6.8', arch="32", identifier='cp36-win32', url=None), + PythonConfiguration(version='3.6.8', arch="64", identifier='cp36-win_amd64', url=None), + PythonConfiguration(version='3.7.6', arch="32", identifier='cp37-win32', url=None), + PythonConfiguration(version='3.7.6', arch="64", identifier='cp37-win_amd64', url=None), + PythonConfiguration(version='3.8.1', arch="32", identifier='cp38-win32', url=None), + PythonConfiguration(version='3.8.1', arch="64", identifier='cp38-win_amd64', url=None), + PythonConfiguration(version='2.7-v7.2.0', arch="32", identifier='pp272-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.2.0-win32.zip'), + PythonConfiguration(version='3.6-v7.2.0', arch="32", identifier='pp372-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.2.0-win32.zip'), ] if IS_RUNNING_ON_TRAVIS: # cannot install VCForPython27.msi which is needed for compiling C software # try with (and similar): msiexec /i VCForPython27.msi ALLUSERS=1 ACCEPT=YES /passive - python_configurations = [c for c in python_configurations if not c.version.startswith('2.7.')] + python_configurations = [c for c in python_configurations if not c.version.startswith('2.7')] # skip builds as required python_configurations = [c for c in python_configurations if build_selector(c.identifier)] @@ -60,6 +68,10 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef args = ['cmd', '/E:ON', '/V:ON', '/C'] + args return subprocess.check_call(' '.join(args), env=env, cwd=cwd) + def extract_zip(zip_src, dest): + with ZipFile(zip_src) as zip: + zip.extractall(dest) + if IS_RUNNING_ON_AZURE or IS_RUNNING_ON_TRAVIS: shell = simple_shell else: @@ -89,7 +101,17 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef for config in python_configurations: # install Python config_python_path = get_python_path(config) - simple_shell([nuget, "install"] + get_nuget_args(config)) + if config.identifier.startswith('cp'): + simple_shell([nuget, "install"] + get_nuget_args(config)) + elif config.identifier.startswith('pp') and not os.path.exists(config_python_path): + pypy_zip = config.url.rsplit('/', 1)[-1] + download(config.url, pypy_zip) + # Extract to the parent of config_python_path because the zip file still contains a directory + extract_zip(pypy_zip, os.path.dirname(config_python_path)) + pypy_exe = 'pypy.exe' if config.version[0] == '2' else 'pypy3.exe' + simple_shell(['mklink', os.path.join(config_python_path, 'python.exe'), os.path.join(config_python_path, pypy_exe)]) + simple_shell(['mklink', '/d', os.path.join(config_python_path, 'Scripts'), os.path.join(config_python_path, 'bin')]) + simple_shell(['chcp', '437']) # Workaround for https://bitbucket.org/pypy/pypy/issues/3081/ assert os.path.exists(os.path.join(config_python_path, 'python.exe')) # set up PATH and environment variables for run_with_env diff --git a/test/shared/utils.py b/test/shared/utils.py index fcacceb9..28b5dcf7 100644 --- a/test/shared/utils.py +++ b/test/shared/utils.py @@ -98,9 +98,13 @@ def expected_wheels(package_name, package_version, manylinux_versions=['manylinu for manylinux_version in manylinux_versions for architecture in architectures] elif platform == 'windows': - + # Second step: adding PyPy support on Windows + python_abi_tags.extend(['pp272-pypy_41', 'pp372-pp372']) def get_platform_tags(python_abi_tag): - return ['win32', 'win_amd64'] + if python_abi_tag.startswith('pp'): + return ['win32'] + else: + return ['win32', 'win_amd64'] elif platform == 'macos': @@ -120,7 +124,7 @@ def expected_wheels(package_name, package_version, manylinux_versions=['manylinu if IS_WINDOWS_RUNNING_ON_TRAVIS: # Python 2.7 isn't supported on Travis. - templates = [t for t in templates if '-cp27-' not in t] + templates = [t for t in templates if '-cp27-' not in t and '-pp2' not in t] return templates From 743051a4530f4c414c2e497937877c5fc021653b Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Sun, 10 Nov 2019 21:56:34 +0100 Subject: [PATCH 05/26] Using C:\cibw as base folder for PyPy installation (cfr. #197) --- cibuildwheel/windows.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 01e1a8a2..d1f90281 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -24,7 +24,7 @@ def get_python_path(config): elif config.identifier.startswith('pp'): # Inside the PyPy zip file is a directory with the same name filename = config.url.rsplit('/', 1)[-1] - return os.path.join("C:\\PyPy", os.path.splitext(filename)[0]) + return os.path.join("C:\\cibw", os.path.splitext(filename)[0]) def get_nuget_args(configuration): @@ -104,7 +104,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef if config.identifier.startswith('cp'): simple_shell([nuget, "install"] + get_nuget_args(config)) elif config.identifier.startswith('pp') and not os.path.exists(config_python_path): - pypy_zip = config.url.rsplit('/', 1)[-1] + pypy_zip = os.path.join("C:\\cibw", config.url.rsplit('/', 1)[-1]) download(config.url, pypy_zip) # Extract to the parent of config_python_path because the zip file still contains a directory extract_zip(pypy_zip, os.path.dirname(config_python_path)) From ae6832e18d7565cff603d4880cf95ee36bfb422b Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Sun, 17 Nov 2019 17:56:07 +0100 Subject: [PATCH 06/26] Building PyPy wheels on macOS --- cibuildwheel/macos.py | 55 ++++++++++++++++++++++++++++++++----------- test/shared/utils.py | 18 +++++++------- 2 files changed, 51 insertions(+), 22 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index c79c5a95..267824d9 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -21,6 +21,8 @@ def get_python_configurations(build_selector): PythonConfiguration(version='3.6', identifier='cp36-macosx_x86_64', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg'), PythonConfiguration(version='3.7', identifier='cp37-macosx_x86_64', url='https://www.python.org/ftp/python/3.7.6/python-3.7.6-macosx10.9.pkg'), PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.1/python-3.8.1-macosx10.9.pkg'), + PythonConfiguration(version='2.7-v7.2.0', identifier='pp272-macosx_x86_64', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.2.0-osx64.tar.bz2'), + PythonConfiguration(version='3.6-v7.2.0', identifier='pp372-macosx_x86_64', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.2.0-osx64.tar.bz2'), ] # skip builds as required @@ -54,20 +56,42 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef download(get_pip_url, get_pip_script) for config in python_configurations: - # if this version of python isn't installed, get it from python.org and install - python_package_identifier = 'org.python.Python.PythonFramework-%s' % config.version - if python_package_identifier not in installed_system_packages: - # download the pkg - download(config.url, '/tmp/Python.pkg') - # install - call(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/']) - # patch open ssl - if config.version == '3.5': - open_ssl_patch_url = 'https://github.com/mayeut/patch-macos-python-openssl/releases/download/v1.0.2t/patch-macos-python-%s-openssl-v1.0.2t.tar.gz' % config.version - download(open_ssl_patch_url, '/tmp/python-patch.tar.gz') - call(['sudo', 'tar', '-C', '/Library/Frameworks/Python.framework/Versions/%s/' % config.version, '-xmf', '/tmp/python-patch.tar.gz']) + if config.identifier.startswith('cp'): + # if this version of python isn't installed, get it from python.org and install + python_package_identifier = 'org.python.Python.PythonFramework-%s' % config.version + if python_package_identifier not in installed_system_packages: + # download the pkg + download(config.url, '/tmp/Python.pkg') + # install + call(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/']) + # patch open ssl + if config.version == '3.5': + open_ssl_patch_url = 'https://github.com/mayeut/patch-macos-python-openssl/releases/download/v1.0.2t/patch-macos-python-%s-openssl-v1.0.2t.tar.gz' % config.version + download(open_ssl_patch_url, '/tmp/python-patch.tar.gz') + call(['sudo', 'tar', '-C', '/Library/Frameworks/Python.framework/Versions/%s/' % config.version, '-xmf', '/tmp/python-patch.tar.gz']) + + installation_bin_path = '/Library/Frameworks/Python.framework/Versions/{}/bin'.format(config.version) + elif config.identifier.startswith('pp'): + pypy_tar_bz2 = config.url.rsplit('/', 1)[-1] + assert pypy_tar_bz2.endswith(".tar.bz2") + pypy_base_filename = os.path.splitext(os.path.splitext(pypy_tar_bz2)[0])[0] + installation_bin_path = os.path.join('/tmp', pypy_base_filename, 'bin') + if not os.path.exists(installation_bin_path): + download(config.url, os.path.join("/tmp", pypy_tar_bz2)) + call(['tar', '-C', '/tmp', '-xf', os.path.join("/tmp", pypy_tar_bz2)]) + pypy_executable = 'pypy' if config.version[0] == '2' else 'pypy3' + python_symlink = 'python' if config.version[0] == '2' else 'python3' + os.symlink(os.path.join(installation_bin_path, pypy_executable), os.path.join(installation_bin_path, python_symlink)) + + # Workaround for a too high 'MACOSX_DEPLOYMENT_TARGET' in PyPy 2.7, v7.2.0; fixed in HEAD and next release + if config.version[0] == '2': + sysconfig_pypy_path = os.path.join('/tmp', pypy_base_filename, 'lib-python', '2.7', 'distutils', 'sysconfig_pypy.py') + with open(sysconfig_pypy_path, 'r') as f: + sysconfig_pypy_contents = f.read() + sysconfig_pypy_contents = sysconfig_pypy_contents.replace("g['MACOSX_DEPLOYMENT_TARGET'] = '10.14'", "g['MACOSX_DEPLOYMENT_TARGET'] = '10.7'") + with open(sysconfig_pypy_path, 'w') as f: + f.write(sysconfig_pypy_contents) - installation_bin_path = '/Library/Frameworks/Python.framework/Versions/{}/bin'.format(config.version) assert os.path.exists(os.path.join(installation_bin_path, 'python3' if config.version[0] == '3' else 'python')) # Python bin folders on Mac don't symlink python3 to python, so we do that @@ -87,6 +111,9 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef installation_bin_path, env['PATH'], ]) + # PyPy 3.6, v7.2.0 doesn't set MACOSX_DEPLOYMENT_TARGET in distutils.sysconfig, so let's set it ourselves until this gets settled + if config.identifier.startswith('pp') and config.version[0] == '3' and 'MACOSX_DEPLOYMENT_TARGET' not in env: + env['MACOSX_DEPLOYMENT_TARGET'] = '10.13' env = environment.as_dictionary(prev_environment=env) # check what version we're on @@ -94,7 +121,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef call(['python', '--version'], env=env) # install pip & wheel - call(['python', get_pip_script, '--no-setuptools', '--no-wheel'], env=env, cwd="/tmp") + call(['python', get_pip_script], env=env, cwd="/tmp") assert os.path.exists(os.path.join(installation_bin_path, 'pip')) call(['pip', '--version'], env=env) call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate'], env=env) diff --git a/test/shared/utils.py b/test/shared/utils.py index 28b5dcf7..3095eaa4 100644 --- a/test/shared/utils.py +++ b/test/shared/utils.py @@ -79,11 +79,10 @@ def expected_wheels(package_name, package_version, manylinux_versions=['manylinu # {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl # {python tag} and {abi tag} are closely related to the python interpreter used to build the wheel # so we'll merge them below as python_abi_tag - python_abi_tags = ['cp27-cp27m', 'cp35-cp35m', 'cp36-cp36m', 'cp37-cp37m', 'cp38-cp38'] + python_abi_tags = ['cp27-cp27m', 'cp35-cp35m', 'cp36-cp36m', 'cp37-cp37m', 'cp38-cp38', + 'pp272-pypy_41', 'pp372-pypy3_72'] if platform == 'linux': python_abi_tags.append('cp27-cp27mu') # python 2.7 has 2 different ABI on manylinux - # Starting out by adding PyPy support on Linux - python_abi_tags.extend(['pp272-pypy_41', 'pp372-pypy3_72']) cp_architectures = ['x86_64', 'i686'] pp_architectures = ['x86_64'] @@ -98,8 +97,8 @@ def expected_wheels(package_name, package_version, manylinux_versions=['manylinu for manylinux_version in manylinux_versions for architecture in architectures] elif platform == 'windows': - # Second step: adding PyPy support on Windows - python_abi_tags.extend(['pp272-pypy_41', 'pp372-pp372']) + # The PyPy3 ABI tag for Windows will be consistent in the 7.3.0 release + python_abi_tags[python_abi_tags.index('pp372-pypy3_72')] = 'pp372-pp372' def get_platform_tags(python_abi_tag): if python_abi_tag.startswith('pp'): return ['win32'] @@ -107,10 +106,13 @@ def expected_wheels(package_name, package_version, manylinux_versions=['manylinu return ['win32', 'win_amd64'] elif platform == 'macos': - def get_platform_tags(python_abi_tag): - return ['macosx_' + (macosx_deployment_target or "10.9").replace(".", "_") + '_x86_64'] - + default_version = '10.9' + if python_abi_tag == 'pp272-pypy_41': + default_version = '10.7' + elif python_abi_tag == 'pp372-pypy3_72': + default_version = '10.13' + return ['macosx_{}_x86_64'.format((macosx_deployment_target or default_version).replace('.', '_'))] else: raise Exception('unsupported platform') From 186e09d5439c317c1ff6d88afd547fde56f8818a Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Fri, 13 Dec 2019 20:33:20 +0100 Subject: [PATCH 07/26] Implementing @mayeut's suggestion for alternative way of generating CPython/PyPy manylinux and Windows platform tags in test/shared/utils.py --- test/shared/utils.py | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/test/shared/utils.py b/test/shared/utils.py index 3095eaa4..f78ac0e3 100644 --- a/test/shared/utils.py +++ b/test/shared/utils.py @@ -83,27 +83,23 @@ def expected_wheels(package_name, package_version, manylinux_versions=['manylinu 'pp272-pypy_41', 'pp372-pypy3_72'] if platform == 'linux': python_abi_tags.append('cp27-cp27mu') # python 2.7 has 2 different ABI on manylinux - cp_architectures = ['x86_64', 'i686'] - pp_architectures = ['x86_64'] - + architectures = {'cp': ['x86_64', 'i686'], 'pp': ['x86_64']} + platform_tags = {} + for python_implemention in architectures: + platform_tags[python_implemention] = [ + '{manylinux_version}_{architecture}'.format( + manylinux_version=manylinux_version, architecture=architecture) + for architecture in architectures[python_implemention] + for manylinux_version in manylinux_versions + ] def get_platform_tags(python_abi_tag): - if python_abi_tag.startswith('pp'): - architectures = pp_architectures - else: - architectures = cp_architectures - return ['{manylinux_version}_{architecture}'.format( - manylinux_version=manylinux_version, architecture=architecture - ) - for manylinux_version in manylinux_versions - for architecture in architectures] + return platform_tags[python_abi_tag[:2]] elif platform == 'windows': # The PyPy3 ABI tag for Windows will be consistent in the 7.3.0 release python_abi_tags[python_abi_tags.index('pp372-pypy3_72')] = 'pp372-pp372' + platform_tags = {'cp': ['win32', 'win_amd64'], 'pp': ['win32']} def get_platform_tags(python_abi_tag): - if python_abi_tag.startswith('pp'): - return ['win32'] - else: - return ['win32', 'win_amd64'] + return platform_tags[python_abi_tag[:2]] elif platform == 'macos': def get_platform_tags(python_abi_tag): From e80f7f477409360b3b140795478d6d72fd1f8171 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Fri, 13 Dec 2019 21:52:40 +0100 Subject: [PATCH 08/26] Trying out PyPy 7.3.0rc1 on macOS and Windows --- cibuildwheel/linux.py | 4 ++-- cibuildwheel/macos.py | 16 ++-------------- cibuildwheel/windows.py | 5 ++--- test/06_docker_images/cibuildwheel_test.py | 2 +- test/shared/utils.py | 11 ++++++----- 5 files changed, 13 insertions(+), 25 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 848a2b94..aa6b7655 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -27,8 +27,8 @@ def get_python_configurations(build_selector): PythonConfiguration(identifier='cp36-manylinux_i686', path='/opt/python/cp36-cp36m'), PythonConfiguration(identifier='cp37-manylinux_i686', path='/opt/python/cp37-cp37m'), PythonConfiguration(identifier='cp38-manylinux_i686', path='/opt/python/cp38-cp38'), - PythonConfiguration(identifier='pp272-manylinux_pypy_x86_64', path='/opt/python/pp272-pypy_41'), - PythonConfiguration(identifier='pp372-manylinux_pypy_x86_64', path='/opt/python/pp372-pypy3_72'), + PythonConfiguration(identifier='pp27_72-manylinux_pypy_x86_64', path='/opt/python/pp272-pypy_41'), + PythonConfiguration(identifier='pp36_72-manylinux_pypy_x86_64', path='/opt/python/pp372-pypy3_72'), ] # skip builds as required diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 267824d9..feed2e84 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -21,8 +21,8 @@ def get_python_configurations(build_selector): PythonConfiguration(version='3.6', identifier='cp36-macosx_x86_64', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg'), PythonConfiguration(version='3.7', identifier='cp37-macosx_x86_64', url='https://www.python.org/ftp/python/3.7.6/python-3.7.6-macosx10.9.pkg'), PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.1/python-3.8.1-macosx10.9.pkg'), - PythonConfiguration(version='2.7-v7.2.0', identifier='pp272-macosx_x86_64', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.2.0-osx64.tar.bz2'), - PythonConfiguration(version='3.6-v7.2.0', identifier='pp372-macosx_x86_64', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.2.0-osx64.tar.bz2'), + PythonConfiguration(version='2.7-v7.3.0rc1', identifier='pp27_73-macosx_x86_64', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.0rc1-osx64.tar.bz2'), + PythonConfiguration(version='3.6-v7.3.0rc1', identifier='pp36_73-macosx_x86_64', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.3.0rc1-osx64.tar.bz2'), ] # skip builds as required @@ -83,15 +83,6 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef python_symlink = 'python' if config.version[0] == '2' else 'python3' os.symlink(os.path.join(installation_bin_path, pypy_executable), os.path.join(installation_bin_path, python_symlink)) - # Workaround for a too high 'MACOSX_DEPLOYMENT_TARGET' in PyPy 2.7, v7.2.0; fixed in HEAD and next release - if config.version[0] == '2': - sysconfig_pypy_path = os.path.join('/tmp', pypy_base_filename, 'lib-python', '2.7', 'distutils', 'sysconfig_pypy.py') - with open(sysconfig_pypy_path, 'r') as f: - sysconfig_pypy_contents = f.read() - sysconfig_pypy_contents = sysconfig_pypy_contents.replace("g['MACOSX_DEPLOYMENT_TARGET'] = '10.14'", "g['MACOSX_DEPLOYMENT_TARGET'] = '10.7'") - with open(sysconfig_pypy_path, 'w') as f: - f.write(sysconfig_pypy_contents) - assert os.path.exists(os.path.join(installation_bin_path, 'python3' if config.version[0] == '3' else 'python')) # Python bin folders on Mac don't symlink python3 to python, so we do that @@ -111,9 +102,6 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef installation_bin_path, env['PATH'], ]) - # PyPy 3.6, v7.2.0 doesn't set MACOSX_DEPLOYMENT_TARGET in distutils.sysconfig, so let's set it ourselves until this gets settled - if config.identifier.startswith('pp') and config.version[0] == '3' and 'MACOSX_DEPLOYMENT_TARGET' not in env: - env['MACOSX_DEPLOYMENT_TARGET'] = '10.13' env = environment.as_dictionary(prev_environment=env) # check what version we're on diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index d1f90281..68f1493d 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -47,8 +47,8 @@ def get_python_configurations(build_selector): PythonConfiguration(version='3.7.6', arch="64", identifier='cp37-win_amd64', url=None), PythonConfiguration(version='3.8.1', arch="32", identifier='cp38-win32', url=None), PythonConfiguration(version='3.8.1', arch="64", identifier='cp38-win_amd64', url=None), - PythonConfiguration(version='2.7-v7.2.0', arch="32", identifier='pp272-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.2.0-win32.zip'), - PythonConfiguration(version='3.6-v7.2.0', arch="32", identifier='pp372-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.2.0-win32.zip'), + PythonConfiguration(version='2.7-v7.3.0rc1', arch="32", identifier='pp27_73-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.0rc1-win32.zip'), + PythonConfiguration(version='3.6-v7.3.0rc1', arch="32", identifier='pp36_73-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.3.0rc1-win32.zip'), ] if IS_RUNNING_ON_TRAVIS: @@ -111,7 +111,6 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef pypy_exe = 'pypy.exe' if config.version[0] == '2' else 'pypy3.exe' simple_shell(['mklink', os.path.join(config_python_path, 'python.exe'), os.path.join(config_python_path, pypy_exe)]) simple_shell(['mklink', '/d', os.path.join(config_python_path, 'Scripts'), os.path.join(config_python_path, 'bin')]) - simple_shell(['chcp', '437']) # Workaround for https://bitbucket.org/pypy/pypy/issues/3081/ assert os.path.exists(os.path.join(config_python_path, 'python.exe')) # set up PATH and environment variables for run_with_env diff --git a/test/06_docker_images/cibuildwheel_test.py b/test/06_docker_images/cibuildwheel_test.py index 4dbf40bd..4b081e98 100644 --- a/test/06_docker_images/cibuildwheel_test.py +++ b/test/06_docker_images/cibuildwheel_test.py @@ -19,5 +19,5 @@ def test(): # also check that we got the right wheels built expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0') - if 'pypy' not in w] + if '-pp' not in w] assert set(actual_wheels) == set(expected_wheels) diff --git a/test/shared/utils.py b/test/shared/utils.py index f78ac0e3..47bfa6e8 100644 --- a/test/shared/utils.py +++ b/test/shared/utils.py @@ -80,9 +80,12 @@ def expected_wheels(package_name, package_version, manylinux_versions=['manylinu # {python tag} and {abi tag} are closely related to the python interpreter used to build the wheel # so we'll merge them below as python_abi_tag python_abi_tags = ['cp27-cp27m', 'cp35-cp35m', 'cp36-cp36m', 'cp37-cp37m', 'cp38-cp38', - 'pp272-pypy_41', 'pp372-pypy3_72'] + 'pp273-pypy_73', 'pp373-pypy36_pp73'] if platform == 'linux': python_abi_tags.append('cp27-cp27mu') # python 2.7 has 2 different ABI on manylinux + # We don't have PyPy 7.3.0rc1 yet on the manylinux image + python_abi_tags[python_abi_tags.index('pp273-pypy_73')] = 'pp272-pypy_41' + python_abi_tags[python_abi_tags.index('pp373-pypy36_pp73')] = 'pp372-pypy3_72' architectures = {'cp': ['x86_64', 'i686'], 'pp': ['x86_64']} platform_tags = {} for python_implemention in architectures: @@ -95,8 +98,6 @@ def expected_wheels(package_name, package_version, manylinux_versions=['manylinu def get_platform_tags(python_abi_tag): return platform_tags[python_abi_tag[:2]] elif platform == 'windows': - # The PyPy3 ABI tag for Windows will be consistent in the 7.3.0 release - python_abi_tags[python_abi_tags.index('pp372-pypy3_72')] = 'pp372-pp372' platform_tags = {'cp': ['win32', 'win_amd64'], 'pp': ['win32']} def get_platform_tags(python_abi_tag): return platform_tags[python_abi_tag[:2]] @@ -104,9 +105,9 @@ def expected_wheels(package_name, package_version, manylinux_versions=['manylinu elif platform == 'macos': def get_platform_tags(python_abi_tag): default_version = '10.9' - if python_abi_tag == 'pp272-pypy_41': + if python_abi_tag == 'pp273-pypy_73': default_version = '10.7' - elif python_abi_tag == 'pp372-pypy3_72': + elif python_abi_tag == 'pp373-pypy36_pp73': default_version = '10.13' return ['macosx_{}_x86_64'.format((macosx_deployment_target or default_version).replace('.', '_'))] else: From 08d3ee1d911faedd4df65955f4756da47ea2e480 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Mon, 23 Dec 2019 14:27:40 +0100 Subject: [PATCH 09/26] Updating PyPy to v7.3.0rc3 on macOS and Windows --- cibuildwheel/macos.py | 4 ++-- cibuildwheel/windows.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index feed2e84..44e45f9c 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -21,8 +21,8 @@ def get_python_configurations(build_selector): PythonConfiguration(version='3.6', identifier='cp36-macosx_x86_64', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg'), PythonConfiguration(version='3.7', identifier='cp37-macosx_x86_64', url='https://www.python.org/ftp/python/3.7.6/python-3.7.6-macosx10.9.pkg'), PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.1/python-3.8.1-macosx10.9.pkg'), - PythonConfiguration(version='2.7-v7.3.0rc1', identifier='pp27_73-macosx_x86_64', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.0rc1-osx64.tar.bz2'), - PythonConfiguration(version='3.6-v7.3.0rc1', identifier='pp36_73-macosx_x86_64', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.3.0rc1-osx64.tar.bz2'), + PythonConfiguration(version='2.7-v7.3.0rc3', identifier='pp27_73-macosx_x86_64', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.0rc3-osx64.tar.bz2'), + PythonConfiguration(version='3.6-v7.3.0rc3', identifier='pp36_73-macosx_x86_64', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.3.0rc3-osx64.tar.bz2'), ] # skip builds as required diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 68f1493d..ea7a1b3f 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -47,8 +47,8 @@ def get_python_configurations(build_selector): PythonConfiguration(version='3.7.6', arch="64", identifier='cp37-win_amd64', url=None), PythonConfiguration(version='3.8.1', arch="32", identifier='cp38-win32', url=None), PythonConfiguration(version='3.8.1', arch="64", identifier='cp38-win_amd64', url=None), - PythonConfiguration(version='2.7-v7.3.0rc1', arch="32", identifier='pp27_73-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.0rc1-win32.zip'), - PythonConfiguration(version='3.6-v7.3.0rc1', arch="32", identifier='pp36_73-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.3.0rc1-win32.zip'), + PythonConfiguration(version='2.7-v7.3.0rc3', arch="32", identifier='pp27_73-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.0rc3-win32.zip'), + PythonConfiguration(version='3.6-v7.3.0rc3', arch="32", identifier='pp36_73-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.3.0rc3-win32.zip'), ] if IS_RUNNING_ON_TRAVIS: From 4c718706c8694095e12a99beccf9778ef8476b82 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Mon, 23 Dec 2019 15:42:39 +0100 Subject: [PATCH 10/26] Trying out PyPy 7.3.0rc3 on manylinux2010 as well, with a temporary newer version of manylinux2010-pypy_x86_64 image --- cibuildwheel/__main__.py | 3 ++- cibuildwheel/linux.py | 4 ++-- test/shared/utils.py | 3 --- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 1e5b249b..f220c03c 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -167,7 +167,8 @@ def main(): default_manylinux_images_i686 = {'manylinux1': 'quay.io/pypa/manylinux1_i686', 'manylinux2010': 'quay.io/pypa/manylinux2010_i686', 'manylinux2014': 'quay.io/pypa/manylinux2014_i686'} - default_manylinux_images_pypy_x86_64 = {'manylinux2010': 'pypywheels/manylinux2010-pypy_x86_64'} + # Testing 7.3.0rc3. Before merging PR, reset to 'pypywheels/manylinux2010-pypy_x86_64'! + default_manylinux_images_pypy_x86_64 = {'manylinux2010': 'yannickjadoul/manylinux2010-pypy_x86_64:7.3.0rc3'} build_options.update( manylinux_images={'x86_64': default_manylinux_images_x86_64.get(manylinux_x86_64_image) or manylinux_x86_64_image, diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index aa6b7655..c99ee388 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -27,8 +27,8 @@ def get_python_configurations(build_selector): PythonConfiguration(identifier='cp36-manylinux_i686', path='/opt/python/cp36-cp36m'), PythonConfiguration(identifier='cp37-manylinux_i686', path='/opt/python/cp37-cp37m'), PythonConfiguration(identifier='cp38-manylinux_i686', path='/opt/python/cp38-cp38'), - PythonConfiguration(identifier='pp27_72-manylinux_pypy_x86_64', path='/opt/python/pp272-pypy_41'), - PythonConfiguration(identifier='pp36_72-manylinux_pypy_x86_64', path='/opt/python/pp372-pypy3_72'), + PythonConfiguration(identifier='pp27_73-manylinux_pypy_x86_64', path='/opt/python/pp273-pypy_73'), + PythonConfiguration(identifier='pp36_73-manylinux_pypy_x86_64', path='/opt/python/pp373-pypy36_pp73'), ] # skip builds as required diff --git a/test/shared/utils.py b/test/shared/utils.py index 47bfa6e8..45961651 100644 --- a/test/shared/utils.py +++ b/test/shared/utils.py @@ -83,9 +83,6 @@ def expected_wheels(package_name, package_version, manylinux_versions=['manylinu 'pp273-pypy_73', 'pp373-pypy36_pp73'] if platform == 'linux': python_abi_tags.append('cp27-cp27mu') # python 2.7 has 2 different ABI on manylinux - # We don't have PyPy 7.3.0rc1 yet on the manylinux image - python_abi_tags[python_abi_tags.index('pp273-pypy_73')] = 'pp272-pypy_41' - python_abi_tags[python_abi_tags.index('pp373-pypy36_pp73')] = 'pp372-pypy3_72' architectures = {'cp': ['x86_64', 'i686'], 'pp': ['x86_64']} platform_tags = {} for python_implemention in architectures: From fea54e25ab61f32cef1d988397fd574ac8152739 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Tue, 24 Dec 2019 15:55:32 +0100 Subject: [PATCH 11/26] PyPy 7.3.0 released! --- cibuildwheel/__main__.py | 4 ++-- cibuildwheel/macos.py | 4 ++-- cibuildwheel/windows.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index f220c03c..6f944d2e 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -167,8 +167,8 @@ def main(): default_manylinux_images_i686 = {'manylinux1': 'quay.io/pypa/manylinux1_i686', 'manylinux2010': 'quay.io/pypa/manylinux2010_i686', 'manylinux2014': 'quay.io/pypa/manylinux2014_i686'} - # Testing 7.3.0rc3. Before merging PR, reset to 'pypywheels/manylinux2010-pypy_x86_64'! - default_manylinux_images_pypy_x86_64 = {'manylinux2010': 'yannickjadoul/manylinux2010-pypy_x86_64:7.3.0rc3'} + # Testing 7.3.0. Before merging PR, reset to 'pypywheels/manylinux2010-pypy_x86_64'! + default_manylinux_images_pypy_x86_64 = {'manylinux2010': 'yannickjadoul/manylinux2010-pypy_x86_64:7.3.0'} build_options.update( manylinux_images={'x86_64': default_manylinux_images_x86_64.get(manylinux_x86_64_image) or manylinux_x86_64_image, diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 44e45f9c..6e41ac95 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -21,8 +21,8 @@ def get_python_configurations(build_selector): PythonConfiguration(version='3.6', identifier='cp36-macosx_x86_64', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg'), PythonConfiguration(version='3.7', identifier='cp37-macosx_x86_64', url='https://www.python.org/ftp/python/3.7.6/python-3.7.6-macosx10.9.pkg'), PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.1/python-3.8.1-macosx10.9.pkg'), - PythonConfiguration(version='2.7-v7.3.0rc3', identifier='pp27_73-macosx_x86_64', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.0rc3-osx64.tar.bz2'), - PythonConfiguration(version='3.6-v7.3.0rc3', identifier='pp36_73-macosx_x86_64', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.3.0rc3-osx64.tar.bz2'), + PythonConfiguration(version='2.7-v7.3.0', identifier='pp27_73-macosx_x86_64', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.0-osx64.tar.bz2'), + PythonConfiguration(version='3.6-v7.3.0', identifier='pp36_73-macosx_x86_64', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.3.0-osx64.tar.bz2'), ] # skip builds as required diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index ea7a1b3f..f4c499f7 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -47,8 +47,8 @@ def get_python_configurations(build_selector): PythonConfiguration(version='3.7.6', arch="64", identifier='cp37-win_amd64', url=None), PythonConfiguration(version='3.8.1', arch="32", identifier='cp38-win32', url=None), PythonConfiguration(version='3.8.1', arch="64", identifier='cp38-win_amd64', url=None), - PythonConfiguration(version='2.7-v7.3.0rc3', arch="32", identifier='pp27_73-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.0rc3-win32.zip'), - PythonConfiguration(version='3.6-v7.3.0rc3', arch="32", identifier='pp36_73-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.3.0rc3-win32.zip'), + PythonConfiguration(version='2.7-v7.3.0', arch="32", identifier='pp27_73-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.0-win32.zip'), + PythonConfiguration(version='3.6-v7.3.0', arch="32", identifier='pp36_73-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.3.0-win32.zip'), ] if IS_RUNNING_ON_TRAVIS: From bafb4fd29445e6fd51dacd3192a96cb399002b51 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Thu, 26 Dec 2019 13:51:57 +0100 Subject: [PATCH 12/26] Adding PyPy to the README and docs --- README.md | 22 +++++++++++++--------- docs/faq.md | 2 +- docs/options.md | 35 +++++++++++++++++++++-------------- 3 files changed, 35 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 2989eac2..b00b5c8a 100644 --- a/README.md +++ b/README.md @@ -15,17 +15,21 @@ Python wheels are great. Building them across **Mac, Linux, Windows**, on **mult What does it do? ---------------- -| | macOS 10.9+ x86_64 | manylinux i686 | manylinux x86_64 | Windows 32bit | Windows 64bit | +| | macOS 10.9+ x86_64 | manylinux i686 | manylinux x86_64 | Windows 32bit | Windows 64bit | |---|---|---|---|---|---| -| Python 2.7 | ✅ | ✅ | ✅ | ✅¹ | ✅¹ | -| Python 3.5 | ✅ | ✅ | ✅ | ✅ | ✅ | -| Python 3.6 | ✅ | ✅ | ✅ | ✅ | ✅ | -| Python 3.7 | ✅ | ✅ | ✅ | ✅ | ✅ | -| Python 3.8 | ✅ | ✅ | ✅ | ✅ | ✅ | +| Python 2.7 | ✅ | ✅ | ✅ | ✅¹ | ✅¹ | +| Python 3.5 | ✅ | ✅ | ✅ | ✅ | ✅ | +| Python 3.6 | ✅ | ✅ | ✅ | ✅ | ✅ | +| Python 3.7 | ✅ | ✅ | ✅ | ✅ | ✅ | +| Python 3.8 | ✅ | ✅ | ✅ | ✅ | ✅ | +| PyPy 2.7 v7.3.0 | ✅² | | ✅ | ✅ | | +| PyPy 3.6 v7.3.0 | ✅² | | ✅ | ✅ | | > ¹ Not supported on Travis -- Builds manylinux, macOS and Windows (32 and 64bit) wheels using Azure Pipelines, Travis CI, AppVeyor, and CircleCI +> ² For PyPy 2.7, the minimally supported OS X/macOS version is 10.7; for PyPy 3.6, macOS 10.13 or higher is required. + +- Builds manylinux, macOS and Windows (32 and 64bit) wheels for CPython and PyPy using Azure Pipelines, Travis CI, AppVeyor, and CircleCI - Bundles shared library dependencies on Linux and macOS through [auditwheel](https://github.com/pypa/auditwheel) and [delocate](https://github.com/matthew-brett/delocate) - Runs the library test suite against the wheel-installed version of your library @@ -48,7 +52,7 @@ Usage Example setup ------------- -To build manylinux, macOS, and Windows wheels on Travis CI and upload them to PyPI whenever you tag a version, you could use this `.travis.yml`: +To build manylinux, macOS, and Windows wheels on Travis CI and upload them to PyPI whenever you tag a version, you could use this `.travis.yml`: ```yaml language: python @@ -96,7 +100,7 @@ Options | **Build environment** | [`CIBW_ENVIRONMENT`](https://cibuildwheel.readthedocs.io/en/stable/options/#environment) | Set environment variables needed during the build | | | [`CIBW_BEFORE_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-build) | Execute a shell command preparing each wheel's build | | | [`CIBW_REPAIR_WHEEL_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#repair-wheel-command) | Execute a shell command to repair each (non-pure Python) built wheel | -| | [`CIBW_MANYLINUX_X86_64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) [`CIBW_MANYLINUX_I686_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) | Specify alternative manylinux docker images | +| | [`CIBW_MANYLINUX_X86_64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) [`CIBW_MANYLINUX_I686_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) [`CIBW_MANYLINUX_PYPY_X86_64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) | Specify alternative manylinux docker images | | **Testing** | [`CIBW_TEST_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-command) | Execute a shell command to test each built wheel | | | [`CIBW_TEST_REQUIRES`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-requires) | Install Python dependencies before running the tests | | | [`CIBW_TEST_EXTRAS`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-extras) | Install your wheel for testing using extras_require | diff --git a/docs/faq.md b/docs/faq.md index dc424980..5f1981b8 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -22,7 +22,7 @@ Linux wheels are built in the [`manylinux` docker images](https://github.com/pyp - The project directory is mounted in the running Docker instance as `/project`, the output directory for the wheels as `/output`. In general, this is handled transparently by `cibuildwheel`. For a more finegrained level of control however, the root of the host file system is mounted as `/host`, allowing for example to access shared files, caches, etc. on the host file system. Note that this is not available on CircleCI due to their Docker policies. -- Alternative dockers images can be specified with the `CIBW_MANYLINUX_X86_64_IMAGE` and `CIBW_MANYLINUX_I686_IMAGE` options to allow for a custom, preconfigured build environment for the Linux builds. See [options](options.md#manylinux-image) for more details. +- Alternative dockers images can be specified with the `CIBW_MANYLINUX_X86_64_IMAGE`, `CIBW_MANYLINUX_I686_IMAGE`, and `CIBW_MANYLINUX_PYPY_X86_64_IMAGE` options to allow for a custom, preconfigured build environment for the Linux builds. See [options](options.md#manylinux-image) for more details. ### Building packages with optional C extensions diff --git a/docs/options.md b/docs/options.md index 82c1f435..087076d0 100644 --- a/docs/options.md +++ b/docs/options.md @@ -56,7 +56,7 @@ Default: `auto` `auto` will auto-detect platform using environment variables, such as `TRAVIS_OS_NAME`/`APPVEYOR`/`CIRCLECI`. -For `linux` you need Docker running, on Mac or Linux. For `macos`, you need a Mac machine, and note that this script is going to automatically install MacPython on your system, so don't run on your development machine. For `windows`, you need to run in Windows, and it will build and test for all versions of Python at `C:\PythonXX[-x64]`. +For `linux` you need Docker running, on macOS or Linux. For `macos`, you need a Mac machine, and note that this script is going to automatically install MacPython on your system, so don't run on your development machine. For `windows`, you need to run in Windows, and it will build and test for all versions of Python under `C:\cibw\python`. This option can also be set using the command-line option `--platform`. @@ -68,17 +68,19 @@ Space-separated list of builds to build and skip. Each build has an identifier l When both options are specified, both conditions are applied and only builds with a tag that matches `CIBW_BUILD` and does not match `CIBW_SKIP` will be built. -When setting the options, you can use shell-style globbing syntax (as per `fnmatch`). All the build identifiers supported by cibuildwheel are shown below: +When setting the options, you can use shell-style globbing syntax (as per [`fnmatch`](https://docs.python.org/3/library/fnmatch.html)). All the build identifiers supported by cibuildwheel are shown below:
-| | macOS 64bit | Manylinux 64bit | Manylinux 32bit | Windows 64bit | Windows 32bit | -|------------|--------------------|------------------------|----------------------|-----------------|----------------| -| Python 2.7 | cp27-macosx_x86_64 | cp27-manylinux_x86_64 | cp27-manylinux_i686 | cp27-win_amd64 | cp27-win32 | -| Python 3.5 | cp35-macosx_x86_64 | cp35-manylinux_x86_64 | cp35-manylinux_i686 | cp35-win_amd64 | cp35-win32 | -| Python 3.6 | cp36-macosx_x86_64 | cp36-manylinux_x86_64 | cp36-manylinux_i686 | cp36-win_amd64 | cp36-win32 | -| Python 3.7 | cp37-macosx_x86_64 | cp37-manylinux_x86_64 | cp37-manylinux_i686 | cp37-win_amd64 | cp37-win32 | -| Python 3.8 | cp38-macosx_x86_64 | cp38-manylinux_x86_64 | cp38-manylinux_i686 | cp38-win_amd64 | cp38-win32 | +| | macOS 64bit | Manylinux 64bit | Manylinux 32bit | Windows 64bit | Windows 32bit | +|-----------------|------------------------|--------------------------------|----------------------|-----------------|----------------| +| Python 2.7 | cp27-macosx_x86_64 | cp27-manylinux_x86_64 | cp27-manylinux_i686 | cp27-win_amd64 | cp27-win32 | +| Python 3.5 | cp35-macosx_x86_64 | cp35-manylinux_x86_64 | cp35-manylinux_i686 | cp35-win_amd64 | cp35-win32 | +| Python 3.6 | cp36-macosx_x86_64 | cp36-manylinux_x86_64 | cp36-manylinux_i686 | cp36-win_amd64 | cp36-win32 | +| Python 3.7 | cp37-macosx_x86_64 | cp37-manylinux_x86_64 | cp37-manylinux_i686 | cp37-win_amd64 | cp37-win32 | +| Python 3.8 | cp38-macosx_x86_64 | cp38-manylinux_x86_64 | cp38-manylinux_i686 | cp38-win_amd64 | cp38-win32 | +| PyPy 2.7 v7.3.0 | pp27_73-macosx_x86_64 | pp27_73-manylinux_pypy_x86_64 | | | pp27_73-win32 | +| PyPy 3.6 v7.3.0 | pp36_73-macosx_x86_64 | pp36_73-manylinux_pypy_x86_64 | | | pp36_73-win32 | The list of supported and currently selected build identifiers can also be retrieved by passing the `--print-build-identifiers` flag to `cibuildwheel`. The format is `python_tag-platform_tag`, with tags similar to those in [PEP 425](https://www.python.org/dev/peps/pep-0425/#details). @@ -113,6 +115,9 @@ CIBW_SKIP: cp36-manylinux* # Only build on Python 3 and skip 32-bit builds CIBW_BUILD: cp3?-* CIBW_SKIP: "*-win32 *-manylinux_i686" + +# Only build PyPy and CPython 3 +CIBW_BUILD: pp??_??-* cp3?-* ```