Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into deterministic-builds

Browse Source
This commit is contained in:
Joe Rickerby
2020-02-02 15:21:42 +00:00
parent 8ead985cfd 17a224c525
commit bad72cbb15
13 changed files with 142 additions and 88 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.editorconfig
+14
View File
@@ -0,0 +1,14 @@
# http://editorconfig.org
root = true
[*]
indent_style = space
indent_size = 2
end_of_line = lf
charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
# Use 4 spaces the Python files
[*.py]
indent_size = 4
.gitignore
+2
View File
@@ -86,9 +86,11 @@ celerybeat-schedule
.venv
venv/
venv3/
venv2/
ENV/
env/
env3/
env2/
# Spyder project settings
.spyderproject
README.md
+7 -7
View File
@@ -15,13 +15,13 @@ Python wheels are great. Building them across **Mac, Linux, Windows**, on **mult
What does it do?
----------------
| | macOS 10.6+ intel | macOS 10.9+ x86_64 | manylinux i686 | manylinux x86_64 | Windows 32bit | Windows 64bit |
|---|---|---|---|---|---|---|
| Python 2.7 | ✅ | | ✅ | ✅ | ✅¹ | ✅¹ |
| Python 3.5 | ✅ | | ✅ | ✅ | ✅ | ✅ |
| Python 3.6 | ✅ | | ✅ | ✅ | ✅ | ✅ |
| Python 3.7 | ✅ | | ✅ | ✅ | ✅ | ✅ |
| Python 3.8 | | ✅ | ✅ | ✅ | ✅ | ✅ |
| | macOS 10.9+ x86_64 | manylinux i686 | manylinux x86_64 | Windows 32bit | Windows 64bit |
|---|---|---|---|---|---|
| Python 2.7 | ✅ | ✅ | ✅ | ✅¹ | ✅¹ |
| Python 3.5 | ✅ | ✅ | ✅ | ✅ | ✅ |
| Python 3.6 | ✅ | ✅ | ✅ | ✅ | ✅ |
| Python 3.7 | ✅ | ✅ | ✅ | ✅ | ✅ |
| Python 3.8 | ✅ | ✅ | ✅ | ✅ | ✅ |
> ¹ Not supported on Travis
cibuildwheel/__main__.py
+14 -11
View File
@@ -61,7 +61,7 @@ def main():
args = parser.parse_args()
detect_obsolete_options()
if args.platform != 'auto':
platform = args.platform
else:
@@ -97,7 +97,7 @@ def main():
if platform == 'linux':
repair_command_default = 'auditwheel repair -w {dest_dir} {wheel}'
elif platform == 'macos':
repair_command_default = 'delocate-listdeps {wheel} && delocate-wheel -w {dest_dir} {wheel}'
repair_command_default = 'delocate-listdeps {wheel} && delocate-wheel --require-archs x86_64 -w {dest_dir} {wheel}'
else:
repair_command_default = ''
repair_command = get_option_from_environment('CIBW_REPAIR_WHEEL_COMMAND', platform=platform, default=repair_command_default)
@@ -208,15 +208,18 @@ def detect_obsolete_options():
print("Option '{}' is not empty. Please unset '{}'".format(alternative, deprecated))
exit(2)
# Check for 'manylinux1' in the 'CIBW_BUILD' and 'CIBW_SKIP' options
for deprecated in ['CIBW_BUILD', 'CIBW_SKIP']:
if deprecated in os.environ and 'manylinux1' in os.environ[deprecated]:
print("Build identifiers with 'manylinux1' have been deprecated. Replacing all occurences of 'manylinux1' by 'manylinux' in the option '{}'".format(deprecated))
os.environ[deprecated] = os.environ[deprecated].replace('manylinux1', 'manylinux')
if deprecated in os.environ and ("macosx_10_6" in os.environ[deprecated] or "macosx_10_9" in os.environ[deprecated]):
print("Build identifiers with 'macosx_10_6' or 'macosx_10_9' have been deprecated. Replacing all occurences with 'macosx' in the option '{}'".format(deprecated))
os.environ[deprecated] = os.environ[deprecated].replace('macosx_10_6', 'macosx').replace('macosx_10_9', 'macosx')
# Check for deprecated identifiers in 'CIBW_BUILD' and 'CIBW_SKIP' options
for option in ['CIBW_BUILD', 'CIBW_SKIP']:
for deprecated, alternative in [('manylinux1', 'manylinux'),
('macosx_10_6_intel', 'macosx_x86_64'),
('macosx_10_9_x86_64', 'macosx_x86_64')]:
if option in os.environ and deprecated in os.environ[option]:
print("Build identifiers with '{deprecated}' have been deprecated. Replacing all occurences of '{deprecated}' with '{alternative}' in the option '{option}'".format(
deprecated=deprecated,
alternative=alternative,
option=option,
))
os.environ[option] = os.environ[option].replace(deprecated, alternative)
def print_preamble(platform, build_options):
print(textwrap.dedent('''
cibuildwheel/macos.py
+20 -8
View File
@@ -8,17 +8,17 @@ try:
except ImportError:
from pipes import quote as shlex_quote
from .util import prepare_command, get_build_verbosity_extra_flags, get_pip_script
from .util import prepare_command, get_build_verbosity_extra_flags, download, get_pip_script
def get_python_configurations(build_selector):
PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'url'])
python_configurations = [
PythonConfiguration(version='2.7', identifier='cp27-macosx_intel', url='https://www.python.org/ftp/python/2.7.17/python-2.7.17-macosx10.6.pkg'),
PythonConfiguration(version='3.5', identifier='cp35-macosx_intel', url='https://www.python.org/ftp/python/3.5.4/python-3.5.4-macosx10.6.pkg'),
PythonConfiguration(version='3.6', identifier='cp36-macosx_intel', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.6.pkg'),
PythonConfiguration(version='3.7', identifier='cp37-macosx_intel', url='https://www.python.org/ftp/python/3.7.5/python-3.7.5-macosx10.6.pkg'),
PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.0/python-3.8.0-macosx10.9.pkg'),
PythonConfiguration(version='2.7', identifier='cp27-macosx_x86_64', url='https://www.python.org/ftp/python/2.7.17/python-2.7.17-macosx10.9.pkg'),
PythonConfiguration(version='3.5', identifier='cp35-macosx_x86_64', url='https://www.python.org/ftp/python/3.5.4/python-3.5.4-macosx10.6.pkg'),
PythonConfiguration(version='3.6', identifier='cp36-macosx_x86_64', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg'),
PythonConfiguration(version='3.7', identifier='cp37-macosx_x86_64', url='https://www.python.org/ftp/python/3.7.6/python-3.7.6-macosx10.9.pkg'),
PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.1/python-3.8.1-macosx10.9.pkg'),
]
# skip builds as required
@@ -52,12 +52,13 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
python_package_identifier = 'org.python.Python.PythonFramework-%s' % config.version
if python_package_identifier not in installed_system_packages:
# download the pkg
call(['curl', '-L', '-o', '/tmp/Python.pkg', config.url])
download(config.url, '/tmp/Python.pkg')
# install
call(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/'])
# patch open ssl
if config.version == '3.5':
call(['curl', '-fsSLo', '/tmp/python-patch.tar.gz', 'https://github.com/mayeut/patch-macos-python-openssl/releases/download/v1.0.2t/patch-macos-python-%s-openssl-v1.0.2t.tar.gz' % config.version])
open_ssl_patch_url = 'https://github.com/mayeut/patch-macos-python-openssl/releases/download/v1.0.2t/patch-macos-python-%s-openssl-v1.0.2t.tar.gz' % config.version
download(open_ssl_patch_url, '/tmp/python-patch.tar.gz')
call(['sudo', 'tar', '-C', '/Library/Frameworks/Python.framework/Versions/%s/' % config.version, '-xmf', '/tmp/python-patch.tar.gz'])
installation_bin_path = '/Library/Frameworks/Python.framework/Versions/{}/bin'.format(config.version)
@@ -94,6 +95,17 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
call(['pip', '--version'], env=env)
call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate'] + dependency_constraint_flags, env=env)
# setup target platform, only required for python 3.5
if config.version == '3.5':
if '_PYTHON_HOST_PLATFORM' not in env:
# cross-compilation platform override
env['_PYTHON_HOST_PLATFORM'] = 'macosx-10.9-x86_64'
if 'ARCHFLAGS' not in env:
# https://github.com/python/cpython/blob/a5ed2fe0eedefa1649aa93ee74a0bafc8e628a10/Lib/_osx_support.py#L260
env['ARCHFLAGS'] = '-arch x86_64'
if 'MACOSX_DEPLOYMENT_TARGET' not in env:
env['MACOSX_DEPLOYMENT_TARGET'] = '10.9'
# run the before_build command
if before_build:
before_build_prepared = prepare_command(before_build, project=abs_project_dir)
cibuildwheel/util.py
+30
View File
@@ -1,6 +1,12 @@
from fnmatch import fnmatch
import warnings
import os
from time import sleep
try:
from urllib.request import urlopen
except ImportError:
from urllib2 import urlopen
def prepare_command(command, **kwargs):
@@ -53,4 +59,28 @@ class Unbuffered(object):
return getattr(self.stream, attr)
def download(url, dest):
print('+ Download ' + url + ' to ' + dest)
dest_dir = os.path.dirname(dest)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
repeat_num = 3
for i in range(repeat_num):
try:
response = urlopen(url)
except:
if i == repeat_num - 1:
raise
sleep(3)
continue
break
try:
with open(dest, 'wb') as file:
file.write(response.read())
finally:
response.close()
get_pip_script = os.path.abspath(os.path.join(os.path.dirname(__file__), 'resources', 'get-pip.py'))
cibuildwheel/windows.py
+1 -29
View File
@@ -1,6 +1,5 @@
from __future__ import print_function
import os, tempfile, subprocess, shutil, sys
from time import sleep
from collections import namedtuple
from glob import glob
@@ -9,12 +8,7 @@ try:
except ImportError:
from pipes import quote as shlex_quote
try:
from urllib.request import urlopen
except ImportError:
from urllib2 import urlopen
from .util import prepare_command, get_build_verbosity_extra_flags, get_pip_script
from .util import prepare_command, get_build_verbosity_extra_flags, download, get_pip_script
IS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache')
@@ -64,28 +58,6 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
args = ['cmd', '/E:ON', '/V:ON', '/C'] + args
return subprocess.check_call(' '.join(args), env=env, cwd=cwd)
def download(url, dest):
print('+ Download ' + url + ' to ' + dest)
dest_dir = os.path.dirname(dest)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
repeat_num = 3
for i in range(repeat_num):
try:
response = urlopen(url)
except:
if i == repeat_num - 1:
raise
sleep(3)
continue
break
try:
with open(dest, 'wb') as file:
file.write(response.read())
finally:
response.close()
if IS_RUNNING_ON_AZURE or IS_RUNNING_ON_TRAVIS:
shell = simple_shell
else:
docs/options.md
+10 -10
View File
@@ -64,7 +64,7 @@ This option can also be set using the command-line option `--platform`.
> Choose the Python versions to build
Space-separated list of builds to build and skip. Each build has an identifier like `cp27-manylinux_x86_64` or `cp35-macosx_intel` - you can list specific ones to build and `cibuildwheel` will only build those, and/or list ones to skip and `cibuildwheel` won't try to build them.
Space-separated list of builds to build and skip. Each build has an identifier like `cp27-manylinux_x86_64` or `cp35-macosx_x86_64` - you can list specific ones to build and `cibuildwheel` will only build those, and/or list ones to skip and `cibuildwheel` won't try to build them.
When both options are specified, both conditions are applied and only builds with a tag that matches `CIBW_BUILD` and does not match `CIBW_SKIP` will be built.
@@ -72,13 +72,13 @@ When setting the options, you can use shell-style globbing syntax (as per `fnmat
<div class="build-id-table-marker"></div>
| | macOS 64bit | macOS 32/64bit | Manylinux 64bit | Manylinux 32bit | Windows 64bit | Windows 32bit |
|------------|---------------------|--------------------|------------------------|----------------------|-----------------|----------------|
| Python 2.7 | | cp27-macosx_intel | cp27-manylinux_x86_64 | cp27-manylinux_i686 | cp27-win_amd64 | cp27-win32 |
| Python 3.5 | | cp35-macosx_intel | cp35-manylinux_x86_64 | cp35-manylinux_i686 | cp35-win_amd64 | cp35-win32 |
| Python 3.6 | | cp36-macosx_intel | cp36-manylinux_x86_64 | cp36-manylinux_i686 | cp36-win_amd64 | cp36-win32 |
| Python 3.7 | | cp37-macosx_intel | cp37-manylinux_x86_64 | cp37-manylinux_i686 | cp37-win_amd64 | cp37-win32 |
| Python 3.8 | cp38-macosx_x86_64 | | cp38-manylinux_x86_64 | cp38-manylinux_i686 | cp38-win_amd64 | cp38-win32 |
| | macOS 64bit | Manylinux 64bit | Manylinux 32bit | Windows 64bit | Windows 32bit |
|------------|--------------------|------------------------|----------------------|-----------------|----------------|
| Python 2.7 | cp27-macosx_x86_64 | cp27-manylinux_x86_64 | cp27-manylinux_i686 | cp27-win_amd64 | cp27-win32 |
| Python 3.5 | cp35-macosx_x86_64 | cp35-manylinux_x86_64 | cp35-manylinux_i686 | cp35-win_amd64 | cp35-win32 |
| Python 3.6 | cp36-macosx_x86_64 | cp36-manylinux_x86_64 | cp36-manylinux_i686 | cp36-win_amd64 | cp36-win32 |
| Python 3.7 | cp37-macosx_x86_64 | cp37-manylinux_x86_64 | cp37-manylinux_i686 | cp37-win_amd64 | cp37-win32 |
| Python 3.8 | cp38-macosx_x86_64 | cp38-manylinux_x86_64 | cp38-manylinux_i686 | cp38-win_amd64 | cp38-win32 |
The list of supported and currently selected build identifiers can also be retrieved by passing the `--print-build-identifiers` flag to `cibuildwheel`.
The format is `python_tag-platform_tag`, with tags similar to those in [PEP 425](https://www.python.org/dev/peps/pep-0425/#details).
@@ -90,7 +90,7 @@ The format is `python_tag-platform_tag`, with tags similar to those in [PEP 425]
CIBW_BUILD: cp36-*
# Skip building on Python 2.7 on the Mac
CIBW_SKIP: cp27-macosx_intel
CIBW_SKIP: cp27-macosx_x86_64
# Skip building on Python 3.8 on the Mac
CIBW_SKIP: cp38-macosx_x86_64
@@ -208,7 +208,7 @@ CIBW_BEFORE_BUILD: yum install -y libffi-dev && pip install .
Default:
- on Linux: `'auditwheel repair -w {dest_dir} {wheel}'`
- on macOS: `'delocate-listdeps {wheel} && delocate-wheel -w {dest_dir} {wheel}'`
- on macOS: `'delocate-listdeps {wheel} && delocate-wheel --require-archs x86_64 -w {dest_dir} {wheel}'`
- on Windows: `''`
A shell command to repair a built wheel by copying external library dependencies into the wheel tree and relinking them.
docs/setup.md
+3
View File
@@ -11,6 +11,9 @@ Using Azure pipelines, you can build all three platforms on the same service. Cr
{% include "../examples/azure-pipelines-minimal.yml" %}
```
!!! note
To support Python 3.5 on Windows, make sure to specify the use of `{vmImage: 'vs2017-win2016'}` on Windows, to ensure the required toolchain is available.
Commit this file, enable building of your repo on Azure Pipelines, and push.
Wheels will be stored for you and available through the Pipelines interface. For more info on this file, check out the [docs](https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema).
test/06_docker_images/cibuildwheel_test.py
+2 -5
View File
@@ -9,12 +9,9 @@ def test():
actual_wheels = utils.cibuildwheel_run(project_dir, add_env={
'CIBW_MANYLINUX_X86_64_IMAGE': 'dockcross/manylinux2010-x64',
'CIBW_MANYLINUX_I686_IMAGE': 'dockcross/manylinux1-x86',
'CIBW_BEFORE_BUILD': '/opt/python/cp36-cp36m/bin/pip install -U auditwheel', # Currently necessary on dockcross images to get auditwheel 2.1 supporting AUDITWHEEL_PLAT
'CIBW_ENVIRONMENT': 'AUDITWHEEL_PLAT=`if [ $(uname -i) == "x86_64" ]; then echo "manylinux2010_x86_64"; else echo "manylinux1_i686"; fi`',
'CIBW_MANYLINUX_I686_IMAGE': 'dockcross/manylinux2010-x86',
})
# also check that we got the right wheels built
expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0')
if '-manylinux2010_i686' not in w]
expected_wheels = utils.expected_wheels('spam', '0.1.0')
assert set(actual_wheels) == set(expected_wheels)
test/shared/utils.py
+6 -8
View File
@@ -86,22 +86,20 @@ def expected_wheels(package_name, package_version, manylinux_versions=['manylinu
platform_tags.append('{manylinux_version}_{architecture}'.format(
manylinux_version=manylinux_version, architecture=architecture
))
def get_platform_tags(python_abi_tag):
return platform_tags
elif platform == 'windows':
def get_platform_tags(python_abi_tag):
return ['win32', 'win_amd64']
elif platform == 'macos':
def get_platform_tags(python_abi_tag):
if python_abi_tag == 'cp38-cp38':
return ['macosx_' + (macosx_deployment_target or "10.9").replace(".", "_") + '_x86_64']
else:
return ['macosx_' + (macosx_deployment_target or "10.6").replace(".", "_") + '_intel']
return ['macosx_' + (macosx_deployment_target or "10.9").replace(".", "_") + '_x86_64']
else:
raise Exception('unsupported platform')
unit_test/main_options_test.py
+31 -8
View File
@@ -50,7 +50,7 @@ def test_build_selector(platform, intercepted_build_args, monkeypatch):
monkeypatch.setenv('CIBW_SKIP', SKIP)
main()
intercepted_build_selector = intercepted_build_args.kwargs['build_selector']
assert isinstance(intercepted_build_selector, BuildSelector)
assert intercepted_build_selector('build-this')
@@ -60,12 +60,12 @@ def test_build_selector(platform, intercepted_build_args, monkeypatch):
@pytest.mark.parametrize('architecture, image, full_image', [
('x86_64', None, 'quay.io/pypa/manylinux2010_x86_64'),
('x86_64', None, 'quay.io/pypa/manylinux2010_x86_64'),
('x86_64', 'manylinux1', 'quay.io/pypa/manylinux1_x86_64'),
('x86_64', 'manylinux2010', 'quay.io/pypa/manylinux2010_x86_64'),
('x86_64', 'manylinux2014', 'quay.io/pypa/manylinux2014_x86_64'),
('x86_64', 'custom_image', 'custom_image'),
('i686', None, 'quay.io/pypa/manylinux2010_i686'),
('i686', None, 'quay.io/pypa/manylinux2010_i686'),
('i686', 'manylinux1', 'quay.io/pypa/manylinux1_i686'),
('i686', 'manylinux2010', 'quay.io/pypa/manylinux2010_i686'),
('i686', 'manylinux2014', 'quay.io/pypa/manylinux2014_i686'),
@@ -87,7 +87,7 @@ def get_default_repair_command(platform):
if platform == 'linux':
return 'auditwheel repair -w {dest_dir} {wheel}'
elif platform == 'macos':
return 'delocate-listdeps {wheel} && delocate-wheel -w {dest_dir} {wheel}'
return 'delocate-listdeps {wheel} && delocate-wheel --require-archs x86_64 -w {dest_dir} {wheel}'
elif platform == 'windows':
return ''
else:
@@ -139,9 +139,9 @@ def test_test_requires(test_requires, platform_specific, platform, intercepted_b
monkeypatch.setenv('CIBW_TEST_REQUIRES', 'overwritten')
else:
monkeypatch.setenv('CIBW_TEST_REQUIRES', test_requires)
main()
assert intercepted_build_args.kwargs['test_requires'] == (test_requires or '').split()
@@ -154,9 +154,9 @@ def test_test_extras(test_extras, platform_specific, platform, intercepted_build
monkeypatch.setenv('CIBW_TEST_EXTRAS', 'overwritten')
else:
monkeypatch.setenv('CIBW_TEST_EXTRAS', test_extras)
main()
assert intercepted_build_args.kwargs['test_extras'] == ('[' + test_extras + ']' if test_extras else '')
@@ -204,3 +204,26 @@ def test_build_verbosity(build_verbosity, platform_specific, platform, intercept
expected_verbosity = max(-3, min(3, int(build_verbosity or 0)))
assert intercepted_build_args.kwargs['build_verbosity'] == expected_verbosity
@pytest.mark.parametrize('option_name', ['CIBW_BUILD', 'CIBW_SKIP'])
@pytest.mark.parametrize('option_value, build_selector_patterns', [
('*-manylinux1_*', ['*-manylinux_*']),
('*-macosx_10_6_intel', ['*-macosx_x86_64']),
('*-macosx_10_9_x86_64', ['*-macosx_x86_64']),
('cp37-macosx_10_9_x86_64', ['cp37-macosx_x86_64']),
])
def test_build_selector_migrations(intercepted_build_args, monkeypatch, option_name, option_value, build_selector_patterns):
monkeypatch.setenv(option_name, option_value)
main()
intercepted_build_selector = intercepted_build_args.kwargs['build_selector']
assert isinstance(intercepted_build_selector, BuildSelector)
if option_name == 'CIBW_BUILD':
assert intercepted_build_selector.build_patterns == build_selector_patterns
else:
assert intercepted_build_selector.skip_patterns == build_selector_patterns
unit_test/main_util_fixtures.py
+2 -2
View File
@@ -4,7 +4,7 @@ import sys
import os
import subprocess
from cibuildwheel import linux, macos, windows
from cibuildwheel import linux, macos, windows, util
class ArgsInterceptor(object):
@@ -26,7 +26,7 @@ def mock_protection(monkeypatch):
raise RuntimeError("This should never be called")
monkeypatch.setattr(subprocess, 'Popen', fail_on_call)
monkeypatch.setattr(windows, 'urlopen', fail_on_call)
monkeypatch.setattr(util, 'download', fail_on_call)
monkeypatch.setattr(windows, 'build', fail_on_call)
monkeypatch.setattr(linux, 'build', fail_on_call)
monkeypatch.setattr(macos, 'build', fail_on_call)