Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #220 from mayeut/macos-platform

Browse Source 
Only build x86_64 / Intel 10.9+ on macOS
This commit is contained in:
Joe Rickerby
2020-02-01 23:07:47 +00:00
committed by GitHub
parent 68d688c016 c3f628d03d
commit 8236877f02
9 changed files with 124 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+7 -7
View File
@@ -15,13 +15,13 @@ Python wheels are great. Building them across **Mac, Linux, Windows**, on **mult
What does it do? What does it do?
---------------- ----------------
| | macOS 10.6+ intel | macOS 10.9+ x86_64 | manylinux i686 | manylinux x86_64 | Windows 32bit | Windows 64bit | | | macOS 10.9+ x86_64 | manylinux i686 | manylinux x86_64 | Windows 32bit | Windows 64bit |
|---|---|---|---|---|---|---| |---|---|---|---|---|---|
| Python 2.7 | ✅ | | ✅ | ✅ | ✅¹ | ✅¹ | | Python 2.7 | ✅ | ✅ | ✅ | ✅¹ | ✅¹ |
| Python 3.5 | ✅ | | ✅ | ✅ | ✅ | ✅ | | Python 3.5 | ✅ | ✅ | ✅ | ✅ | ✅ |
| Python 3.6 | ✅ | | ✅ | ✅ | ✅ | ✅ | | Python 3.6 | ✅ | ✅ | ✅ | ✅ | ✅ |
| Python 3.7 | ✅ | | ✅ | ✅ | ✅ | ✅ | | Python 3.7 | ✅ | ✅ | ✅ | ✅ | ✅ |
| Python 3.8 | | ✅ | ✅ | ✅ | ✅ | ✅ | | Python 3.8 | ✅ | ✅ | ✅ | ✅ | ✅ |
> ¹ Not supported on Travis > ¹ Not supported on Travis
cibuildwheel/__main__.py
+13 -10
View File
@@ -97,7 +97,7 @@ def main():
if platform == 'linux': if platform == 'linux':
repair_command_default = 'auditwheel repair -w {dest_dir} {wheel}' repair_command_default = 'auditwheel repair -w {dest_dir} {wheel}'
elif platform == 'macos': elif platform == 'macos':
repair_command_default = 'delocate-listdeps {wheel} && delocate-wheel -w {dest_dir} {wheel}' repair_command_default = 'delocate-listdeps {wheel} && delocate-wheel --require-archs x86_64 -w {dest_dir} {wheel}'
else: else:
repair_command_default = '' repair_command_default = ''
repair_command = get_option_from_environment('CIBW_REPAIR_WHEEL_COMMAND', platform=platform, default=repair_command_default) repair_command = get_option_from_environment('CIBW_REPAIR_WHEEL_COMMAND', platform=platform, default=repair_command_default)
@@ -198,15 +198,18 @@ def detect_obsolete_options():
print("Option '{}' is not empty. Please unset '{}'".format(alternative, deprecated)) print("Option '{}' is not empty. Please unset '{}'".format(alternative, deprecated))
exit(2) exit(2)
# Check for 'manylinux1' in the 'CIBW_BUILD' and 'CIBW_SKIP' options # Check for deprecated identifiers in 'CIBW_BUILD' and 'CIBW_SKIP' options
for deprecated in ['CIBW_BUILD', 'CIBW_SKIP']: for option in ['CIBW_BUILD', 'CIBW_SKIP']:
if deprecated in os.environ and 'manylinux1' in os.environ[deprecated]: for deprecated, alternative in [('manylinux1', 'manylinux'),
print("Build identifiers with 'manylinux1' have been deprecated. Replacing all occurences of 'manylinux1' by 'manylinux' in the option '{}'".format(deprecated)) ('macosx_10_6_intel', 'macosx_x86_64'),
os.environ[deprecated] = os.environ[deprecated].replace('manylinux1', 'manylinux') ('macosx_10_9_x86_64', 'macosx_x86_64')]:
if deprecated in os.environ and ("macosx_10_6" in os.environ[deprecated] or "macosx_10_9" in os.environ[deprecated]): if option in os.environ and deprecated in os.environ[option]:
print("Build identifiers with 'macosx_10_6' or 'macosx_10_9' have been deprecated. Replacing all occurences with 'macosx' in the option '{}'".format(deprecated)) print("Build identifiers with '{deprecated}' have been deprecated. Replacing all occurences of '{deprecated}' with '{alternative}' in the option '{option}'".format(
os.environ[deprecated] = os.environ[deprecated].replace('macosx_10_6', 'macosx').replace('macosx_10_9', 'macosx') deprecated=deprecated,
alternative=alternative,
option=option,
))
os.environ[option] = os.environ[option].replace(deprecated, alternative)
def print_preamble(platform, build_options): def print_preamble(platform, build_options):
print(textwrap.dedent(''' print(textwrap.dedent('''
cibuildwheel/macos.py
+22 -10
View File
@@ -8,17 +8,17 @@ try:
except ImportError: except ImportError:
from pipes import quote as shlex_quote from pipes import quote as shlex_quote
from .util import prepare_command, get_build_verbosity_extra_flags from .util import prepare_command, get_build_verbosity_extra_flags, download
def get_python_configurations(build_selector): def get_python_configurations(build_selector):
PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'url']) PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'url'])
python_configurations = [ python_configurations = [
PythonConfiguration(version='2.7', identifier='cp27-macosx_intel', url='https://www.python.org/ftp/python/2.7.17/python-2.7.17-macosx10.6.pkg'), PythonConfiguration(version='2.7', identifier='cp27-macosx_x86_64', url='https://www.python.org/ftp/python/2.7.17/python-2.7.17-macosx10.9.pkg'),
PythonConfiguration(version='3.5', identifier='cp35-macosx_intel', url='https://www.python.org/ftp/python/3.5.4/python-3.5.4-macosx10.6.pkg'), PythonConfiguration(version='3.5', identifier='cp35-macosx_x86_64', url='https://www.python.org/ftp/python/3.5.4/python-3.5.4-macosx10.6.pkg'),
PythonConfiguration(version='3.6', identifier='cp36-macosx_intel', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.6.pkg'), PythonConfiguration(version='3.6', identifier='cp36-macosx_x86_64', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg'),
PythonConfiguration(version='3.7', identifier='cp37-macosx_intel', url='https://www.python.org/ftp/python/3.7.5/python-3.7.5-macosx10.6.pkg'), PythonConfiguration(version='3.7', identifier='cp37-macosx_x86_64', url='https://www.python.org/ftp/python/3.7.6/python-3.7.6-macosx10.9.pkg'),
PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.0/python-3.8.0-macosx10.9.pkg'), PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.1/python-3.8.1-macosx10.9.pkg'),
] ]
# skip builds as required # skip builds as required
@@ -32,6 +32,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel') repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel')
python_configurations = get_python_configurations(build_selector) python_configurations = get_python_configurations(build_selector)
get_pip_url = 'https://bootstrap.pypa.io/get-pip.py' get_pip_url = 'https://bootstrap.pypa.io/get-pip.py'
get_pip_script = '/tmp/get-pip.py' get_pip_script = '/tmp/get-pip.py'
@@ -50,20 +51,20 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
return subprocess.check_call(args, env=env, cwd=cwd, shell=shell) return subprocess.check_call(args, env=env, cwd=cwd, shell=shell)
# get latest pip once and for all # get latest pip once and for all
download(get_pip_url, get_pip_script)
call(['curl', '-L', '--retry', '3', '--retry-delay', '3', '-o', get_pip_script, get_pip_url])
for config in python_configurations: for config in python_configurations:
# if this version of python isn't installed, get it from python.org and install # if this version of python isn't installed, get it from python.org and install
python_package_identifier = 'org.python.Python.PythonFramework-%s' % config.version python_package_identifier = 'org.python.Python.PythonFramework-%s' % config.version
if python_package_identifier not in installed_system_packages: if python_package_identifier not in installed_system_packages:
# download the pkg # download the pkg
call(['curl', '-L', '-o', '/tmp/Python.pkg', config.url]) download(config.url, '/tmp/Python.pkg')
# install # install
call(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/']) call(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/'])
# patch open ssl # patch open ssl
if config.version == '3.5': if config.version == '3.5':
call(['curl', '-fsSLo', '/tmp/python-patch.tar.gz', 'https://github.com/mayeut/patch-macos-python-openssl/releases/download/v1.0.2t/patch-macos-python-%s-openssl-v1.0.2t.tar.gz' % config.version]) open_ssl_patch_url = 'https://github.com/mayeut/patch-macos-python-openssl/releases/download/v1.0.2t/patch-macos-python-%s-openssl-v1.0.2t.tar.gz' % config.version
download(open_ssl_patch_url, '/tmp/python-patch.tar.gz')
call(['sudo', 'tar', '-C', '/Library/Frameworks/Python.framework/Versions/%s/' % config.version, '-xmf', '/tmp/python-patch.tar.gz']) call(['sudo', 'tar', '-C', '/Library/Frameworks/Python.framework/Versions/%s/' % config.version, '-xmf', '/tmp/python-patch.tar.gz'])
installation_bin_path = '/Library/Frameworks/Python.framework/Versions/{}/bin'.format(config.version) installation_bin_path = '/Library/Frameworks/Python.framework/Versions/{}/bin'.format(config.version)
@@ -98,6 +99,17 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
call(['pip', '--version'], env=env) call(['pip', '--version'], env=env)
call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate'], env=env) call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate'], env=env)
# setup target platform, only required for python 3.5
if config.version == '3.5':
if '_PYTHON_HOST_PLATFORM' not in env:
# cross-compilation platform override
env['_PYTHON_HOST_PLATFORM'] = 'macosx-10.9-x86_64'
if 'ARCHFLAGS' not in env:
# https://github.com/python/cpython/blob/a5ed2fe0eedefa1649aa93ee74a0bafc8e628a10/Lib/_osx_support.py#L260
env['ARCHFLAGS'] = '-arch x86_64'
if 'MACOSX_DEPLOYMENT_TARGET' not in env:
env['MACOSX_DEPLOYMENT_TARGET'] = '10.9'
# run the before_build command # run the before_build command
if before_build: if before_build:
before_build_prepared = prepare_command(before_build, project=abs_project_dir) before_build_prepared = prepare_command(before_build, project=abs_project_dir)
cibuildwheel/util.py
+31
View File
@@ -1,5 +1,12 @@
from fnmatch import fnmatch from fnmatch import fnmatch
import warnings import warnings
import os
from time import sleep
try:
from urllib.request import urlopen
except ImportError:
from urllib2 import urlopen
def prepare_command(command, **kwargs): def prepare_command(command, **kwargs):
@@ -50,3 +57,27 @@ class Unbuffered(object):
def __getattr__(self, attr): def __getattr__(self, attr):
return getattr(self.stream, attr) return getattr(self.stream, attr)
def download(url, dest):
print('+ Download ' + url + ' to ' + dest)
dest_dir = os.path.dirname(dest)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
repeat_num = 3
for i in range(repeat_num):
try:
response = urlopen(url)
except:
if i == repeat_num - 1:
raise
sleep(3)
continue
break
try:
with open(dest, 'wb') as file:
file.write(response.read())
finally:
response.close()
cibuildwheel/windows.py
+1 -29
View File
@@ -1,6 +1,5 @@
from __future__ import print_function from __future__ import print_function
import os, tempfile, subprocess, shutil, sys import os, tempfile, subprocess, shutil, sys
from time import sleep
from collections import namedtuple from collections import namedtuple
from glob import glob from glob import glob
@@ -9,12 +8,7 @@ try:
except ImportError: except ImportError:
from pipes import quote as shlex_quote from pipes import quote as shlex_quote
try: from .util import prepare_command, get_build_verbosity_extra_flags, download
from urllib.request import urlopen
except ImportError:
from urllib2 import urlopen
from .util import prepare_command, get_build_verbosity_extra_flags
IS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache') IS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache')
@@ -64,28 +58,6 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
args = ['cmd', '/E:ON', '/V:ON', '/C'] + args args = ['cmd', '/E:ON', '/V:ON', '/C'] + args
return subprocess.check_call(' '.join(args), env=env, cwd=cwd) return subprocess.check_call(' '.join(args), env=env, cwd=cwd)
def download(url, dest):
print('+ Download ' + url + ' to ' + dest)
dest_dir = os.path.dirname(dest)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
repeat_num = 3
for i in range(repeat_num):
try:
response = urlopen(url)
except:
if i == repeat_num - 1:
raise
sleep(3)
continue
break
try:
with open(dest, 'wb') as file:
file.write(response.read())
finally:
response.close()
if IS_RUNNING_ON_AZURE or IS_RUNNING_ON_TRAVIS: if IS_RUNNING_ON_AZURE or IS_RUNNING_ON_TRAVIS:
shell = simple_shell shell = simple_shell
else: else:
docs/options.md
+10 -10
View File
@@ -64,7 +64,7 @@ This option can also be set using the command-line option `--platform`.
> Choose the Python versions to build > Choose the Python versions to build
Space-separated list of builds to build and skip. Each build has an identifier like `cp27-manylinux_x86_64` or `cp35-macosx_intel` - you can list specific ones to build and `cibuildwheel` will only build those, and/or list ones to skip and `cibuildwheel` won't try to build them. Space-separated list of builds to build and skip. Each build has an identifier like `cp27-manylinux_x86_64` or `cp35-macosx_x86_64` - you can list specific ones to build and `cibuildwheel` will only build those, and/or list ones to skip and `cibuildwheel` won't try to build them.
When both options are specified, both conditions are applied and only builds with a tag that matches `CIBW_BUILD` and does not match `CIBW_SKIP` will be built. When both options are specified, both conditions are applied and only builds with a tag that matches `CIBW_BUILD` and does not match `CIBW_SKIP` will be built.
@@ -72,13 +72,13 @@ When setting the options, you can use shell-style globbing syntax (as per `fnmat
<div class="build-id-table-marker"></div> <div class="build-id-table-marker"></div>
| | macOS 64bit | macOS 32/64bit | Manylinux 64bit | Manylinux 32bit | Windows 64bit | Windows 32bit | | | macOS 64bit | Manylinux 64bit | Manylinux 32bit | Windows 64bit | Windows 32bit |
|------------|---------------------|--------------------|------------------------|----------------------|-----------------|----------------| |------------|--------------------|------------------------|----------------------|-----------------|----------------|
| Python 2.7 | | cp27-macosx_intel | cp27-manylinux_x86_64 | cp27-manylinux_i686 | cp27-win_amd64 | cp27-win32 | | Python 2.7 | cp27-macosx_x86_64 | cp27-manylinux_x86_64 | cp27-manylinux_i686 | cp27-win_amd64 | cp27-win32 |
| Python 3.5 | | cp35-macosx_intel | cp35-manylinux_x86_64 | cp35-manylinux_i686 | cp35-win_amd64 | cp35-win32 | | Python 3.5 | cp35-macosx_x86_64 | cp35-manylinux_x86_64 | cp35-manylinux_i686 | cp35-win_amd64 | cp35-win32 |
| Python 3.6 | | cp36-macosx_intel | cp36-manylinux_x86_64 | cp36-manylinux_i686 | cp36-win_amd64 | cp36-win32 | | Python 3.6 | cp36-macosx_x86_64 | cp36-manylinux_x86_64 | cp36-manylinux_i686 | cp36-win_amd64 | cp36-win32 |
| Python 3.7 | | cp37-macosx_intel | cp37-manylinux_x86_64 | cp37-manylinux_i686 | cp37-win_amd64 | cp37-win32 | | Python 3.7 | cp37-macosx_x86_64 | cp37-manylinux_x86_64 | cp37-manylinux_i686 | cp37-win_amd64 | cp37-win32 |
| Python 3.8 | cp38-macosx_x86_64 | | cp38-manylinux_x86_64 | cp38-manylinux_i686 | cp38-win_amd64 | cp38-win32 | | Python 3.8 | cp38-macosx_x86_64 | cp38-manylinux_x86_64 | cp38-manylinux_i686 | cp38-win_amd64 | cp38-win32 |
The list of supported and currently selected build identifiers can also be retrieved by passing the `--print-build-identifiers` flag to `cibuildwheel`. The list of supported and currently selected build identifiers can also be retrieved by passing the `--print-build-identifiers` flag to `cibuildwheel`.
The format is `python_tag-platform_tag`, with tags similar to those in [PEP 425](https://www.python.org/dev/peps/pep-0425/#details). The format is `python_tag-platform_tag`, with tags similar to those in [PEP 425](https://www.python.org/dev/peps/pep-0425/#details).
@@ -90,7 +90,7 @@ The format is `python_tag-platform_tag`, with tags similar to those in [PEP 425]
CIBW_BUILD: cp36-* CIBW_BUILD: cp36-*
# Skip building on Python 2.7 on the Mac # Skip building on Python 2.7 on the Mac
CIBW_SKIP: cp27-macosx_intel CIBW_SKIP: cp27-macosx_x86_64
# Skip building on Python 3.8 on the Mac # Skip building on Python 3.8 on the Mac
CIBW_SKIP: cp38-macosx_x86_64 CIBW_SKIP: cp38-macosx_x86_64
@@ -208,7 +208,7 @@ CIBW_BEFORE_BUILD: yum install -y libffi-dev && pip install .
Default: Default:
- on Linux: `'auditwheel repair -w {dest_dir} {wheel}'` - on Linux: `'auditwheel repair -w {dest_dir} {wheel}'`
- on macOS: `'delocate-listdeps {wheel} && delocate-wheel -w {dest_dir} {wheel}'` - on macOS: `'delocate-listdeps {wheel} && delocate-wheel --require-archs x86_64 -w {dest_dir} {wheel}'`
- on Windows: `''` - on Windows: `''`
A shell command to repair a built wheel by copying external library dependencies into the wheel tree and relinking them. A shell command to repair a built wheel by copying external library dependencies into the wheel tree and relinking them.
test/shared/utils.py
+1 -3
View File
@@ -98,10 +98,8 @@ def expected_wheels(package_name, package_version, manylinux_versions=['manylinu
elif platform == 'macos': elif platform == 'macos':
def get_platform_tags(python_abi_tag): def get_platform_tags(python_abi_tag):
if python_abi_tag == 'cp38-cp38':
return ['macosx_' + (macosx_deployment_target or "10.9").replace(".", "_") + '_x86_64'] return ['macosx_' + (macosx_deployment_target or "10.9").replace(".", "_") + '_x86_64']
else:
return ['macosx_' + (macosx_deployment_target or "10.6").replace(".", "_") + '_intel']
else: else:
raise Exception('unsupported platform') raise Exception('unsupported platform')
unit_test/main_options_test.py
+24 -1
View File
@@ -87,7 +87,7 @@ def get_default_repair_command(platform):
if platform == 'linux': if platform == 'linux':
return 'auditwheel repair -w {dest_dir} {wheel}' return 'auditwheel repair -w {dest_dir} {wheel}'
elif platform == 'macos': elif platform == 'macos':
return 'delocate-listdeps {wheel} && delocate-wheel -w {dest_dir} {wheel}' return 'delocate-listdeps {wheel} && delocate-wheel --require-archs x86_64 -w {dest_dir} {wheel}'
elif platform == 'windows': elif platform == 'windows':
return '' return ''
else: else:
@@ -204,3 +204,26 @@ def test_build_verbosity(build_verbosity, platform_specific, platform, intercept
expected_verbosity = max(-3, min(3, int(build_verbosity or 0))) expected_verbosity = max(-3, min(3, int(build_verbosity or 0)))
assert intercepted_build_args.kwargs['build_verbosity'] == expected_verbosity assert intercepted_build_args.kwargs['build_verbosity'] == expected_verbosity
@pytest.mark.parametrize('option_name', ['CIBW_BUILD', 'CIBW_SKIP'])
@pytest.mark.parametrize('option_value, build_selector_patterns', [
('*-manylinux1_*', ['*-manylinux_*']),
('*-macosx_10_6_intel', ['*-macosx_x86_64']),
('*-macosx_10_9_x86_64', ['*-macosx_x86_64']),
('cp37-macosx_10_9_x86_64', ['cp37-macosx_x86_64']),
])
def test_build_selector_migrations(intercepted_build_args, monkeypatch, option_name, option_value, build_selector_patterns):
monkeypatch.setenv(option_name, option_value)
main()
intercepted_build_selector = intercepted_build_args.kwargs['build_selector']
assert isinstance(intercepted_build_selector, BuildSelector)
if option_name == 'CIBW_BUILD':
assert intercepted_build_selector.build_patterns == build_selector_patterns
else:
assert intercepted_build_selector.skip_patterns == build_selector_patterns
unit_test/main_util_fixtures.py
+2 -2
View File
@@ -4,7 +4,7 @@ import sys
import os import os
import subprocess import subprocess
from cibuildwheel import linux, macos, windows from cibuildwheel import linux, macos, windows, util
class ArgsInterceptor(object): class ArgsInterceptor(object):
@@ -26,7 +26,7 @@ def mock_protection(monkeypatch):
raise RuntimeError("This should never be called") raise RuntimeError("This should never be called")
monkeypatch.setattr(subprocess, 'Popen', fail_on_call) monkeypatch.setattr(subprocess, 'Popen', fail_on_call)
monkeypatch.setattr(windows, 'urlopen', fail_on_call) monkeypatch.setattr(util, 'download', fail_on_call)
monkeypatch.setattr(windows, 'build', fail_on_call) monkeypatch.setattr(windows, 'build', fail_on_call)
monkeypatch.setattr(linux, 'build', fail_on_call) monkeypatch.setattr(linux, 'build', fail_on_call)
monkeypatch.setattr(macos, 'build', fail_on_call) monkeypatch.setattr(macos, 'build', fail_on_call)