Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge branch 'master' into returncode

Browse Source
This commit is contained in:
Gertjan van den Burg
2019-11-05 17:09:17 +00:00
parent e0eb375449 89e2ac7ebf
commit 560c1adeb7
15 changed files with 361 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cibuildwheel/__main__.py
+33 -3
View File
@@ -54,6 +54,8 @@ def main():
args = parser.parse_args()
detect_obsolete_options()
if args.platform != 'auto':
platform = args.platform
else:
@@ -63,6 +65,8 @@ def main():
platform = 'linux'
elif os.environ.get('TRAVIS_OS_NAME') == 'osx':
platform = 'macos'
elif os.environ.get('TRAVIS_OS_NAME') == 'windows':
platform = 'windows'
elif 'APPVEYOR' in os.environ:
platform = 'windows'
elif 'BITRISE_BUILD_NUMBER' in os.environ:
@@ -141,11 +145,17 @@ def main():
)
if platform == 'linux':
manylinux1_x86_64_image = os.environ.get('CIBW_MANYLINUX1_X86_64_IMAGE', None)
manylinux1_i686_image = os.environ.get('CIBW_MANYLINUX1_I686_IMAGE', None)
manylinux_x86_64_image = os.environ.get('CIBW_MANYLINUX_X86_64_IMAGE', 'manylinux2010')
manylinux_i686_image = os.environ.get('CIBW_MANYLINUX_I686_IMAGE', 'manylinux2010')
default_manylinux_images_x86_64 = {'manylinux1': 'quay.io/pypa/manylinux1_x86_64',
'manylinux2010': 'quay.io/pypa/manylinux2010_x86_64'}
default_manylinux_images_i686 = {'manylinux1': 'quay.io/pypa/manylinux1_i686',
'manylinux2010': 'quay.io/pypa/manylinux2010_i686'}
build_options.update(
manylinux1_images={'x86_64': manylinux1_x86_64_image, 'i686': manylinux1_i686_image},
manylinux_images={'x86_64': default_manylinux_images_x86_64.get(manylinux_x86_64_image) or manylinux_x86_64_image,
'i686': default_manylinux_images_i686.get(manylinux_i686_image) or manylinux_i686_image},
)
elif platform == 'macos':
pass
@@ -170,6 +180,26 @@ def main():
raise Exception('Unsupported platform')
def detect_obsolete_options():
# Check the old 'MANYLINUX1_*_IMAGE' options
for (deprecated, alternative) in [('CIBW_MANYLINUX1_X86_64_IMAGE', 'CIBW_MANYLINUX_X86_64_IMAGE'),
('CIBW_MANYLINUX1_I686_IMAGE', 'CIBW_MANYLINUX_I686_IMAGE')]:
if deprecated in os.environ:
print("'{}' has been deprecated, and will be removed in a future release. Use the option '{}' instead.".format(deprecated, alternative))
if alternative not in os.environ:
print("Using value of option '{}' as replacement for '{}'".format(deprecated, alternative))
os.environ[alternative] = os.environ[deprecated]
else:
print("Option '{}' is not empty. Please unset '{}'".format(alternative, deprecated))
exit(2)
# Check for 'manylinux1' in the 'CIBW_BUILD' and 'CIBW_SKIP' options
for deprecated in ['CIBW_BUILD', 'CIBW_SKIP']:
if deprecated in os.environ and 'manylinux1' in os.environ[deprecated]:
print("Build identifiers with 'manylinux1' been deprecated. Replacing all occurences of 'manylinux1' by 'manylinux' in the option '{}'".format(deprecated))
os.environ[deprecated] = os.environ[deprecated].replace('manylinux1', 'manylinux')
def print_preamble(platform, build_options):
print(textwrap.dedent('''
_ _ _ _ _ _ _
cibuildwheel/linux.py
+28 -23
View File
@@ -12,25 +12,25 @@ except ImportError:
def get_python_configurations(build_selector):
PythonConfiguration = namedtuple('PythonConfiguration', ['identifier', 'path'])
python_configurations = [
PythonConfiguration(identifier='cp27-manylinux1_x86_64', path='/opt/python/cp27-cp27m'),
PythonConfiguration(identifier='cp27-manylinux1_x86_64', path='/opt/python/cp27-cp27mu'),
PythonConfiguration(identifier='cp34-manylinux1_x86_64', path='/opt/python/cp34-cp34m'),
PythonConfiguration(identifier='cp35-manylinux1_x86_64', path='/opt/python/cp35-cp35m'),
PythonConfiguration(identifier='cp36-manylinux1_x86_64', path='/opt/python/cp36-cp36m'),
PythonConfiguration(identifier='cp37-manylinux1_x86_64', path='/opt/python/cp37-cp37m'),
PythonConfiguration(identifier='cp27-manylinux1_i686', path='/opt/python/cp27-cp27m'),
PythonConfiguration(identifier='cp27-manylinux1_i686', path='/opt/python/cp27-cp27mu'),
PythonConfiguration(identifier='cp34-manylinux1_i686', path='/opt/python/cp34-cp34m'),
PythonConfiguration(identifier='cp35-manylinux1_i686', path='/opt/python/cp35-cp35m'),
PythonConfiguration(identifier='cp36-manylinux1_i686', path='/opt/python/cp36-cp36m'),
PythonConfiguration(identifier='cp37-manylinux1_i686', path='/opt/python/cp37-cp37m'),
PythonConfiguration(identifier='cp27-manylinux_x86_64', path='/opt/python/cp27-cp27m'),
PythonConfiguration(identifier='cp27-manylinux_x86_64', path='/opt/python/cp27-cp27mu'),
PythonConfiguration(identifier='cp35-manylinux_x86_64', path='/opt/python/cp35-cp35m'),
PythonConfiguration(identifier='cp36-manylinux_x86_64', path='/opt/python/cp36-cp36m'),
PythonConfiguration(identifier='cp37-manylinux_x86_64', path='/opt/python/cp37-cp37m'),
PythonConfiguration(identifier='cp38-manylinux_x86_64', path='/opt/python/cp38-cp38'),
PythonConfiguration(identifier='cp27-manylinux_i686', path='/opt/python/cp27-cp27m'),
PythonConfiguration(identifier='cp27-manylinux_i686', path='/opt/python/cp27-cp27mu'),
PythonConfiguration(identifier='cp35-manylinux_i686', path='/opt/python/cp35-cp35m'),
PythonConfiguration(identifier='cp36-manylinux_i686', path='/opt/python/cp36-cp36m'),
PythonConfiguration(identifier='cp37-manylinux_i686', path='/opt/python/cp37-cp37m'),
PythonConfiguration(identifier='cp38-manylinux_i686', path='/opt/python/cp38-cp38'),
]
# skip builds as required
return [c for c in python_configurations if build_selector(c.identifier)]
def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, environment, manylinux1_images):
def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, environment, manylinux_images):
try:
subprocess.check_call(['docker', '--version'])
except:
@@ -42,8 +42,8 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
python_configurations = get_python_configurations(build_selector)
platforms = [
('manylinux1_x86_64', manylinux1_images.get('x86_64') or 'quay.io/pypa/manylinux1_x86_64'),
('manylinux1_i686', manylinux1_images.get('i686') or 'quay.io/pypa/manylinux1_i686'),
('manylinux_x86_64', manylinux_images['x86_64']),
('manylinux_i686', manylinux_images['i686']),
]
for platform_tag, docker_image in platforms:
@@ -62,9 +62,9 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
for PYBIN in {pybin_paths}; do
# Setup
rm -rf /tmp/built_wheel
rm -rf /tmp/delocated_wheel
rm -rf /tmp/delocated_wheels
mkdir /tmp/built_wheel
mkdir /tmp/delocated_wheel
mkdir /tmp/delocated_wheels
if [ ! -z {before_build} ]; then
PATH="$PYBIN:$PATH" sh -c {before_build}
@@ -79,11 +79,11 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
# the first element
if [[ "$built_wheel" == *none-any.whl ]]; then
# pure python wheel - just copy
mv "$built_wheel" /tmp/delocated_wheel
mv "$built_wheel" /tmp/delocated_wheels
else
auditwheel repair "$built_wheel" -w /tmp/delocated_wheel
auditwheel repair "$built_wheel" -w /tmp/delocated_wheels
fi
delocated_wheel=(/tmp/delocated_wheel/*.whl)
delocated_wheels=(/tmp/delocated_wheels/*.whl)
if [ ! -z {test_command} ]; then
# Set up a virtual environment to install and test from, to make sure
@@ -100,7 +100,12 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
echo "Running tests using `which python`"
# Install the wheel we just built
pip install "$delocated_wheel"{test_extras}
# Note: If auditwheel produced two wheels, it's because the earlier produced wheel
# conforms to multiple manylinux standards. These multiple versions of the wheel are
# functionally the same, differing only in name, wheel metadata, and possibly include
# different external shared libraries. so it doesn't matter which one we run the tests on.
# Let's just pick the first one.
pip install "${{delocated_wheels[0]}}"{test_extras}
# Install any requirements to run the tests
if [ ! -z "{test_requires}" ]; then
@@ -122,8 +127,8 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
fi
# we're all done here; move it to output
mv "$delocated_wheel" /output
chown {uid}:{gid} "/output/$(basename "$delocated_wheel")"
mv "${{delocated_wheels[@]}}" /output
for delocated_wheel in "${{delocated_wheels[@]}}"; do chown {uid}:{gid} "/output/$(basename "$delocated_wheel")"; done
done
'''.format(
pybin_paths=' '.join(c.path+'/bin' for c in platform_configs),
cibuildwheel/macos.py
+5 -5
View File
@@ -14,11 +14,11 @@ from .util import prepare_command, get_build_verbosity_extra_flags
def get_python_configurations(build_selector):
PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'url'])
python_configurations = [
PythonConfiguration(version='2.7', identifier='cp27-macosx_10_6_intel', url='https://www.python.org/ftp/python/2.7.16/python-2.7.16-macosx10.6.pkg'),
PythonConfiguration(version='3.4', identifier='cp34-macosx_10_6_intel', url='https://www.python.org/ftp/python/3.4.4/python-3.4.4-macosx10.6.pkg'),
PythonConfiguration(version='2.7', identifier='cp27-macosx_10_6_intel', url='https://www.python.org/ftp/python/2.7.17/python-2.7.17-macosx10.6.pkg'),
PythonConfiguration(version='3.5', identifier='cp35-macosx_10_6_intel', url='https://www.python.org/ftp/python/3.5.4/python-3.5.4-macosx10.6.pkg'),
PythonConfiguration(version='3.6', identifier='cp36-macosx_10_6_intel', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.6.pkg'),
PythonConfiguration(version='3.7', identifier='cp37-macosx_10_6_intel', url='https://www.python.org/ftp/python/3.7.4/python-3.7.4-macosx10.6.pkg'),
PythonConfiguration(version='3.7', identifier='cp37-macosx_10_6_intel', url='https://www.python.org/ftp/python/3.7.5/python-3.7.5-macosx10.6.pkg'),
PythonConfiguration(version='3.8', identifier='cp38-macosx_10_9_x86_64', url='https://www.python.org/ftp/python/3.8.0/python-3.8.0-macosx10.9.pkg'),
]
# skip builds as required
@@ -58,7 +58,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
# install
call(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/'])
# patch open ssl
if config.version in ('3.4', '3.5'):
if config.version == '3.5':
call(['curl', '-fsSLo', '/tmp/python-patch.tar.gz', 'https://github.com/mayeut/patch-macos-python-openssl/releases/download/v1.0.2t/patch-macos-python-%s-openssl-v1.0.2t.tar.gz' % config.version])
call(['sudo', 'tar', '-C', '/Library/Frameworks/Python.framework/Versions/%s/' % config.version, '-xmf', '/tmp/python-patch.tar.gz'])
@@ -152,7 +152,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
# and not the repo code)
test_command_prepared = prepare_command(test_command, project=abs_project_dir)
call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True)
# clean up
shutil.rmtree(venv_dir)
cibuildwheel/windows.py
+66 -47
View File
@@ -1,67 +1,78 @@
from __future__ import print_function
import os, tempfile, subprocess, shutil
import os, tempfile, subprocess, shutil, sys
from collections import namedtuple
from glob import glob
try:
from shlex import quote as shlex_quote
except ImportError:
from pipes import quote as shlex_quote
try:
from urllib.request import urlopen
except ImportError:
from urllib2 import urlopen
from .util import prepare_command, get_build_verbosity_extra_flags
IS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache')
IS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows'
def get_python_path(config):
if IS_RUNNING_ON_AZURE:
# We can't hard-code the paths because on Azure, we don't know which
# bugfix release of Python we are getting so we need to check which
# ones exist. We just use the first one that is found since there should
# only be one.
path_pattern = 'C:\\hostedtoolcache\\windows\\Python\\{version}\\{arch}'.format(
version=config.version.replace('x', '*'),
arch='x86' if config.arch == '32' else 'x64'
)
try:
return glob(path_pattern)[0]
except IndexError:
raise Exception('Could not find a Python install at ' + path_pattern)
else:
# Assume we're running on AppVeyor
major, minor = config.version.split('.')[:2]
return 'C:\\Python{major}{minor}{arch}'.format(
major=major,
minor=minor,
arch = '-x64' if config.arch == '64' else ''
)
nuget_args = get_nuget_args(config)
return os.path.join(nuget_args[-1], nuget_args[0] + "." + config.version, "tools")
def get_nuget_args(configuration):
python_name = "python" if configuration.version[0] == '3' else "python2"
if configuration.arch == "32":
python_name = python_name + "x86"
return [python_name, "-Version", configuration.version, "-OutputDirectory", "C:/python"]
def get_python_configurations(build_selector):
PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier', 'path'])
PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier'])
python_configurations = [
PythonConfiguration(version='2.7.x', arch="32", identifier='cp27-win32', path='C:\Python27'),
PythonConfiguration(version='2.7.x', arch="64", identifier='cp27-win_amd64', path='C:\Python27-x64'),
PythonConfiguration(version='3.4.x', arch="32", identifier='cp34-win32', path='C:\Python34'),
PythonConfiguration(version='3.4.x', arch="64", identifier='cp34-win_amd64', path='C:\Python34-x64'),
PythonConfiguration(version='3.5.x', arch="32", identifier='cp35-win32', path='C:\Python35'),
PythonConfiguration(version='3.5.x', arch="64", identifier='cp35-win_amd64', path='C:\Python35-x64'),
PythonConfiguration(version='3.6.x', arch="32", identifier='cp36-win32', path='C:\Python36'),
PythonConfiguration(version='3.6.x', arch="64", identifier='cp36-win_amd64', path='C:\Python36-x64'),
PythonConfiguration(version='3.7.x', arch="32", identifier='cp37-win32', path='C:\Python37'),
PythonConfiguration(version='3.7.x', arch="64", identifier='cp37-win_amd64', path='C:\Python37-x64'),
PythonConfiguration(version='2.7.17', arch="32", identifier='cp27-win32'),
PythonConfiguration(version='2.7.17', arch="64", identifier='cp27-win_amd64'),
PythonConfiguration(version='3.5.4', arch="32", identifier='cp35-win32'),
PythonConfiguration(version='3.5.4', arch="64", identifier='cp35-win_amd64'),
PythonConfiguration(version='3.6.8', arch="32", identifier='cp36-win32'),
PythonConfiguration(version='3.6.8', arch="64", identifier='cp36-win_amd64'),
PythonConfiguration(version='3.7.5', arch="32", identifier='cp37-win32'),
PythonConfiguration(version='3.7.5', arch="64", identifier='cp37-win_amd64'),
PythonConfiguration(version='3.8.0', arch="32", identifier='cp38-win32'),
PythonConfiguration(version='3.8.0', arch="64", identifier='cp38-win_amd64'),
]
if IS_RUNNING_ON_AZURE:
# Python 3.4 isn't supported on Azure.
# See https://github.com/Microsoft/azure-pipelines-tasks/issues/9674
python_configurations = [c for c in python_configurations if c.version != '3.4.x']
if IS_RUNNING_ON_TRAVIS:
# cannot install VCForPython27.msi which is needed for compiling C software
# try with (and similar): msiexec /i VCForPython27.msi ALLUSERS=1 ACCEPT=YES /passive
python_configurations = [c for c in python_configurations if not c.version.startswith('2.7.')]
# skip builds as required
python_configurations = [c for c in python_configurations if build_selector(c.identifier)]
return python_configurations
# skip builds as required
return [c for c in python_configurations if build_selector(c.identifier)]
def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, environment):
if IS_RUNNING_ON_AZURE:
def shell(args, env=None, cwd=None):
print('+ ' + ' '.join(args))
args = ['cmd', '/E:ON', '/V:ON', '/C'] + args
return subprocess.check_call(' '.join(args), env=env, cwd=cwd)
def simple_shell(args, env=None, cwd=None):
print('+ ' + ' '.join(args))
args = ['cmd', '/E:ON', '/V:ON', '/C'] + args
return subprocess.check_call(' '.join(args), env=env, cwd=cwd)
def download(url, dest):
print('+ Download ' + url + ' to ' + dest)
response = urlopen(url)
try:
with open(dest, 'wb') as file:
file.write(response.read())
finally:
response.close()
if IS_RUNNING_ON_AZURE or IS_RUNNING_ON_TRAVIS:
shell = simple_shell
else:
run_with_env = os.path.abspath(os.path.join(os.path.dirname(__file__), 'resources', 'appveyor_run_with_env.cmd'))
@@ -77,10 +88,19 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
built_wheel_dir = os.path.join(temp_dir, 'built_wheel')
python_configurations = get_python_configurations(build_selector)
# install nuget as best way to provide python
nuget = 'C:\\nuget.exe'
download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget)
# get pip fo this installation which not have.
get_pip_script = 'C:\\get-pip.py'
download('https://bootstrap.pypa.io/get-pip.py', get_pip_script)
python_configurations = get_python_configurations(build_selector)
for config in python_configurations:
config_python_path = get_python_path(config)
simple_shell([nuget, "install"] + get_nuget_args(config))
if not os.path.exists(os.path.join(config_python_path, 'Scripts', 'pip.exe')):
simple_shell([os.path.join(config_python_path, 'python.exe'), get_pip_script ])
# check python & pip exist for this configuration
assert os.path.exists(os.path.join(config_python_path, 'python.exe'))
@@ -107,8 +127,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)\"'], env=env)
# prepare the Python environment
shell(['python', '-m', 'pip', 'install', '--upgrade', 'pip'],
env=env)
shell(['python', '-m', 'pip', 'install', '--upgrade', 'pip'], env=env)
shell(['pip', 'install', '--upgrade', 'setuptools'], env=env)
shell(['pip', 'install', 'wheel'], env=env)