Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into support-subdirectory

Browse Source
This commit is contained in:
Joe Rickerby
2020-04-09 14:05:09 +01:00
parent ed50cfd20a eff997d12e
commit d5010b8242
33 changed files with 23539 additions and 459 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cibuildwheel/__main__.py
+54 -41
View File
@@ -3,6 +3,7 @@ import os
import sys
import textwrap
import traceback
from configparser import ConfigParser
import cibuildwheel
import cibuildwheel.linux
@@ -14,7 +15,9 @@ from cibuildwheel.environment import (
)
from cibuildwheel.util import (
BuildSelector,
Unbuffered
DependencyConstraints,
Unbuffered,
BuildOptions
)
@@ -115,6 +118,14 @@ def main():
environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform, default='')
before_test = get_option_from_environment('CIBW_BEFORE_TEST', platform=platform, default='')
dependency_versions = get_option_from_environment('CIBW_DEPENDENCY_VERSIONS', platform=platform, default='pinned')
if dependency_versions == 'pinned':
dependency_constraints = DependencyConstraints.with_defaults()
elif dependency_versions == 'latest':
dependency_constraints = None
else:
dependency_constraints = DependencyConstraints(dependency_versions)
if test_extras:
test_extras = '[{0}]'.format(test_extras)
@@ -144,7 +155,40 @@ def main():
print_build_identifiers(platform, build_selector)
exit(0)
build_options = dict(
if platform == 'linux':
pinned_docker_images_file = os.path.join(
os.path.dirname(__file__), 'resources', 'pinned_docker_images.cfg'
)
all_pinned_docker_images = ConfigParser()
all_pinned_docker_images.read(pinned_docker_images_file)
# all_pinned_docker_images looks like a dict of dicts, e.g.
# { 'x86_64': {'manylinux1': '...', 'manylinux2010': '...', 'manylinux2014': '...'},
# 'i686': {'manylinux1': '...', 'manylinux2010': '...', 'manylinux2014': '...'},
# 'pypy_x86_64': {'manylinux2010': '...' }
# ... }
manylinux_images = {}
for build_platform in ['x86_64', 'i686', 'pypy_x86_64', 'aarch64', 'ppc64le', 's390x']:
pinned_images = all_pinned_docker_images[build_platform]
config_name = 'CIBW_MANYLINUX_{}_IMAGE'.format(build_platform.upper())
config_value = os.environ.get(config_name)
if config_value is None:
# default to manylinux2010 if it's available, otherwise manylinux2014
image = pinned_images.get('manylinux2010') or pinned_images.get('manylinux2014')
elif config_value in pinned_images:
image = pinned_images[config_value]
else:
image = config_value
manylinux_images[build_platform] = image
else:
manylinux_images = None
build_options = BuildOptions(
project_dir='.',
package_dir=package_dir,
output_dir=output_dir,
@@ -156,42 +200,11 @@ def main():
build_selector=build_selector,
repair_command=repair_command,
environment=environment,
before_test=before_test
before_test=before_test,
dependency_constraints=dependency_constraints,
manylinux_images=manylinux_images,
)
if platform == 'linux':
manylinux_x86_64_image = os.environ.get('CIBW_MANYLINUX_X86_64_IMAGE', 'manylinux2010')
manylinux_i686_image = os.environ.get('CIBW_MANYLINUX_I686_IMAGE', 'manylinux2010')
manylinux_pypy_x86_64_image = os.environ.get('CIBW_MANYLINUX_PYPY_X86_64_IMAGE', 'manylinux2010')
manylinux_aarch64_image = os.environ.get('CIBW_MANYLINUX_AARCH64_IMAGE', 'manylinux2014')
manylinux_ppc64le_image = os.environ.get('CIBW_MANYLINUX_PPC64LE_IMAGE', 'manylinux2014')
manylinux_s390x_image = os.environ.get('CIBW_MANYLINUX_S390X_IMAGE', 'manylinux2014')
default_manylinux_images_x86_64 = {'manylinux1': 'quay.io/pypa/manylinux1_x86_64',
'manylinux2010': 'quay.io/pypa/manylinux2010_x86_64',
'manylinux2014': 'quay.io/pypa/manylinux2014_x86_64'}
default_manylinux_images_i686 = {'manylinux1': 'quay.io/pypa/manylinux1_i686',
'manylinux2010': 'quay.io/pypa/manylinux2010_i686',
'manylinux2014': 'quay.io/pypa/manylinux2014_i686'}
default_manylinux_images_pypy_x86_64 = {'manylinux2010': 'pypywheels/manylinux2010-pypy_x86_64'}
default_manylinux_images_aarch64 = {'manylinux2014': 'quay.io/pypa/manylinux2014_aarch64'}
default_manylinux_images_ppc64le = {'manylinux2014': 'quay.io/pypa/manylinux2014_ppc64le'}
default_manylinux_images_s390x = {'manylinux2014': 'quay.io/pypa/manylinux2014_s390x'}
build_options.update(
manylinux_images={'x86_64': default_manylinux_images_x86_64.get(manylinux_x86_64_image) or manylinux_x86_64_image,
'i686': default_manylinux_images_i686.get(manylinux_i686_image) or manylinux_i686_image,
'pypy_x86_64': default_manylinux_images_pypy_x86_64.get(manylinux_pypy_x86_64_image) or manylinux_pypy_x86_64_image,
'aarch64': default_manylinux_images_aarch64.get(manylinux_aarch64_image) or manylinux_aarch64_image,
'ppc64le': default_manylinux_images_ppc64le.get(manylinux_ppc64le_image) or manylinux_ppc64le_image,
's390x': default_manylinux_images_s390x.get(manylinux_s390x_image) or manylinux_s390x_image,
},
)
elif platform == 'macos':
pass
elif platform == 'windows':
pass
# Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print'
sys.stdout = Unbuffered(sys.stdout)
@@ -201,11 +214,11 @@ def main():
os.makedirs(output_dir)
if platform == 'linux':
cibuildwheel.linux.build(**build_options)
cibuildwheel.linux.build(build_options)
elif platform == 'windows':
cibuildwheel.windows.build(**build_options)
cibuildwheel.windows.build(build_options)
elif platform == 'macos':
cibuildwheel.macos.build(**build_options)
cibuildwheel.macos.build(build_options)
else:
print('cibuildwheel: Unsupported platform: {}'.format(platform), file=sys.stderr)
exit(2)
@@ -250,7 +263,7 @@ def print_preamble(platform, build_options):
print('Build options:')
print(' platform: %r' % platform)
for option, value in sorted(build_options.items()):
for option, value in sorted(build_options._asdict().items()):
print(' %s: %r' % (option, value))
warnings = detect_warnings(platform, build_options)
@@ -281,7 +294,7 @@ def detect_warnings(platform, build_options):
# warn about deprecated {python} and {pip}
for option_name in ['test_command', 'before_build']:
option_value = build_options.get(option_name)
option_value = getattr(build_options, option_name)
if option_value:
if '{python}' in option_value or '{pip}' in option_value:
cibuildwheel/linux.py
+206 -169
View File
@@ -10,9 +10,17 @@ from collections import namedtuple
from .util import (
get_build_verbosity_extra_flags,
prepare_command,
BuildOptions
)
def call(args, input=None, universal_newlines=False):
print('+ ' + ' '.join(shlex.quote(a) for a in args))
subprocess.run(
args, input=input, universal_newlines=universal_newlines, check=True
)
def matches_platform(identifier):
pm = platform.machine()
if pm == "x86_64":
@@ -35,40 +43,40 @@ def matches_platform(identifier):
def get_python_configurations(build_selector):
PythonConfiguration = namedtuple('PythonConfiguration', ['identifier', 'path'])
PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'path'])
python_configurations = [
PythonConfiguration(identifier='cp27-manylinux_x86_64', path='/opt/python/cp27-cp27m'),
PythonConfiguration(identifier='cp27-manylinux_x86_64', path='/opt/python/cp27-cp27mu'),
PythonConfiguration(identifier='cp35-manylinux_x86_64', path='/opt/python/cp35-cp35m'),
PythonConfiguration(identifier='cp36-manylinux_x86_64', path='/opt/python/cp36-cp36m'),
PythonConfiguration(identifier='cp37-manylinux_x86_64', path='/opt/python/cp37-cp37m'),
PythonConfiguration(identifier='cp38-manylinux_x86_64', path='/opt/python/cp38-cp38'),
PythonConfiguration(identifier='cp27-manylinux_i686', path='/opt/python/cp27-cp27m'),
PythonConfiguration(identifier='cp27-manylinux_i686', path='/opt/python/cp27-cp27mu'),
PythonConfiguration(identifier='cp35-manylinux_i686', path='/opt/python/cp35-cp35m'),
PythonConfiguration(identifier='cp36-manylinux_i686', path='/opt/python/cp36-cp36m'),
PythonConfiguration(identifier='cp37-manylinux_i686', path='/opt/python/cp37-cp37m'),
PythonConfiguration(identifier='cp38-manylinux_i686', path='/opt/python/cp38-cp38'),
PythonConfiguration(identifier='pp27-manylinux_x86_64', path='/opt/python/pp27-pypy_73'),
PythonConfiguration(identifier='pp36-manylinux_x86_64', path='/opt/python/pp36-pypy36_pp73'),
PythonConfiguration(identifier='cp35-manylinux_aarch64', path='/opt/python/cp35-cp35m'),
PythonConfiguration(identifier='cp36-manylinux_aarch64', path='/opt/python/cp36-cp36m'),
PythonConfiguration(identifier='cp37-manylinux_aarch64', path='/opt/python/cp37-cp37m'),
PythonConfiguration(identifier='cp38-manylinux_aarch64', path='/opt/python/cp38-cp38'),
PythonConfiguration(identifier='cp35-manylinux_ppc64le', path='/opt/python/cp35-cp35m'),
PythonConfiguration(identifier='cp36-manylinux_ppc64le', path='/opt/python/cp36-cp36m'),
PythonConfiguration(identifier='cp37-manylinux_ppc64le', path='/opt/python/cp37-cp37m'),
PythonConfiguration(identifier='cp38-manylinux_ppc64le', path='/opt/python/cp38-cp38'),
PythonConfiguration(identifier='cp35-manylinux_s390x', path='/opt/python/cp35-cp35m'),
PythonConfiguration(identifier='cp36-manylinux_s390x', path='/opt/python/cp36-cp36m'),
PythonConfiguration(identifier='cp37-manylinux_s390x', path='/opt/python/cp37-cp37m'),
PythonConfiguration(identifier='cp38-manylinux_s390x', path='/opt/python/cp38-cp38'),
PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path='/opt/python/cp27-cp27m'),
PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path='/opt/python/cp27-cp27mu'),
PythonConfiguration(version='3.5', identifier='cp35-manylinux_x86_64', path='/opt/python/cp35-cp35m'),
PythonConfiguration(version='3.6', identifier='cp36-manylinux_x86_64', path='/opt/python/cp36-cp36m'),
PythonConfiguration(version='3.7', identifier='cp37-manylinux_x86_64', path='/opt/python/cp37-cp37m'),
PythonConfiguration(version='3.8', identifier='cp38-manylinux_x86_64', path='/opt/python/cp38-cp38'),
PythonConfiguration(version='2.7', identifier='cp27-manylinux_i686', path='/opt/python/cp27-cp27m'),
PythonConfiguration(version='2.7', identifier='cp27-manylinux_i686', path='/opt/python/cp27-cp27mu'),
PythonConfiguration(version='3.5', identifier='cp35-manylinux_i686', path='/opt/python/cp35-cp35m'),
PythonConfiguration(version='3.6', identifier='cp36-manylinux_i686', path='/opt/python/cp36-cp36m'),
PythonConfiguration(version='3.7', identifier='cp37-manylinux_i686', path='/opt/python/cp37-cp37m'),
PythonConfiguration(version='3.8', identifier='cp38-manylinux_i686', path='/opt/python/cp38-cp38'),
PythonConfiguration(version='2.7', identifier='pp27-manylinux_x86_64', path='/opt/python/pp27-pypy_73'),
PythonConfiguration(version='3.6', identifier='pp36-manylinux_x86_64', path='/opt/python/pp36-pypy36_pp73'),
PythonConfiguration(version='3.5', identifier='cp35-manylinux_aarch64', path='/opt/python/cp35-cp35m'),
PythonConfiguration(version='3.6', identifier='cp36-manylinux_aarch64', path='/opt/python/cp36-cp36m'),
PythonConfiguration(version='3.7', identifier='cp37-manylinux_aarch64', path='/opt/python/cp37-cp37m'),
PythonConfiguration(version='3.8', identifier='cp38-manylinux_aarch64', path='/opt/python/cp38-cp38'),
PythonConfiguration(version='3.5', identifier='cp35-manylinux_ppc64le', path='/opt/python/cp35-cp35m'),
PythonConfiguration(version='3.6', identifier='cp36-manylinux_ppc64le', path='/opt/python/cp36-cp36m'),
PythonConfiguration(version='3.7', identifier='cp37-manylinux_ppc64le', path='/opt/python/cp37-cp37m'),
PythonConfiguration(version='3.8', identifier='cp38-manylinux_ppc64le', path='/opt/python/cp38-cp38'),
PythonConfiguration(version='3.5', identifier='cp35-manylinux_s390x', path='/opt/python/cp35-cp35m'),
PythonConfiguration(version='3.6', identifier='cp36-manylinux_s390x', path='/opt/python/cp36-cp36m'),
PythonConfiguration(version='3.7', identifier='cp37-manylinux_s390x', path='/opt/python/cp37-cp37m'),
PythonConfiguration(version='3.8', identifier='cp38-manylinux_s390x', path='/opt/python/cp38-cp38'),
]
# skip builds as required
return [c for c in python_configurations if matches_platform(c.identifier) and build_selector(c.identifier)]
def build(project_dir, output_dir, test_command, before_test, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, manylinux_images):
def build(options: BuildOptions):
try:
subprocess.check_call(['docker', '--version'])
except Exception:
@@ -78,18 +86,18 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes
file=sys.stderr)
exit(2)
python_configurations = get_python_configurations(build_selector)
python_configurations = get_python_configurations(options.build_selector)
platforms = [
('cp', 'manylinux_x86_64', manylinux_images['x86_64']),
('cp', 'manylinux_i686', manylinux_images['i686']),
('cp', 'manylinux_aarch64', manylinux_images['aarch64']),
('cp', 'manylinux_ppc64le', manylinux_images['ppc64le']),
('cp', 'manylinux_s390x', manylinux_images['s390x']),
('pp', 'manylinux_x86_64', manylinux_images['pypy_x86_64']),
('cp', 'manylinux_x86_64', options.manylinux_images['x86_64']),
('cp', 'manylinux_i686', options.manylinux_images['i686']),
('cp', 'manylinux_aarch64', options.manylinux_images['aarch64']),
('cp', 'manylinux_ppc64le', options.manylinux_images['ppc64le']),
('cp', 'manylinux_s390x', options.manylinux_images['s390x']),
('pp', 'manylinux_x86_64', options.manylinux_images['pypy_x86_64']),
]
abs_project_dir = os.path.abspath(project_dir)
abs_package_dir = os.path.abspath(package_dir)
abs_project_dir = os.path.abspath(options.project_dir)
abs_package_dir = os.path.abspath(options.package_dir)
container_project_dir = '/project'
container_package_dir = os.path.join(container_project_dir, os.path.relpath(abs_package_dir, os.path.commonprefix([abs_project_dir, abs_package_dir]))),
@@ -99,146 +107,175 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes
if not platform_configs:
continue
bash_script = '''
set -o errexit
set -o xtrace
mkdir /output
cd /project
for PYBIN in {pybin_paths}; do (
# Temporary hack/workaround, putting loop body in subshell; fixed in PR #256
export PATH="$PYBIN:$PATH"
{environment_exports}
# check the active python and pip are in PYBIN
if [ "$(which pip)" != "$PYBIN/pip" ]; then
echo "cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it."
exit 1
fi
if [ "$(which python)" != "$PYBIN/python" ]; then
echo "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it."
exit 1
fi
if [ ! -z {before_build} ]; then
sh -c {before_build}
fi
# Build the wheel
rm -rf /tmp/built_wheel
mkdir /tmp/built_wheel
pip wheel {package_dir} -w /tmp/built_wheel --no-deps {build_verbosity_flag}
built_wheel=(/tmp/built_wheel/*.whl)
# repair the wheel
rm -rf /tmp/repaired_wheels
mkdir /tmp/repaired_wheels
# NOTE: 'built_wheel' here is a bash array of glob matches; "$built_wheel" returns
# the first element
if [[ "$built_wheel" == *none-any.whl ]] || [ -z {repair_command} ]; then
# pure Python wheel or empty repair command
mv "$built_wheel" /tmp/repaired_wheels
else
sh -c {repair_command} repair_command "$built_wheel"
fi
repaired_wheels=(/tmp/repaired_wheels/*.whl)
if [ ! -z {test_command} ]; then
# Set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
pip install virtualenv
venv_dir=`mktemp -d`/venv
python -m virtualenv "$venv_dir"
export __CIBW_VIRTUALENV_PATH__=$venv_dir
# run the tests in a subshell to keep that `activate`
# script from polluting the env
(
source "$venv_dir/bin/activate"
echo "Running tests using `which python`"
if [ ! -z {before_test} ]; then
sh -c {before_test}
fi
# Install the wheel we just built
# Note: If auditwheel produced two wheels, it's because the earlier produced wheel
# conforms to multiple manylinux standards. These multiple versions of the wheel are
# functionally the same, differing only in name, wheel metadata, and possibly include
# different external shared libraries. so it doesn't matter which one we run the tests on.
# Let's just pick the first one.
pip install "${{repaired_wheels[0]}}"{test_extras}
# Install any requirements to run the tests
if [ ! -z "{test_requires}" ]; then
pip install {test_requires}
fi
# Run the tests from a different directory
pushd $HOME
sh -c {test_command}
popd
)
# exit if tests failed (needed for older bash versions)
if [ $? -ne 0 ]; then
exit 1;
fi
# clean up
rm -rf "$venv_dir"
fi
# we're all done here; move it to output
mv "${{repaired_wheels[@]}}" /output
for repaired_wheel in "${{repaired_wheels[@]}}"; do chown {uid}:{gid} "/output/$(basename "$repaired_wheel")"; done
) done
'''.format(
pybin_paths=' '.join(c.path + '/bin' for c in platform_configs),
package_dir=container_package_dir,
test_requires=' '.join(test_requires),
test_extras=test_extras,
test_command=shlex.quote(
prepare_command(test_command, project=container_project_dir, package=container_package_dir) if test_command else ''
),
before_build=shlex.quote(
prepare_command(before_build, project=container_project_dir, package=container_package_dir) if before_build else ''
),
build_verbosity_flag=' '.join(get_build_verbosity_extra_flags(build_verbosity)),
repair_command=shlex.quote(
prepare_command(repair_command, wheel='"$1"', dest_dir='/tmp/repaired_wheels') if repair_command else ''
),
environment_exports='\n'.join(environment.as_shell_commands()),
uid=os.getuid(),
gid=os.getgid(),
before_test=shlex.quote(
prepare_command(before_test, project=container_project_dir, package=container_package_dir) if before_test else ''
),
)
container_name = 'cibuildwheel-{}'.format(uuid.uuid4())
try:
subprocess.run(['docker', 'create',
'--env', 'CIBUILDWHEEL',
'--name', container_name,
'-i',
'-v', '/:/host', # ignored on CircleCI
docker_image, '/bin/bash'], check=True)
subprocess.run(['docker', 'cp', os.path.abspath(project_dir) + '/.', container_name + ':/project'], check=True)
subprocess.run(['docker', 'start', '-i', '-a', container_name], input=bash_script, universal_newlines=True, check=True)
subprocess.run(['docker', 'cp', container_name + ':/output/.', os.path.abspath(output_dir)], check=True)
except subprocess.CalledProcessError as error:
troubleshoot(project_dir, error)
call(['docker', 'create',
'--env', 'CIBUILDWHEEL',
'--name', container_name,
'-i',
'-v', '/:/host', # ignored on CircleCI
docker_image,
'/bin/bash'])
call(['docker', 'cp',
os.path.abspath(options.project_dir) + '/.',
container_name + ':/project'])
call(['docker', 'start', container_name])
for config in platform_configs:
if options.dependency_constraints:
constraints_file = options.dependency_constraints.get_for_python_version(config.version)
# `docker cp` causes 'no space left on device' error when
# a container is running and the host filesystem is
# mounted. https://github.com/moby/moby/issues/38995
# Use `docker exec` instead.
with open(constraints_file, 'rb') as f:
call(
['docker', 'exec', '-i', container_name, 'sh', '-c', 'cat > /constraints.txt'],
input=f.read(),
)
call(
['docker', 'exec', '-i', container_name, '/bin/bash'],
universal_newlines=True,
input='''
# give xtrace output an extra level of indent inside docker
PS4=' + '
set -o errexit
set -o xtrace
mkdir -p /output
cd /project
PYBIN="{config_python_bin}"
export PATH="$PYBIN:$PATH"
{environment_exports}
# check the active python and pip are in PYBIN
if [ "$(which pip)" != "$PYBIN/pip" ]; then
echo "cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it."
exit 1
fi
if [ "$(which python)" != "$PYBIN/python" ]; then
echo "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it."
exit 1
fi
if [ ! -z {before_build} ]; then
sh -c {before_build}
fi
# Build the wheel
rm -rf /tmp/built_wheel
mkdir /tmp/built_wheel
pip wheel {package_dir} -w /tmp/built_wheel --no-deps {build_verbosity_flag}
built_wheel=(/tmp/built_wheel/*.whl)
# repair the wheel
rm -rf /tmp/repaired_wheels
mkdir /tmp/repaired_wheels
# NOTE: 'built_wheel' here is a bash array of glob matches; "$built_wheel" returns
# the first element
if [[ "$built_wheel" == *none-any.whl ]] || [ -z {repair_command} ]; then
# pure Python wheel or empty repair command
mv "$built_wheel" /tmp/repaired_wheels
else
sh -c {repair_command} repair_command "$built_wheel"
fi
repaired_wheels=(/tmp/repaired_wheels/*.whl)
if [ ! -z {test_command} ]; then
# Set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
pip install {dependency_install_flags} virtualenv
venv_dir=`mktemp -d`/venv
python -m virtualenv --no-download "$venv_dir"
export __CIBW_VIRTUALENV_PATH__=$venv_dir
# run the tests in a subshell to keep that `activate`
# script from polluting the env
(
source "$venv_dir/bin/activate"
echo "Running tests using `which python`"
if [ ! -z {before_test} ]; then
sh -c {before_test}
fi
# Install the wheel we just built
# Note: If auditwheel produced two wheels, it's because the earlier produced wheel
# conforms to multiple manylinux standards. These multiple versions of the wheel are
# functionally the same, differing only in name, wheel metadata, and possibly include
# different external shared libraries. so it doesn't matter which one we run the tests on.
# Let's just pick the first one.
pip install "${{repaired_wheels[0]}}"{test_extras}
# Install any requirements to run the tests
if [ ! -z "{test_requires}" ]; then
pip install {test_requires}
fi
# Run the tests from a different directory
pushd $HOME
sh -c {test_command}
popd
)
# exit if tests failed (needed for older bash versions)
if [ $? -ne 0 ]; then
exit 1;
fi
# clean up
rm -rf "$venv_dir"
fi
# we're all done here; move it to output
mv "${{repaired_wheels[@]}}" /output
for repaired_wheel in "${{repaired_wheels[@]}}"; do
chown {uid}:{gid} "/output/$(basename "$repaired_wheel")"
done
'''.format(
config_python_bin=config.path + '/bin',
package_dir=container_package_dir,
test_requires=' '.join(options.test_requires),
test_extras=options.test_extras,
test_command=shlex.quote(
prepare_command(options.test_command, project='/project') if options.test_command else ''
),
before_build=shlex.quote(
prepare_command(options.before_build, project='/project') if options.before_build else ''
),
build_verbosity_flag=' '.join(get_build_verbosity_extra_flags(options.build_verbosity)),
repair_command=shlex.quote(
prepare_command(options.repair_command, wheel='"$1"', dest_dir='/tmp/repaired_wheels') if options.repair_command else ''
),
environment_exports='\n'.join(options.environment.as_shell_commands()),
uid=os.getuid(),
gid=os.getgid(),
before_test=shlex.quote(
prepare_command(options.before_test, project='/project') if options.before_test else ''
),
dependency_install_flags='-c /constraints.txt' if options.dependency_constraints else '',
)
)
# copy the output back into the host
call(['docker', 'cp',
container_name + ':/output/.',
os.path.abspath(options.output_dir)])
except subprocess.CalledProcessError:
exit(1)
finally:
# Still gets executed, even when 'exit(1)' gets called
subprocess.run(['docker', 'rm', '--force', '-v', container_name], check=True)
call(['docker', 'rm', '--force', '-v', container_name])
def troubleshoot(project_dir, error):
if (isinstance(error, subprocess.CalledProcessError) and 'start' in error.cmd):
if (isinstance(error, subprocess.CalledProcessError) and 'exec' in error.cmd):
# the bash script failed
print('Checking for common errors...')
so_files = []
cibuildwheel/macos.py
+93 -79
View File
@@ -11,6 +11,8 @@ from .util import (
download,
get_build_verbosity_extra_flags,
prepare_command,
get_pip_script,
BuildOptions
)
@@ -27,13 +29,15 @@ def call(args, env=None, cwd=None, shell=False):
def get_python_configurations(build_selector):
PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'url'])
python_configurations = [
# CPython
PythonConfiguration(version='2.7', identifier='cp27-macosx_x86_64', url='https://www.python.org/ftp/python/2.7.17/python-2.7.17-macosx10.9.pkg'),
PythonConfiguration(version='3.5', identifier='cp35-macosx_x86_64', url='https://www.python.org/ftp/python/3.5.4/python-3.5.4-macosx10.6.pkg'),
PythonConfiguration(version='3.6', identifier='cp36-macosx_x86_64', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg'),
PythonConfiguration(version='3.7', identifier='cp37-macosx_x86_64', url='https://www.python.org/ftp/python/3.7.6/python-3.7.6-macosx10.9.pkg'),
PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.2/python-3.8.2-macosx10.9.pkg'),
PythonConfiguration(version='2.7-v7.3.0', identifier='pp27-macosx_x86_64', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.0-osx64.tar.bz2'),
PythonConfiguration(version='3.6-v7.3.0', identifier='pp36-macosx_x86_64', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.3.0-osx64.tar.bz2'),
# PyPy
PythonConfiguration(version='2.7', identifier='pp27-macosx_x86_64', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.0-osx64.tar.bz2'),
PythonConfiguration(version='3.6', identifier='pp36-macosx_x86_64', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.3.0-osx64.tar.bz2'),
]
# skip builds as required
@@ -92,7 +96,7 @@ def install_pypy(version, url):
call(['tar', '-C', '/tmp', '-xf', os.path.join("/tmp", pypy_tar_bz2)])
# fix PyPy 7.3.0 bug resulting in wrong macOS platform tag
if version.endswith("-v7.3.0") and version[0] == '3':
if "-v7.3.0-" in url and version[0] == '3':
patch_file = os.path.abspath(os.path.join(os.path.dirname(__file__), 'resources', 'pypy3.6.patch'))
sysconfigdata_file = os.path.join(installation_path, 'lib_pypy', '_sysconfigdata.py')
call(['patch', sysconfigdata_file, patch_file, '-N']) # Always has nonzero return code
@@ -105,105 +109,115 @@ def install_pypy(version, url):
return installation_bin_path
def build(project_dir, package_dir, output_dir, test_command, before_test, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment):
abs_project_dir = os.path.abspath(project_dir)
abs_package_dir = os.path.abspath(package_dir)
def setup_python(python_configuration, dependency_constraint_flags, environment):
if python_configuration.identifier.startswith('cp'):
installation_bin_path = install_cpython(python_configuration.version, python_configuration.url)
elif python_configuration.identifier.startswith('pp'):
installation_bin_path = install_pypy(python_configuration.version, python_configuration.url)
else:
raise ValueError("Unknown Python implementation")
env = os.environ.copy()
env['PATH'] = os.pathsep.join([
SYMLINKS_DIR,
installation_bin_path,
env['PATH'],
])
# Fix issue with site.py setting the wrong `sys.prefix`, `sys.exec_prefix`,
# `sys.path`, ... for PyPy: https://foss.heptapod.net/pypy/pypy/issues/3175
# Also fix an issue with the shebang of installed scripts inside the
# testing virtualenv- see https://github.com/theacodes/nox/issues/44 and
# https://github.com/pypa/virtualenv/issues/620
# Also see https://github.com/python/cpython/pull/9516
env.pop('__PYVENV_LAUNCHER__', None)
env = environment.as_dictionary(prev_environment=env)
# check what version we're on
call(['which', 'python'], env=env)
call(['python', '--version'], env=env)
which_python = subprocess.check_output(['which', 'python'], env=env, universal_newlines=True).strip()
if which_python != '/tmp/cibw_bin/python':
print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr)
exit(1)
# install pip & wheel
call(['python', get_pip_script] + dependency_constraint_flags, env=env, cwd="/tmp")
assert os.path.exists(os.path.join(installation_bin_path, 'pip'))
call(['which', 'pip'], env=env)
call(['pip', '--version'], env=env)
which_pip = subprocess.check_output(['which', 'pip'], env=env, universal_newlines=True).strip()
if which_pip != '/tmp/cibw_bin/pip':
print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr)
exit(1)
call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate'] + dependency_constraint_flags, env=env)
# Set MACOSX_DEPLOYMENT_TARGET to 10.9, if the user didn't set it.
# CPython 3.5 defaults to 10.6, and pypy defaults to 10.7, causing
# inconsistencies if it's left unset.
env.setdefault('MACOSX_DEPLOYMENT_TARGET', '10.9')
if python_configuration.version == '3.5':
# Cross-compilation platform override - CPython 3.5 has an
# i386/x86_64 version of Python, but we only want a x64_64 build
env.setdefault('_PYTHON_HOST_PLATFORM', 'macosx-10.9-x86_64')
# https://github.com/python/cpython/blob/a5ed2fe0eedefa1649aa93ee74a0bafc8e628a10/Lib/_osx_support.py#L260
env.setdefault('ARCHFLAGS', '-arch x86_64')
return env
def build(options: BuildOptions):
abs_project_dir = os.path.abspath(options.project_dir)
abs_package_dir = os.path.abspath(options.package_dir)
temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
built_wheel_dir = os.path.join(temp_dir, 'built_wheel')
repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel')
python_configurations = get_python_configurations(build_selector)
get_pip_url = 'https://bootstrap.pypa.io/get-pip.py'
get_pip_script = '/tmp/get-pip.py'
# get latest pip once and for all
download(get_pip_url, get_pip_script)
python_configurations = get_python_configurations(options.build_selector)
for config in python_configurations:
if config.identifier.startswith('cp'):
installation_bin_path = install_cpython(config.version, config.url)
elif config.identifier.startswith('pp'):
installation_bin_path = install_pypy(config.version, config.url)
else:
raise ValueError("Unknown Python implementation")
dependency_constraint_flags = []
if options.dependency_constraints:
dependency_constraint_flags = [
'-c', options.dependency_constraints.get_for_python_version(config.version)
]
env = os.environ.copy()
env['PATH'] = os.pathsep.join([
SYMLINKS_DIR,
installation_bin_path,
env['PATH'],
])
# Fix issue with site.py setting the wrong `sys.prefix`, `sys.exec_prefix`,
# `sys.path`, ... for PyPy: https://foss.heptapod.net/pypy/pypy/issues/3175
# Also fix an issue with the shebang of installed scripts inside the
# testing virtualenv- see https://github.com/theacodes/nox/issues/44 and
# https://github.com/pypa/virtualenv/issues/620
# Also see https://github.com/python/cpython/pull/9516
env.pop('__PYVENV_LAUNCHER__', None)
env = environment.as_dictionary(prev_environment=env)
# check what version we're on
call(['which', 'python'], env=env)
call(['python', '--version'], env=env)
which_python = subprocess.check_output(['which', 'python'], env=env, universal_newlines=True).strip()
if which_python != '/tmp/cibw_bin/python':
print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr)
exit(1)
# install pip & wheel
call(['python', get_pip_script], env=env, cwd="/tmp")
assert os.path.exists(os.path.join(installation_bin_path, 'pip'))
call(['which', 'pip'], env=env)
call(['pip', '--version'], env=env)
which_pip = subprocess.check_output(['which', 'pip'], env=env, universal_newlines=True).strip()
if which_pip != '/tmp/cibw_bin/pip':
print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr)
exit(1)
call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate'], env=env)
# setup target platform, only required for python 3.5
if config.version == '3.5':
if '_PYTHON_HOST_PLATFORM' not in env:
# cross-compilation platform override
env['_PYTHON_HOST_PLATFORM'] = 'macosx-10.9-x86_64'
if 'ARCHFLAGS' not in env:
# https://github.com/python/cpython/blob/a5ed2fe0eedefa1649aa93ee74a0bafc8e628a10/Lib/_osx_support.py#L260
env['ARCHFLAGS'] = '-arch x86_64'
if 'MACOSX_DEPLOYMENT_TARGET' not in env:
env['MACOSX_DEPLOYMENT_TARGET'] = '10.9'
env = setup_python(config, dependency_constraint_flags, options.environment)
# run the before_build command
if before_build:
before_build_prepared = prepare_command(before_build, project=abs_project_dir, package=abs_package_dir)
if options.before_build:
before_build_prepared = prepare_command(options.before_build, project=abs_project_dir, package=abs_package_dir)
call(before_build_prepared, env=env, shell=True)
# build the wheel
if os.path.exists(built_wheel_dir):
shutil.rmtree(built_wheel_dir)
os.makedirs(built_wheel_dir)
call(['pip', 'wheel', abs_package_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(build_verbosity), env=env)
call(['pip', 'wheel', abs_package_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env)
built_wheel = glob(os.path.join(built_wheel_dir, '*.whl'))[0]
# repair the wheel
if os.path.exists(repaired_wheel_dir):
shutil.rmtree(repaired_wheel_dir)
os.makedirs(repaired_wheel_dir)
if built_wheel.endswith('none-any.whl') or not repair_command:
if built_wheel.endswith('none-any.whl') or not options.repair_command:
# pure Python wheel or empty repair command
shutil.move(built_wheel, repaired_wheel_dir)
else:
repair_command_prepared = prepare_command(repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir)
repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir)
call(repair_command_prepared, env=env, shell=True)
repaired_wheel = glob(os.path.join(repaired_wheel_dir, '*.whl'))[0]
if test_command:
if options.test_command:
# set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
call(['pip', 'install', 'virtualenv'], env=env)
call(['pip', 'install', 'virtualenv'] + dependency_constraint_flags, env=env)
venv_dir = tempfile.mkdtemp()
call(['python', '-m', 'virtualenv', venv_dir], env=env)
# Use --no-download to ensure determinism by using seed libraries
# built into virtualenv
call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env)
virtualenv_env = env.copy()
virtualenv_env['PATH'] = os.pathsep.join([
@@ -215,26 +229,26 @@ def build(project_dir, package_dir, output_dir, test_command, before_test, test_
# check that we are using the Python from the virtual environment
call(['which', 'python'], env=virtualenv_env)
if before_test:
before_test_prepared = prepare_command(before_test, project=abs_project_dir, package=abs_package_dir)
if options.before_test:
before_test_prepared = prepare_command(options.before_test, project=abs_project_dir, package=abs_package_dir)
call(before_test_prepared, env=virtualenv_env, shell=True)
# install the wheel
call(['pip', 'install', repaired_wheel + test_extras], env=virtualenv_env)
call(['pip', 'install', repaired_wheel + options.test_extras], env=virtualenv_env)
# test the wheel
if test_requires:
call(['pip', 'install'] + test_requires, env=virtualenv_env)
if options.test_requires:
call(['pip', 'install'] + options.test_requires, env=virtualenv_env)
# run the tests from $HOME, with an absolute path in the command
# (this ensures that Python runs the tests against the installed wheel
# and not the repo code)
test_command_prepared = prepare_command(test_command, project=abs_project_dir, package=abs_package_dir)
test_command_prepared = prepare_command(options.test_command, project=abs_project_dir, package=abs_package_dir)
call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True)
# clean up
shutil.rmtree(venv_dir)
# we're all done here; move it to output (overwrite existing)
dst = os.path.join(output_dir, os.path.basename(repaired_wheel))
dst = os.path.join(options.output_dir, os.path.basename(repaired_wheel))
shutil.move(repaired_wheel, dst)
cibuildwheel/resources/constraints-python27.txt
+26
View File
@@ -0,0 +1,26 @@
#
# This file is autogenerated by pip-compile
# To update, run:
#
# bin/update_constraints
#
appdirs==1.4.3 # via virtualenv
configparser==4.0.2 # via importlib-metadata
contextlib2==0.6.0.post1 # via importlib-metadata, importlib-resources, virtualenv
delocate==0.8.0 # via -r cibuildwheel/resources/constraints.in
distlib==0.3.0 # via virtualenv
filelock==3.0.12 # via virtualenv
importlib-metadata==1.5.0 # via importlib-resources, virtualenv
importlib-resources==1.3.1 # via virtualenv
pathlib2==2.3.5 # via importlib-metadata, importlib-resources, virtualenv
scandir==1.10.0 # via pathlib2
singledispatch==3.4.0.3 # via importlib-resources
six==1.14.0 # via pathlib2, virtualenv
typing==3.7.4.1 # via importlib-resources
virtualenv==20.0.10 # via -r cibuildwheel/resources/constraints.in
wheel==0.34.2 # via -r cibuildwheel/resources/constraints.in, delocate
zipp==1.2.0 # via importlib-metadata, importlib-resources
# The following packages are considered to be unsafe in a requirements file:
pip==20.0.2 # via -r cibuildwheel/resources/constraints.in
setuptools==44.0.0 # via -r cibuildwheel/resources/constraints.in
cibuildwheel/resources/constraints-python35.txt
+20
View File
@@ -0,0 +1,20 @@
#
# This file is autogenerated by pip-compile
# To update, run:
#
# bin/update_constraints
#
appdirs==1.4.3 # via virtualenv
delocate==0.8.0 # via -r cibuildwheel/resources/constraints.in
distlib==0.3.0 # via virtualenv
filelock==3.0.12 # via virtualenv
importlib-metadata==1.5.0 # via importlib-resources, virtualenv
importlib-resources==1.3.1 # via virtualenv
six==1.14.0 # via virtualenv
virtualenv==20.0.10 # via -r cibuildwheel/resources/constraints.in
wheel==0.34.2 # via -r cibuildwheel/resources/constraints.in, delocate
zipp==1.2.0 # via importlib-metadata, importlib-resources
# The following packages are considered to be unsafe in a requirements file:
pip==20.0.2 # via -r cibuildwheel/resources/constraints.in
setuptools==46.0.0 # via -r cibuildwheel/resources/constraints.in
cibuildwheel/resources/constraints-python36.txt
+20
View File
@@ -0,0 +1,20 @@
#
# This file is autogenerated by pip-compile
# To update, run:
#
# bin/update_constraints
#
appdirs==1.4.3 # via virtualenv
delocate==0.8.0 # via -r cibuildwheel/resources/constraints.in
distlib==0.3.0 # via virtualenv
filelock==3.0.12 # via virtualenv
importlib-metadata==1.5.0 # via importlib-resources, virtualenv
importlib-resources==1.3.1 # via virtualenv
six==1.14.0 # via virtualenv
virtualenv==20.0.10 # via -r cibuildwheel/resources/constraints.in
wheel==0.34.2 # via -r cibuildwheel/resources/constraints.in, delocate
zipp==3.1.0 # via importlib-metadata, importlib-resources
# The following packages are considered to be unsafe in a requirements file:
pip==20.0.2 # via -r cibuildwheel/resources/constraints.in
setuptools==46.0.0 # via -r cibuildwheel/resources/constraints.in
cibuildwheel/resources/constraints.in
+5
View File
@@ -0,0 +1,5 @@
pip
setuptools
wheel
delocate
virtualenv
cibuildwheel/resources/constraints.txt
+19
View File
@@ -0,0 +1,19 @@
#
# This file is autogenerated by pip-compile
# To update, run:
#
# bin/update_constraints
#
appdirs==1.4.3 # via virtualenv
delocate==0.8.0 # via -r cibuildwheel/resources/constraints.in
distlib==0.3.0 # via virtualenv
filelock==3.0.12 # via virtualenv
importlib-metadata==1.5.0 # via virtualenv
six==1.14.0 # via virtualenv
virtualenv==20.0.10 # via -r cibuildwheel/resources/constraints.in
wheel==0.34.2 # via -r cibuildwheel/resources/constraints.in, delocate
zipp==3.1.0 # via importlib-metadata
# The following packages are considered to be unsafe in a requirements file:
pip==20.0.2 # via -r cibuildwheel/resources/constraints.in
setuptools==46.0.0 # via -r cibuildwheel/resources/constraints.in
cibuildwheel/resources/get-pip.py
+22318
View File
File diff suppressed because it is too large Load Diff
cibuildwheel/resources/pinned_docker_images.cfg
+22
View File
@@ -0,0 +1,22 @@
[x86_64]
manylinux1 = quay.io/pypa/manylinux1_x86_64:2020-03-07-9c5ba95
manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2020-03-07-1825e8f
manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2020-03-07-06139da
[i686]
manylinux1 = quay.io/pypa/manylinux1_i686:2020-03-07-9c5ba95
manylinux2010 = quay.io/pypa/manylinux2010_i686:2020-03-07-1825e8f
manylinux2014 = quay.io/pypa/manylinux2014_i686:2020-03-07-06139da
[pypy_x86_64]
manylinux2010 = pypywheels/manylinux2010-pypy_x86_64:latest
[aarch64]
manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2020-03-07-06139da
[ppc64le]
manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2020-03-07-06139da
[s390x]
manylinux2014 = quay.io/pypa/manylinux2014_s390x:2020-03-07-06139da
cibuildwheel/util.py
+54
View File
@@ -2,6 +2,9 @@ import os
import urllib.request
from fnmatch import fnmatch
from time import sleep
from typing import NamedTuple, List, Optional, Dict
from .environment import ParsedEnvironment
def prepare_command(command, **kwargs):
@@ -76,3 +79,54 @@ def download(url, dest):
file.write(response.read())
finally:
response.close()
class DependencyConstraints:
def __init__(self, base_file_path):
assert os.path.exists(base_file_path)
self.base_file_path = os.path.abspath(base_file_path)
@classmethod
def with_defaults(cls):
return cls(
base_file_path=os.path.join(os.path.dirname(__file__), 'resources', 'constraints.txt')
)
def get_for_python_version(self, version):
version_parts = version.split('.')
# try to find a version-specific dependency file e.g. if
# ./constraints.txt is the base, look for ./constraints-python27.txt
base, ext = os.path.splitext(self.base_file_path)
specific = base + '-python{}{}'.format(version_parts[0], version_parts[1])
specific_file_path = specific + ext
if os.path.exists(specific_file_path):
return specific_file_path
else:
return self.base_file_path
BuildOptions = NamedTuple("BuildOptions", [
("project_dir", str),
("package_dir", str),
("output_dir", str),
("test_command", Optional[str]),
("test_requires", List[str]),
("test_extras", str),
("before_build", Optional[str]),
("build_verbosity", int),
("build_selector", BuildSelector),
("repair_command", str),
("environment", ParsedEnvironment),
("before_test", str),
("dependency_constraints", Optional[DependencyConstraints]),
("manylinux_images", Optional[Dict[str, str]]),
])
"""
Replace this definition with a class-style NamedTuple in the
PEP526 style when Python 3.5 host support is dropped
"""
resources_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'resources'))
get_pip_script = os.path.join(resources_dir, 'get-pip.py')
cibuildwheel/windows.py
+87 -67
View File
@@ -11,6 +11,8 @@ from .util import (
download,
get_build_verbosity_extra_flags,
prepare_command,
get_pip_script,
BuildOptions
)
@@ -33,6 +35,7 @@ def get_nuget_args(version, arch):
def get_python_configurations(build_selector):
PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier', 'url'])
python_configurations = [
# CPython
PythonConfiguration(version='2.7.17', arch='32', identifier='cp27-win32', url=None),
PythonConfiguration(version='2.7.17', arch='64', identifier='cp27-win_amd64', url=None),
PythonConfiguration(version='3.5.4', arch='32', identifier='cp35-win32', url=None),
@@ -43,8 +46,9 @@ def get_python_configurations(build_selector):
PythonConfiguration(version='3.7.6', arch='64', identifier='cp37-win_amd64', url=None),
PythonConfiguration(version='3.8.2', arch='32', identifier='cp38-win32', url=None),
PythonConfiguration(version='3.8.2', arch='64', identifier='cp38-win_amd64', url=None),
PythonConfiguration(version='2.7-v7.3.0', arch='32', identifier='pp27-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.0-win32.zip'),
PythonConfiguration(version='3.6-v7.3.0', arch='32', identifier='pp36-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.3.0-win32.zip'),
# PyPy
PythonConfiguration(version='2.7', arch='32', identifier='pp27-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.0-win32.zip'),
PythonConfiguration(version='3.6', arch='32', identifier='pp36-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.3.0-win32.zip'),
]
if IS_RUNNING_ON_TRAVIS:
@@ -86,9 +90,61 @@ def install_pypy(version, arch, url):
return installation_path
def build(project_dir, package_dir, output_dir, test_command, before_test, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment):
abs_project_dir = os.path.abspath(project_dir)
abs_package_dir = os.path.abspath(package_dir)
def setup_python(python_configuration, dependency_constraint_flags, environment):
nuget = 'C:\\cibw\\nuget.exe'
if not os.path.exists(nuget):
download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget)
if python_configuration.identifier.startswith('cp'):
installation_path = install_cpython(python_configuration.version, python_configuration.arch, nuget)
elif python_configuration.identifier.startswith('pp'):
installation_path = install_pypy(python_configuration.version, python_configuration.arch, python_configuration.url)
else:
raise ValueError("Unknown Python implementation")
assert os.path.exists(os.path.join(installation_path, 'python.exe'))
# set up PATH and environment variables for run_with_env
env = os.environ.copy()
env['PYTHON_VERSION'] = python_configuration.version
env['PYTHON_ARCH'] = python_configuration.arch
env['PATH'] = os.pathsep.join([
installation_path,
os.path.join(installation_path, 'Scripts'),
env['PATH']
])
# update env with results from CIBW_ENVIRONMENT
env = environment.as_dictionary(prev_environment=env)
# for the logs - check we're running the right version of python
shell(['where', 'python'], env=env)
shell(['python', '--version'], env=env)
shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)"'], env=env)
where_python = subprocess.check_output(['where', 'python'], env=env, universal_newlines=True).splitlines()[0].strip()
if where_python != os.path.join(installation_path, 'python.exe'):
print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr)
exit(1)
# make sure pip is installed
if not os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe')):
shell(['python', get_pip_script] + dependency_constraint_flags, env=env, cwd="C:\\cibw")
assert os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe'))
where_pip = subprocess.check_output(['where', 'pip'], env=env, universal_newlines=True).splitlines()[0].strip()
if where_pip.strip() != os.path.join(installation_path, 'Scripts', 'pip.exe'):
print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr)
exit(1)
# prepare the Python environment
shell(['python', '-m', 'pip', 'install', '--upgrade', 'pip'] + dependency_constraint_flags, env=env)
shell(['pip', '--version'], env=env)
shell(['pip', 'install', '--upgrade', 'setuptools', 'wheel'] + dependency_constraint_flags, env=env)
return env
def build(options: BuildOptions):
abs_project_dir = os.path.abspath(options.project_dir)
abs_package_dir = os.path.abspath(options.package_dir)
temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
built_wheel_dir = os.path.join(temp_dir, 'built_wheel')
repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel')
@@ -96,87 +152,51 @@ def build(project_dir, package_dir, output_dir, test_command, before_test, test_
# install nuget as best way to provide python
nuget = 'C:\\cibw\\nuget.exe'
download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget)
# get pip fo this installation which not have.
get_pip_script = 'C:\\cibw\\get-pip.py'
download('https://bootstrap.pypa.io/get-pip.py', get_pip_script)
python_configurations = get_python_configurations(build_selector)
python_configurations = get_python_configurations(options.build_selector)
for config in python_configurations:
dependency_constraint_flags = []
if options.dependency_constraints:
dependency_constraint_flags = [
'-c', options.dependency_constraints.get_for_python_version(config.version)
]
# install Python
if config.identifier.startswith('cp'):
installation_path = install_cpython(config.version, config.arch, nuget)
elif config.identifier.startswith('pp'):
installation_path = install_pypy(config.version, config.arch, config.url)
else:
raise ValueError("Unknown Python implementation")
assert os.path.exists(os.path.join(installation_path, 'python.exe'))
# set up PATH and environment variables for run_with_env
env = os.environ.copy()
env['PYTHON_VERSION'] = config.version
env['PYTHON_ARCH'] = config.arch
env['PATH'] = os.pathsep.join([
installation_path,
os.path.join(installation_path, 'Scripts'),
env['PATH']
])
# update env with results from CIBW_ENVIRONMENT
env = environment.as_dictionary(prev_environment=env)
# for the logs - check we're running the right version of python
shell(['where', 'python'], env=env)
shell(['python', '--version'], env=env)
shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)"'], env=env)
where_python = subprocess.check_output(['where', 'python'], env=env, universal_newlines=True).splitlines()[0].strip()
if where_python != os.path.join(installation_path, 'python.exe'):
print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr)
exit(1)
# make sure pip is installed
if not os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe')):
shell(['python', get_pip_script], env=env, cwd="C:\\cibw")
assert os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe'))
where_pip = subprocess.check_output(['where', 'pip'], env=env, universal_newlines=True).splitlines()[0].strip()
if where_pip.strip() != os.path.join(installation_path, 'Scripts', 'pip.exe'):
print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr)
exit(1)
# prepare the Python environment
shell(['python', '-m', 'pip', 'install', '--upgrade', 'pip'], env=env)
shell(['pip', '--version'], env=env)
shell(['pip', 'install', '--upgrade', 'setuptools', 'wheel'], env=env)
env = setup_python(config, dependency_constraint_flags, options.environment)
# run the before_build command
if before_build:
before_build_prepared = prepare_command(before_build, project=abs_project_dir, package=abs_package_dir)
if options.before_build:
before_build_prepared = prepare_command(options.before_build, project=abs_project_dir, package=abs_package_dir)
shell([before_build_prepared], env=env)
# build the wheel
if os.path.exists(built_wheel_dir):
shutil.rmtree(built_wheel_dir)
os.makedirs(built_wheel_dir)
shell(['pip', 'wheel', abs_package_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(build_verbosity), env=env)
shell(['pip', 'wheel', abs_package_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env)
built_wheel = glob(os.path.join(built_wheel_dir, '*.whl'))[0]
# repair the wheel
if os.path.exists(repaired_wheel_dir):
shutil.rmtree(repaired_wheel_dir)
os.makedirs(repaired_wheel_dir)
if built_wheel.endswith('none-any.whl') or not repair_command:
if built_wheel.endswith('none-any.whl') or not options.repair_command:
# pure Python wheel or empty repair command
shutil.move(built_wheel, repaired_wheel_dir)
else:
repair_command_prepared = prepare_command(repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir)
repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir)
shell([repair_command_prepared], env=env)
repaired_wheel = glob(os.path.join(repaired_wheel_dir, '*.whl'))[0]
if test_command:
if options.test_command:
# set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
shell(['pip', 'install', 'virtualenv'], env=env)
shell(['pip', 'install', 'virtualenv'] + dependency_constraint_flags, env=env)
venv_dir = tempfile.mkdtemp()
shell(['python', '-m', 'virtualenv', venv_dir], env=env)
# Use --no-download to ensure determinism by using seed libraries
# built into virtualenv
shell(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env)
virtualenv_env = env.copy()
@@ -193,28 +213,28 @@ def build(project_dir, package_dir, output_dir, test_command, before_test, test_
# check that we are using the Python from the virtual environment
shell(['which', 'python'], env=virtualenv_env)
if before_test:
before_test_prepared = prepare_command(before_test, project=abs_project_dir, package=abs_package_dir)
if options.before_test:
before_test_prepared = prepare_command(options.before_test, project=abs_project_dir, package=abs_package_dir)
shell([before_test_prepared], env=virtualenv_env)
# install the wheel
shell(['pip', 'install', repaired_wheel + test_extras], env=virtualenv_env)
shell(['pip', 'install', repaired_wheel + options.test_extras], env=virtualenv_env)
# test the wheel
if test_requires:
shell(['pip', 'install'] + test_requires, env=virtualenv_env)
if options.test_requires:
shell(['pip', 'install'] + options.test_requires, env=virtualenv_env)
# run the tests from c:\, with an absolute path in the command
# (this ensures that Python runs the tests against the installed wheel
# and not the repo code)
test_command_prepared = prepare_command(test_command, project=abs_project_dir, package=abs_package_dir)
test_command_prepared = prepare_command(options.test_command, project=abs_project_dir, package=abs_package_dir)
shell([test_command_prepared], cwd='c:\\', env=virtualenv_env)
# clean up
shutil.rmtree(venv_dir)
# we're all done here; move it to output (remove if already exists)
dst = os.path.join(output_dir, os.path.basename(repaired_wheel))
dst = os.path.join(options.output_dir, os.path.basename(repaired_wheel))
if os.path.isfile(dst):
os.remove(dst)
shutil.move(repaired_wheel, dst)