import os import platform import shlex import subprocess import sys import textwrap import uuid from collections import namedtuple from .util import ( get_build_verbosity_extra_flags, prepare_command, ) def matches_platform(identifier): pm = platform.machine() if pm == "x86_64": # x86_64 machines can run i686 docker containers if identifier.endswith('x86_64') or identifier.endswith('i686'): return True elif pm == "i686": if identifier.endswith('i686'): return True elif pm == "aarch64": if identifier.endswith('aarch64'): return True elif pm == "ppc64le": if identifier.endswith('ppc64le'): return True elif pm == "s390x": if identifier.endswith('s390x'): return True return False def get_python_configurations(build_selector): PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'path']) python_configurations = [ PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path='/opt/python/cp27-cp27m'), PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path='/opt/python/cp27-cp27mu'), PythonConfiguration(version='3.5', identifier='cp35-manylinux_x86_64', path='/opt/python/cp35-cp35m'), PythonConfiguration(version='3.6', identifier='cp36-manylinux_x86_64', path='/opt/python/cp36-cp36m'), PythonConfiguration(version='3.7', identifier='cp37-manylinux_x86_64', path='/opt/python/cp37-cp37m'), PythonConfiguration(version='3.8', identifier='cp38-manylinux_x86_64', path='/opt/python/cp38-cp38'), PythonConfiguration(version='2.7', identifier='cp27-manylinux_i686', path='/opt/python/cp27-cp27m'), PythonConfiguration(version='2.7', identifier='cp27-manylinux_i686', path='/opt/python/cp27-cp27mu'), PythonConfiguration(version='3.5', identifier='cp35-manylinux_i686', path='/opt/python/cp35-cp35m'), PythonConfiguration(version='3.6', identifier='cp36-manylinux_i686', path='/opt/python/cp36-cp36m'), PythonConfiguration(version='3.7', identifier='cp37-manylinux_i686', path='/opt/python/cp37-cp37m'), PythonConfiguration(version='3.8', identifier='cp38-manylinux_i686', path='/opt/python/cp38-cp38'), PythonConfiguration(version='2.7', identifier='pp27-manylinux_x86_64', path='/opt/python/pp27-pypy_73'), PythonConfiguration(version='3.6', identifier='pp36-manylinux_x86_64', path='/opt/python/pp36-pypy36_pp73'), PythonConfiguration(version='3.5', identifier='cp35-manylinux_aarch64', path='/opt/python/cp35-cp35m'), PythonConfiguration(version='3.6', identifier='cp36-manylinux_aarch64', path='/opt/python/cp36-cp36m'), PythonConfiguration(version='3.7', identifier='cp37-manylinux_aarch64', path='/opt/python/cp37-cp37m'), PythonConfiguration(version='3.8', identifier='cp38-manylinux_aarch64', path='/opt/python/cp38-cp38'), PythonConfiguration(version='3.5', identifier='cp35-manylinux_ppc64le', path='/opt/python/cp35-cp35m'), PythonConfiguration(version='3.6', identifier='cp36-manylinux_ppc64le', path='/opt/python/cp36-cp36m'), PythonConfiguration(version='3.7', identifier='cp37-manylinux_ppc64le', path='/opt/python/cp37-cp37m'), PythonConfiguration(version='3.8', identifier='cp38-manylinux_ppc64le', path='/opt/python/cp38-cp38'), PythonConfiguration(version='3.5', identifier='cp35-manylinux_s390x', path='/opt/python/cp35-cp35m'), PythonConfiguration(version='3.6', identifier='cp36-manylinux_s390x', path='/opt/python/cp36-cp36m'), PythonConfiguration(version='3.7', identifier='cp37-manylinux_s390x', path='/opt/python/cp37-cp37m'), PythonConfiguration(version='3.8', identifier='cp38-manylinux_s390x', path='/opt/python/cp38-cp38'), ] # skip builds as required return [c for c in python_configurations if matches_platform(c.identifier) and build_selector(c.identifier)] def build(project_dir, output_dir, test_command, before_test, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, manylinux_images, dependency_constraints): try: subprocess.check_call(['docker', '--version']) except Exception: print('cibuildwheel: Docker not found. Docker is required to run Linux builds. ' 'If you\'re building on Travis CI, add `services: [docker]` to your .travis.yml.' 'If you\'re building on Circle CI in Linux, add a `setup_remote_docker` step to your .circleci/config.yml', file=sys.stderr) exit(2) python_configurations = get_python_configurations(build_selector) platforms = [ ('cp', 'manylinux_x86_64', manylinux_images['x86_64']), ('cp', 'manylinux_i686', manylinux_images['i686']), ('cp', 'manylinux_aarch64', manylinux_images['aarch64']), ('cp', 'manylinux_ppc64le', manylinux_images['ppc64le']), ('cp', 'manylinux_s390x', manylinux_images['s390x']), ('pp', 'manylinux_x86_64', manylinux_images['pypy_x86_64']), ] for implementation, platform_tag, docker_image in platforms: platform_configs = [c for c in python_configurations if c.identifier.startswith(implementation) and c.identifier.endswith(platform_tag)] if not platform_configs: continue container_name = 'cibuildwheel-{}'.format(uuid.uuid4()) try: subprocess.run(['docker', 'create', '--env', 'CIBUILDWHEEL', '--name', container_name, '-i', '-v', '/:/host', # ignored on CircleCI docker_image, '/bin/bash'], check=True) subprocess.run(['docker', 'cp', os.path.abspath(project_dir) + '/.', container_name + ':/project'], check=True) subprocess.run(['docker', 'start', container_name], check=True) for config in platform_configs: if dependency_constraints: constraints_file = dependency_constraints.get_for_python_version(config.version) # `docker cp` causes 'no space left on device' error when # a container is running and the host filesystem is # mounted. https://github.com/moby/moby/issues/38995 # Use `docker exec` instead. with open(constraints_file, 'rb') as f: subprocess.run( ['docker', 'exec', container_name, 'sh', '-c', 'cat > /constraints.txt'], input=f.read(), ) subprocess.run( ['docker', 'exec', '-i', container_name, '/bin/bash'], universal_newlines=True, check=True, input=''' set -o errexit set -o xtrace mkdir -p /output cd /project PYBIN="{config_python_bin}" export PATH="$PYBIN:$PATH" {environment_exports} # check the active python and pip are in PYBIN if [ "$(which pip)" != "$PYBIN/pip" ]; then echo "cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it." exit 1 fi if [ "$(which python)" != "$PYBIN/python" ]; then echo "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it." exit 1 fi if [ ! -z {before_build} ]; then sh -c {before_build} fi # Build the wheel rm -rf /tmp/built_wheel mkdir /tmp/built_wheel pip wheel . -w /tmp/built_wheel --no-deps {build_verbosity_flag} built_wheel=(/tmp/built_wheel/*.whl) # repair the wheel rm -rf /tmp/repaired_wheels mkdir /tmp/repaired_wheels # NOTE: 'built_wheel' here is a bash array of glob matches; "$built_wheel" returns # the first element if [[ "$built_wheel" == *none-any.whl ]] || [ -z {repair_command} ]; then # pure Python wheel or empty repair command mv "$built_wheel" /tmp/repaired_wheels else sh -c {repair_command} repair_command "$built_wheel" fi repaired_wheels=(/tmp/repaired_wheels/*.whl) if [ ! -z {test_command} ]; then # Set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. pip install {dependency_install_flags} virtualenv venv_dir=`mktemp -d`/venv python -m virtualenv --no-download "$venv_dir" export __CIBW_VIRTUALENV_PATH__=$venv_dir # run the tests in a subshell to keep that `activate` # script from polluting the env ( source "$venv_dir/bin/activate" echo "Running tests using `which python`" if [ ! -z {before_test} ]; then sh -c {before_test} fi # Install the wheel we just built # Note: If auditwheel produced two wheels, it's because the earlier produced wheel # conforms to multiple manylinux standards. These multiple versions of the wheel are # functionally the same, differing only in name, wheel metadata, and possibly include # different external shared libraries. so it doesn't matter which one we run the tests on. # Let's just pick the first one. pip install "${{repaired_wheels[0]}}"{test_extras} # Install any requirements to run the tests if [ ! -z "{test_requires}" ]; then pip install {test_requires} fi # Run the tests from a different directory pushd $HOME sh -c {test_command} popd ) # exit if tests failed (needed for older bash versions) if [ $? -ne 0 ]; then exit 1; fi # clean up rm -rf "$venv_dir" fi # we're all done here; move it to output mv "${{repaired_wheels[@]}}" /output for repaired_wheel in "${{repaired_wheels[@]}}"; do chown {uid}:{gid} "/output/$(basename "$repaired_wheel")" done '''.format( config_python_bin=config.path + '/bin', test_requires=' '.join(test_requires), test_extras=test_extras, test_command=shlex.quote( prepare_command(test_command, project='/project') if test_command else '' ), before_build=shlex.quote( prepare_command(before_build, project='/project') if before_build else '' ), build_verbosity_flag=' '.join(get_build_verbosity_extra_flags(build_verbosity)), repair_command=shlex.quote( prepare_command(repair_command, wheel='"$1"', dest_dir='/tmp/repaired_wheels') if repair_command else '' ), environment_exports='\n'.join(environment.as_shell_commands()), uid=os.getuid(), gid=os.getgid(), before_test=shlex.quote( prepare_command(before_test, project='/project') if before_test else '' ), dependency_install_flags='-c /constraints.txt' if dependency_constraints else '', ) ) # copy the output back into the host subprocess.run(['docker', 'cp', container_name + ':/output/.', os.path.abspath(output_dir)], check=True) subprocess.run(['docker', 'stop', container_name], check=True) except subprocess.CalledProcessError: exit(1) finally: # Still gets executed, even when 'exit(1)' gets called subprocess.run(['docker', 'rm', '--force', '-v', container_name], check=True) def troubleshoot(project_dir, error): if (isinstance(error, subprocess.CalledProcessError) and 'start' in error.cmd): # the bash script failed print('Checking for common errors...') so_files = [] for root, dirs, files in os.walk(project_dir): for name in files: _, ext = os.path.splitext(name) if ext == '.so': so_files.append(os.path.join(root, name)) if so_files: print(textwrap.dedent(''' NOTE: Shared object (.so) files found in this project. These files might be built against the wrong OS, causing problems with auditwheel. If you're using Cython and have previously done an in-place build, remove those build files (*.so and *.c) before starting cibuildwheel. ''')) print(' Files detected:') print('\n'.join([' ' + f for f in so_files])) print('')