From 076987cb7e072246a4af3f7ab3b1b05f37213e35 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 2 Feb 2020 16:03:33 +0000 Subject: [PATCH] Move the config flow control of the linux build into Python The use of separate bash processes for each build means we can be a bit freer with environment pollution --- cibuildwheel/linux.py | 245 ++++++++++++++++++++++-------------------- 1 file changed, 129 insertions(+), 116 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 3f5e0487..ebbd9dc4 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -30,6 +30,22 @@ def get_python_configurations(build_selector): return [c for c in python_configurations if build_selector(c.identifier)] +def run_docker(command, stdin_str=None): + print('docker command: docker {}'.format(' '.join(map(shlex_quote, command)))) + if stdin_str is None: + subprocess.check_call(['docker'] + command) + else: + args = ['docker'] + command + process = subprocess.Popen(args, stdin=subprocess.PIPE, universal_newlines=True) + try: + process.communicate(stdin_str) + except KeyboardInterrupt: + process.kill() + process.wait() + if process.returncode != 0: + raise subprocess.CalledProcessError(process.returncode, args) + + def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, manylinux_images, dependency_constraints): try: subprocess.check_call(['docker', '--version']) @@ -51,133 +67,130 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef if not platform_configs: continue - bash_script = ''' - set -o errexit - set -o xtrace - mkdir /output - cd /project - - {environment_exports} - - for PYBIN in {pybin_paths}; do - if [ ! -z {before_build} ]; then - PATH="$PYBIN:$PATH" sh -c {before_build} - fi - - # Build the wheel - rm -rf /tmp/built_wheel - mkdir /tmp/built_wheel - PATH="$PYBIN:$PATH" "$PYBIN/pip" wheel . -w /tmp/built_wheel --no-deps {build_verbosity_flag} - built_wheel=(/tmp/built_wheel/*.whl) - - # repair the wheel - rm -rf /tmp/repaired_wheels - mkdir /tmp/repaired_wheels - # NOTE: 'built_wheel' here is a bash array of glob matches; "$built_wheel" returns - # the first element - if [[ "$built_wheel" == *none-any.whl ]] || [ -z {repair_command} ]; then - # pure Python wheel or empty repair command - mv "$built_wheel" /tmp/repaired_wheels - else - sh -c {repair_command} repair_command "$built_wheel" - fi - repaired_wheels=(/tmp/repaired_wheels/*.whl) - - if [ ! -z {test_command} ]; then - # Set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - "$PYBIN/pip" install {dependency_install_flags} virtualenv - venv_dir=`mktemp -d`/venv - "$PYBIN/python" -m virtualenv "$venv_dir" - - # run the tests in a subshell to keep that `activate` - # script from polluting the env - ( - source "$venv_dir/bin/activate" - - echo "Running tests using `which python`" - - # Install the wheel we just built - # Note: If auditwheel produced two wheels, it's because the earlier produced wheel - # conforms to multiple manylinux standards. These multiple versions of the wheel are - # functionally the same, differing only in name, wheel metadata, and possibly include - # different external shared libraries. so it doesn't matter which one we run the tests on. - # Let's just pick the first one. - pip install "${{repaired_wheels[0]}}"{test_extras} - - # Install any requirements to run the tests - if [ ! -z "{test_requires}" ]; then - pip install {test_requires} - fi - - # Run the tests from a different directory - pushd $HOME - sh -c {test_command} - popd - ) - # exit if tests failed (needed for older bash versions) - if [ $? -ne 0 ]; then - exit 1; - fi - - # clean up - rm -rf "$venv_dir" - fi - - # we're all done here; move it to output - mv "${{repaired_wheels[@]}}" /output - for repaired_wheel in "${{repaired_wheels[@]}}"; do chown {uid}:{gid} "/output/$(basename "$repaired_wheel")"; done - done - '''.format( - pybin_paths=' '.join(c.path+'/bin' for c in platform_configs), - test_requires=' '.join(test_requires), - test_extras=test_extras, - test_command=shlex_quote( - prepare_command(test_command, project='/project') if test_command else '' - ), - before_build=shlex_quote( - prepare_command(before_build, project='/project') if before_build else '' - ), - build_verbosity_flag=' '.join(get_build_verbosity_extra_flags(build_verbosity)), - repair_command=shlex_quote( - prepare_command(repair_command, wheel='"$1"', dest_dir='/tmp/repaired_wheels') if repair_command else '' - ), - environment_exports='\n'.join(environment.as_shell_commands()), - uid=os.getuid(), - gid=os.getgid(), - dependency_install_flags='-c /constraints.txt' if dependency_constraints else '', - ) - - def run_docker(command, stdin_str=None): - print('docker command: docker {}'.format(' '.join(map(shlex_quote, command)))) - if stdin_str is None: - subprocess.check_call(['docker'] + command) - else: - args = ['docker'] + command - process = subprocess.Popen(args, stdin=subprocess.PIPE, universal_newlines=True) - try: - process.communicate(stdin_str) - except KeyboardInterrupt: - process.kill() - process.wait() - if process.returncode != 0: - raise subprocess.CalledProcessError(process.returncode, args) - container_name = 'cibuildwheel-{}'.format(uuid.uuid4()) try: run_docker(['create', '--env', 'CIBUILDWHEEL', '--name', container_name, '-i', - '-v', '/:/host', # ignored on Circle + '-v', '/:/host', # ignored on CircleCI docker_image, '/bin/bash']) run_docker(['cp', os.path.abspath(project_dir) + '/.', container_name + ':/project']) - if dependency_constraints: - run_docker(['cp', os.path.abspath(dependency_constraints), container_name + ':/constraints.txt']) - run_docker(['start', '-i', '-a', container_name], stdin_str=bash_script) + + for config in platform_configs: + if dependency_constraints: + run_docker(['cp', os.path.abspath(dependency_constraints), container_name + ':/constraints.txt']) + run_docker(['start', '-i', '-a', container_name], stdin_str=''' + set -o errexit + set -o xtrace + mkdir -p /output + cd /project + + PYBIN="{config_python_bin}" + export PATH="$PYBIN:$PATH" + + # check the active python and pip are in PYBIN + # if `test` returns false, the script will exit due to errexit + test "$(which pip)" = "$PYBIN/pip" + test "$(which python)" = "$PYBIN/python" + + {environment_exports} + + if [ ! -z {before_build} ]; then + sh -c {before_build} + fi + + # Build the wheel + rm -rf /tmp/built_wheel + mkdir /tmp/built_wheel + pip wheel . -w /tmp/built_wheel --no-deps {build_verbosity_flag} + built_wheel=(/tmp/built_wheel/*.whl) + + # repair the wheel + rm -rf /tmp/repaired_wheels + mkdir /tmp/repaired_wheels + # NOTE: 'built_wheel' here is a bash array of glob matches; "$built_wheel" returns + # the first element + if [[ "$built_wheel" == *none-any.whl ]] || [ -z {repair_command} ]; then + # pure Python wheel or empty repair command + mv "$built_wheel" /tmp/repaired_wheels + else + sh -c {repair_command} repair_command "$built_wheel" + fi + repaired_wheels=(/tmp/repaired_wheels/*.whl) + + if [ ! -z {test_command} ]; then + # Set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + pip install {dependency_install_flags} virtualenv + venv_dir=`mktemp -d`/venv + python -m virtualenv "$venv_dir" + + # run the tests in a subshell to keep that `activate` + # script from polluting the env + ( + source "$venv_dir/bin/activate" + + echo "Running tests using `which python`" + + # Install the wheel we just built + # Note: If auditwheel produced two wheels, it's because the earlier produced wheel + # conforms to multiple manylinux standards. These multiple versions of the wheel are + # functionally the same, differing only in name, wheel metadata, and possibly include + # different external shared libraries. so it doesn't matter which one we run the tests on. + # Let's just pick the first one. + pip install "${{repaired_wheels[0]}}"{test_extras} + + # Install any requirements to run the tests + if [ ! -z "{test_requires}" ]; then + pip install {test_requires} + fi + + # Run the tests from a different directory + pushd $HOME + sh -c {test_command} + popd + ) + # exit if tests failed (needed for older bash versions) + if [ $? -ne 0 ]; then + exit 1; + fi + + # clean up + rm -rf "$venv_dir" + fi + + # we're all done here; move it to output + mv "${{repaired_wheels[@]}}" /output + for repaired_wheel in "${{repaired_wheels[@]}}"; do + chown {uid}:{gid} "/output/$(basename "$repaired_wheel")" + done + '''.format( + config_python_bin=config.path + '/bin', + test_requires=' '.join(test_requires), + test_extras=test_extras, + test_command=shlex_quote( + prepare_command(test_command, project='/project') if test_command else '' + ), + before_build=shlex_quote( + prepare_command(before_build, project='/project') if before_build else '' + ), + build_verbosity_flag=' '.join(get_build_verbosity_extra_flags(build_verbosity)), + repair_command=shlex_quote( + prepare_command(repair_command, wheel='"$1"', dest_dir='/tmp/repaired_wheels') if repair_command else '' + ), + environment_exports='\n'.join(environment.as_shell_commands()), + uid=os.getuid(), + gid=os.getgid(), + dependency_install_flags='-c /constraints.txt' if dependency_constraints else '', + )) + + # copy the output back into the host run_docker(['cp', container_name + ':/output/.', os.path.abspath(output_dir)]) except subprocess.CalledProcessError: exit(1) finally: # Still gets executed, even when 'exit(1)' gets called run_docker(['rm', '--force', '-v', container_name]) + +