Move the config flow control of the linux build into Python
The use of separate bash processes for each build means we can be a bit freer with environment pollution
This commit is contained in:
Joe Rickerby
+129
-116
@@ -30,6 +30,22 @@ def get_python_configurations(build_selector):
|
|||||||
return [c for c in python_configurations if build_selector(c.identifier)]
|
return [c for c in python_configurations if build_selector(c.identifier)]
|
||||||
|
|
||||||
|
|
||||||
|
def run_docker(command, stdin_str=None):
|
||||||
|
print('docker command: docker {}'.format(' '.join(map(shlex_quote, command))))
|
||||||
|
if stdin_str is None:
|
||||||
|
subprocess.check_call(['docker'] + command)
|
||||||
|
else:
|
||||||
|
args = ['docker'] + command
|
||||||
|
process = subprocess.Popen(args, stdin=subprocess.PIPE, universal_newlines=True)
|
||||||
|
try:
|
||||||
|
process.communicate(stdin_str)
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
process.kill()
|
||||||
|
process.wait()
|
||||||
|
if process.returncode != 0:
|
||||||
|
raise subprocess.CalledProcessError(process.returncode, args)
|
||||||
|
|
||||||
|
|
||||||
def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, manylinux_images, dependency_constraints):
|
def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, manylinux_images, dependency_constraints):
|
||||||
try:
|
try:
|
||||||
subprocess.check_call(['docker', '--version'])
|
subprocess.check_call(['docker', '--version'])
|
||||||
@@ -51,133 +67,130 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
|
|||||||
if not platform_configs:
|
if not platform_configs:
|
||||||
continue
|
continue
|
||||||
|
|
||||||
bash_script = '''
|
|
||||||
set -o errexit
|
|
||||||
set -o xtrace
|
|
||||||
mkdir /output
|
|
||||||
cd /project
|
|
||||||
|
|
||||||
{environment_exports}
|
|
||||||
|
|
||||||
for PYBIN in {pybin_paths}; do
|
|
||||||
if [ ! -z {before_build} ]; then
|
|
||||||
PATH="$PYBIN:$PATH" sh -c {before_build}
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Build the wheel
|
|
||||||
rm -rf /tmp/built_wheel
|
|
||||||
mkdir /tmp/built_wheel
|
|
||||||
PATH="$PYBIN:$PATH" "$PYBIN/pip" wheel . -w /tmp/built_wheel --no-deps {build_verbosity_flag}
|
|
||||||
built_wheel=(/tmp/built_wheel/*.whl)
|
|
||||||
|
|
||||||
# repair the wheel
|
|
||||||
rm -rf /tmp/repaired_wheels
|
|
||||||
mkdir /tmp/repaired_wheels
|
|
||||||
# NOTE: 'built_wheel' here is a bash array of glob matches; "$built_wheel" returns
|
|
||||||
# the first element
|
|
||||||
if [[ "$built_wheel" == *none-any.whl ]] || [ -z {repair_command} ]; then
|
|
||||||
# pure Python wheel or empty repair command
|
|
||||||
mv "$built_wheel" /tmp/repaired_wheels
|
|
||||||
else
|
|
||||||
sh -c {repair_command} repair_command "$built_wheel"
|
|
||||||
fi
|
|
||||||
repaired_wheels=(/tmp/repaired_wheels/*.whl)
|
|
||||||
|
|
||||||
if [ ! -z {test_command} ]; then
|
|
||||||
# Set up a virtual environment to install and test from, to make sure
|
|
||||||
# there are no dependencies that were pulled in at build time.
|
|
||||||
"$PYBIN/pip" install {dependency_install_flags} virtualenv
|
|
||||||
venv_dir=`mktemp -d`/venv
|
|
||||||
"$PYBIN/python" -m virtualenv "$venv_dir"
|
|
||||||
|
|
||||||
# run the tests in a subshell to keep that `activate`
|
|
||||||
# script from polluting the env
|
|
||||||
(
|
|
||||||
source "$venv_dir/bin/activate"
|
|
||||||
|
|
||||||
echo "Running tests using `which python`"
|
|
||||||
|
|
||||||
# Install the wheel we just built
|
|
||||||
# Note: If auditwheel produced two wheels, it's because the earlier produced wheel
|
|
||||||
# conforms to multiple manylinux standards. These multiple versions of the wheel are
|
|
||||||
# functionally the same, differing only in name, wheel metadata, and possibly include
|
|
||||||
# different external shared libraries. so it doesn't matter which one we run the tests on.
|
|
||||||
# Let's just pick the first one.
|
|
||||||
pip install "${{repaired_wheels[0]}}"{test_extras}
|
|
||||||
|
|
||||||
# Install any requirements to run the tests
|
|
||||||
if [ ! -z "{test_requires}" ]; then
|
|
||||||
pip install {test_requires}
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Run the tests from a different directory
|
|
||||||
pushd $HOME
|
|
||||||
sh -c {test_command}
|
|
||||||
popd
|
|
||||||
)
|
|
||||||
# exit if tests failed (needed for older bash versions)
|
|
||||||
if [ $? -ne 0 ]; then
|
|
||||||
exit 1;
|
|
||||||
fi
|
|
||||||
|
|
||||||
# clean up
|
|
||||||
rm -rf "$venv_dir"
|
|
||||||
fi
|
|
||||||
|
|
||||||
# we're all done here; move it to output
|
|
||||||
mv "${{repaired_wheels[@]}}" /output
|
|
||||||
for repaired_wheel in "${{repaired_wheels[@]}}"; do chown {uid}:{gid} "/output/$(basename "$repaired_wheel")"; done
|
|
||||||
done
|
|
||||||
'''.format(
|
|
||||||
pybin_paths=' '.join(c.path+'/bin' for c in platform_configs),
|
|
||||||
test_requires=' '.join(test_requires),
|
|
||||||
test_extras=test_extras,
|
|
||||||
test_command=shlex_quote(
|
|
||||||
prepare_command(test_command, project='/project') if test_command else ''
|
|
||||||
),
|
|
||||||
before_build=shlex_quote(
|
|
||||||
prepare_command(before_build, project='/project') if before_build else ''
|
|
||||||
),
|
|
||||||
build_verbosity_flag=' '.join(get_build_verbosity_extra_flags(build_verbosity)),
|
|
||||||
repair_command=shlex_quote(
|
|
||||||
prepare_command(repair_command, wheel='"$1"', dest_dir='/tmp/repaired_wheels') if repair_command else ''
|
|
||||||
),
|
|
||||||
environment_exports='\n'.join(environment.as_shell_commands()),
|
|
||||||
uid=os.getuid(),
|
|
||||||
gid=os.getgid(),
|
|
||||||
dependency_install_flags='-c /constraints.txt' if dependency_constraints else '',
|
|
||||||
)
|
|
||||||
|
|
||||||
def run_docker(command, stdin_str=None):
|
|
||||||
print('docker command: docker {}'.format(' '.join(map(shlex_quote, command))))
|
|
||||||
if stdin_str is None:
|
|
||||||
subprocess.check_call(['docker'] + command)
|
|
||||||
else:
|
|
||||||
args = ['docker'] + command
|
|
||||||
process = subprocess.Popen(args, stdin=subprocess.PIPE, universal_newlines=True)
|
|
||||||
try:
|
|
||||||
process.communicate(stdin_str)
|
|
||||||
except KeyboardInterrupt:
|
|
||||||
process.kill()
|
|
||||||
process.wait()
|
|
||||||
if process.returncode != 0:
|
|
||||||
raise subprocess.CalledProcessError(process.returncode, args)
|
|
||||||
|
|
||||||
container_name = 'cibuildwheel-{}'.format(uuid.uuid4())
|
container_name = 'cibuildwheel-{}'.format(uuid.uuid4())
|
||||||
try:
|
try:
|
||||||
run_docker(['create',
|
run_docker(['create',
|
||||||
'--env', 'CIBUILDWHEEL',
|
'--env', 'CIBUILDWHEEL',
|
||||||
'--name', container_name,
|
'--name', container_name,
|
||||||
'-i',
|
'-i',
|
||||||
'-v', '/:/host', # ignored on Circle
|
'-v', '/:/host', # ignored on CircleCI
|
||||||
docker_image, '/bin/bash'])
|
docker_image, '/bin/bash'])
|
||||||
run_docker(['cp', os.path.abspath(project_dir) + '/.', container_name + ':/project'])
|
run_docker(['cp', os.path.abspath(project_dir) + '/.', container_name + ':/project'])
|
||||||
if dependency_constraints:
|
|
||||||
run_docker(['cp', os.path.abspath(dependency_constraints), container_name + ':/constraints.txt'])
|
for config in platform_configs:
|
||||||
run_docker(['start', '-i', '-a', container_name], stdin_str=bash_script)
|
if dependency_constraints:
|
||||||
|
run_docker(['cp', os.path.abspath(dependency_constraints), container_name + ':/constraints.txt'])
|
||||||
|
run_docker(['start', '-i', '-a', container_name], stdin_str='''
|
||||||
|
set -o errexit
|
||||||
|
set -o xtrace
|
||||||
|
mkdir -p /output
|
||||||
|
cd /project
|
||||||
|
|
||||||
|
PYBIN="{config_python_bin}"
|
||||||
|
export PATH="$PYBIN:$PATH"
|
||||||
|
|
||||||
|
# check the active python and pip are in PYBIN
|
||||||
|
# if `test` returns false, the script will exit due to errexit
|
||||||
|
test "$(which pip)" = "$PYBIN/pip"
|
||||||
|
test "$(which python)" = "$PYBIN/python"
|
||||||
|
|
||||||
|
{environment_exports}
|
||||||
|
|
||||||
|
if [ ! -z {before_build} ]; then
|
||||||
|
sh -c {before_build}
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Build the wheel
|
||||||
|
rm -rf /tmp/built_wheel
|
||||||
|
mkdir /tmp/built_wheel
|
||||||
|
pip wheel . -w /tmp/built_wheel --no-deps {build_verbosity_flag}
|
||||||
|
built_wheel=(/tmp/built_wheel/*.whl)
|
||||||
|
|
||||||
|
# repair the wheel
|
||||||
|
rm -rf /tmp/repaired_wheels
|
||||||
|
mkdir /tmp/repaired_wheels
|
||||||
|
# NOTE: 'built_wheel' here is a bash array of glob matches; "$built_wheel" returns
|
||||||
|
# the first element
|
||||||
|
if [[ "$built_wheel" == *none-any.whl ]] || [ -z {repair_command} ]; then
|
||||||
|
# pure Python wheel or empty repair command
|
||||||
|
mv "$built_wheel" /tmp/repaired_wheels
|
||||||
|
else
|
||||||
|
sh -c {repair_command} repair_command "$built_wheel"
|
||||||
|
fi
|
||||||
|
repaired_wheels=(/tmp/repaired_wheels/*.whl)
|
||||||
|
|
||||||
|
if [ ! -z {test_command} ]; then
|
||||||
|
# Set up a virtual environment to install and test from, to make sure
|
||||||
|
# there are no dependencies that were pulled in at build time.
|
||||||
|
pip install {dependency_install_flags} virtualenv
|
||||||
|
venv_dir=`mktemp -d`/venv
|
||||||
|
python -m virtualenv "$venv_dir"
|
||||||
|
|
||||||
|
# run the tests in a subshell to keep that `activate`
|
||||||
|
# script from polluting the env
|
||||||
|
(
|
||||||
|
source "$venv_dir/bin/activate"
|
||||||
|
|
||||||
|
echo "Running tests using `which python`"
|
||||||
|
|
||||||
|
# Install the wheel we just built
|
||||||
|
# Note: If auditwheel produced two wheels, it's because the earlier produced wheel
|
||||||
|
# conforms to multiple manylinux standards. These multiple versions of the wheel are
|
||||||
|
# functionally the same, differing only in name, wheel metadata, and possibly include
|
||||||
|
# different external shared libraries. so it doesn't matter which one we run the tests on.
|
||||||
|
# Let's just pick the first one.
|
||||||
|
pip install "${{repaired_wheels[0]}}"{test_extras}
|
||||||
|
|
||||||
|
# Install any requirements to run the tests
|
||||||
|
if [ ! -z "{test_requires}" ]; then
|
||||||
|
pip install {test_requires}
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Run the tests from a different directory
|
||||||
|
pushd $HOME
|
||||||
|
sh -c {test_command}
|
||||||
|
popd
|
||||||
|
)
|
||||||
|
# exit if tests failed (needed for older bash versions)
|
||||||
|
if [ $? -ne 0 ]; then
|
||||||
|
exit 1;
|
||||||
|
fi
|
||||||
|
|
||||||
|
# clean up
|
||||||
|
rm -rf "$venv_dir"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# we're all done here; move it to output
|
||||||
|
mv "${{repaired_wheels[@]}}" /output
|
||||||
|
for repaired_wheel in "${{repaired_wheels[@]}}"; do
|
||||||
|
chown {uid}:{gid} "/output/$(basename "$repaired_wheel")"
|
||||||
|
done
|
||||||
|
'''.format(
|
||||||
|
config_python_bin=config.path + '/bin',
|
||||||
|
test_requires=' '.join(test_requires),
|
||||||
|
test_extras=test_extras,
|
||||||
|
test_command=shlex_quote(
|
||||||
|
prepare_command(test_command, project='/project') if test_command else ''
|
||||||
|
),
|
||||||
|
before_build=shlex_quote(
|
||||||
|
prepare_command(before_build, project='/project') if before_build else ''
|
||||||
|
),
|
||||||
|
build_verbosity_flag=' '.join(get_build_verbosity_extra_flags(build_verbosity)),
|
||||||
|
repair_command=shlex_quote(
|
||||||
|
prepare_command(repair_command, wheel='"$1"', dest_dir='/tmp/repaired_wheels') if repair_command else ''
|
||||||
|
),
|
||||||
|
environment_exports='\n'.join(environment.as_shell_commands()),
|
||||||
|
uid=os.getuid(),
|
||||||
|
gid=os.getgid(),
|
||||||
|
dependency_install_flags='-c /constraints.txt' if dependency_constraints else '',
|
||||||
|
))
|
||||||
|
|
||||||
|
# copy the output back into the host
|
||||||
run_docker(['cp', container_name + ':/output/.', os.path.abspath(output_dir)])
|
run_docker(['cp', container_name + ':/output/.', os.path.abspath(output_dir)])
|
||||||
except subprocess.CalledProcessError:
|
except subprocess.CalledProcessError:
|
||||||
exit(1)
|
exit(1)
|
||||||
finally:
|
finally:
|
||||||
# Still gets executed, even when 'exit(1)' gets called
|
# Still gets executed, even when 'exit(1)' gets called
|
||||||
run_docker(['rm', '--force', '-v', container_name])
|
run_docker(['rm', '--force', '-v', container_name])
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user