Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Move the config flow control of the linux build into Python

Browse Source 
The use of separate bash processes for each build means we can be a bit
freer with environment pollution
This commit is contained in:
Joe Rickerby
2020-02-02 16:03:33 +00:00
parent bad72cbb15
commit 076987cb7e
1 changed files with 129 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cibuildwheel/linux.py
+50 -37
View File
@@ -30,6 +30,22 @@ def get_python_configurations(build_selector):
return [c for c in python_configurations if build_selector(c.identifier)]
def run_docker(command, stdin_str=None):
print('docker command: docker {}'.format(' '.join(map(shlex_quote, command))))
if stdin_str is None:
subprocess.check_call(['docker'] + command)
else:
args = ['docker'] + command
process = subprocess.Popen(args, stdin=subprocess.PIPE, universal_newlines=True)
try:
process.communicate(stdin_str)
except KeyboardInterrupt:
process.kill()
process.wait()
if process.returncode != 0:
raise subprocess.CalledProcessError(process.returncode, args)
def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, manylinux_images, dependency_constraints):
try:
subprocess.check_call(['docker', '--version'])
@@ -51,23 +67,43 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
if not platform_configs:
continue
bash_script = '''
container_name = 'cibuildwheel-{}'.format(uuid.uuid4())
try:
run_docker(['create',
'--env', 'CIBUILDWHEEL',
'--name', container_name,
'-i',
'-v', '/:/host', # ignored on CircleCI
docker_image, '/bin/bash'])
run_docker(['cp', os.path.abspath(project_dir) + '/.', container_name + ':/project'])
for config in platform_configs:
if dependency_constraints:
run_docker(['cp', os.path.abspath(dependency_constraints), container_name + ':/constraints.txt'])
run_docker(['start', '-i', '-a', container_name], stdin_str='''
set -o errexit
set -o xtrace
mkdir /output
mkdir -p /output
cd /project
PYBIN="{config_python_bin}"
export PATH="$PYBIN:$PATH"
# check the active python and pip are in PYBIN
# if `test` returns false, the script will exit due to errexit
test "$(which pip)" = "$PYBIN/pip"
test "$(which python)" = "$PYBIN/python"
{environment_exports}
for PYBIN in {pybin_paths}; do
if [ ! -z {before_build} ]; then
PATH="$PYBIN:$PATH" sh -c {before_build}
sh -c {before_build}
fi
# Build the wheel
rm -rf /tmp/built_wheel
mkdir /tmp/built_wheel
PATH="$PYBIN:$PATH" "$PYBIN/pip" wheel . -w /tmp/built_wheel --no-deps {build_verbosity_flag}
pip wheel . -w /tmp/built_wheel --no-deps {build_verbosity_flag}
built_wheel=(/tmp/built_wheel/*.whl)
# repair the wheel
@@ -86,9 +122,9 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
if [ ! -z {test_command} ]; then
# Set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
"$PYBIN/pip" install {dependency_install_flags} virtualenv
pip install {dependency_install_flags} virtualenv
venv_dir=`mktemp -d`/venv
"$PYBIN/python" -m virtualenv "$venv_dir"
python -m virtualenv "$venv_dir"
# run the tests in a subshell to keep that `activate`
# script from polluting the env
@@ -126,10 +162,11 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
# we're all done here; move it to output
mv "${{repaired_wheels[@]}}" /output
for repaired_wheel in "${{repaired_wheels[@]}}"; do chown {uid}:{gid} "/output/$(basename "$repaired_wheel")"; done
for repaired_wheel in "${{repaired_wheels[@]}}"; do
chown {uid}:{gid} "/output/$(basename "$repaired_wheel")"
done
'''.format(
pybin_paths=' '.join(c.path+'/bin' for c in platform_configs),
config_python_bin=config.path + '/bin',
test_requires=' '.join(test_requires),
test_extras=test_extras,
test_command=shlex_quote(
@@ -146,38 +183,14 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
uid=os.getuid(),
gid=os.getgid(),
dependency_install_flags='-c /constraints.txt' if dependency_constraints else '',
)
))
def run_docker(command, stdin_str=None):
print('docker command: docker {}'.format(' '.join(map(shlex_quote, command))))
if stdin_str is None:
subprocess.check_call(['docker'] + command)
else:
args = ['docker'] + command
process = subprocess.Popen(args, stdin=subprocess.PIPE, universal_newlines=True)
try:
process.communicate(stdin_str)
except KeyboardInterrupt:
process.kill()
process.wait()
if process.returncode != 0:
raise subprocess.CalledProcessError(process.returncode, args)
container_name = 'cibuildwheel-{}'.format(uuid.uuid4())
try:
run_docker(['create',
'--env', 'CIBUILDWHEEL',
'--name', container_name,
'-i',
'-v', '/:/host', # ignored on Circle
docker_image, '/bin/bash'])
run_docker(['cp', os.path.abspath(project_dir) + '/.', container_name + ':/project'])
if dependency_constraints:
run_docker(['cp', os.path.abspath(dependency_constraints), container_name + ':/constraints.txt'])
run_docker(['start', '-i', '-a', container_name], stdin_str=bash_script)
# copy the output back into the host
run_docker(['cp', container_name + ':/output/.', os.path.abspath(output_dir)])
except subprocess.CalledProcessError:
exit(1)
finally:
# Still gets executed, even when 'exit(1)' gets called
run_docker(['rm', '--force', '-v', container_name])