Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add CIBW_REPAIR_COMMAND env variable

Browse Source 
to allow different options for auditwheel/delocate, alternative
commands and a future Windows equivalent.

Fix https://github.com/joerick/cibuildwheel/issues/191 .
This commit is contained in:
Nicola Soranzo
2019-11-14 12:10:55 +00:00
parent 34515b78c6
commit 0e04fafd3a
7 changed files with 118 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cibuildwheel/__main__.py
+8
View File
@@ -89,6 +89,13 @@ def main():
before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform)
build_verbosity = get_option_from_environment('CIBW_BUILD_VERBOSITY', platform=platform, default='')
build_config, skip_config = os.environ.get('CIBW_BUILD', '*'), os.environ.get('CIBW_SKIP', '')
if platform == 'linux':
repair_command_default = 'auditwheel repair -w {dest_dir} {wheel}'
elif platform == 'macos':
repair_command_default = 'delocate-listdeps {wheel} && delocate-wheel -w {dest_dir} {wheel}'
else:
repair_command_default = ''
repair_command = get_option_from_environment('CIBW_REPAIR_COMMAND', platform=platform, default=repair_command_default)
environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform, default='')
if test_extras:
@@ -130,6 +137,7 @@ def main():
before_build=before_build,
build_verbosity=build_verbosity,
build_selector=build_selector,
repair_command=repair_command,
environment=environment,
)
cibuildwheel/linux.py
+18 -17
View File
@@ -30,7 +30,7 @@ def get_python_configurations(build_selector):
return [c for c in python_configurations if build_selector(c.identifier)]
def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, environment, manylinux_images):
def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, manylinux_images):
try:
subprocess.check_call(['docker', '--version'])
except:
@@ -60,30 +60,28 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
{environment_exports}
for PYBIN in {pybin_paths}; do
# Setup
rm -rf /tmp/built_wheel
rm -rf /tmp/delocated_wheels
mkdir /tmp/built_wheel
mkdir /tmp/delocated_wheels
if [ ! -z {before_build} ]; then
PATH="$PYBIN:$PATH" sh -c {before_build}
fi
# Build that wheel
# Build the wheel
rm -rf /tmp/built_wheel
mkdir /tmp/built_wheel
PATH="$PYBIN:$PATH" "$PYBIN/pip" wheel . -w /tmp/built_wheel --no-deps {build_verbosity_flag}
built_wheel=(/tmp/built_wheel/*.whl)
# Delocate the wheel
# repair the wheel
rm -rf /tmp/repaired_wheels
mkdir /tmp/repaired_wheels
# NOTE: 'built_wheel' here is a bash array of glob matches; "$built_wheel" returns
# the first element
if [[ "$built_wheel" == *none-any.whl ]]; then
# pure python wheel - just copy
mv "$built_wheel" /tmp/delocated_wheels
if [[ "$built_wheel" == *none-any.whl ]] || [ -z {repair_command} ]; then
# pure Python wheel or empty repair command
mv "$built_wheel" /tmp/repaired_wheels
else
auditwheel repair "$built_wheel" -w /tmp/delocated_wheels
built_wheel=$built_wheel sh -c {repair_command}
fi
delocated_wheels=(/tmp/delocated_wheels/*.whl)
repaired_wheels=(/tmp/repaired_wheels/*.whl)
if [ ! -z {test_command} ]; then
# Set up a virtual environment to install and test from, to make sure
@@ -105,7 +103,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
# functionally the same, differing only in name, wheel metadata, and possibly include
# different external shared libraries. so it doesn't matter which one we run the tests on.
# Let's just pick the first one.
pip install "${{delocated_wheels[0]}}"{test_extras}
pip install "${{repaired_wheels[0]}}"{test_extras}
# Install any requirements to run the tests
if [ ! -z "{test_requires}" ]; then
@@ -127,8 +125,8 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
fi
# we're all done here; move it to output
mv "${{delocated_wheels[@]}}" /output
for delocated_wheel in "${{delocated_wheels[@]}}"; do chown {uid}:{gid} "/output/$(basename "$delocated_wheel")"; done
mv "${{repaired_wheels[@]}}" /output
for repaired_wheel in "${{repaired_wheels[@]}}"; do chown {uid}:{gid} "/output/$(basename "$repaired_wheel")"; done
done
'''.format(
pybin_paths=' '.join(c.path+'/bin' for c in platform_configs),
@@ -141,6 +139,9 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
prepare_command(before_build, project='/project') if before_build else ''
),
build_verbosity_flag=' '.join(get_build_verbosity_extra_flags(build_verbosity)),
repair_command=shlex_quote(
prepare_command(repair_command, wheel='"$built_wheel"', dest_dir='/tmp/repaired_wheels') if repair_command else ''
),
environment_exports='\n'.join(environment.as_shell_commands()),
uid=os.getuid(),
gid=os.getgid(),
cibuildwheel/macos.py
+24 -24
View File
@@ -25,7 +25,12 @@ def get_python_configurations(build_selector):
return [c for c in python_configurations if build_selector(c.identifier)]
def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, environment):
def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment):
abs_project_dir = os.path.abspath(project_dir)
temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
built_wheel_dir = os.path.join(temp_dir, 'built_wheel')
repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel')
python_configurations = get_python_configurations(build_selector)
get_pip_url = 'https://bootstrap.pypa.io/get-pip.py'
get_pip_script = '/tmp/get-pip.py'
@@ -44,8 +49,6 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
return subprocess.check_call(args, env=env, cwd=cwd, shell=shell)
abs_project_dir = os.path.abspath(project_dir)
# get latest pip once and for all
call(['curl', '-L', '-o', get_pip_script, get_pip_url])
@@ -94,32 +97,29 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
call(['pip', '--version'], env=env)
call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate'], env=env)
# setup dirs
if os.path.exists('/tmp/built_wheel'):
shutil.rmtree('/tmp/built_wheel')
os.makedirs('/tmp/built_wheel')
if os.path.exists('/tmp/delocated_wheel'):
shutil.rmtree('/tmp/delocated_wheel')
os.makedirs('/tmp/delocated_wheel')
# run the before_build command
if before_build:
before_build_prepared = prepare_command(before_build, project=abs_project_dir)
call(before_build_prepared, env=env, shell=True)
# build the wheel
call(['pip', 'wheel', abs_project_dir, '-w', '/tmp/built_wheel', '--no-deps'] + get_build_verbosity_extra_flags(build_verbosity), env=env)
built_wheel = glob('/tmp/built_wheel/*.whl')[0]
if os.path.exists(built_wheel_dir):
shutil.rmtree(built_wheel_dir)
os.makedirs(built_wheel_dir)
call(['pip', 'wheel', abs_project_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(build_verbosity), env=env)
built_wheel = glob(os.path.join(built_wheel_dir, '*.whl'))[0]
if built_wheel.endswith('none-any.whl'):
# pure python wheel - just move
shutil.move(built_wheel, '/tmp/delocated_wheel')
# repair the wheel
if os.path.exists(repaired_wheel_dir):
shutil.rmtree(repaired_wheel_dir)
os.makedirs(repaired_wheel_dir)
if built_wheel.endswith('none-any.whl') or not repair_command:
# pure Python wheel or empty repair command
shutil.move(built_wheel, repaired_wheel_dir)
else:
# list the dependencies
call(['delocate-listdeps', built_wheel], env=env)
# rebuild the wheel with shared libraries included and place in output dir
call(['delocate-wheel', '-w', '/tmp/delocated_wheel', built_wheel], env=env)
delocated_wheel = glob('/tmp/delocated_wheel/*.whl')[0]
repair_command_prepared = prepare_command(repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir)
call(repair_command_prepared, env=env, shell=True)
repaired_wheel = glob(os.path.join(repaired_wheel_dir, '*.whl'))[0]
if test_command:
# set up a virtual environment to install and test from, to make sure
@@ -141,7 +141,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
call(['which', 'python'], env=virtualenv_env)
# install the wheel
call(['pip', 'install', delocated_wheel + test_extras], env=virtualenv_env)
call(['pip', 'install', repaired_wheel + test_extras], env=virtualenv_env)
# test the wheel
if test_requires:
@@ -157,5 +157,5 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
shutil.rmtree(venv_dir)
# we're all done here; move it to output (overwrite existing)
dst = os.path.join(output_dir, os.path.basename(delocated_wheel))
shutil.move(delocated_wheel, dst)
dst = os.path.join(output_dir, os.path.basename(repaired_wheel))
shutil.move(repaired_wheel, dst)
cibuildwheel/util.py
+5 -5
View File
@@ -2,14 +2,14 @@ from fnmatch import fnmatch
import warnings
def prepare_command(command, project):
def prepare_command(command, **kwargs):
'''
Preprocesses a command by expanding variables like {project}.
Preprocesses a command by expanding variables like {python}.
For example, used in the test_command option, to specify the path to the
tests directory.
For example, used in the test_command option to specify the path to the
project's root.
'''
return command.format(python='python', pip='pip', project=project)
return command.format(python='python', pip='pip', **kwargs)
def get_build_verbosity_extra_flags(level):
cibuildwheel/windows.py
+21 -11
View File
@@ -57,8 +57,7 @@ def get_python_configurations(build_selector):
return python_configurations
def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, environment):
def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment):
def simple_shell(args, env=None, cwd=None):
print('+ ' + ' '.join(args))
args = ['cmd', '/E:ON', '/V:ON', '/C'] + args
@@ -90,6 +89,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
abs_project_dir = os.path.abspath(project_dir)
temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
built_wheel_dir = os.path.join(temp_dir, 'built_wheel')
repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel')
# install nuget as best way to provide python
nuget = 'C:\\cibw\\nuget.exe'
@@ -132,19 +132,29 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
simple_shell(['pip', '--version'], env=env)
simple_shell(['pip', 'install', '--upgrade', 'setuptools', 'wheel'], env=env)
# setup dirs
if os.path.exists(built_wheel_dir):
shutil.rmtree(built_wheel_dir)
os.makedirs(built_wheel_dir)
# run the before_build command
if before_build:
before_build_prepared = prepare_command(before_build, project=abs_project_dir)
shell([before_build_prepared], env=env)
# build the wheel
if os.path.exists(built_wheel_dir):
shutil.rmtree(built_wheel_dir)
os.makedirs(built_wheel_dir)
shell(['pip', 'wheel', abs_project_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(build_verbosity), env=env)
built_wheel = glob(built_wheel_dir+'/*.whl')[0]
built_wheel = glob(os.path.join(built_wheel_dir, '*.whl'))[0]
# repair the wheel
if os.path.exists(repaired_wheel_dir):
shutil.rmtree(repaired_wheel_dir)
os.makedirs(repaired_wheel_dir)
if built_wheel.endswith('none-any.whl') or not repair_command:
# pure Python wheel or empty repair command
shutil.move(built_wheel, repaired_wheel_dir)
else:
repair_command_prepared = prepare_command(repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir)
shell([repair_command_prepared], env=env)
repaired_wheel = glob(os.path.join(repaired_wheel_dir, '*.whl'))[0]
if test_command:
# set up a virtual environment to install and test from, to make sure
@@ -163,7 +173,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
shell(['which', 'python'], env=virtualenv_env)
# install the wheel
shell(['pip', 'install', built_wheel + test_extras], env=virtualenv_env)
shell(['pip', 'install', repaired_wheel + test_extras], env=virtualenv_env)
# test the wheel
if test_requires:
@@ -179,7 +189,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
shutil.rmtree(venv_dir)
# we're all done here; move it to output (remove if already exists)
dst = os.path.join(output_dir, os.path.basename(built_wheel))
dst = os.path.join(output_dir, os.path.basename(repaired_wheel))
if os.path.isfile(dst):
os.remove(dst)
shutil.move(built_wheel, dst)
shutil.move(repaired_wheel, dst)