From 0e04fafd3a726d7ce5225410e44f6c54f6601ea6 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 12 Nov 2019 23:34:59 +0000 Subject: [PATCH 1/4] Add `CIBW_REPAIR_COMMAND` env variable to allow different options for auditwheel/delocate, alternative commands and a future Windows equivalent. Fix https://github.com/joerick/cibuildwheel/issues/191 . --- README.md | 5 ++-- cibuildwheel/__main__.py | 8 ++++++ cibuildwheel/linux.py | 35 ++++++++++++------------- cibuildwheel/macos.py | 48 +++++++++++++++++------------------ cibuildwheel/util.py | 10 ++++---- cibuildwheel/windows.py | 32 +++++++++++++++-------- docs/options.md | 55 ++++++++++++++++++++++++++++------------ 7 files changed, 118 insertions(+), 75 deletions(-) diff --git a/README.md b/README.md index 84e9cc44..5e06b880 100644 --- a/README.md +++ b/README.md @@ -95,8 +95,9 @@ Options | | [`CIBW_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip) [`CIBW_SKIP`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip) | Choose the Python versions to build | | **Build environment** | [`CIBW_ENVIRONMENT`](https://cibuildwheel.readthedocs.io/en/stable/options/#environment) | Set environment variables needed during the build | | | [`CIBW_BEFORE_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-build) | Execute a shell command preparing each wheel's build | +| | [`CIBW_REPAIR_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#repair-command) | Execute a shell command to repair each (non-pure Python) built wheel | | | [`CIBW_MANYLINUX_X86_64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) [`CIBW_MANYLINUX_I686_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) | Specify alternative manylinux docker images | -| **Testing** | [`CIBW_TEST_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-command) | Execute a shell command to test all built wheels | +| **Testing** | [`CIBW_TEST_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-command) | Execute a shell command to test each built wheel | | | [`CIBW_TEST_REQUIRES`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-requires) | Install Python dependencies before running the tests | | | [`CIBW_TEST_EXTRAS`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-extras) | Install your wheel for testing using extras_require | | **Other** | [`CIBW_BUILD_VERBOSITY`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-extras) | Increase/decrease the output of pip wheel | @@ -122,7 +123,7 @@ Here are some repos that use cibuildwheel. Legal note ---------- -Since `cibuildwheel` runs the wheel through delocate or auditwheel, it might automatically bundle dynamically linked libraries from the build machine. +Since `cibuildwheel` repairs the wheel with `delocate` or `auditwheel`, it might automatically bundle dynamically linked libraries from the build machine. It helps ensure that the library can run without any dependencies outside of the pip toolchain. diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index a10bcee4..a4be4f52 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -89,6 +89,13 @@ def main(): before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform) build_verbosity = get_option_from_environment('CIBW_BUILD_VERBOSITY', platform=platform, default='') build_config, skip_config = os.environ.get('CIBW_BUILD', '*'), os.environ.get('CIBW_SKIP', '') + if platform == 'linux': + repair_command_default = 'auditwheel repair -w {dest_dir} {wheel}' + elif platform == 'macos': + repair_command_default = 'delocate-listdeps {wheel} && delocate-wheel -w {dest_dir} {wheel}' + else: + repair_command_default = '' + repair_command = get_option_from_environment('CIBW_REPAIR_COMMAND', platform=platform, default=repair_command_default) environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform, default='') if test_extras: @@ -130,6 +137,7 @@ def main(): before_build=before_build, build_verbosity=build_verbosity, build_selector=build_selector, + repair_command=repair_command, environment=environment, ) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 7605c943..1a3575cc 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -30,7 +30,7 @@ def get_python_configurations(build_selector): return [c for c in python_configurations if build_selector(c.identifier)] -def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, environment, manylinux_images): +def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, manylinux_images): try: subprocess.check_call(['docker', '--version']) except: @@ -60,30 +60,28 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef {environment_exports} for PYBIN in {pybin_paths}; do - # Setup - rm -rf /tmp/built_wheel - rm -rf /tmp/delocated_wheels - mkdir /tmp/built_wheel - mkdir /tmp/delocated_wheels - if [ ! -z {before_build} ]; then PATH="$PYBIN:$PATH" sh -c {before_build} fi - # Build that wheel + # Build the wheel + rm -rf /tmp/built_wheel + mkdir /tmp/built_wheel PATH="$PYBIN:$PATH" "$PYBIN/pip" wheel . -w /tmp/built_wheel --no-deps {build_verbosity_flag} built_wheel=(/tmp/built_wheel/*.whl) - # Delocate the wheel + # repair the wheel + rm -rf /tmp/repaired_wheels + mkdir /tmp/repaired_wheels # NOTE: 'built_wheel' here is a bash array of glob matches; "$built_wheel" returns # the first element - if [[ "$built_wheel" == *none-any.whl ]]; then - # pure python wheel - just copy - mv "$built_wheel" /tmp/delocated_wheels + if [[ "$built_wheel" == *none-any.whl ]] || [ -z {repair_command} ]; then + # pure Python wheel or empty repair command + mv "$built_wheel" /tmp/repaired_wheels else - auditwheel repair "$built_wheel" -w /tmp/delocated_wheels + built_wheel=$built_wheel sh -c {repair_command} fi - delocated_wheels=(/tmp/delocated_wheels/*.whl) + repaired_wheels=(/tmp/repaired_wheels/*.whl) if [ ! -z {test_command} ]; then # Set up a virtual environment to install and test from, to make sure @@ -105,7 +103,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef # functionally the same, differing only in name, wheel metadata, and possibly include # different external shared libraries. so it doesn't matter which one we run the tests on. # Let's just pick the first one. - pip install "${{delocated_wheels[0]}}"{test_extras} + pip install "${{repaired_wheels[0]}}"{test_extras} # Install any requirements to run the tests if [ ! -z "{test_requires}" ]; then @@ -127,8 +125,8 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef fi # we're all done here; move it to output - mv "${{delocated_wheels[@]}}" /output - for delocated_wheel in "${{delocated_wheels[@]}}"; do chown {uid}:{gid} "/output/$(basename "$delocated_wheel")"; done + mv "${{repaired_wheels[@]}}" /output + for repaired_wheel in "${{repaired_wheels[@]}}"; do chown {uid}:{gid} "/output/$(basename "$repaired_wheel")"; done done '''.format( pybin_paths=' '.join(c.path+'/bin' for c in platform_configs), @@ -141,6 +139,9 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef prepare_command(before_build, project='/project') if before_build else '' ), build_verbosity_flag=' '.join(get_build_verbosity_extra_flags(build_verbosity)), + repair_command=shlex_quote( + prepare_command(repair_command, wheel='"$built_wheel"', dest_dir='/tmp/repaired_wheels') if repair_command else '' + ), environment_exports='\n'.join(environment.as_shell_commands()), uid=os.getuid(), gid=os.getgid(), diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 40e59a86..1344c9ff 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -25,7 +25,12 @@ def get_python_configurations(build_selector): return [c for c in python_configurations if build_selector(c.identifier)] -def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, environment): +def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment): + abs_project_dir = os.path.abspath(project_dir) + temp_dir = tempfile.mkdtemp(prefix='cibuildwheel') + built_wheel_dir = os.path.join(temp_dir, 'built_wheel') + repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel') + python_configurations = get_python_configurations(build_selector) get_pip_url = 'https://bootstrap.pypa.io/get-pip.py' get_pip_script = '/tmp/get-pip.py' @@ -44,8 +49,6 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef return subprocess.check_call(args, env=env, cwd=cwd, shell=shell) - abs_project_dir = os.path.abspath(project_dir) - # get latest pip once and for all call(['curl', '-L', '-o', get_pip_script, get_pip_url]) @@ -94,32 +97,29 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef call(['pip', '--version'], env=env) call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate'], env=env) - # setup dirs - if os.path.exists('/tmp/built_wheel'): - shutil.rmtree('/tmp/built_wheel') - os.makedirs('/tmp/built_wheel') - if os.path.exists('/tmp/delocated_wheel'): - shutil.rmtree('/tmp/delocated_wheel') - os.makedirs('/tmp/delocated_wheel') - # run the before_build command if before_build: before_build_prepared = prepare_command(before_build, project=abs_project_dir) call(before_build_prepared, env=env, shell=True) # build the wheel - call(['pip', 'wheel', abs_project_dir, '-w', '/tmp/built_wheel', '--no-deps'] + get_build_verbosity_extra_flags(build_verbosity), env=env) - built_wheel = glob('/tmp/built_wheel/*.whl')[0] + if os.path.exists(built_wheel_dir): + shutil.rmtree(built_wheel_dir) + os.makedirs(built_wheel_dir) + call(['pip', 'wheel', abs_project_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(build_verbosity), env=env) + built_wheel = glob(os.path.join(built_wheel_dir, '*.whl'))[0] - if built_wheel.endswith('none-any.whl'): - # pure python wheel - just move - shutil.move(built_wheel, '/tmp/delocated_wheel') + # repair the wheel + if os.path.exists(repaired_wheel_dir): + shutil.rmtree(repaired_wheel_dir) + os.makedirs(repaired_wheel_dir) + if built_wheel.endswith('none-any.whl') or not repair_command: + # pure Python wheel or empty repair command + shutil.move(built_wheel, repaired_wheel_dir) else: - # list the dependencies - call(['delocate-listdeps', built_wheel], env=env) - # rebuild the wheel with shared libraries included and place in output dir - call(['delocate-wheel', '-w', '/tmp/delocated_wheel', built_wheel], env=env) - delocated_wheel = glob('/tmp/delocated_wheel/*.whl')[0] + repair_command_prepared = prepare_command(repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) + call(repair_command_prepared, env=env, shell=True) + repaired_wheel = glob(os.path.join(repaired_wheel_dir, '*.whl'))[0] if test_command: # set up a virtual environment to install and test from, to make sure @@ -141,7 +141,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef call(['which', 'python'], env=virtualenv_env) # install the wheel - call(['pip', 'install', delocated_wheel + test_extras], env=virtualenv_env) + call(['pip', 'install', repaired_wheel + test_extras], env=virtualenv_env) # test the wheel if test_requires: @@ -157,5 +157,5 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef shutil.rmtree(venv_dir) # we're all done here; move it to output (overwrite existing) - dst = os.path.join(output_dir, os.path.basename(delocated_wheel)) - shutil.move(delocated_wheel, dst) + dst = os.path.join(output_dir, os.path.basename(repaired_wheel)) + shutil.move(repaired_wheel, dst) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 160e8c20..ccbe57c0 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -2,14 +2,14 @@ from fnmatch import fnmatch import warnings -def prepare_command(command, project): +def prepare_command(command, **kwargs): ''' - Preprocesses a command by expanding variables like {project}. + Preprocesses a command by expanding variables like {python}. - For example, used in the test_command option, to specify the path to the - tests directory. + For example, used in the test_command option to specify the path to the + project's root. ''' - return command.format(python='python', pip='pip', project=project) + return command.format(python='python', pip='pip', **kwargs) def get_build_verbosity_extra_flags(level): diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 4b1bea8b..e5d3240d 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -57,8 +57,7 @@ def get_python_configurations(build_selector): return python_configurations - -def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, environment): +def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment): def simple_shell(args, env=None, cwd=None): print('+ ' + ' '.join(args)) args = ['cmd', '/E:ON', '/V:ON', '/C'] + args @@ -90,6 +89,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef abs_project_dir = os.path.abspath(project_dir) temp_dir = tempfile.mkdtemp(prefix='cibuildwheel') built_wheel_dir = os.path.join(temp_dir, 'built_wheel') + repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel') # install nuget as best way to provide python nuget = 'C:\\cibw\\nuget.exe' @@ -132,19 +132,29 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef simple_shell(['pip', '--version'], env=env) simple_shell(['pip', 'install', '--upgrade', 'setuptools', 'wheel'], env=env) - # setup dirs - if os.path.exists(built_wheel_dir): - shutil.rmtree(built_wheel_dir) - os.makedirs(built_wheel_dir) - # run the before_build command if before_build: before_build_prepared = prepare_command(before_build, project=abs_project_dir) shell([before_build_prepared], env=env) # build the wheel + if os.path.exists(built_wheel_dir): + shutil.rmtree(built_wheel_dir) + os.makedirs(built_wheel_dir) shell(['pip', 'wheel', abs_project_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(build_verbosity), env=env) - built_wheel = glob(built_wheel_dir+'/*.whl')[0] + built_wheel = glob(os.path.join(built_wheel_dir, '*.whl'))[0] + + # repair the wheel + if os.path.exists(repaired_wheel_dir): + shutil.rmtree(repaired_wheel_dir) + os.makedirs(repaired_wheel_dir) + if built_wheel.endswith('none-any.whl') or not repair_command: + # pure Python wheel or empty repair command + shutil.move(built_wheel, repaired_wheel_dir) + else: + repair_command_prepared = prepare_command(repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) + shell([repair_command_prepared], env=env) + repaired_wheel = glob(os.path.join(repaired_wheel_dir, '*.whl'))[0] if test_command: # set up a virtual environment to install and test from, to make sure @@ -163,7 +173,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef shell(['which', 'python'], env=virtualenv_env) # install the wheel - shell(['pip', 'install', built_wheel + test_extras], env=virtualenv_env) + shell(['pip', 'install', repaired_wheel + test_extras], env=virtualenv_env) # test the wheel if test_requires: @@ -179,7 +189,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef shutil.rmtree(venv_dir) # we're all done here; move it to output (remove if already exists) - dst = os.path.join(output_dir, os.path.basename(built_wheel)) + dst = os.path.join(output_dir, os.path.basename(repaired_wheel)) if os.path.isfile(dst): os.remove(dst) - shutil.move(built_wheel, dst) + shutil.move(repaired_wheel, dst) diff --git a/docs/options.md b/docs/options.md index edbc9939..110c175a 100644 --- a/docs/options.md +++ b/docs/options.md @@ -4,7 +4,7 @@ ## Setting options -cibuildwheel is configured using environment variables, that can be set using +cibuildwheel is configured using environment variables, that can be set using your CI config. For example, to configure cibuildwheel to run tests, add the following YAML to @@ -120,7 +120,7 @@ CIBW_SKIP: "*-win32 *-manylinux_i686" font-size: 90%; white-space: nowrap; } - .rst-content .build-id-table-marker + table td, + .rst-content .build-id-table-marker + table td, .rst-content .build-id-table-marker + table th { padding: 4px 4px; } @@ -163,7 +163,7 @@ CIBW_ENVIRONMENT: "BUILD_TIME=$(date)" CIBW_ENVIRONMENT: "PIP_EXTRA_INDEX_URL=https://pypi.myorg.com/simple" ``` -Platform-specific variants also available: +Platform-specific variants also available:
`CIBW_ENVIRONMENT_MACOS` | `CIBW_ENVIRONMENT_WINDOWS` | `CIBW_ENVIRONMENT_LINUX` !!! note @@ -177,7 +177,9 @@ A shell command to run before building the wheel. This option allows you to run If dependencies are required to build your wheel (for example if you include a header from a Python module), set this to `pip install .`, and the dependencies will be installed automatically by pip. However, this means your package will be built twice - if your package takes a long time to build, you might wish to manually list the dependencies here instead. -The active Python binary can be accessed using `python`, and pip with `pip`; `cibuildwheel` makes sure the right version of Python and pip will be executed. `{project}` can be used as a placeholder for the absolute path to the project's root. +The active Python binary can be accessed using `python`, and pip with `pip`; `cibuildwheel` makes sure the right version of Python and pip will be executed. `{project}` can be used as a placeholder for the absolute path to the project's root and will be replaced by `cibuildwheel`. + +On Linux and macOS, the command is run in a shell, so you can write things like `cmd1 && cmd2`. #### Examples ```yaml @@ -191,12 +193,33 @@ CIBW_BEFORE_BUILD: pip install pybind11 CIBW_BEFORE_BUILD: yum install -y libffi-dev && pip install . ``` -Platform-specific variants also available: +Platform-specific variants also available:
`CIBW_BEFORE_BUILD_MACOS` | `CIBW_BEFORE_BUILD_WINDOWS` | `CIBW_BEFORE_BUILD_LINUX` +### `CIBW_REPAIR_COMMAND` {: #repair-command} +> Execute a shell command to repair each (non-pure Python) built wheel + +Default: +- on Linux: `'auditwheel repair -w {dest_dir} {wheel}'` +- on macOS: `'delocate-listdeps {wheel} && delocate-wheel -w {dest_dir} {wheel}'` +- on Windows: `''` + +A shell command to repair a built wheel by copying external library dependencies into the wheel tree and relinking them. +The command is run on each built wheel (except for pure Python ones) before testing it. + +The following placeholders must be used inside the command and will be replaced by `cibuildwheel`: +- `{wheel}` for the absolute path to the built wheel +- `{dest_dir}` for the absolute path of the directory where to create the repaired wheel. + +On Linux and macOS, the command is run in a shell, so you can write things like `cmd1 && cmd2`. + +Platform-specific variants also available:
+`CIBW_REPAIR_COMMAND_MACOS` | `CIBW_REPAIR_COMMAND_WINDOWS` | `CIBW_REPAIR_COMMAND_LINUX` + + ### `CIBW_MANYLINUX_X86_64_IMAGE`, `CIBW_MANYLINUX_I686_IMAGE` {: #manylinux-image} -> Specify alternative manylinux docker images +> Specify alternative manylinux docker images An alternative Docker image to be used for building [`manylinux`](https://github.com/pypa/manylinux) wheels. `cibuildwheel` will then pull these instead of the default images, [`quay.io/pypa/manylinux2010_x86_64`](https://quay.io/pypa/manylinux2010_x86_64) and [`quay.io/pypa/manylinux2010_i686`](https://quay.io/pypa/manylinux2010_i686). @@ -221,11 +244,11 @@ CIBW_MANYLINUX_I686_IMAGE: dockcross/manylinux-x86 ## Testing ### `CIBW_TEST_COMMAND` {: #test-command} -> Execute a shell command to test all built wheels +> Execute a shell command to test each built wheel Shell command to run tests after the build. The wheel will be installed automatically and available for import from the tests. `{project}` can be used as a placeholder for the absolute path to the project's root and will be replaced by `cibuildwheel`. -On Linux and Mac, the command runs in a shell, so you can write things like `cmd1 && cmd2`. +On Linux and macOS, the command is run in a shell, so you can write things like `cmd1 && cmd2`. #### Examples @@ -237,7 +260,7 @@ CIBW_TEST_COMMAND: nosetests {project}/tests CIBW_TEST_COMMAND: nosetests {project}/tests ``` -Platform-specific variants also available: +Platform-specific variants also available:
`CIBW_TEST_COMMAND_MACOS` | `CIBW_TEST_COMMAND_WINDOWS` | `CIBW_TEST_COMMAND_LINUX` @@ -250,13 +273,13 @@ Space-separated list of dependencies required for running the tests. ```yaml # install pytest before running CIBW_TEST_COMMAND -CIBW_TEST_REQUIRES: pytest +CIBW_TEST_REQUIRES: pytest # install specific versions of test dependencies CIBW_TEST_REQUIRES: nose==1.3.7 moto==0.4.31 ``` -Platform-specific variants also available: +Platform-specific variants also available:
`CIBW_TEST_REQUIRES_MACOS` | `CIBW_TEST_REQUIRES_WINDOWS` | `CIBW_TEST_REQUIRES_LINUX` ### `CIBW_TEST_EXTRAS` {: #test-extras} @@ -273,10 +296,10 @@ tests. This can be used to avoid having to redefine test dependencies in ```yaml # will cause the wheel to be installed with `pip install [test,qt]` -CIBW_TEST_EXTRAS: test,qt +CIBW_TEST_EXTRAS: test,qt ``` -Platform-specific variants also available: +Platform-specific variants also available:
`CIBW_TEST_EXTRAS_MACOS` | `CIBW_TEST_EXTRAS_WINDOWS` | `CIBW_TEST_EXTRAS_LINUX` ## Other @@ -293,7 +316,7 @@ An number from 1 to 3 to increase the level of verbosity (corresponding to invok CIBW_BUILD_VERBOSITY: 1 ``` -Platform-specific variants also available: +Platform-specific variants also available:
`CIBW_BUILD_VERBOSITY_MACOS` | `CIBW_BUILD_VERBOSITY_WINDOWS` | `CIBW_BUILD_VERBOSITY_LINUX` ## Command line options @@ -302,7 +325,7 @@ Platform-specific variants also available: usage: cibuildwheel [-h] [--platform {auto,linux,macos,windows}] [--output-dir OUTPUT_DIR] [--print-build-identifiers] [project_dir] - + Build wheels for all the platforms. positional arguments: @@ -381,7 +404,7 @@ optional arguments: options[header].push({name: optionName, description, id}); }); - + // write the table of contents var tocTable = $('.options-toc'); From e9fd9eaeadf9edbd398861799c14506faa244b82 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 15 Nov 2019 02:27:29 +0000 Subject: [PATCH 2/4] Address @joerick's comments --- README.md | 2 +- cibuildwheel/__main__.py | 2 +- docs/options.md | 16 ++++++++++++++-- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 5e06b880..050b8a55 100644 --- a/README.md +++ b/README.md @@ -95,7 +95,7 @@ Options | | [`CIBW_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip) [`CIBW_SKIP`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip) | Choose the Python versions to build | | **Build environment** | [`CIBW_ENVIRONMENT`](https://cibuildwheel.readthedocs.io/en/stable/options/#environment) | Set environment variables needed during the build | | | [`CIBW_BEFORE_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-build) | Execute a shell command preparing each wheel's build | -| | [`CIBW_REPAIR_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#repair-command) | Execute a shell command to repair each (non-pure Python) built wheel | +| | [`CIBW_REPAIR_WHEEL_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#repair-wheel-command) | Execute a shell command to repair each (non-pure Python) built wheel | | | [`CIBW_MANYLINUX_X86_64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) [`CIBW_MANYLINUX_I686_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) | Specify alternative manylinux docker images | | **Testing** | [`CIBW_TEST_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-command) | Execute a shell command to test each built wheel | | | [`CIBW_TEST_REQUIRES`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-requires) | Install Python dependencies before running the tests | diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index a4be4f52..9b237f76 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -95,7 +95,7 @@ def main(): repair_command_default = 'delocate-listdeps {wheel} && delocate-wheel -w {dest_dir} {wheel}' else: repair_command_default = '' - repair_command = get_option_from_environment('CIBW_REPAIR_COMMAND', platform=platform, default=repair_command_default) + repair_command = get_option_from_environment('CIBW_REPAIR_WHEEL_COMMAND', platform=platform, default=repair_command_default) environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform, default='') if test_extras: diff --git a/docs/options.md b/docs/options.md index 110c175a..45a42b72 100644 --- a/docs/options.md +++ b/docs/options.md @@ -197,10 +197,11 @@ Platform-specific variants also available:
`CIBW_BEFORE_BUILD_MACOS` | `CIBW_BEFORE_BUILD_WINDOWS` | `CIBW_BEFORE_BUILD_LINUX` -### `CIBW_REPAIR_COMMAND` {: #repair-command} +### `CIBW_REPAIR_WHEEL_COMMAND` {: #repair-wheel-command} > Execute a shell command to repair each (non-pure Python) built wheel Default: + - on Linux: `'auditwheel repair -w {dest_dir} {wheel}'` - on macOS: `'delocate-listdeps {wheel} && delocate-wheel -w {dest_dir} {wheel}'` - on Windows: `''` @@ -209,13 +210,24 @@ A shell command to repair a built wheel by copying external library dependencies The command is run on each built wheel (except for pure Python ones) before testing it. The following placeholders must be used inside the command and will be replaced by `cibuildwheel`: + - `{wheel}` for the absolute path to the built wheel - `{dest_dir}` for the absolute path of the directory where to create the repaired wheel. On Linux and macOS, the command is run in a shell, so you can write things like `cmd1 && cmd2`. +#### Examples + +```yaml +# don't repair macOS wheels +CIBW_REPAIR_WHEEL_COMMAND_MACOS: "" + +# pass the `--lib-sdir .` flag to auditwheel on Linux +CIBW_REPAIR_WHEEL_COMMAND_LINUX: "auditwheel repair --lib-sdir . -w {dest_dir} {wheel}" +``` + Platform-specific variants also available:
-`CIBW_REPAIR_COMMAND_MACOS` | `CIBW_REPAIR_COMMAND_WINDOWS` | `CIBW_REPAIR_COMMAND_LINUX` +`CIBW_REPAIR_WHEEL_COMMAND_MACOS` | `CIBW_REPAIR_WHEEL_COMMAND_WINDOWS` | `CIBW_REPAIR_WHEEL_COMMAND_LINUX` ### `CIBW_MANYLINUX_X86_64_IMAGE`, `CIBW_MANYLINUX_I686_IMAGE` {: #manylinux-image} From 402ec9e4ac059a7356ce307665d7272a3d13b090 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Fri, 15 Nov 2019 02:56:30 +0000 Subject: [PATCH 3/4] Address one more review comment --- cibuildwheel/linux.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 1a3575cc..eb420516 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -79,7 +79,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef # pure Python wheel or empty repair command mv "$built_wheel" /tmp/repaired_wheels else - built_wheel=$built_wheel sh -c {repair_command} + sh -c {repair_command} repair_command "$built_wheel" fi repaired_wheels=(/tmp/repaired_wheels/*.whl) @@ -140,7 +140,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef ), build_verbosity_flag=' '.join(get_build_verbosity_extra_flags(build_verbosity)), repair_command=shlex_quote( - prepare_command(repair_command, wheel='"$built_wheel"', dest_dir='/tmp/repaired_wheels') if repair_command else '' + prepare_command(repair_command, wheel='"$1"', dest_dir='/tmp/repaired_wheels') if repair_command else '' ), environment_exports='\n'.join(environment.as_shell_commands()), uid=os.getuid(), From 85cba9b1ebe49cb7cb49a7610f42c0a0cabcb438 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Sun, 17 Nov 2019 19:08:22 +0000 Subject: [PATCH 4/4] Move `Platform-specific variants` before `Examples` --- docs/options.md | 39 +++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/docs/options.md b/docs/options.md index 45a42b72..f703d063 100644 --- a/docs/options.md +++ b/docs/options.md @@ -148,6 +148,9 @@ You must set this variable to pass variables to Linux builds (since they execute You can use `$PATH` syntax to insert other variables, or the `$(pwd)` syntax to insert the output of other shell commands. +Platform-specific variants also available:
+`CIBW_ENVIRONMENT_MACOS` | `CIBW_ENVIRONMENT_WINDOWS` | `CIBW_ENVIRONMENT_LINUX` + #### Examples ```yaml # Set some compiler flags @@ -163,9 +166,6 @@ CIBW_ENVIRONMENT: "BUILD_TIME=$(date)" CIBW_ENVIRONMENT: "PIP_EXTRA_INDEX_URL=https://pypi.myorg.com/simple" ``` -Platform-specific variants also available:
-`CIBW_ENVIRONMENT_MACOS` | `CIBW_ENVIRONMENT_WINDOWS` | `CIBW_ENVIRONMENT_LINUX` - !!! note `cibuildwheel` always defines the environment variable `CIBUILDWHEEL=1`. This can be useful for [building wheels with optional extensions](faq.md#building-packages-with-optional-c-extensions). @@ -181,6 +181,9 @@ The active Python binary can be accessed using `python`, and pip with `pip`; `ci On Linux and macOS, the command is run in a shell, so you can write things like `cmd1 && cmd2`. +Platform-specific variants also available:
+ `CIBW_BEFORE_BUILD_MACOS` | `CIBW_BEFORE_BUILD_WINDOWS` | `CIBW_BEFORE_BUILD_LINUX` + #### Examples ```yaml # install your project and dependencies before building @@ -193,9 +196,6 @@ CIBW_BEFORE_BUILD: pip install pybind11 CIBW_BEFORE_BUILD: yum install -y libffi-dev && pip install . ``` -Platform-specific variants also available:
- `CIBW_BEFORE_BUILD_MACOS` | `CIBW_BEFORE_BUILD_WINDOWS` | `CIBW_BEFORE_BUILD_LINUX` - ### `CIBW_REPAIR_WHEEL_COMMAND` {: #repair-wheel-command} > Execute a shell command to repair each (non-pure Python) built wheel @@ -216,6 +216,9 @@ The following placeholders must be used inside the command and will be replaced On Linux and macOS, the command is run in a shell, so you can write things like `cmd1 && cmd2`. +Platform-specific variants also available:
+`CIBW_REPAIR_WHEEL_COMMAND_MACOS` | `CIBW_REPAIR_WHEEL_COMMAND_WINDOWS` | `CIBW_REPAIR_WHEEL_COMMAND_LINUX` + #### Examples ```yaml @@ -226,9 +229,6 @@ CIBW_REPAIR_WHEEL_COMMAND_MACOS: "" CIBW_REPAIR_WHEEL_COMMAND_LINUX: "auditwheel repair --lib-sdir . -w {dest_dir} {wheel}" ``` -Platform-specific variants also available:
-`CIBW_REPAIR_WHEEL_COMMAND_MACOS` | `CIBW_REPAIR_WHEEL_COMMAND_WINDOWS` | `CIBW_REPAIR_WHEEL_COMMAND_LINUX` - ### `CIBW_MANYLINUX_X86_64_IMAGE`, `CIBW_MANYLINUX_I686_IMAGE` {: #manylinux-image} > Specify alternative manylinux docker images @@ -262,6 +262,9 @@ Shell command to run tests after the build. The wheel will be installed automati On Linux and macOS, the command is run in a shell, so you can write things like `cmd1 && cmd2`. +Platform-specific variants also available:
+`CIBW_TEST_COMMAND_MACOS` | `CIBW_TEST_COMMAND_WINDOWS` | `CIBW_TEST_COMMAND_LINUX` + #### Examples ```yaml @@ -272,15 +275,15 @@ CIBW_TEST_COMMAND: nosetests {project}/tests CIBW_TEST_COMMAND: nosetests {project}/tests ``` -Platform-specific variants also available:
-`CIBW_TEST_COMMAND_MACOS` | `CIBW_TEST_COMMAND_WINDOWS` | `CIBW_TEST_COMMAND_LINUX` - ### `CIBW_TEST_REQUIRES` {: #test-requires} > Install Python dependencies before running the tests Space-separated list of dependencies required for running the tests. +Platform-specific variants also available:
+`CIBW_TEST_REQUIRES_MACOS` | `CIBW_TEST_REQUIRES_WINDOWS` | `CIBW_TEST_REQUIRES_LINUX` + #### Examples ```yaml @@ -291,8 +294,6 @@ CIBW_TEST_REQUIRES: pytest CIBW_TEST_REQUIRES: nose==1.3.7 moto==0.4.31 ``` -Platform-specific variants also available:
-`CIBW_TEST_REQUIRES_MACOS` | `CIBW_TEST_REQUIRES_WINDOWS` | `CIBW_TEST_REQUIRES_LINUX` ### `CIBW_TEST_EXTRAS` {: #test-extras} > Install your wheel for testing using `extras_require` @@ -304,6 +305,9 @@ tests. This can be used to avoid having to redefine test dependencies in `CIBW_TEST_REQUIRES` if they are already defined in `setup.py` or `setup.cfg`. +Platform-specific variants also available:
+`CIBW_TEST_EXTRAS_MACOS` | `CIBW_TEST_EXTRAS_WINDOWS` | `CIBW_TEST_EXTRAS_LINUX` + #### Examples ```yaml @@ -311,8 +315,6 @@ tests. This can be used to avoid having to redefine test dependencies in CIBW_TEST_EXTRAS: test,qt ``` -Platform-specific variants also available:
-`CIBW_TEST_EXTRAS_MACOS` | `CIBW_TEST_EXTRAS_WINDOWS` | `CIBW_TEST_EXTRAS_LINUX` ## Other @@ -321,6 +323,9 @@ Platform-specific variants also available:
An number from 1 to 3 to increase the level of verbosity (corresponding to invoking pip with `-v`, `-vv`, and `-vvv`), between -1 and -3 (`-q`, `-qq`, and `-qqq`), or just 0 (default verbosity). These flags are useful while debugging a build when the output of the actual build invoked by `pip wheel` is required. +Platform-specific variants also available:
+`CIBW_BUILD_VERBOSITY_MACOS` | `CIBW_BUILD_VERBOSITY_WINDOWS` | `CIBW_BUILD_VERBOSITY_LINUX` + #### Examples ```yaml @@ -328,8 +333,6 @@ An number from 1 to 3 to increase the level of verbosity (corresponding to invok CIBW_BUILD_VERBOSITY: 1 ``` -Platform-specific variants also available:
-`CIBW_BUILD_VERBOSITY_MACOS` | `CIBW_BUILD_VERBOSITY_WINDOWS` | `CIBW_BUILD_VERBOSITY_LINUX` ## Command line options