diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 137bc19e..4e25befa 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -128,6 +128,7 @@ def main() -> None: build_config, skip_config = os.environ.get('CIBW_BUILD', '*'), os.environ.get('CIBW_SKIP', '') environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform, default='') + before_all = get_option_from_environment('CIBW_BEFORE_ALL', platform=platform, default='') before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform) repair_command = get_option_from_environment('CIBW_REPAIR_WHEEL_COMMAND', platform=platform, default=repair_command_default) dependency_versions = get_option_from_environment('CIBW_DEPENDENCY_VERSIONS', platform=platform, default='pinned') @@ -212,6 +213,7 @@ def main() -> None: test_extras=test_extras, before_test=before_test, before_build=before_build, + before_all=before_all, build_verbosity=build_verbosity, build_selector=build_selector, repair_command=repair_command, diff --git a/cibuildwheel/environment.py b/cibuildwheel/environment.py index 086a253c..9f7ea2c4 100644 --- a/cibuildwheel/environment.py +++ b/cibuildwheel/environment.py @@ -1,6 +1,6 @@ import bashlex # type: ignore -from typing import Dict, List +from typing import Dict, List, Mapping from . import bashlex_eval @@ -61,8 +61,8 @@ class ParsedEnvironment: def __init__(self, assignments: List[EnvironmentAssignment]): self.assignments = assignments - def as_dictionary(self, prev_environment: Dict[str, str]) -> Dict[str, str]: - environment = prev_environment.copy() + def as_dictionary(self, prev_environment: Mapping[str, str]) -> Dict[str, str]: + environment = dict(**prev_environment) for assignment in self.assignments: value = assignment.evaluated_value(environment=environment) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 99e779c4..6e42f47f 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -133,6 +133,30 @@ def build(options: BuildOptions) -> None: call(['docker', 'start', container_name]) + if options.before_all: + call( + ['docker', 'exec', '-i', container_name] + shell_cmd, + universal_newlines=True, + input=''' + PS4=' + ' + + set -o errexit + set -o xtrace + + # add a modern Python interpreter to PATH so it can be used by BEFORE_ALL + # commands + export PATH=/opt/python/cp38-cp38:$PATH + + {environment_exports} + + sh -c {before_all} + + '''.format( + environment_exports='\n'.join(options.environment.as_shell_commands()), + before_all=shlex.quote(prepare_command(options.before_all, project='/project', package=container_package_dir)) + ) + ) + for config in platform_configs: if options.dependency_constraints: constraints_file = options.dependency_constraints.get_for_python_version(config.version) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 2fd28d20..d9d892b9 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -176,6 +176,11 @@ def build(options: BuildOptions) -> None: built_wheel_dir = temp_dir / 'built_wheel' repaired_wheel_dir = temp_dir / 'repaired_wheel' + if options.before_all: + env = options.environment.as_dictionary(prev_environment=os.environ) + before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) + call([before_all_prepared], shell=True, env=env) + python_configurations = get_python_configurations(options.build_selector) for config in python_configurations: diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index feef6185..012afa1a 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -112,6 +112,7 @@ class BuildOptions(NamedTuple): output_dir: Path build_selector: BuildSelector environment: ParsedEnvironment + before_all: str before_build: Optional[str] repair_command: str manylinux_images: Optional[Dict[str, str]] diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index e81fe301..bae831de 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -162,6 +162,11 @@ def build(options: BuildOptions) -> None: nuget = Path('C:\\cibw\\nuget.exe') download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget) + if options.before_all: + env = options.environment.as_dictionary(prev_environment=os.environ) + before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) + shell([before_all_prepared], env=env) + python_configurations = get_python_configurations(options.build_selector) for config in python_configurations: dependency_constraint_flags = [] diff --git a/docs/options.md b/docs/options.md index 18d5417b..aba121db 100644 --- a/docs/options.md +++ b/docs/options.md @@ -190,6 +190,28 @@ CIBW_ENVIRONMENT: "BUILD_TIME=$(date) SAMPLE_TEXT=\"sample text\"" !!! note `cibuildwheel` always defines the environment variable `CIBUILDWHEEL=1`. This can be useful for [building wheels with optional extensions](faq.md#building-packages-with-optional-c-extensions). +### `CIBW_BEFORE_ALL` {: #before-all} +> Execute a shell command on the build system before any wheels are built. + +Shell command to prepare a common part of the project (e.g. build or install libraries which does not depend on the specific version of Python). + +This option is very useful for the Linux build, where builds take place in isolated Docker containers managed by cibuildwheel. This command will run inside the container before the wheel builds start. Note, if you're building both x86_64 and i686 wheels (the default), your build uses two different Docker images. In that case, this command will execute twice - once per build container. + +The placeholder `{package}` can be used here; it will be replaced by the path to the package being built by `cibuildwheel`. + +On Windows and macOS, the version of Python available inside `CIBW_BEFORE_ALL` is whatever is available on the host machine. On Linux, a modern Python version is available on PATH. + +Platform-specific variants also available:
+ `CIBW_BEFORE_ALL_MACOS` | `CIBW_BEFORE_ALL_WINDOWS` | `CIBW_BEFORE_ALL_LINUX` + +#### Examples +```yaml +# build third party library +CIBW_BEFORE_ALL: make -C third_party_lib + +# install system library +CIBW_BEFORE_ALL_LINUX: yum install -y libffi-dev +``` ### `CIBW_BEFORE_BUILD` {: #before-build} > Execute a shell command preparing each wheel's build diff --git a/test/test_before_all.py b/test/test_before_all.py new file mode 100644 index 00000000..177ec555 --- /dev/null +++ b/test/test_before_all.py @@ -0,0 +1,38 @@ +import os +import textwrap + +from . import utils +from . import test_projects + +project_with_before_build_asserts = test_projects.new_c_project( + setup_py_add=textwrap.dedent(r''' + # assert that the Python version as written to text_info.txt in the CIBW_BEFORE_ALL step + # is the same one as is currently running. + with open("text_info.txt") as f: + stored_text = f.read() + + print("## stored text: " + stored_text) + assert stored_text == "sample text 123" + ''') +) + + +def test(tmp_path): + project_dir = tmp_path / 'project' + project_with_before_build_asserts.generate(project_dir) + + with open(os.path.join(project_dir, "text_info.txt"), mode='w') as ff: + print("dummy text", file=ff) + + # build the wheels + actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ + # write python version information to a temporary file, this is + # checked in setup.py + 'CIBW_BEFORE_ALL': '''python -c "import os;open('{project}/text_info.txt', 'w').write('sample text '+os.environ.get('TEST_VAL', ''))"''', + 'CIBW_ENVIRONMENT': "TEST_VAL='123'" + }) + + # also check that we got the right wheels + os.remove(os.path.join(project_dir, "text_info.txt")) + expected_wheels = utils.expected_wheels('spam', '0.1.0') + assert set(actual_wheels) == set(expected_wheels) diff --git a/unit_test/main_tests/main_options_test.py b/unit_test/main_tests/main_options_test.py index 14252e67..c7e6bfdd 100644 --- a/unit_test/main_tests/main_options_test.py +++ b/unit_test/main_tests/main_options_test.py @@ -233,3 +233,21 @@ def test_build_selector_migrations(intercepted_build_args, monkeypatch, option_n assert intercepted_build_selector.build_patterns == build_selector_patterns else: assert intercepted_build_selector.skip_patterns == build_selector_patterns + + +@pytest.mark.parametrize('before_all', ["", None, 'test text']) +@pytest.mark.parametrize('platform_specific', [False, True]) +def test_before_all(before_all, platform_specific, platform, intercepted_build_args, monkeypatch): + if before_all is not None: + if platform_specific: + monkeypatch.setenv('CIBW_BEFORE_ALL_' + platform.upper(), before_all) + monkeypatch.setenv('CIBW_BEFORE_ALL', 'overwritten') + else: + monkeypatch.setenv('CIBW_BEFORE_ALL', before_all) + + main() + + if before_all is None: + before_all = "" + + assert intercepted_build_args.args[0].before_all == before_all