From f45de3f24e60c5ca7d7b7f0dcc4509453ca1c206 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sun, 21 Nov 2021 15:19:45 -0500 Subject: [PATCH] feat: support passenv (#914) * feat: support passenv * refactor: environment-pass and spaces * docs: document environment pass * Add unit test for passing through awkward env variable values * Remove unused as_shell_commands method from environment * Fix passthrough edge case by avoiding bash for passthrough assignments * Fix mypy error in test * Fix unit test * Apply suggestions from code review Co-authored-by: Joe Rickerby --- README.md | 1 + cibuildwheel/environment.py | 55 +++++++++++++++++++++++----- cibuildwheel/options.py | 9 +++++ cibuildwheel/resources/defaults.toml | 1 + docs/options.md | 35 ++++++++++++++++++ unit_test/docker_container_test.py | 4 +- unit_test/environment_test.py | 14 ------- unit_test/options_test.py | 48 ++++++++++++++++++++++++ 8 files changed, 141 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 9dffb47d..19ef9174 100644 --- a/README.md +++ b/README.md @@ -112,6 +112,7 @@ Options | | [`CIBW_PRERELEASE_PYTHONS`](https://cibuildwheel.readthedocs.io/en/stable/options/#prerelease-pythons) | Enable building with pre-release versions of Python if available | | **Build customization** | [`CIBW_BUILD_FRONTEND`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-frontend) | Set the tool to use to build, either "pip" (default for now) or "build" | | | [`CIBW_ENVIRONMENT`](https://cibuildwheel.readthedocs.io/en/stable/options/#environment) | Set environment variables needed during the build | +| | [`CIBW_ENVIRONMENT_PASS_LINUX`](https://cibuildwheel.readthedocs.io/en/stable/options/#environment-pass) | Set environment variables on the host to pass-through to the container during the build. | | | [`CIBW_BEFORE_ALL`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-all) | Execute a shell command on the build system before any wheels are built. | | | [`CIBW_BEFORE_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-build) | Execute a shell command preparing each wheel's build | | | [`CIBW_REPAIR_WHEEL_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#repair-wheel-command) | Execute a shell command to repair each (non-pure Python) built wheel | diff --git a/cibuildwheel/environment.py b/cibuildwheel/environment.py index 748398c5..ec8a8e36 100644 --- a/cibuildwheel/environment.py +++ b/cibuildwheel/environment.py @@ -1,8 +1,10 @@ import dataclasses -from typing import Dict, List, Mapping, Optional +from typing import Any, Dict, List, Mapping, Optional, Sequence import bashlex +from cibuildwheel.typing import Protocol + from . import bashlex_eval @@ -39,7 +41,41 @@ def split_env_items(env_string: str) -> List[str]: return result -class EnvironmentAssignment: +class EnvironmentAssignment(Protocol): + name: str + + def evaluated_value( + self, + *, + environment: Dict[str, str], + executor: Optional[bashlex_eval.EnvironmentExecutor] = None, + ) -> str: + """Returns the value of this assignment, as evaluated in the environment""" + ... + + +class EnvironmentAssignmentRaw: + """ + An environment variable - a simple name/value pair + """ + + def __init__(self, name: str, value: str): + self.name = name + self.value = value + + def __repr__(self) -> str: + return f"{self.name}: {self.value}" + + def evaluated_value(self, **kwargs: Any) -> str: + return self.value + + +class EnvironmentAssignmentBash: + """ + An environment variable, in bash syntax. The value can use bash constructs + like "$OTHER_VAR" and "$(command arg1 arg2)". + """ + def __init__(self, assignment: str): name, equals, value = assignment.partition("=") if not equals: @@ -52,17 +88,13 @@ class EnvironmentAssignment: environment: Dict[str, str], executor: Optional[bashlex_eval.EnvironmentExecutor] = None, ) -> str: - """Returns the value of this assignment, as evaluated in the environment""" return bashlex_eval.evaluate(self.value, environment=environment, executor=executor) - def as_shell_assignment(self) -> str: - return f"export {self.name}={self.value}" - def __repr__(self) -> str: return f"{self.name}={self.value}" def __eq__(self, other: object) -> bool: - if isinstance(other, EnvironmentAssignment): + if isinstance(other, EnvironmentAssignmentBash): return self.name == other.name and self.value == other.value return False @@ -71,6 +103,9 @@ class EnvironmentAssignment: class ParsedEnvironment: assignments: List[EnvironmentAssignment] + def __init__(self, assignments: Sequence[EnvironmentAssignment]) -> None: + self.assignments = list(assignments) + def as_dictionary( self, prev_environment: Mapping[str, str], @@ -84,8 +119,8 @@ class ParsedEnvironment: return environment - def as_shell_commands(self) -> List[str]: - return [a.as_shell_assignment() for a in self.assignments] + def add(self, name: str, value: str) -> None: + self.assignments.append(EnvironmentAssignmentRaw(name=name, value=value)) def __repr__(self) -> str: return f"{self.__class__.__name__}({[repr(a) for a in self.assignments]!r})" @@ -93,5 +128,5 @@ class ParsedEnvironment: def parse_environment(env_string: str) -> ParsedEnvironment: env_items = split_env_items(env_string) - assignments = [EnvironmentAssignment(item) for item in env_items] + assignments = [EnvironmentAssignmentBash(item) for item in env_items] return ParsedEnvironment(assignments=assignments) diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index 9a207e9e..98bfe756 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -408,6 +408,7 @@ class Options: environment_config = self.reader.get( "environment", table={"item": '{k}="{v}"', "sep": " "} ) + environment_pass = self.reader.get("environment-pass", sep=" ").split() before_build = self.reader.get("before-build", sep=" && ") repair_command = self.reader.get("repair-wheel-command", sep=" && ") @@ -438,6 +439,14 @@ class Options: traceback.print_exc(None, sys.stderr) sys.exit(2) + # Pass through environment variables + if self.platform == "linux": + for env_var_name in environment_pass: + try: + environment.add(env_var_name, os.environ[env_var_name]) + except KeyError: + pass + if dependency_versions == "pinned": dependency_constraints: Optional[ DependencyConstraints diff --git a/cibuildwheel/resources/defaults.toml b/cibuildwheel/resources/defaults.toml index 3c7d3583..81b91c03 100644 --- a/cibuildwheel/resources/defaults.toml +++ b/cibuildwheel/resources/defaults.toml @@ -7,6 +7,7 @@ archs = ["auto"] build-frontend = "pip" dependency-versions = "pinned" environment = {} +environment-pass = [] build-verbosity = "" before-all = "" diff --git a/docs/options.md b/docs/options.md index 53ff1c4f..cd562ebb 100644 --- a/docs/options.md +++ b/docs/options.md @@ -605,6 +605,41 @@ Platform-specific environment variables are also available:
!!! note cibuildwheel always defines the environment variable `CIBUILDWHEEL=1`. This can be useful for [building wheels with optional extensions](faq.md#building-packages-with-optional-c-extensions). +### `CIBW_ENVIRONMENT_PASS_LINUX` {: #environment-pass} +> Set environment variables on the host to pass-through to the container during the build. + +A list of environment variables to pass into the linux container during the build. It has no affect on the other platforms, which can already access all environment variables directly. + +To specify more than one environment variable, separate the variable names by spaces. + +#### Examples + +!!! tab examples "Environment passthrough" + + ```yaml + # Export a variable + CIBW_ENVIRONMENT_PASS_LINUX: CFLAGS + + # Set two flags variables + CIBW_ENVIRONMENT_PASS_LINUX: BUILD_TIME SAMPLE_TEXT + ``` + + Separate multiple values with a space. + +!!! tab examples "pyproject.toml" + + ```toml + [tool.cibuildwheel.linux] + + # Export a variable + environment-pass = ["CFLAGS"] + + # Set two flags variables + environment-pass = ["BUILD_TIME", "SAMPLE_TEXT"] + ``` + + In configuration mode, you can use a [TOML][] list instead of a raw string as shown above. + ### `CIBW_BEFORE_ALL` {: #before-all} > Execute a shell command on the build system before any wheels are built. diff --git a/unit_test/docker_container_test.py b/unit_test/docker_container_test.py index 37c5cca7..3b250ad3 100644 --- a/unit_test/docker_container_test.py +++ b/unit_test/docker_container_test.py @@ -8,7 +8,7 @@ from pathlib import Path, PurePath import pytest from cibuildwheel.docker_container import DockerContainer -from cibuildwheel.environment import EnvironmentAssignment +from cibuildwheel.environment import EnvironmentAssignmentBash # for these tests we use manylinux2014 images, because they're available on # multi architectures and include python3.8 @@ -198,5 +198,5 @@ def test_dir_operations(tmp_path: Path): @pytest.mark.docker def test_environment_executor(): with DockerContainer(docker_image=DEFAULT_IMAGE) as container: - assignment = EnvironmentAssignment("TEST=$(echo 42)") + assignment = EnvironmentAssignmentBash("TEST=$(echo 42)") assert assignment.evaluated_value({}, container.environment_executor) == "42" diff --git a/unit_test/environment_test.py b/unit_test/environment_test.py index c5162aea..c1d0a4a1 100644 --- a/unit_test/environment_test.py +++ b/unit_test/environment_test.py @@ -7,30 +7,24 @@ def test_basic_parsing(): environment_recipe = parse_environment("VAR=1 VBR=2") environment_dict = environment_recipe.as_dictionary(prev_environment={}) - environment_cmds = environment_recipe.as_shell_commands() assert environment_dict == {"VAR": "1", "VBR": "2"} - assert environment_cmds == ["export VAR=1", "export VBR=2"] def test_quotes(): environment_recipe = parse_environment("A=1 VAR=\"1 NOT_A_VAR=2\" VBR='vbr'") environment_dict = environment_recipe.as_dictionary(prev_environment={}) - environment_cmds = environment_recipe.as_shell_commands() assert environment_dict == {"A": "1", "VAR": "1 NOT_A_VAR=2", "VBR": "vbr"} - assert environment_cmds == ["export A=1", 'export VAR="1 NOT_A_VAR=2"', "export VBR='vbr'"] def test_inheritance(): environment_recipe = parse_environment("PATH=$PATH:/usr/local/bin") environment_dict = environment_recipe.as_dictionary(prev_environment={"PATH": "/usr/bin"}) - environment_cmds = environment_recipe.as_shell_commands() assert environment_dict == {"PATH": "/usr/bin:/usr/local/bin"} - assert environment_cmds == ["export PATH=$PATH:/usr/local/bin"] def test_shell_eval(): @@ -40,40 +34,32 @@ def test_shell_eval(): env_copy.pop("VAR", None) environment_dict = environment_recipe.as_dictionary(prev_environment=env_copy) - environment_cmds = environment_recipe.as_shell_commands() assert environment_dict["VAR"] == "a test string" - assert environment_cmds == ['export VAR="$(echo "a test" string)"'] def test_shell_eval_and_env(): environment_recipe = parse_environment('VAR="$(echo "$PREV_VAR" string)"') environment_dict = environment_recipe.as_dictionary(prev_environment={"PREV_VAR": "1 2 3"}) - environment_cmds = environment_recipe.as_shell_commands() assert environment_dict == {"PREV_VAR": "1 2 3", "VAR": "1 2 3 string"} - assert environment_cmds == ['export VAR="$(echo "$PREV_VAR" string)"'] def test_empty_var(): environment_recipe = parse_environment("CFLAGS=") environment_dict = environment_recipe.as_dictionary(prev_environment={"CFLAGS": "-Wall"}) - environment_cmds = environment_recipe.as_shell_commands() assert environment_dict == {"CFLAGS": ""} - assert environment_cmds == ["export CFLAGS="] def test_no_vars(): environment_recipe = parse_environment("") environment_dict = environment_recipe.as_dictionary(prev_environment={}) - environment_cmds = environment_recipe.as_shell_commands() assert environment_dict == {} - assert environment_cmds == [] def test_no_vars_pass_through(): diff --git a/unit_test/options_test.py b/unit_test/options_test.py index abaf04c9..01a98f3b 100644 --- a/unit_test/options_test.py +++ b/unit_test/options_test.py @@ -1,5 +1,7 @@ import platform as platform_module +import pytest + from cibuildwheel.__main__ import get_build_identifiers from cibuildwheel.environment import parse_environment from cibuildwheel.options import Options, _get_pinned_docker_images @@ -15,6 +17,8 @@ test-command = "pyproject" manylinux-x86_64-image = "manylinux1" +environment-pass = ["EXAMPLE_ENV"] + [tool.cibuildwheel.macos] test-requires = "else" @@ -66,3 +70,47 @@ test_command: 'pyproject' assert local.manylinux_images is not None assert local.test_command == "pyproject-override" assert local.manylinux_images["x86_64"] == pinned_x86_64_docker_image["manylinux2014"] + + +def test_passthrough(tmp_path, monkeypatch): + with tmp_path.joinpath("pyproject.toml").open("w") as f: + f.write(PYPROJECT_1) + + args = get_default_command_line_arguments() + args.package_dir = str(tmp_path) + + monkeypatch.setattr(platform_module, "machine", lambda: "x86_64") + monkeypatch.setenv("EXAMPLE_ENV", "ONE") + + options = Options(platform="linux", command_line_arguments=args) + + default_build_options = options.build_options(identifier=None) + + assert default_build_options.environment.as_dictionary(prev_environment={}) == { + "FOO": "BAR", + "EXAMPLE_ENV": "ONE", + } + + +@pytest.mark.parametrize( + "env_var_value", + [ + "normal value", + '"value wrapped in quotes"', + "an unclosed single-quote: '", + 'an unclosed double-quote: "', + "string\nwith\ncarriage\nreturns\n", + "a trailing backslash \\", + ], +) +def test_passthrough_evil(tmp_path, monkeypatch, env_var_value): + args = get_default_command_line_arguments() + args.package_dir = str(tmp_path) + + monkeypatch.setattr(platform_module, "machine", lambda: "x86_64") + monkeypatch.setenv("CIBW_ENVIRONMENT_PASS_LINUX", "ENV_VAR") + options = Options(platform="linux", command_line_arguments=args) + + monkeypatch.setenv("ENV_VAR", env_var_value) + parsed_environment = options.build_options(identifier=None).environment + assert parsed_environment.as_dictionary(prev_environment={}) == {"ENV_VAR": env_var_value}