feat: support passenv (#914)
* feat: support passenv * refactor: environment-pass and spaces * docs: document environment pass * Add unit test for passing through awkward env variable values * Remove unused as_shell_commands method from environment * Fix passthrough edge case by avoiding bash for passthrough assignments * Fix mypy error in test * Fix unit test * Apply suggestions from code review Co-authored-by: Joe Rickerby <joerick@mac.com>
This commit is contained in:
Henry Schreiner
co-authored by
Joe Rickerby
Joe Rickerby
parent
1176024473
commit
f45de3f24e
@@ -112,6 +112,7 @@ Options
|
||||
| | [`CIBW_PRERELEASE_PYTHONS`](https://cibuildwheel.readthedocs.io/en/stable/options/#prerelease-pythons) | Enable building with pre-release versions of Python if available |
|
||||
| **Build customization** | [`CIBW_BUILD_FRONTEND`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-frontend) | Set the tool to use to build, either "pip" (default for now) or "build" |
|
||||
| | [`CIBW_ENVIRONMENT`](https://cibuildwheel.readthedocs.io/en/stable/options/#environment) | Set environment variables needed during the build |
|
||||
| | [`CIBW_ENVIRONMENT_PASS_LINUX`](https://cibuildwheel.readthedocs.io/en/stable/options/#environment-pass) | Set environment variables on the host to pass-through to the container during the build. |
|
||||
| | [`CIBW_BEFORE_ALL`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-all) | Execute a shell command on the build system before any wheels are built. |
|
||||
| | [`CIBW_BEFORE_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-build) | Execute a shell command preparing each wheel's build |
|
||||
| | [`CIBW_REPAIR_WHEEL_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#repair-wheel-command) | Execute a shell command to repair each (non-pure Python) built wheel |
|
||||
|
||||
+45
-10
@@ -1,8 +1,10 @@
|
||||
import dataclasses
|
||||
from typing import Dict, List, Mapping, Optional
|
||||
from typing import Any, Dict, List, Mapping, Optional, Sequence
|
||||
|
||||
import bashlex
|
||||
|
||||
from cibuildwheel.typing import Protocol
|
||||
|
||||
from . import bashlex_eval
|
||||
|
||||
|
||||
@@ -39,7 +41,41 @@ def split_env_items(env_string: str) -> List[str]:
|
||||
return result
|
||||
|
||||
|
||||
class EnvironmentAssignment:
|
||||
class EnvironmentAssignment(Protocol):
|
||||
name: str
|
||||
|
||||
def evaluated_value(
|
||||
self,
|
||||
*,
|
||||
environment: Dict[str, str],
|
||||
executor: Optional[bashlex_eval.EnvironmentExecutor] = None,
|
||||
) -> str:
|
||||
"""Returns the value of this assignment, as evaluated in the environment"""
|
||||
...
|
||||
|
||||
|
||||
class EnvironmentAssignmentRaw:
|
||||
"""
|
||||
An environment variable - a simple name/value pair
|
||||
"""
|
||||
|
||||
def __init__(self, name: str, value: str):
|
||||
self.name = name
|
||||
self.value = value
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"{self.name}: {self.value}"
|
||||
|
||||
def evaluated_value(self, **kwargs: Any) -> str:
|
||||
return self.value
|
||||
|
||||
|
||||
class EnvironmentAssignmentBash:
|
||||
"""
|
||||
An environment variable, in bash syntax. The value can use bash constructs
|
||||
like "$OTHER_VAR" and "$(command arg1 arg2)".
|
||||
"""
|
||||
|
||||
def __init__(self, assignment: str):
|
||||
name, equals, value = assignment.partition("=")
|
||||
if not equals:
|
||||
@@ -52,17 +88,13 @@ class EnvironmentAssignment:
|
||||
environment: Dict[str, str],
|
||||
executor: Optional[bashlex_eval.EnvironmentExecutor] = None,
|
||||
) -> str:
|
||||
"""Returns the value of this assignment, as evaluated in the environment"""
|
||||
return bashlex_eval.evaluate(self.value, environment=environment, executor=executor)
|
||||
|
||||
def as_shell_assignment(self) -> str:
|
||||
return f"export {self.name}={self.value}"
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"{self.name}={self.value}"
|
||||
|
||||
def __eq__(self, other: object) -> bool:
|
||||
if isinstance(other, EnvironmentAssignment):
|
||||
if isinstance(other, EnvironmentAssignmentBash):
|
||||
return self.name == other.name and self.value == other.value
|
||||
return False
|
||||
|
||||
@@ -71,6 +103,9 @@ class EnvironmentAssignment:
|
||||
class ParsedEnvironment:
|
||||
assignments: List[EnvironmentAssignment]
|
||||
|
||||
def __init__(self, assignments: Sequence[EnvironmentAssignment]) -> None:
|
||||
self.assignments = list(assignments)
|
||||
|
||||
def as_dictionary(
|
||||
self,
|
||||
prev_environment: Mapping[str, str],
|
||||
@@ -84,8 +119,8 @@ class ParsedEnvironment:
|
||||
|
||||
return environment
|
||||
|
||||
def as_shell_commands(self) -> List[str]:
|
||||
return [a.as_shell_assignment() for a in self.assignments]
|
||||
def add(self, name: str, value: str) -> None:
|
||||
self.assignments.append(EnvironmentAssignmentRaw(name=name, value=value))
|
||||
|
||||
def __repr__(self) -> str:
|
||||
return f"{self.__class__.__name__}({[repr(a) for a in self.assignments]!r})"
|
||||
@@ -93,5 +128,5 @@ class ParsedEnvironment:
|
||||
|
||||
def parse_environment(env_string: str) -> ParsedEnvironment:
|
||||
env_items = split_env_items(env_string)
|
||||
assignments = [EnvironmentAssignment(item) for item in env_items]
|
||||
assignments = [EnvironmentAssignmentBash(item) for item in env_items]
|
||||
return ParsedEnvironment(assignments=assignments)
|
||||
|
||||
@@ -408,6 +408,7 @@ class Options:
|
||||
environment_config = self.reader.get(
|
||||
"environment", table={"item": '{k}="{v}"', "sep": " "}
|
||||
)
|
||||
environment_pass = self.reader.get("environment-pass", sep=" ").split()
|
||||
before_build = self.reader.get("before-build", sep=" && ")
|
||||
repair_command = self.reader.get("repair-wheel-command", sep=" && ")
|
||||
|
||||
@@ -438,6 +439,14 @@ class Options:
|
||||
traceback.print_exc(None, sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
# Pass through environment variables
|
||||
if self.platform == "linux":
|
||||
for env_var_name in environment_pass:
|
||||
try:
|
||||
environment.add(env_var_name, os.environ[env_var_name])
|
||||
except KeyError:
|
||||
pass
|
||||
|
||||
if dependency_versions == "pinned":
|
||||
dependency_constraints: Optional[
|
||||
DependencyConstraints
|
||||
|
||||
@@ -7,6 +7,7 @@ archs = ["auto"]
|
||||
build-frontend = "pip"
|
||||
dependency-versions = "pinned"
|
||||
environment = {}
|
||||
environment-pass = []
|
||||
build-verbosity = ""
|
||||
|
||||
before-all = ""
|
||||
|
||||
@@ -605,6 +605,41 @@ Platform-specific environment variables are also available:<br/>
|
||||
!!! note
|
||||
cibuildwheel always defines the environment variable `CIBUILDWHEEL=1`. This can be useful for [building wheels with optional extensions](faq.md#building-packages-with-optional-c-extensions).
|
||||
|
||||
### `CIBW_ENVIRONMENT_PASS_LINUX` {: #environment-pass}
|
||||
> Set environment variables on the host to pass-through to the container during the build.
|
||||
|
||||
A list of environment variables to pass into the linux container during the build. It has no affect on the other platforms, which can already access all environment variables directly.
|
||||
|
||||
To specify more than one environment variable, separate the variable names by spaces.
|
||||
|
||||
#### Examples
|
||||
|
||||
!!! tab examples "Environment passthrough"
|
||||
|
||||
```yaml
|
||||
# Export a variable
|
||||
CIBW_ENVIRONMENT_PASS_LINUX: CFLAGS
|
||||
|
||||
# Set two flags variables
|
||||
CIBW_ENVIRONMENT_PASS_LINUX: BUILD_TIME SAMPLE_TEXT
|
||||
```
|
||||
|
||||
Separate multiple values with a space.
|
||||
|
||||
!!! tab examples "pyproject.toml"
|
||||
|
||||
```toml
|
||||
[tool.cibuildwheel.linux]
|
||||
|
||||
# Export a variable
|
||||
environment-pass = ["CFLAGS"]
|
||||
|
||||
# Set two flags variables
|
||||
environment-pass = ["BUILD_TIME", "SAMPLE_TEXT"]
|
||||
```
|
||||
|
||||
In configuration mode, you can use a [TOML][] list instead of a raw string as shown above.
|
||||
|
||||
### `CIBW_BEFORE_ALL` {: #before-all}
|
||||
> Execute a shell command on the build system before any wheels are built.
|
||||
|
||||
|
||||
@@ -8,7 +8,7 @@ from pathlib import Path, PurePath
|
||||
import pytest
|
||||
|
||||
from cibuildwheel.docker_container import DockerContainer
|
||||
from cibuildwheel.environment import EnvironmentAssignment
|
||||
from cibuildwheel.environment import EnvironmentAssignmentBash
|
||||
|
||||
# for these tests we use manylinux2014 images, because they're available on
|
||||
# multi architectures and include python3.8
|
||||
@@ -198,5 +198,5 @@ def test_dir_operations(tmp_path: Path):
|
||||
@pytest.mark.docker
|
||||
def test_environment_executor():
|
||||
with DockerContainer(docker_image=DEFAULT_IMAGE) as container:
|
||||
assignment = EnvironmentAssignment("TEST=$(echo 42)")
|
||||
assignment = EnvironmentAssignmentBash("TEST=$(echo 42)")
|
||||
assert assignment.evaluated_value({}, container.environment_executor) == "42"
|
||||
|
||||
@@ -7,30 +7,24 @@ def test_basic_parsing():
|
||||
environment_recipe = parse_environment("VAR=1 VBR=2")
|
||||
|
||||
environment_dict = environment_recipe.as_dictionary(prev_environment={})
|
||||
environment_cmds = environment_recipe.as_shell_commands()
|
||||
|
||||
assert environment_dict == {"VAR": "1", "VBR": "2"}
|
||||
assert environment_cmds == ["export VAR=1", "export VBR=2"]
|
||||
|
||||
|
||||
def test_quotes():
|
||||
environment_recipe = parse_environment("A=1 VAR=\"1 NOT_A_VAR=2\" VBR='vbr'")
|
||||
|
||||
environment_dict = environment_recipe.as_dictionary(prev_environment={})
|
||||
environment_cmds = environment_recipe.as_shell_commands()
|
||||
|
||||
assert environment_dict == {"A": "1", "VAR": "1 NOT_A_VAR=2", "VBR": "vbr"}
|
||||
assert environment_cmds == ["export A=1", 'export VAR="1 NOT_A_VAR=2"', "export VBR='vbr'"]
|
||||
|
||||
|
||||
def test_inheritance():
|
||||
environment_recipe = parse_environment("PATH=$PATH:/usr/local/bin")
|
||||
|
||||
environment_dict = environment_recipe.as_dictionary(prev_environment={"PATH": "/usr/bin"})
|
||||
environment_cmds = environment_recipe.as_shell_commands()
|
||||
|
||||
assert environment_dict == {"PATH": "/usr/bin:/usr/local/bin"}
|
||||
assert environment_cmds == ["export PATH=$PATH:/usr/local/bin"]
|
||||
|
||||
|
||||
def test_shell_eval():
|
||||
@@ -40,40 +34,32 @@ def test_shell_eval():
|
||||
env_copy.pop("VAR", None)
|
||||
|
||||
environment_dict = environment_recipe.as_dictionary(prev_environment=env_copy)
|
||||
environment_cmds = environment_recipe.as_shell_commands()
|
||||
|
||||
assert environment_dict["VAR"] == "a test string"
|
||||
assert environment_cmds == ['export VAR="$(echo "a test" string)"']
|
||||
|
||||
|
||||
def test_shell_eval_and_env():
|
||||
environment_recipe = parse_environment('VAR="$(echo "$PREV_VAR" string)"')
|
||||
|
||||
environment_dict = environment_recipe.as_dictionary(prev_environment={"PREV_VAR": "1 2 3"})
|
||||
environment_cmds = environment_recipe.as_shell_commands()
|
||||
|
||||
assert environment_dict == {"PREV_VAR": "1 2 3", "VAR": "1 2 3 string"}
|
||||
assert environment_cmds == ['export VAR="$(echo "$PREV_VAR" string)"']
|
||||
|
||||
|
||||
def test_empty_var():
|
||||
environment_recipe = parse_environment("CFLAGS=")
|
||||
|
||||
environment_dict = environment_recipe.as_dictionary(prev_environment={"CFLAGS": "-Wall"})
|
||||
environment_cmds = environment_recipe.as_shell_commands()
|
||||
|
||||
assert environment_dict == {"CFLAGS": ""}
|
||||
assert environment_cmds == ["export CFLAGS="]
|
||||
|
||||
|
||||
def test_no_vars():
|
||||
environment_recipe = parse_environment("")
|
||||
|
||||
environment_dict = environment_recipe.as_dictionary(prev_environment={})
|
||||
environment_cmds = environment_recipe.as_shell_commands()
|
||||
|
||||
assert environment_dict == {}
|
||||
assert environment_cmds == []
|
||||
|
||||
|
||||
def test_no_vars_pass_through():
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import platform as platform_module
|
||||
|
||||
import pytest
|
||||
|
||||
from cibuildwheel.__main__ import get_build_identifiers
|
||||
from cibuildwheel.environment import parse_environment
|
||||
from cibuildwheel.options import Options, _get_pinned_docker_images
|
||||
@@ -15,6 +17,8 @@ test-command = "pyproject"
|
||||
|
||||
manylinux-x86_64-image = "manylinux1"
|
||||
|
||||
environment-pass = ["EXAMPLE_ENV"]
|
||||
|
||||
[tool.cibuildwheel.macos]
|
||||
test-requires = "else"
|
||||
|
||||
@@ -66,3 +70,47 @@ test_command: 'pyproject'
|
||||
assert local.manylinux_images is not None
|
||||
assert local.test_command == "pyproject-override"
|
||||
assert local.manylinux_images["x86_64"] == pinned_x86_64_docker_image["manylinux2014"]
|
||||
|
||||
|
||||
def test_passthrough(tmp_path, monkeypatch):
|
||||
with tmp_path.joinpath("pyproject.toml").open("w") as f:
|
||||
f.write(PYPROJECT_1)
|
||||
|
||||
args = get_default_command_line_arguments()
|
||||
args.package_dir = str(tmp_path)
|
||||
|
||||
monkeypatch.setattr(platform_module, "machine", lambda: "x86_64")
|
||||
monkeypatch.setenv("EXAMPLE_ENV", "ONE")
|
||||
|
||||
options = Options(platform="linux", command_line_arguments=args)
|
||||
|
||||
default_build_options = options.build_options(identifier=None)
|
||||
|
||||
assert default_build_options.environment.as_dictionary(prev_environment={}) == {
|
||||
"FOO": "BAR",
|
||||
"EXAMPLE_ENV": "ONE",
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"env_var_value",
|
||||
[
|
||||
"normal value",
|
||||
'"value wrapped in quotes"',
|
||||
"an unclosed single-quote: '",
|
||||
'an unclosed double-quote: "',
|
||||
"string\nwith\ncarriage\nreturns\n",
|
||||
"a trailing backslash \\",
|
||||
],
|
||||
)
|
||||
def test_passthrough_evil(tmp_path, monkeypatch, env_var_value):
|
||||
args = get_default_command_line_arguments()
|
||||
args.package_dir = str(tmp_path)
|
||||
|
||||
monkeypatch.setattr(platform_module, "machine", lambda: "x86_64")
|
||||
monkeypatch.setenv("CIBW_ENVIRONMENT_PASS_LINUX", "ENV_VAR")
|
||||
options = Options(platform="linux", command_line_arguments=args)
|
||||
|
||||
monkeypatch.setenv("ENV_VAR", env_var_value)
|
||||
parsed_environment = options.build_options(identifier=None).environment
|
||||
assert parsed_environment.as_dictionary(prev_environment={}) == {"ENV_VAR": env_var_value}
|
||||
|
||||
Reference in New Issue
Block a user