feat: support config-settings (#1244)

* feat: support config-settings Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> feat: support config-settings Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * refactor: use shlex.quote Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>
2022-09-06 00:56:20 -04:00
parent 6a9b39af49
commit 3597095948
10 changed files with 170 additions and 16 deletions
@@ -20,6 +20,7 @@ from .util import (
    get_build_verbosity_extra_flags,
    prepare_command,
    read_python_configs,
+    split_config_settings,
    unwrap,
 )

@@ -212,8 +213,10 @@ def build_in_container(
            container.call(["mkdir", "-p", built_wheel_dir])

            verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity)
+            extra_flags = split_config_settings(build_options.config_settings)

            if build_options.build_frontend == "pip":
+                extra_flags += verbosity_flags
                container.call(
                    [
                        "python",
@@ -223,12 +226,13 @@ def build_in_container(
                        container_package_dir,
                        f"--wheel-dir={built_wheel_dir}",
                        "--no-deps",
-                        *verbosity_flags,
+                        *extra_flags,
                    ],
                    env=env,
                )
            elif build_options.build_frontend == "build":
-                config_setting = " ".join(verbosity_flags)
+                verbosity_setting = " ".join(verbosity_flags)
+                extra_flags += (f"--config-setting={verbosity_setting}",)
                container.call(
                    [
                        "python",
@@ -237,7 +241,7 @@ def build_in_container(
                        container_package_dir,
                        "--wheel",
                        f"--outdir={built_wheel_dir}",
-                        f"--config-setting={config_setting}",
+                        *extra_flags,
                    ],
                    env=env,
                )
@@ -34,6 +34,7 @@ from .util import (
    prepare_command,
    read_python_configs,
    shell,
+    split_config_settings,
    unwrap,
    virtualenv,
 )
@@ -345,8 +346,10 @@ def build(options: Options, tmp_path: Path) -> None:
                built_wheel_dir.mkdir()

                verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity)
+                extra_flags = split_config_settings(build_options.config_settings)

                if build_options.build_frontend == "pip":
+                    extra_flags += verbosity_flags
                    # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
                    # see https://github.com/pypa/cibuildwheel/pull/369
                    call(
@@ -357,11 +360,12 @@ def build(options: Options, tmp_path: Path) -> None:
                        build_options.package_dir.resolve(),
                        f"--wheel-dir={built_wheel_dir}",
                        "--no-deps",
-                        *verbosity_flags,
+                        *extra_flags,
                        env=env,
                    )
                elif build_options.build_frontend == "build":
-                    config_setting = " ".join(verbosity_flags)
+                    verbosity_setting = " ".join(verbosity_flags)
+                    extra_flags += (f"--config-setting={verbosity_setting}",)
                    build_env = env.copy()
                    if build_options.dependency_constraints:
                        constraint_path = (
@@ -378,7 +382,7 @@ def build(options: Options, tmp_path: Path) -> None:
                        build_options.package_dir,
                        "--wheel",
                        f"--outdir={built_wheel_dir}",
-                        f"--config-setting={config_setting}",
+                        *extra_flags,
                        env=build_env,
                    )
                else:
@@ -3,13 +3,14 @@ from __future__ import annotations
 import difflib
 import functools
 import os
+import shlex
 import sys
 import traceback
 from configparser import ConfigParser
 from contextlib import contextmanager
 from dataclasses import asdict, dataclass
 from pathlib import Path
-from typing import Any, Dict, Generator, List, Mapping, Union, cast
+from typing import Any, Dict, Generator, Iterator, List, Mapping, Union, cast

 if sys.version_info >= (3, 11):
    import tomllib
@@ -77,6 +78,7 @@ class BuildOptions:
    test_extras: str
    build_verbosity: int
    build_frontend: BuildFrontend
+    config_settings: str

    @property
    def package_dir(self) -> Path:
@@ -293,8 +295,9 @@ class OptionsReader:
        accept platform versions of the environment variable. If this is an
        array it will be merged with "sep" before returning. If it is a table,
        it will be formatted with "table['item']" using {k} and {v} and merged
-        with "table['sep']". Empty variables will not override if ignore_empty
-        is True.
+        with "table['sep']". If sep is also given, it will be used for arrays
+        inside the table (must match table['sep']). Empty variables will not
+        override if ignore_empty is True.
        """

        if name not in self.default_options and name not in self.default_platform_options:
@@ -324,7 +327,9 @@ class OptionsReader:
        if isinstance(result, dict):
            if table is None:
                raise ConfigOptionError(f"{name!r} does not accept a table")
-            return table["sep"].join(table["item"].format(k=k, v=v) for k, v in result.items())
+            return table["sep"].join(
+                item for k, v in result.items() for item in _inner_fmt(k, v, table["item"])
+            )

        if isinstance(result, list):
            if sep is None:
@@ -337,6 +342,16 @@ class OptionsReader:
        return result


+def _inner_fmt(k: str, v: Any, table_item: str) -> Iterator[str]:
+    if isinstance(v, list):
+        for inner_v in v:
+            qv = shlex.quote(inner_v)
+            yield table_item.format(k=k, v=qv)
+    else:
+        qv = shlex.quote(v)
+        yield table_item.format(k=k, v=qv)
+
+
 class Options:
    def __init__(self, platform: PlatformName, command_line_arguments: CommandLineArguments):
        self.platform = platform
@@ -427,11 +442,14 @@ class Options:

            build_frontend_str = self.reader.get("build-frontend", env_plat=False)
            environment_config = self.reader.get(
-                "environment", table={"item": '{k}="{v}"', "sep": " "}
+                "environment", table={"item": "{k}={v}", "sep": " "}
            )
            environment_pass = self.reader.get("environment-pass", sep=" ").split()
            before_build = self.reader.get("before-build", sep=" && ")
            repair_command = self.reader.get("repair-wheel-command", sep=" && ")
+            config_settings = self.reader.get(
+                "config-settings", table={"item": "{k}={v}", "sep": " "}
+            )

            dependency_versions = self.reader.get("dependency-versions")
            test_command = self.reader.get("test-command", sep=" && ")
@@ -537,6 +555,7 @@ class Options:
                manylinux_images=manylinux_images or None,
                musllinux_images=musllinux_images or None,
                build_frontend=build_frontend,
+                config_settings=config_settings,
            )

    def check_for_invalid_configuration(self, identifiers: list[str]) -> None:
@@ -5,6 +5,7 @@ test-skip = ""

 archs = ["auto"]
 build-frontend = "pip"
+config-settings = {}
 dependency-versions = "pinned"
 environment = {}
 environment-pass = []
@@ -60,6 +60,7 @@ __all__ = [
    "strtobool",
    "cached_property",
    "chdir",
+    "split_config_settings",
 ]

 resources_dir: Final = Path(__file__).parent / "resources"
@@ -205,6 +206,11 @@ def get_build_verbosity_extra_flags(level: int) -> list[str]:
        return []


+def split_config_settings(config_settings: str) -> list[str]:
+    config_settings_list = shlex.split(config_settings)
+    return [f"--config-setting={setting}" for setting in config_settings_list]
+
+
 def read_python_configs(config: PlatformName) -> list[dict[str, str]]:
    input_file = resources_dir / "build-platforms.toml"
    with input_file.open("rb") as f:
@@ -32,6 +32,7 @@ from .util import (
    prepare_command,
    read_python_configs,
    shell,
+    split_config_settings,
    virtualenv,
 )

@@ -302,8 +303,10 @@ def build(options: Options, tmp_path: Path) -> None:
                built_wheel_dir.mkdir()

                verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity)
+                extra_flags = split_config_settings(build_options.config_settings)

                if build_options.build_frontend == "pip":
+                    extra_flags += verbosity_flags
                    # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
                    # see https://github.com/pypa/cibuildwheel/pull/369
                    call(
@@ -314,11 +317,12 @@ def build(options: Options, tmp_path: Path) -> None:
                        options.globals.package_dir.resolve(),
                        f"--wheel-dir={built_wheel_dir}",
                        "--no-deps",
-                        *get_build_verbosity_extra_flags(build_options.build_verbosity),
+                        *extra_flags,
                        env=env,
                    )
                elif build_options.build_frontend == "build":
-                    config_setting = " ".join(verbosity_flags)
+                    verbosity_setting = " ".join(verbosity_flags)
+                    extra_flags += (f"--config-setting={verbosity_setting}",)
                    build_env = env.copy()
                    if build_options.dependency_constraints:
                        constraints_path = (
@@ -345,7 +349,7 @@ def build(options: Options, tmp_path: Path) -> None:
                            build_options.package_dir,
                            "--wheel",
                            f"--outdir={built_wheel_dir}",
-                            f"--config-setting={config_setting}",
+                            *extra_flags,
                            env=build_env,
                        )
                else:
@@ -529,6 +529,36 @@ Choose which build backend to use. Can either be "pip", which will run
    build-frontend = "pip"
    ```

+### `CIBW_CONFIG_SETTINGS` {: #config-settings}
+> Specify config-settings for the build backend.
+
+Specify config settings for the build backend. Each space separated
+item will be passed via `--config-setting`. In TOML, you can specify
+a table of items, including arrays.
+
+!!! tip
+    Currently, "build" supports arrays for options, but "pip" only supports
+    single values.
+
+Platform-specific environment variables also available:<br/>
+`CIBW_BEFORE_ALL_MACOS` | `CIBW_BEFORE_ALL_WINDOWS` | `CIBW_BEFORE_ALL_LINUX`
+
+
+#### Examples
+
+!!! tab examples "Environment variables"
+
+    ```yaml
+    CIBW_CONFIG_SETTINGS: "--build-option=--use-mypyc"
+    ```
+
+!!! tab examples "pyproject.toml"
+
+    ```toml
+    [tool.cibuildwheel.config-settings]
+    --build-option = "--use-mypyc"
+    ```
+

 ### `CIBW_ENVIRONMENT` {: #environment}
 > Set environment variables needed during the build
@@ -14,7 +14,7 @@ else:
 from cibuildwheel.__main__ import main
 from cibuildwheel.environment import ParsedEnvironment
 from cibuildwheel.options import BuildOptions, _get_pinned_container_images
-from cibuildwheel.util import BuildSelector, resources_dir
+from cibuildwheel.util import BuildSelector, resources_dir, split_config_settings

 # CIBW_PLATFORM is tested in main_platform_test.py

@@ -263,6 +263,27 @@ def test_build_verbosity(
    assert build_options.build_verbosity == expected_verbosity


+@pytest.mark.parametrize("platform_specific", [False, True])
+def test_config_settings(platform_specific, platform, intercepted_build_args, monkeypatch):
+    config_settings = 'setting=value setting=value2 other="something else"'
+    if platform_specific:
+        monkeypatch.setenv("CIBW_CONFIG_SETTINGS_" + platform.upper(), config_settings)
+        monkeypatch.setenv("CIBW_CONFIG_SETTIGNS", "a=b")
+    else:
+        monkeypatch.setenv("CIBW_CONFIG_SETTINGS", config_settings)
+
+    main()
+    build_options = intercepted_build_args.args[0].build_options(identifier=None)
+
+    assert build_options.config_settings == config_settings
+
+    assert split_config_settings(config_settings) == [
+        "--config-setting=setting=value",
+        "--config-setting=setting=value2",
+        "--config-setting=other=something else",
+    ]
+
+
@pytest.mark.parametrize(
    "selector",
    [
@@ -1,6 +1,7 @@
 from __future__ import annotations

 import platform as platform_module
+import textwrap

 import pytest

@@ -58,7 +59,7 @@ test_command: 'pyproject'

    default_build_options = options.build_options(identifier=None)

-    assert default_build_options.environment == parse_environment('FOO="BAR"')
+    assert default_build_options.environment == parse_environment("FOO=BAR")

    all_pinned_container_images = _get_pinned_container_images()
    pinned_x86_64_container_image = all_pinned_container_images["x86_64"]
@@ -116,3 +117,32 @@ def test_passthrough_evil(tmp_path, monkeypatch, env_var_value):
    monkeypatch.setenv("ENV_VAR", env_var_value)
    parsed_environment = options.build_options(identifier=None).environment
    assert parsed_environment.as_dictionary(prev_environment={}) == {"ENV_VAR": env_var_value}
+
+
+@pytest.mark.parametrize(
+    "env_var_value",
+    [
+        "normal value",
+        '"value wrapped in quotes"',
+        'an unclosed double-quote: "',
+        "string\nwith\ncarriage\nreturns\n",
+        "a trailing backslash \\",
+    ],
+)
+def test_toml_environment_evil(tmp_path, monkeypatch, env_var_value):
+    args = get_default_command_line_arguments()
+    args.package_dir = tmp_path
+
+    with tmp_path.joinpath("pyproject.toml").open("w") as f:
+        f.write(
+            textwrap.dedent(
+                f"""\
+        [tool.cibuildwheel.environment]
+        EXAMPLE='''{env_var_value}'''
+        """
+            )
+        )
+
+    options = Options(platform="linux", command_line_arguments=args)
+    parsed_environment = options.build_options(identifier=None).environment
+    assert parsed_environment.as_dictionary(prev_environment={}) == {"EXAMPLE": env_var_value}
@@ -331,3 +331,38 @@ test-command = "pyproject-override"

    with pytest.raises(ConfigOptionError):
        OptionsReader(config_file_path=pyproject_toml, platform=platform)
+
+
+def test_config_settings(tmp_path):
+    pyproject_toml: Path = tmp_path / "pyproject.toml"
+    pyproject_toml.write_text(
+        """\
+[tool.cibuildwheel.config-settings]
+example = "one"
+other = ["two", "three"]
+"""
+    )
+
+    options_reader = OptionsReader(config_file_path=pyproject_toml, platform="linux")
+    assert (
+        options_reader.get("config-settings", table={"item": '{k}="{v}"', "sep": " "})
+        == 'example="one" other="two" other="three"'
+    )
+
+
+def test_pip_config_settings(tmp_path):
+    pyproject_toml: Path = tmp_path / "pyproject.toml"
+    pyproject_toml.write_text(
+        """\
+[tool.cibuildwheel.config-settings]
+--build-option="--use-mypyc"
+"""
+    )
+
+    options_reader = OptionsReader(config_file_path=pyproject_toml, platform="linux")
+    assert (
+        options_reader.get(
+            "config-settings", table={"item": "--config-settings='{k}=\"{v}\"'", "sep": " "}
+        )
+        == "--config-settings='--build-option=\"--use-mypyc\"'"
+    )