Merge pull request #1499 from pypa/docker-flags

Add create_args suboption to CIBW_CONTAINER_ENGINE

cibuildwheel/linux.py

@@ -379,7 +379,9 @@ def build(options: Options, tmp_path: Path) -> None:  # noqa: ARG001
     try:
         # check the container engine is installed
         subprocess.run(
-            [options.globals.container_engine, "--version"], check=True, stdout=subprocess.DEVNULL
+            [options.globals.container_engine.name, "--version"],
+            check=True,
+            stdout=subprocess.DEVNULL,
         )
     except subprocess.CalledProcessError:
         print(

cibuildwheel/oci_container.py

@@ -11,15 +11,44 @@ import sys
 import typing
 import uuid
 from collections.abc import Mapping, Sequence
+from dataclasses import dataclass
 from pathlib import Path, PurePath, PurePosixPath
 from types import TracebackType
 from typing import IO, Dict
 
 from ._compat.typing import Literal
 from .typing import PathOrStr, PopenBytes
-from .util import CIProvider, detect_ci_provider
+from .util import CIProvider, detect_ci_provider, parse_key_value_string
 
-ContainerEngine = Literal["docker", "podman"]
+ContainerEngineName = Literal["docker", "podman"]
+
+
+@dataclass(frozen=True)
+class OCIContainerEngineConfig:
+    name: ContainerEngineName
+    create_args: Sequence[str] = ()
+
+    @staticmethod
+    def from_config_string(config_string: str) -> OCIContainerEngineConfig:
+        config_dict = parse_key_value_string(config_string, ["name"])
+        name = " ".join(config_dict["name"])
+        if name not in {"docker", "podman"}:
+            msg = f"unknown container engine {name}"
+            raise ValueError(msg)
+
+        name = typing.cast(ContainerEngineName, name)
+        # some flexibility in the option name to cope with TOML conventions
+        create_args = config_dict.get("create_args") or config_dict.get("create-args") or []
+        return OCIContainerEngineConfig(name=name, create_args=create_args)
+
+    def options_summary(self) -> str | dict[str, str]:
+        if not self.create_args:
+            return self.name
+        else:
+            return {"name": self.name, "create_args": repr(self.create_args)}
+
+
+DEFAULT_ENGINE = OCIContainerEngineConfig("docker")
 
 
 class OCIContainer:
@@ -57,7 +86,7 @@ class OCIContainer:
         image: str,
         simulate_32_bit: bool = False,
         cwd: PathOrStr | None = None,
-        engine: ContainerEngine = "docker",
+        engine: OCIContainerEngineConfig = DEFAULT_ENGINE,
     ):
         if not image:
             msg = "Must have a non-empty image to run."
@@ -84,13 +113,14 @@ class OCIContainer:
 
         subprocess.run(
             [
-                self.engine,
+                self.engine.name,
                 "create",
                 "--env=CIBUILDWHEEL",
                 f"--name={self.name}",
                 "--interactive",
                 "--volume=/:/host",  # ignored on CircleCI
                 *network_args,
+                *self.engine.create_args,
                 self.image,
                 *shell_args,
             ],
@@ -99,7 +129,7 @@ class OCIContainer:
 
         self.process = subprocess.Popen(
             [
-                self.engine,
+                self.engine.name,
                 "start",
                 "--attach",
                 "--interactive",
@@ -137,7 +167,7 @@ class OCIContainer:
         self.bash_stdin.close()
         self.bash_stdout.close()
 
-        if self.engine == "podman":
+        if self.engine.name == "podman":
             # This works around what seems to be a race condition in the podman
             # backend. The full reason is not understood. See PR #966 for a
             # discussion on possible causes and attempts to remove this line.
@@ -147,7 +177,7 @@ class OCIContainer:
         assert isinstance(self.name, str)
 
         subprocess.run(
-            [self.engine, "rm", "--force", "-v", self.name],
+            [self.engine.name, "rm", "--force", "-v", self.name],
             stdout=subprocess.DEVNULL,
             check=False,
         )
@@ -162,7 +192,7 @@ class OCIContainer:
         if from_path.is_dir():
             self.call(["mkdir", "-p", to_path])
             subprocess.run(
-                f"tar cf - . | {self.engine} exec -i {self.name} tar --no-same-owner -xC {shell_quote(to_path)} -f -",
+                f"tar cf - . | {self.engine.name} exec -i {self.name} tar --no-same-owner -xC {shell_quote(to_path)} -f -",
                 shell=True,
                 check=True,
                 cwd=from_path,
@@ -171,7 +201,7 @@ class OCIContainer:
             exec_process: subprocess.Popen[bytes]
             with subprocess.Popen(
                 [
-                    self.engine,
+                    self.engine.name,
                     "exec",
                     "-i",
                     str(self.name),
@@ -198,10 +228,10 @@ class OCIContainer:
         # note: we assume from_path is a dir
         to_path.mkdir(parents=True, exist_ok=True)
 
-        if self.engine == "podman":
+        if self.engine.name == "podman":
             subprocess.run(
                 [
-                    self.engine,
+                    self.engine.name,
                     "cp",
                     f"{self.name}:{from_path}/.",
                     str(to_path),
@@ -209,10 +239,10 @@ class OCIContainer:
                 check=True,
                 cwd=to_path,
             )
-        elif self.engine == "docker":
+        elif self.engine.name == "docker":
             # There is a bug in docker that prevents a simple 'cp' invocation
             # from working https://github.com/moby/moby/issues/38995
-            command = f"{self.engine} exec -i {self.name} tar -cC {shell_quote(from_path)} -f - . | tar -xf -"
+            command = f"{self.engine.name} exec -i {self.name} tar -cC {shell_quote(from_path)} -f - . | tar -xf -"
             subprocess.run(
                 command,
                 shell=True,
@@ -220,7 +250,7 @@ class OCIContainer:
                 cwd=to_path,
             )
         else:
-            raise KeyError(self.engine)
+            raise KeyError(self.engine.name)
 
     def glob(self, path: PurePosixPath, pattern: str) -> list[PurePosixPath]:
         glob_pattern = path.joinpath(pattern)
@@ -338,10 +368,10 @@ class OCIContainer:
         return self.call(command, env=environment, capture_output=True)
 
     def debug_info(self) -> str:
-        if self.engine == "podman":
+        if self.engine.name == "podman":
-            command = f"{self.engine} info --debug"
+            command = f"{self.engine.name} info --debug"
         else:
-            command = f"{self.engine} info"
+            command = f"{self.engine.name} info"
         completed = subprocess.run(
             command,
             shell=True,

cibuildwheel/options.py

@@ -10,7 +10,6 @@ import shlex
 import sys
 import textwrap
 import traceback
-import typing
 from collections.abc import Callable, Generator, Iterable, Iterator, Mapping, Set
 from pathlib import Path
 from typing import Any, Dict, List, Union
@@ -22,7 +21,7 @@ from ._compat.typing import Literal, NotRequired, TypedDict
 from .architecture import Architecture
 from .environment import EnvironmentParseError, ParsedEnvironment, parse_environment
 from .logger import log
-from .oci_container import ContainerEngine
+from .oci_container import OCIContainerEngineConfig
 from .projectfiles import get_requires_python_str
 from .typing import PLATFORMS, PlatformName
 from .util import (
@@ -75,7 +74,7 @@ class GlobalOptions:
     build_selector: BuildSelector
     test_selector: TestSelector
     architectures: set[Architecture]
-    container_engine: ContainerEngine
+    container_engine: OCIContainerEngineConfig
 
 
 @dataclasses.dataclass(frozen=True)
@@ -136,8 +135,14 @@ DISALLOWED_OPTIONS = {
 
 
 class TableFmt(TypedDict):
+    # a format string, used with '.format', with `k` and `v` parameters
+    # e.g. "{k}={v}"
     item: str
+    # the string that is inserted between items
+    # e.g. " "
     sep: str
+    # a quoting function that, if supplied, is called to quote each value
+    # e.g. shlex.quote
     quote: NotRequired[Callable[[str], str]]
 
 
@@ -454,15 +459,17 @@ class Options:
         )
         test_selector = TestSelector(skip_config=test_skip)
 
-        container_engine_str = self.reader.get("container-engine")
+        container_engine_str = self.reader.get(
+            "container-engine", table={"item": "{k}:{v}", "sep": "; ", "quote": shlex.quote}
+        )
 
-        if container_engine_str not in ["docker", "podman"]:
+        try:
-            msg = f"cibuildwheel: Unrecognised container_engine {container_engine_str!r}, only 'docker' and 'podman' are supported"
+            container_engine = OCIContainerEngineConfig.from_config_string(container_engine_str)
+        except ValueError as e:
+            msg = f"cibuildwheel: Failed to parse container config. {e}"
             print(msg, file=sys.stderr)
             sys.exit(2)
 
-        container_engine = typing.cast(ContainerEngine, container_engine_str)
-
         return GlobalOptions(
             package_dir=package_dir,
             output_dir=output_dir,

cibuildwheel/util.py

@@ -13,6 +13,7 @@ import textwrap
 import time
 import typing
 import urllib.request
+from collections import defaultdict
 from collections.abc import Generator, Iterable, Mapping, Sequence
 from dataclasses import dataclass
 from enum import Enum
@@ -697,3 +698,40 @@ def fix_ansi_codes_for_github_actions(text: str) -> str:
                     ansi_codes.append(code)
 
     return output
+
+
+def parse_key_value_string(
+    key_value_string: str, positional_arg_names: list[str] | None = None
+) -> dict[str, list[str]]:
+    """
+    Parses a string like "docker; create_args: --some-option=value another-option"
+    """
+    if positional_arg_names is None:
+        positional_arg_names = []
+
+    shlexer = shlex.shlex(key_value_string, posix=True, punctuation_chars=";:")
+    shlexer.commenters = ""
+    parts = list(shlexer)
+    # parts now looks like
+    # ['docker', ';', 'create_args',':', '--some-option=value', 'another-option']
+
+    # split by semicolon
+    fields = [list(group) for k, group in itertools.groupby(parts, lambda x: x == ";") if not k]
+
+    result: dict[str, list[str]] = defaultdict(list)
+    for field_i, field in enumerate(fields):
+        if len(field) > 1 and field[1] == ":":
+            field_name = field[0]
+            values = field[2:]
+        else:
+            try:
+                field_name = positional_arg_names[field_i]
+            except IndexError:
+                msg = f"Failed to parse {key_value_string!r}. Too many positional arguments - expected a maximum of {len(positional_arg_names)}"
+                raise ValueError(msg) from None
+
+            values = field
+
+        result[field_name] += values
+
+    return result

docs/options.md

@@ -1048,9 +1048,12 @@ Auditwheel detects the version of the manylinux / musllinux standard in the imag
 
 
 ### `CIBW_CONTAINER_ENGINE` {: #container-engine}
-> Specify which container engine to use when building Linux wheels
+> Specify the container engine to use when building Linux wheels
 
-Options: `docker` `podman`
+Options:
+
+- `docker[;create_args: ...]`
+- `podman[;create_args: ...]`
 
 Default: `docker`
 
@@ -1059,6 +1062,12 @@ Set the container engine to use. Docker is the default, or you can switch to
 running and `docker` available on PATH. To use Podman, it needs to be
 installed and `podman` available on PATH.
 
+Arguments can be supplied to the container engine. Currently, the only option
+that's customisable is 'create_args'. Parameters to create_args are
+space-separated strings, which are passed to the container engine on the
+command line when it's creating the container. If you want to include spaces
+inside a parameter, use shell-style quoting.
+
 !!! tip
 
     While most users will stick with Docker, Podman is available in different
@@ -1073,14 +1082,22 @@ installed and `podman` available on PATH.
 !!! tab examples "Environment variables"
 
     ```yaml
+    # use podman instead of docker
     CIBW_CONTAINER_ENGINE: podman
+
+    # pass command line options to 'docker create'
+    CIBW_CONTAINER_ENGINE: "docker; create_args: --gpus all"
     ```
 
 !!! tab examples "pyproject.toml"
 
     ```toml
     [tool.cibuildwheel]
+    # use podman instead of docker
     container-engine = "podman"
+
+    # pass command line options to 'docker create'
+    container-engine = { name = "docker", create-args = ["--gpus", "all"]}
     ```
 
 

test/test_container_engine.py

@@ -7,7 +7,7 @@ from . import test_projects, utils
 basic_project = test_projects.new_c_project()
 
 
-def test(tmp_path, capfd, request):
+def test_podman(tmp_path, capfd, request):
     if utils.platform != "linux":
         pytest.skip("the test is only relevant to the linux build")
 
@@ -38,3 +38,29 @@ def test(tmp_path, capfd, request):
     # check that stdout is bring passed-though from container correctly
     captured = capfd.readouterr()
     assert "test log statement from before-all" in captured.out
+
+
+def test_create_args(tmp_path, capfd):
+    if utils.platform != "linux":
+        pytest.skip("the test is only relevant to the linux build")
+
+    project_dir = tmp_path / "project"
+    basic_project.generate(project_dir)
+
+    # build a manylinux wheel, using create_args to set an environment variable
+    actual_wheels = utils.cibuildwheel_run(
+        project_dir,
+        add_env={
+            "CIBW_BUILD": "cp310-manylinux_*",
+            "CIBW_BEFORE_ALL": "echo TEST_CREATE_ARGS is set to $TEST_CREATE_ARGS",
+            "CIBW_CONTAINER_ENGINE": "docker; create_args: --env=TEST_CREATE_ARGS=itworks",
+        },
+    )
+
+    expected_wheels = [
+        w for w in utils.expected_wheels("spam", "0.1.0") if ("cp310-manylinux" in w)
+    ]
+    assert set(actual_wheels) == set(expected_wheels)
+
+    captured = capfd.readouterr()
+    assert "TEST_CREATE_ARGS is set to itworks" in captured.out

test/utils.py

@@ -192,7 +192,7 @@ def expected_wheels(
         platform_tags = []
 
         if platform == "linux":
-            architectures = [machine_arch]
+            architectures = [arch_name_for_linux(machine_arch)]
 
             if machine_arch == "x86_64":
                 architectures.append("i686")
@@ -255,3 +255,14 @@ def get_macos_version():
     """
     version_str, _, _ = pm.mac_ver()
     return tuple(map(int, version_str.split(".")[:2]))
+
+
+def arch_name_for_linux(arch: str):
+    """
+    Archs have different names on different platforms, but it's useful to be
+    able to run linux tests on dev machines. This function translates between
+    the different names.
+    """
+    if arch == "arm64":
+        return "aarch64"
+    return arch

unit_test/oci_container_test.py

@@ -12,7 +12,7 @@ import pytest
 import tomli_w
 
 from cibuildwheel.environment import EnvironmentAssignmentBash
-from cibuildwheel.oci_container import OCIContainer
+from cibuildwheel.oci_container import OCIContainer, OCIContainerEngineConfig
 
 # Test utilities
 
@@ -21,7 +21,7 @@ from cibuildwheel.oci_container import OCIContainer
 pm = platform.machine()
 if pm == "x86_64":
     DEFAULT_IMAGE = "quay.io/pypa/manylinux2014_x86_64:2020-05-17-2f8ac3b"
-elif pm == "aarch64":
+elif pm in {"aarch64", "arm64"}:
     DEFAULT_IMAGE = "quay.io/pypa/manylinux2014_aarch64:2020-05-17-2f8ac3b"
 elif pm == "ppc64le":
     DEFAULT_IMAGE = "quay.io/pypa/manylinux2014_ppc64le:2020-05-17-2f8ac3b"
@@ -30,6 +30,8 @@ elif pm == "s390x":
 else:
     DEFAULT_IMAGE = ""
 
+PODMAN = OCIContainerEngineConfig(name="podman")
+
 
 @pytest.fixture(params=["docker", "podman"])
 def container_engine(request):
@@ -37,7 +39,7 @@ def container_engine(request):
         pytest.skip("need --run-docker option to run")
     if request.param == "podman" and not request.config.getoption("--run-podman"):
         pytest.skip("need --run-podman option to run")
-    return request.param
+    return OCIContainerEngineConfig(name=request.param)
 
 
 # Tests
@@ -81,7 +83,7 @@ def test_cwd(container_engine):
 def test_container_removed(container_engine):
     with OCIContainer(engine=container_engine, image=DEFAULT_IMAGE) as container:
         docker_containers_listing = subprocess.run(
-            f"{container.engine} container ls",
+            f"{container.engine.name} container ls",
             shell=True,
             check=True,
             stdout=subprocess.PIPE,
@@ -92,7 +94,7 @@ def test_container_removed(container_engine):
         old_container_name = container.name
 
     docker_containers_listing = subprocess.run(
-        f"{container.engine} container ls",
+        f"{container.engine.name} container ls",
         shell=True,
         check=True,
         stdout=subprocess.PIPE,
@@ -280,7 +282,7 @@ def test_podman_vfs(tmp_path: Path, monkeypatch, request):
     monkeypatch.setenv("CONTAINERS_CONF", str(vfs_containers_conf_fpath))
     monkeypatch.setenv("CONTAINERS_STORAGE_CONF", str(vfs_containers_storage_conf_fpath))
 
-    with OCIContainer(engine="podman", image=DEFAULT_IMAGE) as container:
+    with OCIContainer(engine=PODMAN, image=DEFAULT_IMAGE) as container:
         # test running a command
         assert container.call(["echo", "hello"], capture_output=True) == "hello\n"
 
@@ -296,3 +298,72 @@ def test_podman_vfs(tmp_path: Path, monkeypatch, request):
     # as UID 0. The reason why permission errors occur on podman is documented
     # in https://podman.io/blogs/2018/10/03/podman-remove-content-homedir.html
     subprocess.run(["podman", "unshare", "rm", "-rf", vfs_path], check=True)
+
+
+def test_create_args_volume(tmp_path: Path, request):
+    if not request.config.getoption("--run-docker"):
+        pytest.skip("need --run-docker option to run")
+
+    if "CIRCLECI" in os.environ or "GITLAB_CI" in os.environ:
+        pytest.skip(
+            "Skipping test on CircleCI/GitLab because docker there does not support --volume"
+        )
+
+    test_mount_dir = tmp_path / "test_mount"
+    test_mount_dir.mkdir()
+    (test_mount_dir / "test_file.txt").write_text("1234")
+    container_engine = OCIContainerEngineConfig(
+        name="docker", create_args=[f"--volume={test_mount_dir}:/test_mount"]
+    )
+
+    with OCIContainer(
+        engine=container_engine,
+        image=DEFAULT_IMAGE,
+    ) as container:
+        assert container.call(["cat", "/test_mount/test_file.txt"], capture_output=True) == "1234"
+
+
+@pytest.mark.parametrize(
+    ("config", "name", "create_args"),
+    [
+        (
+            "docker",
+            "docker",
+            [],
+        ),
+        (
+            "docker;create_args:",
+            "docker",
+            [],
+        ),
+        (
+            "docker;create_args:--abc --def",
+            "docker",
+            ["--abc", "--def"],
+        ),
+        (
+            "docker; create_args: --abc --def",
+            "docker",
+            ["--abc", "--def"],
+        ),
+        (
+            "name:docker; create_args: --abc --def",
+            "docker",
+            ["--abc", "--def"],
+        ),
+        (
+            'docker; create_args: --some-option="value with spaces"',
+            "docker",
+            ["--some-option=value with spaces"],
+        ),
+        (
+            'docker; create_args: --some-option="value; with; semicolons" --another-option',
+            "docker",
+            ["--some-option=value; with; semicolons", "--another-option"],
+        ),
+    ],
+)
+def test_parse_engine_config(config, name, create_args):
+    engine_config = OCIContainerEngineConfig.from_config_string(config)
+    assert engine_config.name == name
+    assert engine_config.create_args == create_args

unit_test/options_test.py

@@ -198,3 +198,58 @@ def test_toml_environment_quoting(tmp_path: Path, toml_assignment, result_value)
     )
 
     assert environment_values["TEST_VAR"] == result_value
+
+
+@pytest.mark.parametrize(
+    ("toml_assignment", "result_name", "result_create_args"),
+    [
+        (
+            'container-engine = "podman"',
+            "podman",
+            [],
+        ),
+        (
+            'container-engine = {name = "podman"}',
+            "podman",
+            [],
+        ),
+        (
+            'container-engine = "docker; create_args: --some-option"',
+            "docker",
+            ["--some-option"],
+        ),
+        (
+            'container-engine = {name = "docker", create-args = ["--some-option"]}',
+            "docker",
+            ["--some-option"],
+        ),
+        (
+            'container-engine = {name = "docker", create-args = ["--some-option", "value that contains spaces"]}',
+            "docker",
+            ["--some-option", "value that contains spaces"],
+        ),
+        (
+            'container-engine = {name = "docker", create-args = ["--some-option", "value;that;contains;semicolons"]}',
+            "docker",
+            ["--some-option", "value;that;contains;semicolons"],
+        ),
+    ],
+)
+def test_container_engine_option(tmp_path: Path, toml_assignment, result_name, result_create_args):
+    args = CommandLineArguments.defaults()
+    args.package_dir = tmp_path
+
+    tmp_path.joinpath("pyproject.toml").write_text(
+        textwrap.dedent(
+            f"""\
+            [tool.cibuildwheel]
+            {toml_assignment}
+            """
+        )
+    )
+
+    options = Options(platform="linux", command_line_arguments=args, env={})
+    parsed_container_engine = options.globals.container_engine
+
+    assert parsed_container_engine.name == result_name
+    assert parsed_container_engine.create_args == result_create_args