Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1499 from pypa/docker-flags

Browse Source 
Add create_args suboption to CIBW_CONTAINER_ENGINE
This commit is contained in:
Joe Rickerby
2023-05-26 21:07:09 +01:00
committed by GitHub
parent 754a47388d 9b904777d4
commit b2bc6fdda6
9 changed files with 293 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cibuildwheel/linux.py
+3 -1
View File
@@ -379,7 +379,9 @@ def build(options: Options, tmp_path: Path) -> None: # noqa: ARG001
try:
# check the container engine is installed
subprocess.run(
[options.globals.container_engine, "--version"], check=True, stdout=subprocess.DEVNULL
[options.globals.container_engine.name, "--version"],
check=True,
stdout=subprocess.DEVNULL,
)
except subprocess.CalledProcessError:
print(
cibuildwheel/oci_container.py
+47 -17
View File
@@ -11,15 +11,44 @@ import sys
import typing
import uuid
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path, PurePath, PurePosixPath
from types import TracebackType
from typing import IO, Dict
from ._compat.typing import Literal
from .typing import PathOrStr, PopenBytes
from .util import CIProvider, detect_ci_provider
from .util import CIProvider, detect_ci_provider, parse_key_value_string
ContainerEngine = Literal["docker", "podman"]
ContainerEngineName = Literal["docker", "podman"]
@dataclass(frozen=True)
class OCIContainerEngineConfig:
name: ContainerEngineName
create_args: Sequence[str] = ()
@staticmethod
def from_config_string(config_string: str) -> OCIContainerEngineConfig:
config_dict = parse_key_value_string(config_string, ["name"])
name = " ".join(config_dict["name"])
if name not in {"docker", "podman"}:
msg = f"unknown container engine {name}"
raise ValueError(msg)
name = typing.cast(ContainerEngineName, name)
# some flexibility in the option name to cope with TOML conventions
create_args = config_dict.get("create_args") or config_dict.get("create-args") or []
return OCIContainerEngineConfig(name=name, create_args=create_args)
def options_summary(self) -> str | dict[str, str]:
if not self.create_args:
return self.name
else:
return {"name": self.name, "create_args": repr(self.create_args)}
DEFAULT_ENGINE = OCIContainerEngineConfig("docker")
class OCIContainer:
@@ -57,7 +86,7 @@ class OCIContainer:
image: str,
simulate_32_bit: bool = False,
cwd: PathOrStr | None = None,
engine: ContainerEngine = "docker",
engine: OCIContainerEngineConfig = DEFAULT_ENGINE,
):
if not image:
msg = "Must have a non-empty image to run."
@@ -84,13 +113,14 @@ class OCIContainer:
subprocess.run(
[
self.engine,
self.engine.name,
"create",
"--env=CIBUILDWHEEL",
f"--name={self.name}",
"--interactive",
"--volume=/:/host", # ignored on CircleCI
*network_args,
*self.engine.create_args,
self.image,
*shell_args,
],
@@ -99,7 +129,7 @@ class OCIContainer:
self.process = subprocess.Popen(
[
self.engine,
self.engine.name,
"start",
"--attach",
"--interactive",
@@ -137,7 +167,7 @@ class OCIContainer:
self.bash_stdin.close()
self.bash_stdout.close()
if self.engine == "podman":
if self.engine.name == "podman":
# This works around what seems to be a race condition in the podman
# backend. The full reason is not understood. See PR #966 for a
# discussion on possible causes and attempts to remove this line.
@@ -147,7 +177,7 @@ class OCIContainer:
assert isinstance(self.name, str)
subprocess.run(
[self.engine, "rm", "--force", "-v", self.name],
[self.engine.name, "rm", "--force", "-v", self.name],
stdout=subprocess.DEVNULL,
check=False,
)
@@ -162,7 +192,7 @@ class OCIContainer:
if from_path.is_dir():
self.call(["mkdir", "-p", to_path])
subprocess.run(
f"tar cf - . | {self.engine} exec -i {self.name} tar --no-same-owner -xC {shell_quote(to_path)} -f -",
f"tar cf - . | {self.engine.name} exec -i {self.name} tar --no-same-owner -xC {shell_quote(to_path)} -f -",
shell=True,
check=True,
cwd=from_path,
@@ -171,7 +201,7 @@ class OCIContainer:
exec_process: subprocess.Popen[bytes]
with subprocess.Popen(
[
self.engine,
self.engine.name,
"exec",
"-i",
str(self.name),
@@ -198,10 +228,10 @@ class OCIContainer:
# note: we assume from_path is a dir
to_path.mkdir(parents=True, exist_ok=True)
if self.engine == "podman":
if self.engine.name == "podman":
subprocess.run(
[
self.engine,
self.engine.name,
"cp",
f"{self.name}:{from_path}/.",
str(to_path),
@@ -209,10 +239,10 @@ class OCIContainer:
check=True,
cwd=to_path,
)
elif self.engine == "docker":
elif self.engine.name == "docker":
# There is a bug in docker that prevents a simple 'cp' invocation
# from working https://github.com/moby/moby/issues/38995
command = f"{self.engine} exec -i {self.name} tar -cC {shell_quote(from_path)} -f - . | tar -xf -"
command = f"{self.engine.name} exec -i {self.name} tar -cC {shell_quote(from_path)} -f - . | tar -xf -"
subprocess.run(
command,
shell=True,
@@ -220,7 +250,7 @@ class OCIContainer:
cwd=to_path,
)
else:
raise KeyError(self.engine)
raise KeyError(self.engine.name)
def glob(self, path: PurePosixPath, pattern: str) -> list[PurePosixPath]:
glob_pattern = path.joinpath(pattern)
@@ -338,10 +368,10 @@ class OCIContainer:
return self.call(command, env=environment, capture_output=True)
def debug_info(self) -> str:
if self.engine == "podman":
command = f"{self.engine} info --debug"
if self.engine.name == "podman":
command = f"{self.engine.name} info --debug"
else:
command = f"{self.engine} info"
command = f"{self.engine.name} info"
completed = subprocess.run(
command,
shell=True,
cibuildwheel/options.py
+15 -8
View File
@@ -10,7 +10,6 @@ import shlex
import sys
import textwrap
import traceback
import typing
from collections.abc import Callable, Generator, Iterable, Iterator, Mapping, Set
from pathlib import Path
from typing import Any, Dict, List, Union
@@ -22,7 +21,7 @@ from ._compat.typing import Literal, NotRequired, TypedDict
from .architecture import Architecture
from .environment import EnvironmentParseError, ParsedEnvironment, parse_environment
from .logger import log
from .oci_container import ContainerEngine
from .oci_container import OCIContainerEngineConfig
from .projectfiles import get_requires_python_str
from .typing import PLATFORMS, PlatformName
from .util import (
@@ -75,7 +74,7 @@ class GlobalOptions:
build_selector: BuildSelector
test_selector: TestSelector
architectures: set[Architecture]
container_engine: ContainerEngine
container_engine: OCIContainerEngineConfig
@dataclasses.dataclass(frozen=True)
@@ -136,8 +135,14 @@ DISALLOWED_OPTIONS = {
class TableFmt(TypedDict):
# a format string, used with '.format', with `k` and `v` parameters
# e.g. "{k}={v}"
item: str
# the string that is inserted between items
# e.g. " "
sep: str
# a quoting function that, if supplied, is called to quote each value
# e.g. shlex.quote
quote: NotRequired[Callable[[str], str]]
@@ -454,15 +459,17 @@ class Options:
)
test_selector = TestSelector(skip_config=test_skip)
container_engine_str = self.reader.get("container-engine")
container_engine_str = self.reader.get(
"container-engine", table={"item": "{k}:{v}", "sep": "; ", "quote": shlex.quote}
)
if container_engine_str not in ["docker", "podman"]:
msg = f"cibuildwheel: Unrecognised container_engine {container_engine_str!r}, only 'docker' and 'podman' are supported"
try:
container_engine = OCIContainerEngineConfig.from_config_string(container_engine_str)
except ValueError as e:
msg = f"cibuildwheel: Failed to parse container config. {e}"
print(msg, file=sys.stderr)
sys.exit(2)
container_engine = typing.cast(ContainerEngine, container_engine_str)
return GlobalOptions(
package_dir=package_dir,
output_dir=output_dir,
cibuildwheel/util.py
+38
View File
@@ -13,6 +13,7 @@ import textwrap
import time
import typing
import urllib.request
from collections import defaultdict
from collections.abc import Generator, Iterable, Mapping, Sequence
from dataclasses import dataclass
from enum import Enum
@@ -697,3 +698,40 @@ def fix_ansi_codes_for_github_actions(text: str) -> str:
ansi_codes.append(code)
return output
def parse_key_value_string(
key_value_string: str, positional_arg_names: list[str] | None = None
) -> dict[str, list[str]]:
"""
Parses a string like "docker; create_args: --some-option=value another-option"
"""
if positional_arg_names is None:
positional_arg_names = []
shlexer = shlex.shlex(key_value_string, posix=True, punctuation_chars=";:")
shlexer.commenters = ""
parts = list(shlexer)
# parts now looks like
# ['docker', ';', 'create_args',':', '--some-option=value', 'another-option']
# split by semicolon
fields = [list(group) for k, group in itertools.groupby(parts, lambda x: x == ";") if not k]
result: dict[str, list[str]] = defaultdict(list)
for field_i, field in enumerate(fields):
if len(field) > 1 and field[1] == ":":
field_name = field[0]
values = field[2:]
else:
try:
field_name = positional_arg_names[field_i]
except IndexError:
msg = f"Failed to parse {key_value_string!r}. Too many positional arguments - expected a maximum of {len(positional_arg_names)}"
raise ValueError(msg) from None
values = field
result[field_name] += values
return result
docs/options.md
+19 -2
View File
@@ -1048,9 +1048,12 @@ Auditwheel detects the version of the manylinux / musllinux standard in the imag
### `CIBW_CONTAINER_ENGINE` {: #container-engine}
> Specify which container engine to use when building Linux wheels
> Specify the container engine to use when building Linux wheels
Options: `docker` `podman`
Options:
- `docker[;create_args: ...]`
- `podman[;create_args: ...]`
Default: `docker`
@@ -1059,6 +1062,12 @@ Set the container engine to use. Docker is the default, or you can switch to
running and `docker` available on PATH. To use Podman, it needs to be
installed and `podman` available on PATH.
Arguments can be supplied to the container engine. Currently, the only option
that's customisable is 'create_args'. Parameters to create_args are
space-separated strings, which are passed to the container engine on the
command line when it's creating the container. If you want to include spaces
inside a parameter, use shell-style quoting.
!!! tip
While most users will stick with Docker, Podman is available in different
@@ -1073,14 +1082,22 @@ installed and `podman` available on PATH.
!!! tab examples "Environment variables"
```yaml
# use podman instead of docker
CIBW_CONTAINER_ENGINE: podman
# pass command line options to 'docker create'
CIBW_CONTAINER_ENGINE: "docker; create_args: --gpus all"
```
!!! tab examples "pyproject.toml"
```toml
[tool.cibuildwheel]
# use podman instead of docker
container-engine = "podman"
# pass command line options to 'docker create'
container-engine = { name = "docker", create-args = ["--gpus", "all"]}
```
test/test_podman.py → test/test_container_engine.py
+27 -1
View File
@@ -7,7 +7,7 @@ from . import test_projects, utils
basic_project = test_projects.new_c_project()
def test(tmp_path, capfd, request):
def test_podman(tmp_path, capfd, request):
if utils.platform != "linux":
pytest.skip("the test is only relevant to the linux build")
@@ -38,3 +38,29 @@ def test(tmp_path, capfd, request):
# check that stdout is bring passed-though from container correctly
captured = capfd.readouterr()
assert "test log statement from before-all" in captured.out
def test_create_args(tmp_path, capfd):
if utils.platform != "linux":
pytest.skip("the test is only relevant to the linux build")
project_dir = tmp_path / "project"
basic_project.generate(project_dir)
# build a manylinux wheel, using create_args to set an environment variable
actual_wheels = utils.cibuildwheel_run(
project_dir,
add_env={
"CIBW_BUILD": "cp310-manylinux_*",
"CIBW_BEFORE_ALL": "echo TEST_CREATE_ARGS is set to $TEST_CREATE_ARGS",
"CIBW_CONTAINER_ENGINE": "docker; create_args: --env=TEST_CREATE_ARGS=itworks",
},
)
expected_wheels = [
w for w in utils.expected_wheels("spam", "0.1.0") if ("cp310-manylinux" in w)
]
assert set(actual_wheels) == set(expected_wheels)
captured = capfd.readouterr()
assert "TEST_CREATE_ARGS is set to itworks" in captured.out
test/utils.py
+12 -1
View File
@@ -192,7 +192,7 @@ def expected_wheels(
platform_tags = []
if platform == "linux":
architectures = [machine_arch]
architectures = [arch_name_for_linux(machine_arch)]
if machine_arch == "x86_64":
architectures.append("i686")
@@ -255,3 +255,14 @@ def get_macos_version():
"""
version_str, _, _ = pm.mac_ver()
return tuple(map(int, version_str.split(".")[:2]))
def arch_name_for_linux(arch: str):
"""
Archs have different names on different platforms, but it's useful to be
able to run linux tests on dev machines. This function translates between
the different names.
"""
if arch == "arm64":
return "aarch64"
return arch
unit_test/oci_container_test.py
+77 -6
View File
@@ -12,7 +12,7 @@ import pytest
import tomli_w
from cibuildwheel.environment import EnvironmentAssignmentBash
from cibuildwheel.oci_container import OCIContainer
from cibuildwheel.oci_container import OCIContainer, OCIContainerEngineConfig
# Test utilities
@@ -21,7 +21,7 @@ from cibuildwheel.oci_container import OCIContainer
pm = platform.machine()
if pm == "x86_64":
DEFAULT_IMAGE = "quay.io/pypa/manylinux2014_x86_64:2020-05-17-2f8ac3b"
elif pm == "aarch64":
elif pm in {"aarch64", "arm64"}:
DEFAULT_IMAGE = "quay.io/pypa/manylinux2014_aarch64:2020-05-17-2f8ac3b"
elif pm == "ppc64le":
DEFAULT_IMAGE = "quay.io/pypa/manylinux2014_ppc64le:2020-05-17-2f8ac3b"
@@ -30,6 +30,8 @@ elif pm == "s390x":
else:
DEFAULT_IMAGE = ""
PODMAN = OCIContainerEngineConfig(name="podman")
@pytest.fixture(params=["docker", "podman"])
def container_engine(request):
@@ -37,7 +39,7 @@ def container_engine(request):
pytest.skip("need --run-docker option to run")
if request.param == "podman" and not request.config.getoption("--run-podman"):
pytest.skip("need --run-podman option to run")
return request.param
return OCIContainerEngineConfig(name=request.param)
# Tests
@@ -81,7 +83,7 @@ def test_cwd(container_engine):
def test_container_removed(container_engine):
with OCIContainer(engine=container_engine, image=DEFAULT_IMAGE) as container:
docker_containers_listing = subprocess.run(
f"{container.engine} container ls",
f"{container.engine.name} container ls",
shell=True,
check=True,
stdout=subprocess.PIPE,
@@ -92,7 +94,7 @@ def test_container_removed(container_engine):
old_container_name = container.name
docker_containers_listing = subprocess.run(
f"{container.engine} container ls",
f"{container.engine.name} container ls",
shell=True,
check=True,
stdout=subprocess.PIPE,
@@ -280,7 +282,7 @@ def test_podman_vfs(tmp_path: Path, monkeypatch, request):
monkeypatch.setenv("CONTAINERS_CONF", str(vfs_containers_conf_fpath))
monkeypatch.setenv("CONTAINERS_STORAGE_CONF", str(vfs_containers_storage_conf_fpath))
with OCIContainer(engine="podman", image=DEFAULT_IMAGE) as container:
with OCIContainer(engine=PODMAN, image=DEFAULT_IMAGE) as container:
# test running a command
assert container.call(["echo", "hello"], capture_output=True) == "hello\n"
@@ -296,3 +298,72 @@ def test_podman_vfs(tmp_path: Path, monkeypatch, request):
# as UID 0. The reason why permission errors occur on podman is documented
# in https://podman.io/blogs/2018/10/03/podman-remove-content-homedir.html
subprocess.run(["podman", "unshare", "rm", "-rf", vfs_path], check=True)
def test_create_args_volume(tmp_path: Path, request):
if not request.config.getoption("--run-docker"):
pytest.skip("need --run-docker option to run")
if "CIRCLECI" in os.environ or "GITLAB_CI" in os.environ:
pytest.skip(
"Skipping test on CircleCI/GitLab because docker there does not support --volume"
)
test_mount_dir = tmp_path / "test_mount"
test_mount_dir.mkdir()
(test_mount_dir / "test_file.txt").write_text("1234")
container_engine = OCIContainerEngineConfig(
name="docker", create_args=[f"--volume={test_mount_dir}:/test_mount"]
)
with OCIContainer(
engine=container_engine,
image=DEFAULT_IMAGE,
) as container:
assert container.call(["cat", "/test_mount/test_file.txt"], capture_output=True) == "1234"
@pytest.mark.parametrize(
("config", "name", "create_args"),
[
(
"docker",
"docker",
[],
),
(
"docker;create_args:",
"docker",
[],
),
(
"docker;create_args:--abc --def",
"docker",
["--abc", "--def"],
),
(
"docker; create_args: --abc --def",
"docker",
["--abc", "--def"],
),
(
"name:docker; create_args: --abc --def",
"docker",
["--abc", "--def"],
),
(
'docker; create_args: --some-option="value with spaces"',
"docker",
["--some-option=value with spaces"],
),
(
'docker; create_args: --some-option="value; with; semicolons" --another-option',
"docker",
["--some-option=value; with; semicolons", "--another-option"],
),
],
)
def test_parse_engine_config(config, name, create_args):
engine_config = OCIContainerEngineConfig.from_config_string(config)
assert engine_config.name == name
assert engine_config.create_args == create_args
unit_test/options_test.py
+55
View File
@@ -198,3 +198,58 @@ def test_toml_environment_quoting(tmp_path: Path, toml_assignment, result_value)
)
assert environment_values["TEST_VAR"] == result_value
@pytest.mark.parametrize(
("toml_assignment", "result_name", "result_create_args"),
[
(
'container-engine = "podman"',
"podman",
[],
),
(
'container-engine = {name = "podman"}',
"podman",
[],
),
(
'container-engine = "docker; create_args: --some-option"',
"docker",
["--some-option"],
),
(
'container-engine = {name = "docker", create-args = ["--some-option"]}',
"docker",
["--some-option"],
),
(
'container-engine = {name = "docker", create-args = ["--some-option", "value that contains spaces"]}',
"docker",
["--some-option", "value that contains spaces"],
),
(
'container-engine = {name = "docker", create-args = ["--some-option", "value;that;contains;semicolons"]}',
"docker",
["--some-option", "value;that;contains;semicolons"],
),
],
)
def test_container_engine_option(tmp_path: Path, toml_assignment, result_name, result_create_args):
args = CommandLineArguments.defaults()
args.package_dir = tmp_path
tmp_path.joinpath("pyproject.toml").write_text(
textwrap.dedent(
f"""\
[tool.cibuildwheel]
{toml_assignment}
"""
)
)
options = Options(platform="linux", command_line_arguments=args, env={})
parsed_container_engine = options.globals.container_engine
assert parsed_container_engine.name == result_name
assert parsed_container_engine.create_args == result_create_args