Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #1499 from pypa/docker-flags

Browse Source 
Add create_args suboption to CIBW_CONTAINER_ENGINE
This commit is contained in:
Joe Rickerby
2023-05-26 21:07:09 +01:00
committed by GitHub
parent 754a47388d 9b904777d4
commit b2bc6fdda6
9 changed files with 293 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cibuildwheel/linux.py
+3 -1
View File
@@ -379,7 +379,9 @@ def build(options: Options, tmp_path: Path) -> None: # noqa: ARG001
try:
# check the container engine is installed
subprocess.run(
[options.globals.container_engine, "--version"], check=True, stdout=subprocess.DEVNULL
[options.globals.container_engine.name, "--version"],
check=True,
stdout=subprocess.DEVNULL,
)
except subprocess.CalledProcessError:
print(
cibuildwheel/oci_container.py
+47 -17
View File
@@ -11,15 +11,44 @@ import sys
import typing
import uuid
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from pathlib import Path, PurePath, PurePosixPath
from types import TracebackType
from typing import IO, Dict
from ._compat.typing import Literal
from .typing import PathOrStr, PopenBytes
from .util import CIProvider, detect_ci_provider
from .util import CIProvider, detect_ci_provider, parse_key_value_string
ContainerEngine = Literal["docker", "podman"]
ContainerEngineName = Literal["docker", "podman"]
@dataclass(frozen=True)
class OCIContainerEngineConfig:
name: ContainerEngineName
create_args: Sequence[str] = ()
@staticmethod
def from_config_string(config_string: str) -> OCIContainerEngineConfig:
config_dict = parse_key_value_string(config_string, ["name"])
name = " ".join(config_dict["name"])
if name not in {"docker", "podman"}:
msg = f"unknown container engine {name}"
raise ValueError(msg)
name = typing.cast(ContainerEngineName, name)
# some flexibility in the option name to cope with TOML conventions
create_args = config_dict.get("create_args") or config_dict.get("create-args") or []
return OCIContainerEngineConfig(name=name, create_args=create_args)
def options_summary(self) -> str | dict[str, str]:
if not self.create_args:
return self.name
else:
return {"name": self.name, "create_args": repr(self.create_args)}
DEFAULT_ENGINE = OCIContainerEngineConfig("docker")
class OCIContainer:
@@ -57,7 +86,7 @@ class OCIContainer:
image: str,
simulate_32_bit: bool = False,
cwd: PathOrStr | None = None,
engine: ContainerEngine = "docker",
engine: OCIContainerEngineConfig = DEFAULT_ENGINE,
):
if not image:
msg = "Must have a non-empty image to run."
@@ -84,13 +113,14 @@ class OCIContainer:
subprocess.run(
[
self.engine,
self.engine.name,
"create",
"--env=CIBUILDWHEEL",
f"--name={self.name}",
"--interactive",
"--volume=/:/host", # ignored on CircleCI
*network_args,
*self.engine.create_args,
self.image,
*shell_args,
],
@@ -99,7 +129,7 @@ class OCIContainer:
self.process = subprocess.Popen(
[
self.engine,
self.engine.name,
"start",
"--attach",
"--interactive",
@@ -137,7 +167,7 @@ class OCIContainer:
self.bash_stdin.close()
self.bash_stdout.close()
if self.engine == "podman":
if self.engine.name == "podman":
# This works around what seems to be a race condition in the podman
# backend. The full reason is not understood. See PR #966 for a
# discussion on possible causes and attempts to remove this line.
@@ -147,7 +177,7 @@ class OCIContainer:
assert isinstance(self.name, str)
subprocess.run(
[self.engine, "rm", "--force", "-v", self.name],
[self.engine.name, "rm", "--force", "-v", self.name],
stdout=subprocess.DEVNULL,
check=False,
)
@@ -162,7 +192,7 @@ class OCIContainer:
if from_path.is_dir():
self.call(["mkdir", "-p", to_path])
subprocess.run(
f"tar cf - . | {self.engine} exec -i {self.name} tar --no-same-owner -xC {shell_quote(to_path)} -f -",
f"tar cf - . | {self.engine.name} exec -i {self.name} tar --no-same-owner -xC {shell_quote(to_path)} -f -",
shell=True,
check=True,
cwd=from_path,
@@ -171,7 +201,7 @@ class OCIContainer:
exec_process: subprocess.Popen[bytes]
with subprocess.Popen(
[
self.engine,
self.engine.name,
"exec",
"-i",
str(self.name),
@@ -198,10 +228,10 @@ class OCIContainer:
# note: we assume from_path is a dir
to_path.mkdir(parents=True, exist_ok=True)
if self.engine == "podman":
if self.engine.name == "podman":
subprocess.run(
[
self.engine,
self.engine.name,
"cp",
f"{self.name}:{from_path}/.",
str(to_path),
@@ -209,10 +239,10 @@ class OCIContainer:
check=True,
cwd=to_path,
)
elif self.engine == "docker":
elif self.engine.name == "docker":
# There is a bug in docker that prevents a simple 'cp' invocation
# from working https://github.com/moby/moby/issues/38995
command = f"{self.engine} exec -i {self.name} tar -cC {shell_quote(from_path)} -f - . | tar -xf -"
command = f"{self.engine.name} exec -i {self.name} tar -cC {shell_quote(from_path)} -f - . | tar -xf -"
subprocess.run(
command,
shell=True,
@@ -220,7 +250,7 @@ class OCIContainer:
cwd=to_path,
)
else:
raise KeyError(self.engine)
raise KeyError(self.engine.name)
def glob(self, path: PurePosixPath, pattern: str) -> list[PurePosixPath]:
glob_pattern = path.joinpath(pattern)
@@ -338,10 +368,10 @@ class OCIContainer:
return self.call(command, env=environment, capture_output=True)
def debug_info(self) -> str:
if self.engine == "podman":
command = f"{self.engine} info --debug"
if self.engine.name == "podman":
command = f"{self.engine.name} info --debug"
else:
command = f"{self.engine} info"
command = f"{self.engine.name} info"
completed = subprocess.run(
command,
shell=True,
cibuildwheel/options.py
+15 -8
View File
@@ -10,7 +10,6 @@ import shlex
import sys
import textwrap
import traceback
import typing
from collections.abc import Callable, Generator, Iterable, Iterator, Mapping, Set
from pathlib import Path
from typing import Any, Dict, List, Union
@@ -22,7 +21,7 @@ from ._compat.typing import Literal, NotRequired, TypedDict
from .architecture import Architecture
from .environment import EnvironmentParseError, ParsedEnvironment, parse_environment
from .logger import log
from .oci_container import ContainerEngine
from .oci_container import OCIContainerEngineConfig
from .projectfiles import get_requires_python_str
from .typing import PLATFORMS, PlatformName
from .util import (
@@ -75,7 +74,7 @@ class GlobalOptions:
build_selector: BuildSelector
test_selector: TestSelector
architectures: set[Architecture]
container_engine: ContainerEngine
container_engine: OCIContainerEngineConfig
@dataclasses.dataclass(frozen=True)
@@ -136,8 +135,14 @@ DISALLOWED_OPTIONS = {
class TableFmt(TypedDict):
# a format string, used with '.format', with `k` and `v` parameters
# e.g. "{k}={v}"
item: str
# the string that is inserted between items
# e.g. " "
sep: str
# a quoting function that, if supplied, is called to quote each value
# e.g. shlex.quote
quote: NotRequired[Callable[[str], str]]
@@ -454,15 +459,17 @@ class Options:
)
test_selector = TestSelector(skip_config=test_skip)
container_engine_str = self.reader.get("container-engine")
container_engine_str = self.reader.get(
"container-engine", table={"item": "{k}:{v}", "sep": "; ", "quote": shlex.quote}
)
if container_engine_str not in ["docker", "podman"]:
msg = f"cibuildwheel: Unrecognised container_engine {container_engine_str!r}, only 'docker' and 'podman' are supported"
try:
container_engine = OCIContainerEngineConfig.from_config_string(container_engine_str)
except ValueError as e:
msg = f"cibuildwheel: Failed to parse container config. {e}"
print(msg, file=sys.stderr)
sys.exit(2)
container_engine = typing.cast(ContainerEngine, container_engine_str)
return GlobalOptions(
package_dir=package_dir,
output_dir=output_dir,
cibuildwheel/util.py
+38
View File
@@ -13,6 +13,7 @@ import textwrap
import time
import typing
import urllib.request
from collections import defaultdict
from collections.abc import Generator, Iterable, Mapping, Sequence
from dataclasses import dataclass
from enum import Enum
@@ -697,3 +698,40 @@ def fix_ansi_codes_for_github_actions(text: str) -> str:
ansi_codes.append(code)
return output
def parse_key_value_string(
key_value_string: str, positional_arg_names: list[str] | None = None
) -> dict[str, list[str]]:
"""
Parses a string like "docker; create_args: --some-option=value another-option"
"""
if positional_arg_names is None:
positional_arg_names = []
shlexer = shlex.shlex(key_value_string, posix=True, punctuation_chars=";:")
shlexer.commenters = ""
parts = list(shlexer)
# parts now looks like
# ['docker', ';', 'create_args',':', '--some-option=value', 'another-option']
# split by semicolon
fields = [list(group) for k, group in itertools.groupby(parts, lambda x: x == ";") if not k]
result: dict[str, list[str]] = defaultdict(list)
for field_i, field in enumerate(fields):
if len(field) > 1 and field[1] == ":":
field_name = field[0]
values = field[2:]
else:
try:
field_name = positional_arg_names[field_i]
except IndexError:
msg = f"Failed to parse {key_value_string!r}. Too many positional arguments - expected a maximum of {len(positional_arg_names)}"
raise ValueError(msg) from None
values = field
result[field_name] += values
return result