Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Flattened podman support commits

Browse Source
This commit is contained in:
joncrall
2022-06-23 11:00:10 -04:00
parent 0bf4f7abc3
commit df317a9000
9 changed files with 339 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cibuildwheel/docker_container.py
+116 -16
View File
@@ -1,4 +1,5 @@
import io
import os
import json
import platform
import shlex
@@ -25,6 +26,18 @@ class DockerContainer:
A bash shell is running in the remote container. When `call()` is invoked,
the command is relayed to the remote shell, and the results are streamed
back to cibuildwheel.
TODO:
- [ ] Rename to Container as this now generalizes docker and podman?
Example:
>>> from cibuildwheel.docker_container import * # NOQA
>>> docker_image = "quay.io/pypa/manylinux2014_x86_64:2020-05-17-2f8ac3b"
>>> # Test the default container
>>> with DockerContainer(docker_image=docker_image) as self:
... self.call(["echo", "hello world"])
... self.call(["cat", "/proc/1/cgroup"])
... print(self.get_environment())
"""
UTILITY_PYTHON = "/opt/python/cp38-cp38/bin/python"
@@ -34,7 +47,13 @@ class DockerContainer:
bash_stdout: IO[bytes]
def __init__(
self, *, docker_image: str, simulate_32_bit: bool = False, cwd: Optional[PathOrStr] = None
self,
*,
docker_image: str,
simulate_32_bit: bool = False,
cwd: Optional[PathOrStr] = None,
container_engine: str = "docker",
env: Optional[Dict[str, str]] = None,
):
if not docker_image:
raise ValueError("Must have a non-empty docker image to run.")
@@ -43,10 +62,12 @@ class DockerContainer:
self.simulate_32_bit = simulate_32_bit
self.cwd = cwd
self.name: Optional[str] = None
self.container_engine = container_engine
self.env = env # If specified, overwrite environment variables
def __enter__(self) -> "DockerContainer":
self.name = f"cibuildwheel-{uuid.uuid4()}"
cwd_args = ["-w", str(self.cwd)] if self.cwd else []
# work-around for Travis-CI PPC64le Docker runs since 2021:
# this avoids network splits
@@ -57,24 +78,27 @@ class DockerContainer:
network_args = ["--network=host"]
shell_args = ["linux32", "/bin/bash"] if self.simulate_32_bit else ["/bin/bash"]
subprocess.run(
[
"docker",
self.container_engine,
"create",
"--env=CIBUILDWHEEL",
f"--name={self.name}",
"--interactive",
"--volume=/:/host", # ignored on CircleCI
*network_args,
*cwd_args,
# Z-flags is for SELinux
"--volume=/:/host:Z", # ignored on CircleCI
self.docker_image,
*shell_args,
],
env=self.env,
check=True,
)
self.process = subprocess.Popen(
[
"docker",
self.container_engine,
"start",
"--attach",
"--interactive",
@@ -82,6 +106,7 @@ class DockerContainer:
],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
env=self.env,
)
assert self.process.stdin and self.process.stdout
@@ -89,7 +114,13 @@ class DockerContainer:
self.bash_stdout = self.process.stdout
# run a noop command to block until the container is responding
self.call(["/bin/true"])
self.call(["/bin/true"], cwd="")
if self.cwd:
# Although `docker create -w` does create the working dir if it
# does not exist, podman does not. There does not seem to be a way
# to setup a workdir for a container running in podman.
self.call(["mkdir", "-p", str(self.cwd)], cwd="")
return self
@@ -106,10 +137,20 @@ class DockerContainer:
self.bash_stdin.close()
self.bash_stdout.close()
if self.container_engine == "podman":
# This works around what seems to be a race condition in the podman
# backend. The full reason is not understood. See PR #966 for a
# discussion on possible causes and attempts to remove this line.
# For now, this seems to work "well enough".
self.process.wait()
assert isinstance(self.name, str)
subprocess.run(
["docker", "rm", "--force", "-v", self.name], stdout=subprocess.DEVNULL, check=False
[self.container_engine, "rm", "--force", "-v", self.name],
stdout=subprocess.DEVNULL,
env=self.env,
check=False,
)
self.name = None
@@ -122,15 +163,16 @@ class DockerContainer:
if from_path.is_dir():
self.call(["mkdir", "-p", to_path])
subprocess.run(
f"tar cf - . | docker exec -i {self.name} tar --no-same-owner -xC {shell_quote(to_path)} -f -",
f"tar cf - . | {self.container_engine} exec -i {self.name} tar --no-same-owner -xC {shell_quote(to_path)} -f -",
shell=True,
check=True,
cwd=from_path,
env=self.env,
)
else:
with subprocess.Popen(
[
"docker",
"{self.container_engine}",
"exec",
"-i",
str(self.name),
@@ -138,6 +180,7 @@ class DockerContainer:
"-c",
f"cat > {shell_quote(to_path)}",
],
env=self.env,
stdin=subprocess.PIPE,
) as docker:
docker.stdin = cast(IO[bytes], docker.stdin)
@@ -155,12 +198,47 @@ class DockerContainer:
# note: we assume from_path is a dir
to_path.mkdir(parents=True, exist_ok=True)
subprocess.run(
f"docker exec -i {self.name} tar -cC {shell_quote(from_path)} -f - . | tar -xf -",
shell=True,
check=True,
cwd=to_path,
)
if self.container_engine == "podman":
# The copy out logic that works for docker does not seem to
# translate to podman, which seems to need the steps spelled out
# more explicitly.
command = f"{self.container_engine} exec -i {self.name} tar -cC {shell_quote(from_path)} -f /tmp/output-{self.name}.tar ."
subprocess.run(
command,
shell=True,
check=True,
cwd=to_path,
env=self.env,
)
command = f"{self.container_engine} cp {self.name}:/tmp/output-{self.name}.tar output-{self.name}.tar"
subprocess.run(
command,
shell=True,
check=True,
cwd=to_path,
env=self.env,
)
command = f"tar -xvf output-{self.name}.tar"
subprocess.run(
command,
shell=True,
check=True,
cwd=to_path,
env=self.env,
)
os.unlink(to_path / f"output-{self.name}.tar")
elif self.container_engine == "docker":
command = f"{self.container_engine} exec -i {self.name} tar -cC {shell_quote(from_path)} -f - . | tar -xf -"
subprocess.run(
command,
shell=True,
check=True,
cwd=to_path,
env=self.env,
)
else:
raise KeyError(self.container_engine)
def glob(self, path: PurePosixPath, pattern: str) -> List[PurePosixPath]:
glob_pattern = path.joinpath(pattern)
@@ -186,6 +264,11 @@ class DockerContainer:
cwd: Optional[PathOrStr] = None,
) -> str:
if cwd is None:
# Podman does not start the a container in a specific working dir
# so we always need to specify it when making calls.
cwd = self.cwd
chdir = f"cd {cwd}" if cwd else ""
env_assignments = (
" ".join(f"{shlex.quote(k)}={shlex.quote(v)}" for k, v in env.items())
@@ -271,6 +354,23 @@ class DockerContainer:
# used as an EnvironmentExecutor to evaluate commands and capture output
return self.call(command, env=environment, capture_output=True)
def debug_info(self) -> str:
if self.container_engine == "podman":
command = f"{self.container_engine} info --debug"
else:
command = f"{self.container_engine} info"
completed = subprocess.run(
command,
shell=True,
check=True,
cwd=self.cwd,
env=self.env,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
)
output = str(completed.stdout, encoding="utf8", errors="surrogateescape")
return output
def shell_quote(path: PurePath) -> str:
return shlex.quote(str(path))
cibuildwheel/linux.py
+7 -2
View File
@@ -320,12 +320,16 @@ def build_on_docker(
def build(options: Options, tmp_path: Path) -> None: # pylint: disable=unused-argument
build_opts = options.build_options(None)
try:
# check docker is installed
subprocess.run(["docker", "--version"], check=True, stdout=subprocess.DEVNULL)
subprocess.run(
[build_opts.container_engine, "--version"], check=True, stdout=subprocess.DEVNULL
)
except subprocess.CalledProcessError:
print(
"cibuildwheel: Docker not found. Docker is required to run Linux builds. "
f"cibuildwheel: {build_opts.container_engine} not found. An OCI exe like Docker or Podman is required to run Linux builds "
"If you're building on Travis CI, add `services: [docker]` to your .travis.yml."
"If you're building on Circle CI in Linux, add a `setup_remote_docker` step to your .circleci/config.yml",
file=sys.stderr,
@@ -355,6 +359,7 @@ def build(options: Options, tmp_path: Path) -> None: # pylint: disable=unused-a
docker_image=build_step.docker_image,
simulate_32_bit=build_step.platform_tag.endswith("i686"),
cwd=container_project_path,
container_engine=build_opts.container_engine,
) as docker:
build_on_docker(
cibuildwheel/options.py
+3
View File
@@ -79,6 +79,7 @@ class BuildOptions(NamedTuple):
test_extras: str
build_verbosity: int
build_frontend: BuildFrontend
container_engine: str
@property
def package_dir(self) -> Path:
@@ -422,6 +423,7 @@ class Options:
test_requires = self.reader.get("test-requires", sep=" ").split()
test_extras = self.reader.get("test-extras", sep=",")
build_verbosity_str = self.reader.get("build-verbosity")
container_engine = self.reader.get("container-engine")
build_frontend: BuildFrontend
if build_frontend_str == "build":
@@ -520,6 +522,7 @@ class Options:
manylinux_images=manylinux_images or None,
musllinux_images=musllinux_images or None,
build_frontend=build_frontend,
container_engine=container_engine,
)
def check_for_invalid_configuration(self, identifiers: List[str]) -> None:
cibuildwheel/resources/defaults.toml
+2
View File
@@ -19,6 +19,8 @@ before-test = ""
test-requires = []
test-extras = []
container-engine = "docker"
manylinux-x86_64-image = "manylinux2014"
manylinux-i686-image = "manylinux2014"
manylinux-aarch64-image = "manylinux2014"