Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Flattened podman support commits

Browse Source
This commit is contained in:
joncrall
2022-06-23 11:00:10 -04:00
parent 0bf4f7abc3
commit df317a9000
9 changed files with 339 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.github/install_latest_podman.sh
Executable
+12
View File
@@ -0,0 +1,12 @@
#!/bin/bash
__doc__="
Based on code in: https://github.com/redhat-actions/podman-login/blob/main/.github/install_latest_podman.sh
"
# https://podman.io/getting-started/installation
# shellcheck source=/dev/null
. /etc/os-release
echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
curl -sSfL "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/Release.key" | sudo apt-key add -
sudo apt-get update
sudo apt-get -y upgrade
sudo apt-get -y install podman
.github/workflows/test.yml
+7
View File
@@ -45,6 +45,13 @@ jobs:
with:
python-version: ${{ matrix.python_version }}
# Install podman on this CI instance for podman tests on linux
# Snippet from: https://github.com/redhat-actions/podman-login/blob/main/.github/workflows/example.yml
- name: Install latest podman
if: runner.os == 'Linux'
run: |
bash .github/install_latest_podman.sh
- name: Install dependencies
run: |
python -m pip install ".[test]"
cibuildwheel/docker_container.py
+116 -16
View File
@@ -1,4 +1,5 @@
import io
import os
import json
import platform
import shlex
@@ -25,6 +26,18 @@ class DockerContainer:
A bash shell is running in the remote container. When `call()` is invoked,
the command is relayed to the remote shell, and the results are streamed
back to cibuildwheel.
TODO:
- [ ] Rename to Container as this now generalizes docker and podman?
Example:
>>> from cibuildwheel.docker_container import * # NOQA
>>> docker_image = "quay.io/pypa/manylinux2014_x86_64:2020-05-17-2f8ac3b"
>>> # Test the default container
>>> with DockerContainer(docker_image=docker_image) as self:
... self.call(["echo", "hello world"])
... self.call(["cat", "/proc/1/cgroup"])
... print(self.get_environment())
"""
UTILITY_PYTHON = "/opt/python/cp38-cp38/bin/python"
@@ -34,7 +47,13 @@ class DockerContainer:
bash_stdout: IO[bytes]
def __init__(
self, *, docker_image: str, simulate_32_bit: bool = False, cwd: Optional[PathOrStr] = None
self,
*,
docker_image: str,
simulate_32_bit: bool = False,
cwd: Optional[PathOrStr] = None,
container_engine: str = "docker",
env: Optional[Dict[str, str]] = None,
):
if not docker_image:
raise ValueError("Must have a non-empty docker image to run.")
@@ -43,10 +62,12 @@ class DockerContainer:
self.simulate_32_bit = simulate_32_bit
self.cwd = cwd
self.name: Optional[str] = None
self.container_engine = container_engine
self.env = env # If specified, overwrite environment variables
def __enter__(self) -> "DockerContainer":
self.name = f"cibuildwheel-{uuid.uuid4()}"
cwd_args = ["-w", str(self.cwd)] if self.cwd else []
# work-around for Travis-CI PPC64le Docker runs since 2021:
# this avoids network splits
@@ -57,24 +78,27 @@ class DockerContainer:
network_args = ["--network=host"]
shell_args = ["linux32", "/bin/bash"] if self.simulate_32_bit else ["/bin/bash"]
subprocess.run(
[
"docker",
self.container_engine,
"create",
"--env=CIBUILDWHEEL",
f"--name={self.name}",
"--interactive",
"--volume=/:/host", # ignored on CircleCI
*network_args,
*cwd_args,
# Z-flags is for SELinux
"--volume=/:/host:Z", # ignored on CircleCI
self.docker_image,
*shell_args,
],
env=self.env,
check=True,
)
self.process = subprocess.Popen(
[
"docker",
self.container_engine,
"start",
"--attach",
"--interactive",
@@ -82,6 +106,7 @@ class DockerContainer:
],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
env=self.env,
)
assert self.process.stdin and self.process.stdout
@@ -89,7 +114,13 @@ class DockerContainer:
self.bash_stdout = self.process.stdout
# run a noop command to block until the container is responding
self.call(["/bin/true"])
self.call(["/bin/true"], cwd="")
if self.cwd:
# Although `docker create -w` does create the working dir if it
# does not exist, podman does not. There does not seem to be a way
# to setup a workdir for a container running in podman.
self.call(["mkdir", "-p", str(self.cwd)], cwd="")
return self
@@ -106,10 +137,20 @@ class DockerContainer:
self.bash_stdin.close()
self.bash_stdout.close()
if self.container_engine == "podman":
# This works around what seems to be a race condition in the podman
# backend. The full reason is not understood. See PR #966 for a
# discussion on possible causes and attempts to remove this line.
# For now, this seems to work "well enough".
self.process.wait()
assert isinstance(self.name, str)
subprocess.run(
["docker", "rm", "--force", "-v", self.name], stdout=subprocess.DEVNULL, check=False
[self.container_engine, "rm", "--force", "-v", self.name],
stdout=subprocess.DEVNULL,
env=self.env,
check=False,
)
self.name = None
@@ -122,15 +163,16 @@ class DockerContainer:
if from_path.is_dir():
self.call(["mkdir", "-p", to_path])
subprocess.run(
f"tar cf - . | docker exec -i {self.name} tar --no-same-owner -xC {shell_quote(to_path)} -f -",
f"tar cf - . | {self.container_engine} exec -i {self.name} tar --no-same-owner -xC {shell_quote(to_path)} -f -",
shell=True,
check=True,
cwd=from_path,
env=self.env,
)
else:
with subprocess.Popen(
[
"docker",
"{self.container_engine}",
"exec",
"-i",
str(self.name),
@@ -138,6 +180,7 @@ class DockerContainer:
"-c",
f"cat > {shell_quote(to_path)}",
],
env=self.env,
stdin=subprocess.PIPE,
) as docker:
docker.stdin = cast(IO[bytes], docker.stdin)
@@ -155,12 +198,47 @@ class DockerContainer:
# note: we assume from_path is a dir
to_path.mkdir(parents=True, exist_ok=True)
subprocess.run(
f"docker exec -i {self.name} tar -cC {shell_quote(from_path)} -f - . | tar -xf -",
shell=True,
check=True,
cwd=to_path,
)
if self.container_engine == "podman":
# The copy out logic that works for docker does not seem to
# translate to podman, which seems to need the steps spelled out
# more explicitly.
command = f"{self.container_engine} exec -i {self.name} tar -cC {shell_quote(from_path)} -f /tmp/output-{self.name}.tar ."
subprocess.run(
command,
shell=True,
check=True,
cwd=to_path,
env=self.env,
)
command = f"{self.container_engine} cp {self.name}:/tmp/output-{self.name}.tar output-{self.name}.tar"
subprocess.run(
command,
shell=True,
check=True,
cwd=to_path,
env=self.env,
)
command = f"tar -xvf output-{self.name}.tar"
subprocess.run(
command,
shell=True,
check=True,
cwd=to_path,
env=self.env,
)
os.unlink(to_path / f"output-{self.name}.tar")
elif self.container_engine == "docker":
command = f"{self.container_engine} exec -i {self.name} tar -cC {shell_quote(from_path)} -f - . | tar -xf -"
subprocess.run(
command,
shell=True,
check=True,
cwd=to_path,
env=self.env,
)
else:
raise KeyError(self.container_engine)
def glob(self, path: PurePosixPath, pattern: str) -> List[PurePosixPath]:
glob_pattern = path.joinpath(pattern)
@@ -186,6 +264,11 @@ class DockerContainer:
cwd: Optional[PathOrStr] = None,
) -> str:
if cwd is None:
# Podman does not start the a container in a specific working dir
# so we always need to specify it when making calls.
cwd = self.cwd
chdir = f"cd {cwd}" if cwd else ""
env_assignments = (
" ".join(f"{shlex.quote(k)}={shlex.quote(v)}" for k, v in env.items())
@@ -271,6 +354,23 @@ class DockerContainer:
# used as an EnvironmentExecutor to evaluate commands and capture output
return self.call(command, env=environment, capture_output=True)
def debug_info(self) -> str:
if self.container_engine == "podman":
command = f"{self.container_engine} info --debug"
else:
command = f"{self.container_engine} info"
completed = subprocess.run(
command,
shell=True,
check=True,
cwd=self.cwd,
env=self.env,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
)
output = str(completed.stdout, encoding="utf8", errors="surrogateescape")
return output
def shell_quote(path: PurePath) -> str:
return shlex.quote(str(path))
cibuildwheel/linux.py
+7 -2
View File
@@ -320,12 +320,16 @@ def build_on_docker(
def build(options: Options, tmp_path: Path) -> None: # pylint: disable=unused-argument
build_opts = options.build_options(None)
try:
# check docker is installed
subprocess.run(["docker", "--version"], check=True, stdout=subprocess.DEVNULL)
subprocess.run(
[build_opts.container_engine, "--version"], check=True, stdout=subprocess.DEVNULL
)
except subprocess.CalledProcessError:
print(
"cibuildwheel: Docker not found. Docker is required to run Linux builds. "
f"cibuildwheel: {build_opts.container_engine} not found. An OCI exe like Docker or Podman is required to run Linux builds "
"If you're building on Travis CI, add `services: [docker]` to your .travis.yml."
"If you're building on Circle CI in Linux, add a `setup_remote_docker` step to your .circleci/config.yml",
file=sys.stderr,
@@ -355,6 +359,7 @@ def build(options: Options, tmp_path: Path) -> None: # pylint: disable=unused-a
docker_image=build_step.docker_image,
simulate_32_bit=build_step.platform_tag.endswith("i686"),
cwd=container_project_path,
container_engine=build_opts.container_engine,
) as docker:
build_on_docker(
cibuildwheel/options.py
+3
View File
@@ -79,6 +79,7 @@ class BuildOptions(NamedTuple):
test_extras: str
build_verbosity: int
build_frontend: BuildFrontend
container_engine: str
@property
def package_dir(self) -> Path:
@@ -422,6 +423,7 @@ class Options:
test_requires = self.reader.get("test-requires", sep=" ").split()
test_extras = self.reader.get("test-extras", sep=",")
build_verbosity_str = self.reader.get("build-verbosity")
container_engine = self.reader.get("container-engine")
build_frontend: BuildFrontend
if build_frontend_str == "build":
@@ -520,6 +522,7 @@ class Options:
manylinux_images=manylinux_images or None,
musllinux_images=musllinux_images or None,
build_frontend=build_frontend,
container_engine=container_engine,
)
def check_for_invalid_configuration(self, identifiers: List[str]) -> None:
cibuildwheel/resources/defaults.toml
+2
View File
@@ -19,6 +19,8 @@ before-test = ""
test-requires = []
test-extras = []
container-engine = "docker"
manylinux-x86_64-image = "manylinux2014"
manylinux-i686-image = "manylinux2014"
manylinux-aarch64-image = "manylinux2014"
setup.py
+1
View File
@@ -14,6 +14,7 @@ extras = {
"pytest-timeout",
"pytest-xdist",
"build",
"toml",
],
"bin": [
"click",
unit_test/conftest.py
+1 -1
View File
@@ -16,7 +16,7 @@ def pytest_configure(config):
def pytest_collection_modifyitems(config, items):
if config.getoption("--run-docker"):
# --run-docker given in cli: do not skip docker tests
# --run-docker given in cli: do not skip container tests
return
skip_docker = pytest.mark.skip(reason="need --run-docker option to run")
for item in items:
unit_test/docker_container_test.py
+190 -24
View File
@@ -1,11 +1,15 @@
import atexit
import os
import platform
import random
import shutil
import subprocess
import tempfile
import textwrap
from pathlib import Path, PurePath, PurePosixPath
import pytest
import toml # type: ignore[import]
from cibuildwheel.docker_container import DockerContainer
from cibuildwheel.environment import EnvironmentAssignmentBash
@@ -21,23 +25,178 @@ elif pm == "ppc64le":
DEFAULT_IMAGE = "quay.io/pypa/manylinux2014_ppc64le:2020-05-17-2f8ac3b"
elif pm == "s390x":
DEFAULT_IMAGE = "quay.io/pypa/manylinux2014_s390x:2020-05-17-2f8ac3b"
else:
DEFAULT_IMAGE = ""
temp_test_dir = None
@atexit.register
def _cleanup_tempdir():
"""
Cleans up any configuration written by :func:`basis_container_kwargs`.
For podman tests, the user is not given write permissions by default in new
directories. As a workaround chown them before trying to delete them.
It may be possible to handle this more cleanly in pytest itself, but using
atexit works well enough for now.
"""
import stat
global temp_test_dir
if temp_test_dir is not None:
print(f"CLEANUP temp_test_dir = {temp_test_dir!r}") # type: ignore[unreachable]
for r, ds, fs in os.walk(temp_test_dir.name):
for d in ds:
dpath = os.path.join(r, d)
if not os.path.islink(dpath):
perms = os.lstat(dpath).st_mode
try:
os.chmod(dpath, stat.S_IWUSR | perms)
except Exception as ex:
print(f"issue with dpath = {dpath!r}, {ex!r}")
for f in fs:
fpath = os.path.join(r, f)
if not os.path.islink(fpath):
perms = os.lstat(fpath).st_mode
try:
os.chmod(fpath, stat.S_IWUSR | perms)
except Exception as ex:
print(f"issue with fpath = {fpath!r}, {ex!r}")
else:
os.unlink(fpath)
try:
temp_test_dir.cleanup()
except Exception as ex:
print(f"Issue cleaning up ex = {ex!r}")
temp_test_dir = None
def basis_container_kwargs():
"""
Generate keyword args that can be passed to to :class:`DockerContainer`.
This is used with :func:`pytest.mark.parametrize` to run each test with
different configuraions of each supported containers engine.
For docker we test the default configuration.
For podman we test the default configuration and a configuration with VFS
(virtual file system) enabled as the storage driver.
"""
global temp_test_dir
if temp_test_dir is None:
# Only setup the temp directory once for all tests
temp_test_dir = tempfile.TemporaryDirectory(prefix="cibw_test_")
HAVE_DOCKER = bool(shutil.which("docker"))
HAVE_PODMAN = bool(shutil.which("podman"))
if HAVE_DOCKER:
# Basic podman configuration
yield {"container_engine": "docker", "docker_image": DEFAULT_IMAGE}
if HAVE_PODMAN:
# Basic podman usage
yield {"container_engine": "podman", "docker_image": DEFAULT_IMAGE}
# VFS Podman usage (for the podman in docker use-case)
dpath = Path(temp_test_dir.name)
# This requires that we write configuration files and point to them
# with environment variables before we run podman
# https://github.com/containers/common/blob/main/docs/containers.conf.5.md
vfs_containers_conf_data = {
"containers": {
"default_capabilities": [
"CHOWN",
"DAC_OVERRIDE",
"FOWNER",
"FSETID",
"KILL",
"NET_BIND_SERVICE",
"SETFCAP",
"SETGID",
"SETPCAP",
"SETUID",
"SYS_CHROOT",
]
},
"engine": {"cgroup_manager": "cgroupfs", "events_logger": "file"},
}
# https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md
storage_root = dpath / ".local/share/containers/vfs-storage"
run_root = dpath / ".local/share/containers/vfs-runroot"
storage_root.mkdir(parents=True, exist_ok=True)
run_root.mkdir(parents=True, exist_ok=True)
vfs_containers_storage_conf_data = {
"storage": {
"driver": "vfs",
"graphroot": str(storage_root),
"runroot": str(run_root),
"rootless_storage_path": str(storage_root),
"options": {
# "remap-user": "containers",
"aufs": {"mountopt": "rw"},
"overlay": {"mountopt": "rw", "force_mask": "shared"},
# "vfs": {"ignore_chown_errors": "true"},
},
}
}
vfs_containers_conf_fpath = dpath / "temp_vfs_containers.conf"
vfs_containers_storage_conf_fpath = dpath / "temp_vfs_containers_storage.conf"
with open(vfs_containers_conf_fpath, "w") as file:
toml.dump(vfs_containers_conf_data, file)
with open(vfs_containers_storage_conf_fpath, "w") as file:
toml.dump(vfs_containers_storage_conf_data, file)
oci_environ = os.environ.copy()
oci_environ.update(
{
"CONTAINERS_CONF": str(vfs_containers_conf_fpath),
"CONTAINERS_STORAGE_CONF": str(vfs_containers_storage_conf_fpath),
}
)
yield {
"container_engine": "podman",
"docker_image": DEFAULT_IMAGE,
"env": oci_environ,
}
@pytest.mark.docker
def test_simple():
with DockerContainer(docker_image=DEFAULT_IMAGE) as container:
@pytest.mark.parametrize("container_kwargs", basis_container_kwargs())
def test_simple(container_kwargs):
with DockerContainer(**container_kwargs) as container:
assert container.call(["echo", "hello"], capture_output=True) == "hello\n"
@pytest.mark.docker
def test_no_lf():
with DockerContainer(docker_image=DEFAULT_IMAGE) as container:
@pytest.mark.parametrize("container_kwargs", basis_container_kwargs())
def test_no_lf(container_kwargs):
with DockerContainer(**container_kwargs) as container:
assert container.call(["printf", "hello"], capture_output=True) == "hello"
@pytest.mark.docker
def test_environment():
with DockerContainer(docker_image=DEFAULT_IMAGE) as container:
@pytest.mark.parametrize("container_kwargs", basis_container_kwargs())
def test_debug_info(container_kwargs):
container = DockerContainer(**container_kwargs)
print(container.debug_info())
with container:
pass
@pytest.mark.docker
@pytest.mark.parametrize("container_kwargs", basis_container_kwargs())
def test_environment(container_kwargs):
with DockerContainer(**container_kwargs) as container:
assert (
container.call(
["sh", "-c", "echo $TEST_VAR"], env={"TEST_VAR": "1"}, capture_output=True
@@ -47,40 +206,43 @@ def test_environment():
@pytest.mark.docker
def test_cwd():
with DockerContainer(
docker_image=DEFAULT_IMAGE, cwd="/cibuildwheel/working_directory"
) as container:
@pytest.mark.parametrize("container_kwargs", basis_container_kwargs())
def test_cwd(container_kwargs):
with DockerContainer(cwd="/cibuildwheel/working_directory", **container_kwargs) as container:
assert container.call(["pwd"], capture_output=True) == "/cibuildwheel/working_directory\n"
assert container.call(["pwd"], capture_output=True, cwd="/opt") == "/opt\n"
@pytest.mark.docker
def test_container_removed():
with DockerContainer(docker_image=DEFAULT_IMAGE) as container:
@pytest.mark.parametrize("container_kwargs", basis_container_kwargs())
def test_container_removed(container_kwargs):
with DockerContainer(**container_kwargs) as container:
docker_containers_listing = subprocess.run(
"docker container ls",
f"{container.container_engine} container ls",
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True,
env=container.env,
).stdout
assert container.name is not None
assert container.name in docker_containers_listing
old_container_name = container.name
docker_containers_listing = subprocess.run(
"docker container ls",
f"{container.container_engine} container ls",
shell=True,
check=True,
stdout=subprocess.PIPE,
universal_newlines=True,
env=container.env,
).stdout
assert old_container_name not in docker_containers_listing
@pytest.mark.docker
def test_large_environment():
@pytest.mark.parametrize("container_kwargs", basis_container_kwargs())
def test_large_environment(container_kwargs):
# max environment variable size is 128kB
long_env_var_length = 127 * 1024
large_environment = {
@@ -90,7 +252,7 @@ def test_large_environment():
"d": "0" * long_env_var_length,
}
with DockerContainer(docker_image=DEFAULT_IMAGE) as container:
with DockerContainer(**container_kwargs) as container:
# check the length of d
assert (
container.call(["sh", "-c", "echo ${#d}"], env=large_environment, capture_output=True)
@@ -99,8 +261,9 @@ def test_large_environment():
@pytest.mark.docker
def test_binary_output():
with DockerContainer(docker_image=DEFAULT_IMAGE) as container:
@pytest.mark.parametrize("container_kwargs", basis_container_kwargs())
def test_binary_output(container_kwargs):
with DockerContainer(**container_kwargs) as container:
# note: the below embedded snippets are in python2
# check that we can pass though arbitrary binary data without erroring
@@ -150,8 +313,9 @@ def test_binary_output():
@pytest.mark.docker
def test_file_operations(tmp_path: Path):
with DockerContainer(docker_image=DEFAULT_IMAGE) as container:
@pytest.mark.parametrize("container_kwargs", basis_container_kwargs())
def test_file_operation(tmp_path: Path, container_kwargs):
with DockerContainer(**container_kwargs) as container:
# test copying a file in
test_binary_data = bytes(random.randrange(256) for _ in range(1000))
original_test_file = tmp_path / "test.dat"
@@ -166,8 +330,9 @@ def test_file_operations(tmp_path: Path):
@pytest.mark.docker
def test_dir_operations(tmp_path: Path):
with DockerContainer(docker_image=DEFAULT_IMAGE) as container:
@pytest.mark.parametrize("container_kwargs", basis_container_kwargs())
def test_dir_operations(tmp_path: Path, container_kwargs):
with DockerContainer(**container_kwargs) as container:
test_binary_data = bytes(random.randrange(256) for _ in range(1000))
original_test_file = tmp_path / "test.dat"
original_test_file.write_bytes(test_binary_data)
@@ -196,7 +361,8 @@ def test_dir_operations(tmp_path: Path):
@pytest.mark.docker
def test_environment_executor():
with DockerContainer(docker_image=DEFAULT_IMAGE) as container:
@pytest.mark.parametrize("container_kwargs", basis_container_kwargs())
def test_environment_executor(container_kwargs):
with DockerContainer(**container_kwargs) as container:
assignment = EnvironmentAssignmentBash("TEST=$(echo 42)")
assert assignment.evaluated_value({}, container.environment_executor) == "42"