Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

reworks

Browse Source
This commit is contained in:
joncrall
2022-06-26 15:53:44 -04:00
parent 65d70682af
commit 90945a38d2
2 changed files with 115 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cibuildwheel/docker_container.py
+19 -5
View File
@@ -80,7 +80,6 @@ class DockerContainer:
network_args = ["--network=host"]
shell_args = ["linux32", "/bin/bash"] if self.simulate_32_bit else ["/bin/bash"]
subprocess.run(
[
self.container_engine,
@@ -89,8 +88,10 @@ class DockerContainer:
f"--name={self.name}",
"--interactive",
*network_args,
# Do we need the hostmout?
# Z-flags is for SELinux
"--volume=/:/host:Z", # ignored on CircleCI
# "--volume=/:/host:Z", # ignored on CircleCI
# "--volume=/:/host",
self.docker_image,
*shell_args,
],
@@ -116,13 +117,13 @@ class DockerContainer:
self.bash_stdout = self.process.stdout
# run a noop command to block until the container is responding
self.call(["/bin/true"], cwd="")
self.call(["/bin/true"], cwd="/")
if self.cwd:
# Although `docker create -w` does create the working dir if it
# does not exist, podman does not. There does not seem to be a way
# to setup a workdir for a container running in podman.
self.call(["mkdir", "-p", str(self.cwd)], cwd="")
self.call(["mkdir", "-p", str(self.cwd)], cwd="/")
return self
@@ -200,7 +201,20 @@ class DockerContainer:
# note: we assume from_path is a dir
to_path.mkdir(parents=True, exist_ok=True)
if self.container_engine == "podman":
TRY_SIMPLE_CP = 0
if TRY_SIMPLE_CP:
# There is a bug in docker that prevents this simple implementation
# from working https://github.com/moby/moby/issues/38995
# It seems to also not workin podman as well
command = f"{self.container_engine} cp {self.name}:{shell_quote(from_path)} {shell_quote(to_path)}"
subprocess.run(
command,
shell=True,
check=True,
cwd=to_path,
env=self.env,
)
elif self.container_engine == "podman":
# The copy out logic that works for docker does not seem to
# translate to podman, which seems to need the steps spelled out
# more explicitly.
unit_test/docker_container_test.py
+96 -90
View File
@@ -28,11 +28,14 @@ elif pm == "s390x":
else:
DEFAULT_IMAGE = ""
temp_test_dir = None
# A dictionary to make it easier to manipulate globals
_STATE = {
'temp_test_dir': None,
'using_podman': False,
}
@atexit.register
# @atexit.register
def _cleanup_tempdir():
"""
Cleans up any configuration written by :func:`basis_container_kwargs`.
@@ -42,37 +45,24 @@ def _cleanup_tempdir():
It may be possible to handle this more cleanly in pytest itself, but using
atexit works well enough for now.
The reason why permission errors occur on podman is documented in
[PodmanStoragePerms]_.
References:
.. [PodmanStoragePerms] https://podman.io/blogs/2018/10/03/podman-remove-content-homedir.html
"""
import stat
global temp_test_dir
temp_test_dir = _STATE['temp_test_dir']
if temp_test_dir is not None:
print(f"CLEANUP temp_test_dir = {temp_test_dir!r}") # type: ignore[unreachable]
for r, ds, fs in os.walk(temp_test_dir.name):
for d in ds:
dpath = os.path.join(r, d)
if not os.path.islink(dpath):
perms = os.lstat(dpath).st_mode
try:
os.chmod(dpath, stat.S_IWUSR | perms)
except Exception as ex:
print(f"issue with dpath = {dpath!r}, {ex!r}")
for f in fs:
fpath = os.path.join(r, f)
if not os.path.islink(fpath):
perms = os.lstat(fpath).st_mode
try:
os.chmod(fpath, stat.S_IWUSR | perms)
except Exception as ex:
print(f"issue with fpath = {fpath!r}, {ex!r}")
else:
os.unlink(fpath)
# When podman creates special directories, they cant be cleaned up
# unless you fake a UID of 0. The package rootlesskit helps with that.
if _STATE['using_podman']:
subprocess.call(['podman', 'unshare', 'rm', '-rf', temp_test_dir.name])
try:
temp_test_dir.cleanup()
except Exception as ex:
print(f"Issue cleaning up ex = {ex!r}")
temp_test_dir = None
_STATE['temp_test_dir'] = None
def basis_container_kwargs():
@@ -80,18 +70,27 @@ def basis_container_kwargs():
Generate keyword args that can be passed to to :class:`DockerContainer`.
This is used with :func:`pytest.mark.parametrize` to run each test with
different configuraions of each supported containers engine.
different configurations of each supported containers engine.
For docker we test the default configuration.
For podman we test the default configuration and a configuration with VFS
(virtual file system) enabled as the storage driver.
Yields:
Dict: a configuration passed as ``container_kwargs`` to each
parameterized test.
"""
global temp_test_dir
if temp_test_dir is None:
if _STATE['temp_test_dir'] is None:
# Only setup the temp directory once for all tests
temp_test_dir = tempfile.TemporaryDirectory(prefix="cibw_test_")
_STATE['temp_test_dir'] = tempfile.TemporaryDirectory(prefix="cibw_test_")
# Register the special cleanup hook after the temp directory is created
# to ensure that it runs before the temp directory logic runs (which
# will not handle cases where there is a fake root UID).
atexit.register(_cleanup_tempdir)
temp_test_dir = _STATE['temp_test_dir']
HAVE_DOCKER = bool(shutil.which("docker"))
HAVE_PODMAN = bool(shutil.which("podman"))
@@ -102,67 +101,11 @@ def basis_container_kwargs():
if HAVE_PODMAN:
# Basic podman usage
_STATE['using_podman'] = True
yield {"container_engine": "podman", "docker_image": DEFAULT_IMAGE}
# VFS Podman usage (for the podman in docker use-case)
dpath = Path(temp_test_dir.name)
# This requires that we write configuration files and point to them
# with environment variables before we run podman
# https://github.com/containers/common/blob/main/docs/containers.conf.5.md
vfs_containers_conf_data = {
"containers": {
"default_capabilities": [
"CHOWN",
"DAC_OVERRIDE",
"FOWNER",
"FSETID",
"KILL",
"NET_BIND_SERVICE",
"SETFCAP",
"SETGID",
"SETPCAP",
"SETUID",
"SYS_CHROOT",
]
},
"engine": {"cgroup_manager": "cgroupfs", "events_logger": "file"},
}
# https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md
storage_root = dpath / ".local/share/containers/vfs-storage"
run_root = dpath / ".local/share/containers/vfs-runroot"
storage_root.mkdir(parents=True, exist_ok=True)
run_root.mkdir(parents=True, exist_ok=True)
vfs_containers_storage_conf_data = {
"storage": {
"driver": "vfs",
"graphroot": str(storage_root),
"runroot": str(run_root),
"rootless_storage_path": str(storage_root),
"options": {
# "remap-user": "containers",
"aufs": {"mountopt": "rw"},
"overlay": {"mountopt": "rw", "force_mask": "shared"},
# "vfs": {"ignore_chown_errors": "true"},
},
}
}
vfs_containers_conf_fpath = dpath / "temp_vfs_containers.conf"
vfs_containers_storage_conf_fpath = dpath / "temp_vfs_containers_storage.conf"
with open(vfs_containers_conf_fpath, "w") as file:
toml.dump(vfs_containers_conf_data, file)
with open(vfs_containers_storage_conf_fpath, "w") as file:
toml.dump(vfs_containers_storage_conf_data, file)
oci_environ = os.environ.copy()
oci_environ.update(
{
"CONTAINERS_CONF": str(vfs_containers_conf_fpath),
"CONTAINERS_STORAGE_CONF": str(vfs_containers_storage_conf_fpath),
}
)
oci_environ = _setup_podman_vfs(temp_test_dir.name)
yield {
"container_engine": "podman",
"docker_image": DEFAULT_IMAGE,
@@ -170,6 +113,69 @@ def basis_container_kwargs():
}
def _setup_podman_vfs(dpath):
"""
Setup the filesystem and environment variables for the VFS podman test
"""
dpath = Path(dpath)
# This requires that we write configuration files and point to them
# with environment variables before we run podman
# https://github.com/containers/common/blob/main/docs/containers.conf.5.md
vfs_containers_conf_data = {
"containers": {
"default_capabilities": [
"CHOWN",
"DAC_OVERRIDE",
"FOWNER",
"FSETID",
"KILL",
"NET_BIND_SERVICE",
"SETFCAP",
"SETGID",
"SETPCAP",
"SETUID",
"SYS_CHROOT",
]
},
"engine": {"cgroup_manager": "cgroupfs", "events_logger": "file"},
}
# https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md
storage_root = dpath / ".local/share/containers/vfs-storage"
run_root = dpath / ".local/share/containers/vfs-runroot"
storage_root.mkdir(parents=True, exist_ok=True)
run_root.mkdir(parents=True, exist_ok=True)
vfs_containers_storage_conf_data = {
"storage": {
"driver": "vfs",
"graphroot": os.fspath(storage_root),
"runroot": os.fspath(run_root),
"rootless_storage_path": os.fspath(storage_root),
"options": {
# "remap-user": "containers",
"aufs": {"mountopt": "rw"},
"overlay": {"mountopt": "rw", "force_mask": "shared"},
# "vfs": {"ignore_chown_errors": "true"},
},
}
}
vfs_containers_conf_fpath = dpath / "temp_vfs_containers.conf"
vfs_containers_storage_conf_fpath = dpath / "temp_vfs_containers_storage.conf"
with open(vfs_containers_conf_fpath, "w") as file:
toml.dump(vfs_containers_conf_data, file)
with open(vfs_containers_storage_conf_fpath, "w") as file:
toml.dump(vfs_containers_storage_conf_data, file)
oci_environ = os.environ.copy()
oci_environ.update(
{
"CONTAINERS_CONF": os.fspath(vfs_containers_conf_fpath),
"CONTAINERS_STORAGE_CONF": os.fspath(vfs_containers_storage_conf_fpath),
}
)
return oci_environ
@pytest.mark.docker
@pytest.mark.parametrize("container_kwargs", basis_container_kwargs())
def test_simple(container_kwargs):