Lots more naming changes to reflect that containers are not necessarily 'docker'
This commit is contained in:
Joe Rickerby
@@ -0,0 +1,296 @@
|
||||
import os
|
||||
import platform
|
||||
import random
|
||||
import shutil
|
||||
import subprocess
|
||||
import textwrap
|
||||
from pathlib import Path, PurePath, PurePosixPath
|
||||
|
||||
import pytest
|
||||
import toml
|
||||
|
||||
from cibuildwheel.environment import EnvironmentAssignmentBash
|
||||
from cibuildwheel.oci_container import OCIContainer
|
||||
|
||||
# Test utilities
|
||||
|
||||
# for these tests we use manylinux2014 images, because they're available on
|
||||
# multi architectures and include python3.8
|
||||
pm = platform.machine()
|
||||
if pm == "x86_64":
|
||||
DEFAULT_IMAGE = "quay.io/pypa/manylinux2014_x86_64:2020-05-17-2f8ac3b"
|
||||
elif pm == "aarch64":
|
||||
DEFAULT_IMAGE = "quay.io/pypa/manylinux2014_aarch64:2020-05-17-2f8ac3b"
|
||||
elif pm == "ppc64le":
|
||||
DEFAULT_IMAGE = "quay.io/pypa/manylinux2014_ppc64le:2020-05-17-2f8ac3b"
|
||||
elif pm == "s390x":
|
||||
DEFAULT_IMAGE = "quay.io/pypa/manylinux2014_s390x:2020-05-17-2f8ac3b"
|
||||
else:
|
||||
DEFAULT_IMAGE = ""
|
||||
|
||||
|
||||
@pytest.fixture(params=["docker", "podman"])
|
||||
def container_engine(request):
|
||||
if request.param == "docker" and not request.config.getoption("--run-docker"):
|
||||
pytest.skip("need --run-docker option to run")
|
||||
if request.param == "podman" and not request.config.getoption("--run-podman"):
|
||||
pytest.skip("need --run-podman option to run")
|
||||
return request.param
|
||||
|
||||
|
||||
# Tests
|
||||
|
||||
|
||||
def test_simple(container_engine):
|
||||
with OCIContainer(engine=container_engine, image=DEFAULT_IMAGE) as container:
|
||||
assert container.call(["echo", "hello"], capture_output=True) == "hello\n"
|
||||
|
||||
|
||||
def test_no_lf(container_engine):
|
||||
with OCIContainer(engine=container_engine, image=DEFAULT_IMAGE) as container:
|
||||
assert container.call(["printf", "hello"], capture_output=True) == "hello"
|
||||
|
||||
|
||||
def test_debug_info(container_engine):
|
||||
container = OCIContainer(engine=container_engine, image=DEFAULT_IMAGE)
|
||||
print(container.debug_info())
|
||||
with container:
|
||||
pass
|
||||
|
||||
|
||||
def test_environment(container_engine):
|
||||
with OCIContainer(engine=container_engine, image=DEFAULT_IMAGE) as container:
|
||||
assert (
|
||||
container.call(
|
||||
["sh", "-c", "echo $TEST_VAR"], env={"TEST_VAR": "1"}, capture_output=True
|
||||
)
|
||||
== "1\n"
|
||||
)
|
||||
|
||||
|
||||
def test_cwd(container_engine):
|
||||
with OCIContainer(
|
||||
engine=container_engine, image=DEFAULT_IMAGE, cwd="/cibuildwheel/working_directory"
|
||||
) as container:
|
||||
assert container.call(["pwd"], capture_output=True) == "/cibuildwheel/working_directory\n"
|
||||
assert container.call(["pwd"], capture_output=True, cwd="/opt") == "/opt\n"
|
||||
|
||||
|
||||
def test_container_removed(container_engine):
|
||||
with OCIContainer(engine=container_engine, image=DEFAULT_IMAGE) as container:
|
||||
docker_containers_listing = subprocess.run(
|
||||
f"{container.engine} container ls",
|
||||
shell=True,
|
||||
check=True,
|
||||
stdout=subprocess.PIPE,
|
||||
universal_newlines=True,
|
||||
).stdout
|
||||
assert container.name is not None
|
||||
assert container.name in docker_containers_listing
|
||||
old_container_name = container.name
|
||||
|
||||
docker_containers_listing = subprocess.run(
|
||||
f"{container.engine} container ls",
|
||||
shell=True,
|
||||
check=True,
|
||||
stdout=subprocess.PIPE,
|
||||
universal_newlines=True,
|
||||
).stdout
|
||||
assert old_container_name not in docker_containers_listing
|
||||
|
||||
|
||||
def test_large_environment(container_engine):
|
||||
# max environment variable size is 128kB
|
||||
long_env_var_length = 127 * 1024
|
||||
large_environment = {
|
||||
"a": "0" * long_env_var_length,
|
||||
"b": "0" * long_env_var_length,
|
||||
"c": "0" * long_env_var_length,
|
||||
"d": "0" * long_env_var_length,
|
||||
}
|
||||
|
||||
with OCIContainer(engine=container_engine, image=DEFAULT_IMAGE) as container:
|
||||
# check the length of d
|
||||
assert (
|
||||
container.call(["sh", "-c", "echo ${#d}"], env=large_environment, capture_output=True)
|
||||
== f"{long_env_var_length}\n"
|
||||
)
|
||||
|
||||
|
||||
def test_binary_output(container_engine):
|
||||
with OCIContainer(engine=container_engine, image=DEFAULT_IMAGE) as container:
|
||||
# note: the below embedded snippets are in python2
|
||||
|
||||
# check that we can pass though arbitrary binary data without erroring
|
||||
container.call(
|
||||
[
|
||||
"/usr/bin/python2",
|
||||
"-c",
|
||||
textwrap.dedent(
|
||||
"""
|
||||
import sys
|
||||
sys.stdout.write(''.join(chr(n) for n in range(0, 256)))
|
||||
"""
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
# check that we can capture arbitrary binary data
|
||||
output = container.call(
|
||||
[
|
||||
"/usr/bin/python2",
|
||||
"-c",
|
||||
textwrap.dedent(
|
||||
"""
|
||||
import sys
|
||||
sys.stdout.write(''.join(chr(n % 256) for n in range(0, 512)))
|
||||
"""
|
||||
),
|
||||
],
|
||||
capture_output=True,
|
||||
)
|
||||
|
||||
data = bytes(output, encoding="utf8", errors="surrogateescape")
|
||||
|
||||
for i in range(512):
|
||||
assert data[i] == i % 256
|
||||
|
||||
# check that environment variables can carry binary data, except null characters
|
||||
# (https://www.gnu.org/software/libc/manual/html_node/Environment-Variables.html)
|
||||
binary_data = bytes(n for n in range(1, 256))
|
||||
binary_data_string = str(binary_data, encoding="utf8", errors="surrogateescape")
|
||||
output = container.call(
|
||||
["python2", "-c", 'import os, sys; sys.stdout.write(os.environ["TEST_VAR"])'],
|
||||
env={"TEST_VAR": binary_data_string},
|
||||
capture_output=True,
|
||||
)
|
||||
assert output == binary_data_string
|
||||
|
||||
|
||||
def test_file_operation(tmp_path: Path, container_engine):
|
||||
with OCIContainer(engine=container_engine, image=DEFAULT_IMAGE) as container:
|
||||
# test copying a file in
|
||||
test_binary_data = bytes(random.randrange(256) for _ in range(1000))
|
||||
original_test_file = tmp_path / "test.dat"
|
||||
original_test_file.write_bytes(test_binary_data)
|
||||
|
||||
dst_file = PurePath("/tmp/test.dat")
|
||||
|
||||
container.copy_into(original_test_file, dst_file)
|
||||
|
||||
output = container.call(["cat", dst_file], capture_output=True)
|
||||
assert test_binary_data == bytes(output, encoding="utf8", errors="surrogateescape")
|
||||
|
||||
|
||||
def test_dir_operations(tmp_path: Path, container_engine):
|
||||
with OCIContainer(engine=container_engine, image=DEFAULT_IMAGE) as container:
|
||||
test_binary_data = bytes(random.randrange(256) for _ in range(1000))
|
||||
original_test_file = tmp_path / "test.dat"
|
||||
original_test_file.write_bytes(test_binary_data)
|
||||
|
||||
# test copying a dir in
|
||||
test_dir = tmp_path / "test_dir"
|
||||
test_dir.mkdir()
|
||||
test_file = test_dir / "test.dat"
|
||||
shutil.copyfile(original_test_file, test_file)
|
||||
|
||||
dst_dir = PurePosixPath("/tmp/test_dir")
|
||||
dst_file = dst_dir / "test.dat"
|
||||
container.copy_into(test_dir, dst_dir)
|
||||
|
||||
output = container.call(["cat", dst_file], capture_output=True)
|
||||
assert test_binary_data == bytes(output, encoding="utf8", errors="surrogateescape")
|
||||
|
||||
# test glob
|
||||
assert container.glob(dst_dir, "*.dat") == [dst_file]
|
||||
|
||||
# test copy dir out
|
||||
new_test_dir = tmp_path / "test_dir_new"
|
||||
container.copy_out(dst_dir, new_test_dir)
|
||||
|
||||
assert test_binary_data == (new_test_dir / "test.dat").read_bytes()
|
||||
|
||||
|
||||
def test_environment_executor(container_engine):
|
||||
with OCIContainer(engine=container_engine, image=DEFAULT_IMAGE) as container:
|
||||
assignment = EnvironmentAssignmentBash("TEST=$(echo 42)")
|
||||
assert assignment.evaluated_value({}, container.environment_executor) == "42"
|
||||
|
||||
|
||||
def test_podman_vfs(container_engine, tmp_path: Path, monkeypatch):
|
||||
# Tests podman VFS, for the podman in docker use-case
|
||||
if container_engine != "podman":
|
||||
pytest.skip("podman is the only supported container engine for this test")
|
||||
|
||||
# create the VFS configuration
|
||||
vfs_path = tmp_path / "podman_vfs"
|
||||
vfs_path.mkdir()
|
||||
|
||||
# This requires that we write configuration files and point to them
|
||||
# with environment variables before we run podman
|
||||
# https://github.com/containers/common/blob/main/docs/containers.conf.5.md
|
||||
vfs_containers_conf_data = {
|
||||
"containers": {
|
||||
"default_capabilities": [
|
||||
"CHOWN",
|
||||
"DAC_OVERRIDE",
|
||||
"FOWNER",
|
||||
"FSETID",
|
||||
"KILL",
|
||||
"NET_BIND_SERVICE",
|
||||
"SETFCAP",
|
||||
"SETGID",
|
||||
"SETPCAP",
|
||||
"SETUID",
|
||||
"SYS_CHROOT",
|
||||
]
|
||||
},
|
||||
"engine": {"cgroup_manager": "cgroupfs", "events_logger": "file"},
|
||||
}
|
||||
# https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md
|
||||
storage_root = vfs_path / ".local/share/containers/vfs-storage"
|
||||
run_root = vfs_path / ".local/share/containers/vfs-runroot"
|
||||
storage_root.mkdir(parents=True, exist_ok=True)
|
||||
run_root.mkdir(parents=True, exist_ok=True)
|
||||
vfs_containers_storage_conf_data = {
|
||||
"storage": {
|
||||
"driver": "vfs",
|
||||
"graphroot": os.fspath(storage_root),
|
||||
"runroot": os.fspath(run_root),
|
||||
"rootless_storage_path": os.fspath(storage_root),
|
||||
"options": {
|
||||
# "remap-user": "containers",
|
||||
"aufs": {"mountopt": "rw"},
|
||||
"overlay": {"mountopt": "rw", "force_mask": "shared"},
|
||||
# "vfs": {"ignore_chown_errors": "true"},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
vfs_containers_conf_fpath = vfs_path / "temp_vfs_containers.conf"
|
||||
vfs_containers_storage_conf_fpath = vfs_path / "temp_vfs_containers_storage.conf"
|
||||
with open(vfs_containers_conf_fpath, "w") as file:
|
||||
toml.dump(vfs_containers_conf_data, file)
|
||||
|
||||
with open(vfs_containers_storage_conf_fpath, "w") as file:
|
||||
toml.dump(vfs_containers_storage_conf_data, file)
|
||||
|
||||
monkeypatch.setenv("CONTAINERS_CONF", str(vfs_containers_conf_fpath))
|
||||
monkeypatch.setenv("CONTAINERS_STORAGE_CONF", str(vfs_containers_storage_conf_fpath))
|
||||
|
||||
with OCIContainer(engine=container_engine, image=DEFAULT_IMAGE) as container:
|
||||
# test running a command
|
||||
assert container.call(["echo", "hello"], capture_output=True) == "hello\n"
|
||||
|
||||
# test copying a file into the container
|
||||
(tmp_path / "some_file.txt").write_text("1234")
|
||||
container.copy_into(tmp_path / "some_file.txt", PurePosixPath("some_file.txt"))
|
||||
assert container.call(["cat", "some_file.txt"], capture_output=True) == "1234"
|
||||
|
||||
# Clean up
|
||||
|
||||
# When using the VFS, user is not given write permissions by default in
|
||||
# new directories. As a workaround we use 'podman unshare' to delete them
|
||||
# as UID 0. The reason why permission errors occur on podman is documented
|
||||
# in https://podman.io/blogs/2018/10/03/podman-remove-content-homedir.html
|
||||
subprocess.run(["podman", "unshare", "rm", "-rf", vfs_path], check=True)
|
||||
Reference in New Issue
Block a user