297 lines
11 KiB
Python
297 lines
11 KiB
Python
import os
|
|
import platform
|
|
import random
|
|
import shutil
|
|
import subprocess
|
|
import textwrap
|
|
from pathlib import Path, PurePath, PurePosixPath
|
|
|
|
import pytest
|
|
import toml
|
|
|
|
from cibuildwheel.environment import EnvironmentAssignmentBash
|
|
from cibuildwheel.oci_container import OCIContainer
|
|
|
|
# Test utilities
|
|
|
|
# for these tests we use manylinux2014 images, because they're available on
|
|
# multi architectures and include python3.8
|
|
pm = platform.machine()
|
|
if pm == "x86_64":
|
|
DEFAULT_IMAGE = "quay.io/pypa/manylinux2014_x86_64:2020-05-17-2f8ac3b"
|
|
elif pm == "aarch64":
|
|
DEFAULT_IMAGE = "quay.io/pypa/manylinux2014_aarch64:2020-05-17-2f8ac3b"
|
|
elif pm == "ppc64le":
|
|
DEFAULT_IMAGE = "quay.io/pypa/manylinux2014_ppc64le:2020-05-17-2f8ac3b"
|
|
elif pm == "s390x":
|
|
DEFAULT_IMAGE = "quay.io/pypa/manylinux2014_s390x:2020-05-17-2f8ac3b"
|
|
else:
|
|
DEFAULT_IMAGE = ""
|
|
|
|
|
|
@pytest.fixture(params=["docker", "podman"])
|
|
def container_engine(request):
|
|
if request.param == "docker" and not request.config.getoption("--run-docker"):
|
|
pytest.skip("need --run-docker option to run")
|
|
if request.param == "podman" and not request.config.getoption("--run-podman"):
|
|
pytest.skip("need --run-podman option to run")
|
|
return request.param
|
|
|
|
|
|
# Tests
|
|
|
|
|
|
def test_simple(container_engine):
|
|
with OCIContainer(engine=container_engine, image=DEFAULT_IMAGE) as container:
|
|
assert container.call(["echo", "hello"], capture_output=True) == "hello\n"
|
|
|
|
|
|
def test_no_lf(container_engine):
|
|
with OCIContainer(engine=container_engine, image=DEFAULT_IMAGE) as container:
|
|
assert container.call(["printf", "hello"], capture_output=True) == "hello"
|
|
|
|
|
|
def test_debug_info(container_engine):
|
|
container = OCIContainer(engine=container_engine, image=DEFAULT_IMAGE)
|
|
print(container.debug_info())
|
|
with container:
|
|
pass
|
|
|
|
|
|
def test_environment(container_engine):
|
|
with OCIContainer(engine=container_engine, image=DEFAULT_IMAGE) as container:
|
|
assert (
|
|
container.call(
|
|
["sh", "-c", "echo $TEST_VAR"], env={"TEST_VAR": "1"}, capture_output=True
|
|
)
|
|
== "1\n"
|
|
)
|
|
|
|
|
|
def test_cwd(container_engine):
|
|
with OCIContainer(
|
|
engine=container_engine, image=DEFAULT_IMAGE, cwd="/cibuildwheel/working_directory"
|
|
) as container:
|
|
assert container.call(["pwd"], capture_output=True) == "/cibuildwheel/working_directory\n"
|
|
assert container.call(["pwd"], capture_output=True, cwd="/opt") == "/opt\n"
|
|
|
|
|
|
def test_container_removed(container_engine):
|
|
with OCIContainer(engine=container_engine, image=DEFAULT_IMAGE) as container:
|
|
docker_containers_listing = subprocess.run(
|
|
f"{container.engine} container ls",
|
|
shell=True,
|
|
check=True,
|
|
stdout=subprocess.PIPE,
|
|
universal_newlines=True,
|
|
).stdout
|
|
assert container.name is not None
|
|
assert container.name in docker_containers_listing
|
|
old_container_name = container.name
|
|
|
|
docker_containers_listing = subprocess.run(
|
|
f"{container.engine} container ls",
|
|
shell=True,
|
|
check=True,
|
|
stdout=subprocess.PIPE,
|
|
universal_newlines=True,
|
|
).stdout
|
|
assert old_container_name not in docker_containers_listing
|
|
|
|
|
|
def test_large_environment(container_engine):
|
|
# max environment variable size is 128kB
|
|
long_env_var_length = 127 * 1024
|
|
large_environment = {
|
|
"a": "0" * long_env_var_length,
|
|
"b": "0" * long_env_var_length,
|
|
"c": "0" * long_env_var_length,
|
|
"d": "0" * long_env_var_length,
|
|
}
|
|
|
|
with OCIContainer(engine=container_engine, image=DEFAULT_IMAGE) as container:
|
|
# check the length of d
|
|
assert (
|
|
container.call(["sh", "-c", "echo ${#d}"], env=large_environment, capture_output=True)
|
|
== f"{long_env_var_length}\n"
|
|
)
|
|
|
|
|
|
def test_binary_output(container_engine):
|
|
with OCIContainer(engine=container_engine, image=DEFAULT_IMAGE) as container:
|
|
# note: the below embedded snippets are in python2
|
|
|
|
# check that we can pass though arbitrary binary data without erroring
|
|
container.call(
|
|
[
|
|
"/usr/bin/python2",
|
|
"-c",
|
|
textwrap.dedent(
|
|
"""
|
|
import sys
|
|
sys.stdout.write(''.join(chr(n) for n in range(0, 256)))
|
|
"""
|
|
),
|
|
]
|
|
)
|
|
|
|
# check that we can capture arbitrary binary data
|
|
output = container.call(
|
|
[
|
|
"/usr/bin/python2",
|
|
"-c",
|
|
textwrap.dedent(
|
|
"""
|
|
import sys
|
|
sys.stdout.write(''.join(chr(n % 256) for n in range(0, 512)))
|
|
"""
|
|
),
|
|
],
|
|
capture_output=True,
|
|
)
|
|
|
|
data = bytes(output, encoding="utf8", errors="surrogateescape")
|
|
|
|
for i in range(512):
|
|
assert data[i] == i % 256
|
|
|
|
# check that environment variables can carry binary data, except null characters
|
|
# (https://www.gnu.org/software/libc/manual/html_node/Environment-Variables.html)
|
|
binary_data = bytes(n for n in range(1, 256))
|
|
binary_data_string = str(binary_data, encoding="utf8", errors="surrogateescape")
|
|
output = container.call(
|
|
["python2", "-c", 'import os, sys; sys.stdout.write(os.environ["TEST_VAR"])'],
|
|
env={"TEST_VAR": binary_data_string},
|
|
capture_output=True,
|
|
)
|
|
assert output == binary_data_string
|
|
|
|
|
|
def test_file_operation(tmp_path: Path, container_engine):
|
|
with OCIContainer(engine=container_engine, image=DEFAULT_IMAGE) as container:
|
|
# test copying a file in
|
|
test_binary_data = bytes(random.randrange(256) for _ in range(1000))
|
|
original_test_file = tmp_path / "test.dat"
|
|
original_test_file.write_bytes(test_binary_data)
|
|
|
|
dst_file = PurePath("/tmp/test.dat")
|
|
|
|
container.copy_into(original_test_file, dst_file)
|
|
|
|
output = container.call(["cat", dst_file], capture_output=True)
|
|
assert test_binary_data == bytes(output, encoding="utf8", errors="surrogateescape")
|
|
|
|
|
|
def test_dir_operations(tmp_path: Path, container_engine):
|
|
with OCIContainer(engine=container_engine, image=DEFAULT_IMAGE) as container:
|
|
test_binary_data = bytes(random.randrange(256) for _ in range(1000))
|
|
original_test_file = tmp_path / "test.dat"
|
|
original_test_file.write_bytes(test_binary_data)
|
|
|
|
# test copying a dir in
|
|
test_dir = tmp_path / "test_dir"
|
|
test_dir.mkdir()
|
|
test_file = test_dir / "test.dat"
|
|
shutil.copyfile(original_test_file, test_file)
|
|
|
|
dst_dir = PurePosixPath("/tmp/test_dir")
|
|
dst_file = dst_dir / "test.dat"
|
|
container.copy_into(test_dir, dst_dir)
|
|
|
|
output = container.call(["cat", dst_file], capture_output=True)
|
|
assert test_binary_data == bytes(output, encoding="utf8", errors="surrogateescape")
|
|
|
|
# test glob
|
|
assert container.glob(dst_dir, "*.dat") == [dst_file]
|
|
|
|
# test copy dir out
|
|
new_test_dir = tmp_path / "test_dir_new"
|
|
container.copy_out(dst_dir, new_test_dir)
|
|
|
|
assert test_binary_data == (new_test_dir / "test.dat").read_bytes()
|
|
|
|
|
|
def test_environment_executor(container_engine):
|
|
with OCIContainer(engine=container_engine, image=DEFAULT_IMAGE) as container:
|
|
assignment = EnvironmentAssignmentBash("TEST=$(echo 42)")
|
|
assert assignment.evaluated_value({}, container.environment_executor) == "42"
|
|
|
|
|
|
def test_podman_vfs(tmp_path: Path, monkeypatch, request):
|
|
# Tests podman VFS, for the podman in docker use-case
|
|
if not request.config.getoption("--run-podman"):
|
|
pytest.skip("need --run-podman option to run")
|
|
|
|
# create the VFS configuration
|
|
vfs_path = tmp_path / "podman_vfs"
|
|
vfs_path.mkdir()
|
|
|
|
# This requires that we write configuration files and point to them
|
|
# with environment variables before we run podman
|
|
# https://github.com/containers/common/blob/main/docs/containers.conf.5.md
|
|
vfs_containers_conf_data = {
|
|
"containers": {
|
|
"default_capabilities": [
|
|
"CHOWN",
|
|
"DAC_OVERRIDE",
|
|
"FOWNER",
|
|
"FSETID",
|
|
"KILL",
|
|
"NET_BIND_SERVICE",
|
|
"SETFCAP",
|
|
"SETGID",
|
|
"SETPCAP",
|
|
"SETUID",
|
|
"SYS_CHROOT",
|
|
]
|
|
},
|
|
"engine": {"cgroup_manager": "cgroupfs", "events_logger": "file"},
|
|
}
|
|
# https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md
|
|
storage_root = vfs_path / ".local/share/containers/vfs-storage"
|
|
run_root = vfs_path / ".local/share/containers/vfs-runroot"
|
|
storage_root.mkdir(parents=True, exist_ok=True)
|
|
run_root.mkdir(parents=True, exist_ok=True)
|
|
vfs_containers_storage_conf_data = {
|
|
"storage": {
|
|
"driver": "vfs",
|
|
"graphroot": os.fspath(storage_root),
|
|
"runroot": os.fspath(run_root),
|
|
"rootless_storage_path": os.fspath(storage_root),
|
|
"options": {
|
|
# "remap-user": "containers",
|
|
"aufs": {"mountopt": "rw"},
|
|
"overlay": {"mountopt": "rw", "force_mask": "shared"},
|
|
# "vfs": {"ignore_chown_errors": "true"},
|
|
},
|
|
}
|
|
}
|
|
|
|
vfs_containers_conf_fpath = vfs_path / "temp_vfs_containers.conf"
|
|
vfs_containers_storage_conf_fpath = vfs_path / "temp_vfs_containers_storage.conf"
|
|
with open(vfs_containers_conf_fpath, "w") as file:
|
|
toml.dump(vfs_containers_conf_data, file)
|
|
|
|
with open(vfs_containers_storage_conf_fpath, "w") as file:
|
|
toml.dump(vfs_containers_storage_conf_data, file)
|
|
|
|
monkeypatch.setenv("CONTAINERS_CONF", str(vfs_containers_conf_fpath))
|
|
monkeypatch.setenv("CONTAINERS_STORAGE_CONF", str(vfs_containers_storage_conf_fpath))
|
|
|
|
with OCIContainer(engine="podman", image=DEFAULT_IMAGE) as container:
|
|
# test running a command
|
|
assert container.call(["echo", "hello"], capture_output=True) == "hello\n"
|
|
|
|
# test copying a file into the container
|
|
(tmp_path / "some_file.txt").write_text("1234")
|
|
container.copy_into(tmp_path / "some_file.txt", PurePosixPath("some_file.txt"))
|
|
assert container.call(["cat", "some_file.txt"], capture_output=True) == "1234"
|
|
|
|
# Clean up
|
|
|
|
# When using the VFS, user is not given write permissions by default in
|
|
# new directories. As a workaround we use 'podman unshare' to delete them
|
|
# as UID 0. The reason why permission errors occur on podman is documented
|
|
# in https://podman.io/blogs/2018/10/03/podman-remove-content-homedir.html
|
|
subprocess.run(["podman", "unshare", "rm", "-rf", vfs_path], check=True)
|