import atexit import os import platform import random import shutil import subprocess import tempfile import textwrap from pathlib import Path, PurePath, PurePosixPath from tabnanny import check import pytest import toml from cibuildwheel.docker_container import DockerContainer from cibuildwheel.environment import EnvironmentAssignmentBash # Test utilities # for these tests we use manylinux2014 images, because they're available on # multi architectures and include python3.8 pm = platform.machine() if pm == "x86_64": DEFAULT_IMAGE = "quay.io/pypa/manylinux2014_x86_64:2020-05-17-2f8ac3b" elif pm == "aarch64": DEFAULT_IMAGE = "quay.io/pypa/manylinux2014_aarch64:2020-05-17-2f8ac3b" elif pm == "ppc64le": DEFAULT_IMAGE = "quay.io/pypa/manylinux2014_ppc64le:2020-05-17-2f8ac3b" elif pm == "s390x": DEFAULT_IMAGE = "quay.io/pypa/manylinux2014_s390x:2020-05-17-2f8ac3b" else: DEFAULT_IMAGE = "" @pytest.fixture(params=["docker", "podman"]) def container_engine(request): if request.param == "docker" and not request.config.getoption("--run-docker"): pytest.skip("need --run-docker option to run") if request.param == "podman" and not request.config.getoption("--run-podman"): pytest.skip("need --run-podman option to run") return request.param # Tests def test_simple(container_engine): with DockerContainer( container_engine=container_engine, docker_image=DEFAULT_IMAGE ) as container: assert container.call(["echo", "hello"], capture_output=True) == "hello\n" def test_no_lf(container_engine): with DockerContainer( container_engine=container_engine, docker_image=DEFAULT_IMAGE ) as container: assert container.call(["printf", "hello"], capture_output=True) == "hello" def test_debug_info(container_engine): container = DockerContainer(container_engine=container_engine, docker_image=DEFAULT_IMAGE) print(container.debug_info()) with container: pass def test_environment(container_engine): with DockerContainer( container_engine=container_engine, docker_image=DEFAULT_IMAGE ) as container: assert ( container.call( ["sh", "-c", "echo $TEST_VAR"], env={"TEST_VAR": "1"}, capture_output=True ) == "1\n" ) def test_cwd(container_engine): with DockerContainer( container_engine=container_engine, docker_image=DEFAULT_IMAGE, cwd="/cibuildwheel/working_directory", ) as container: assert container.call(["pwd"], capture_output=True) == "/cibuildwheel/working_directory\n" assert container.call(["pwd"], capture_output=True, cwd="/opt") == "/opt\n" def test_container_removed(container_engine): with DockerContainer( container_engine=container_engine, docker_image=DEFAULT_IMAGE ) as container: docker_containers_listing = subprocess.run( f"{container.container_engine} container ls", shell=True, check=True, stdout=subprocess.PIPE, universal_newlines=True, ).stdout assert container.name is not None assert container.name in docker_containers_listing old_container_name = container.name docker_containers_listing = subprocess.run( f"{container.container_engine} container ls", shell=True, check=True, stdout=subprocess.PIPE, universal_newlines=True, ).stdout assert old_container_name not in docker_containers_listing def test_large_environment(container_engine): # max environment variable size is 128kB long_env_var_length = 127 * 1024 large_environment = { "a": "0" * long_env_var_length, "b": "0" * long_env_var_length, "c": "0" * long_env_var_length, "d": "0" * long_env_var_length, } with DockerContainer( container_engine=container_engine, docker_image=DEFAULT_IMAGE ) as container: # check the length of d assert ( container.call(["sh", "-c", "echo ${#d}"], env=large_environment, capture_output=True) == f"{long_env_var_length}\n" ) def test_binary_output(container_engine): with DockerContainer( container_engine=container_engine, docker_image=DEFAULT_IMAGE ) as container: # note: the below embedded snippets are in python2 # check that we can pass though arbitrary binary data without erroring container.call( [ "/usr/bin/python2", "-c", textwrap.dedent( """ import sys sys.stdout.write(''.join(chr(n) for n in range(0, 256))) """ ), ] ) # check that we can capture arbitrary binary data output = container.call( [ "/usr/bin/python2", "-c", textwrap.dedent( """ import sys sys.stdout.write(''.join(chr(n % 256) for n in range(0, 512))) """ ), ], capture_output=True, ) data = bytes(output, encoding="utf8", errors="surrogateescape") for i in range(512): assert data[i] == i % 256 # check that environment variables can carry binary data, except null characters # (https://www.gnu.org/software/libc/manual/html_node/Environment-Variables.html) binary_data = bytes(n for n in range(1, 256)) binary_data_string = str(binary_data, encoding="utf8", errors="surrogateescape") output = container.call( ["python2", "-c", 'import os, sys; sys.stdout.write(os.environ["TEST_VAR"])'], env={"TEST_VAR": binary_data_string}, capture_output=True, ) assert output == binary_data_string def test_file_operation(tmp_path: Path, container_engine): with DockerContainer( container_engine=container_engine, docker_image=DEFAULT_IMAGE ) as container: # test copying a file in test_binary_data = bytes(random.randrange(256) for _ in range(1000)) original_test_file = tmp_path / "test.dat" original_test_file.write_bytes(test_binary_data) dst_file = PurePath("/tmp/test.dat") container.copy_into(original_test_file, dst_file) output = container.call(["cat", dst_file], capture_output=True) assert test_binary_data == bytes(output, encoding="utf8", errors="surrogateescape") def test_dir_operations(tmp_path: Path, container_engine): with DockerContainer( container_engine=container_engine, docker_image=DEFAULT_IMAGE ) as container: test_binary_data = bytes(random.randrange(256) for _ in range(1000)) original_test_file = tmp_path / "test.dat" original_test_file.write_bytes(test_binary_data) # test copying a dir in test_dir = tmp_path / "test_dir" test_dir.mkdir() test_file = test_dir / "test.dat" shutil.copyfile(original_test_file, test_file) dst_dir = PurePosixPath("/tmp/test_dir") dst_file = dst_dir / "test.dat" container.copy_into(test_dir, dst_dir) output = container.call(["cat", dst_file], capture_output=True) assert test_binary_data == bytes(output, encoding="utf8", errors="surrogateescape") # test glob assert container.glob(dst_dir, "*.dat") == [dst_file] # test copy dir out new_test_dir = tmp_path / "test_dir_new" container.copy_out(dst_dir, new_test_dir) assert test_binary_data == (new_test_dir / "test.dat").read_bytes() def test_environment_executor(container_engine): with DockerContainer( container_engine=container_engine, docker_image=DEFAULT_IMAGE ) as container: assignment = EnvironmentAssignmentBash("TEST=$(echo 42)") assert assignment.evaluated_value({}, container.environment_executor) == "42" def test_podman_vfs(container_engine, tmp_path: Path, monkeypatch): # Tests podman VFS, for the podman in docker use-case if container_engine != "podman": pytest.skip("podman is the only supported container engine for this test") # create the VFS configuration vfs_path = tmp_path / "podman_vfs" vfs_path.mkdir() # This requires that we write configuration files and point to them # with environment variables before we run podman # https://github.com/containers/common/blob/main/docs/containers.conf.5.md vfs_containers_conf_data = { "containers": { "default_capabilities": [ "CHOWN", "DAC_OVERRIDE", "FOWNER", "FSETID", "KILL", "NET_BIND_SERVICE", "SETFCAP", "SETGID", "SETPCAP", "SETUID", "SYS_CHROOT", ] }, "engine": {"cgroup_manager": "cgroupfs", "events_logger": "file"}, } # https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md storage_root = vfs_path / ".local/share/containers/vfs-storage" run_root = vfs_path / ".local/share/containers/vfs-runroot" storage_root.mkdir(parents=True, exist_ok=True) run_root.mkdir(parents=True, exist_ok=True) vfs_containers_storage_conf_data = { "storage": { "driver": "vfs", "graphroot": os.fspath(storage_root), "runroot": os.fspath(run_root), "rootless_storage_path": os.fspath(storage_root), "options": { # "remap-user": "containers", "aufs": {"mountopt": "rw"}, "overlay": {"mountopt": "rw", "force_mask": "shared"}, # "vfs": {"ignore_chown_errors": "true"}, }, } } vfs_containers_conf_fpath = vfs_path / "temp_vfs_containers.conf" vfs_containers_storage_conf_fpath = vfs_path / "temp_vfs_containers_storage.conf" with open(vfs_containers_conf_fpath, "w") as file: toml.dump(vfs_containers_conf_data, file) with open(vfs_containers_storage_conf_fpath, "w") as file: toml.dump(vfs_containers_storage_conf_data, file) monkeypatch.setenv("CONTAINERS_CONF", str(vfs_containers_conf_fpath)) monkeypatch.setenv("CONTAINERS_STORAGE_CONF", str(vfs_containers_storage_conf_fpath)) with DockerContainer( container_engine=container_engine, docker_image=DEFAULT_IMAGE ) as container: # test running a command assert container.call(["echo", "hello"], capture_output=True) == "hello\n" # test copying a file into the container (tmp_path / "some_file.txt").write_text("1234") container.copy_into(tmp_path / "some_file.txt", PurePosixPath("some_file.txt")) assert container.call(["cat", "some_file.txt"], capture_output=True) == "1234" # Clean up # When using the VFS, user is not given write permissions by default in # new directories. As a workaround we use 'podman unshare' to delete them # as UID 0. The reason why permission errors occur on podman is documented # in https://podman.io/blogs/2018/10/03/podman-remove-content-homedir.html subprocess.run(["podman", "unshare", "rm", "-rf", vfs_path], check=True)