diff --git a/.github/install_latest_podman.sh b/.github/install_latest_podman.sh new file mode 100755 index 00000000..42eca8b9 --- /dev/null +++ b/.github/install_latest_podman.sh @@ -0,0 +1,12 @@ +#!/bin/bash +__doc__=" +Based on code in: https://github.com/redhat-actions/podman-login/blob/main/.github/install_latest_podman.sh +" +# https://podman.io/getting-started/installation +# shellcheck source=/dev/null +. /etc/os-release +echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list +curl -sSfL "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/Release.key" | sudo apt-key add - +sudo apt-get update +sudo apt-get -y upgrade +sudo apt-get -y install podman diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a588b526..63f3f5d4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,6 +45,13 @@ jobs: with: python-version: ${{ matrix.python_version }} + # Install podman on this CI instance for podman tests on linux + # Snippet from: https://github.com/redhat-actions/podman-login/blob/main/.github/workflows/example.yml + - name: Install latest podman + if: runner.os == 'Linux' + run: | + bash .github/install_latest_podman.sh + - name: Install dependencies run: | python -m pip install ".[test]" diff --git a/cibuildwheel/docker_container.py b/cibuildwheel/docker_container.py index 9dc3ab70..ff57b78d 100644 --- a/cibuildwheel/docker_container.py +++ b/cibuildwheel/docker_container.py @@ -1,4 +1,5 @@ import io +import os import json import platform import shlex @@ -25,6 +26,18 @@ class DockerContainer: A bash shell is running in the remote container. When `call()` is invoked, the command is relayed to the remote shell, and the results are streamed back to cibuildwheel. + + TODO: + - [ ] Rename to Container as this now generalizes docker and podman? + + Example: + >>> from cibuildwheel.docker_container import * # NOQA + >>> docker_image = "quay.io/pypa/manylinux2014_x86_64:2020-05-17-2f8ac3b" + >>> # Test the default container + >>> with DockerContainer(docker_image=docker_image) as self: + ... self.call(["echo", "hello world"]) + ... self.call(["cat", "/proc/1/cgroup"]) + ... print(self.get_environment()) """ UTILITY_PYTHON = "/opt/python/cp38-cp38/bin/python" @@ -34,7 +47,13 @@ class DockerContainer: bash_stdout: IO[bytes] def __init__( - self, *, docker_image: str, simulate_32_bit: bool = False, cwd: Optional[PathOrStr] = None + self, + *, + docker_image: str, + simulate_32_bit: bool = False, + cwd: Optional[PathOrStr] = None, + container_engine: str = "docker", + env: Optional[Dict[str, str]] = None, ): if not docker_image: raise ValueError("Must have a non-empty docker image to run.") @@ -43,10 +62,12 @@ class DockerContainer: self.simulate_32_bit = simulate_32_bit self.cwd = cwd self.name: Optional[str] = None + self.container_engine = container_engine + self.env = env # If specified, overwrite environment variables def __enter__(self) -> "DockerContainer": + self.name = f"cibuildwheel-{uuid.uuid4()}" - cwd_args = ["-w", str(self.cwd)] if self.cwd else [] # work-around for Travis-CI PPC64le Docker runs since 2021: # this avoids network splits @@ -57,24 +78,27 @@ class DockerContainer: network_args = ["--network=host"] shell_args = ["linux32", "/bin/bash"] if self.simulate_32_bit else ["/bin/bash"] + subprocess.run( [ - "docker", + self.container_engine, "create", "--env=CIBUILDWHEEL", f"--name={self.name}", "--interactive", - "--volume=/:/host", # ignored on CircleCI *network_args, - *cwd_args, + # Z-flags is for SELinux + "--volume=/:/host:Z", # ignored on CircleCI self.docker_image, *shell_args, ], + env=self.env, check=True, ) + self.process = subprocess.Popen( [ - "docker", + self.container_engine, "start", "--attach", "--interactive", @@ -82,6 +106,7 @@ class DockerContainer: ], stdin=subprocess.PIPE, stdout=subprocess.PIPE, + env=self.env, ) assert self.process.stdin and self.process.stdout @@ -89,7 +114,13 @@ class DockerContainer: self.bash_stdout = self.process.stdout # run a noop command to block until the container is responding - self.call(["/bin/true"]) + self.call(["/bin/true"], cwd="") + + if self.cwd: + # Although `docker create -w` does create the working dir if it + # does not exist, podman does not. There does not seem to be a way + # to setup a workdir for a container running in podman. + self.call(["mkdir", "-p", str(self.cwd)], cwd="") return self @@ -106,10 +137,20 @@ class DockerContainer: self.bash_stdin.close() self.bash_stdout.close() + if self.container_engine == "podman": + # This works around what seems to be a race condition in the podman + # backend. The full reason is not understood. See PR #966 for a + # discussion on possible causes and attempts to remove this line. + # For now, this seems to work "well enough". + self.process.wait() + assert isinstance(self.name, str) subprocess.run( - ["docker", "rm", "--force", "-v", self.name], stdout=subprocess.DEVNULL, check=False + [self.container_engine, "rm", "--force", "-v", self.name], + stdout=subprocess.DEVNULL, + env=self.env, + check=False, ) self.name = None @@ -122,15 +163,16 @@ class DockerContainer: if from_path.is_dir(): self.call(["mkdir", "-p", to_path]) subprocess.run( - f"tar cf - . | docker exec -i {self.name} tar --no-same-owner -xC {shell_quote(to_path)} -f -", + f"tar cf - . | {self.container_engine} exec -i {self.name} tar --no-same-owner -xC {shell_quote(to_path)} -f -", shell=True, check=True, cwd=from_path, + env=self.env, ) else: with subprocess.Popen( [ - "docker", + "{self.container_engine}", "exec", "-i", str(self.name), @@ -138,6 +180,7 @@ class DockerContainer: "-c", f"cat > {shell_quote(to_path)}", ], + env=self.env, stdin=subprocess.PIPE, ) as docker: docker.stdin = cast(IO[bytes], docker.stdin) @@ -155,12 +198,47 @@ class DockerContainer: # note: we assume from_path is a dir to_path.mkdir(parents=True, exist_ok=True) - subprocess.run( - f"docker exec -i {self.name} tar -cC {shell_quote(from_path)} -f - . | tar -xf -", - shell=True, - check=True, - cwd=to_path, - ) + if self.container_engine == "podman": + # The copy out logic that works for docker does not seem to + # translate to podman, which seems to need the steps spelled out + # more explicitly. + command = f"{self.container_engine} exec -i {self.name} tar -cC {shell_quote(from_path)} -f /tmp/output-{self.name}.tar ." + subprocess.run( + command, + shell=True, + check=True, + cwd=to_path, + env=self.env, + ) + + command = f"{self.container_engine} cp {self.name}:/tmp/output-{self.name}.tar output-{self.name}.tar" + subprocess.run( + command, + shell=True, + check=True, + cwd=to_path, + env=self.env, + ) + command = f"tar -xvf output-{self.name}.tar" + subprocess.run( + command, + shell=True, + check=True, + cwd=to_path, + env=self.env, + ) + os.unlink(to_path / f"output-{self.name}.tar") + elif self.container_engine == "docker": + command = f"{self.container_engine} exec -i {self.name} tar -cC {shell_quote(from_path)} -f - . | tar -xf -" + subprocess.run( + command, + shell=True, + check=True, + cwd=to_path, + env=self.env, + ) + else: + raise KeyError(self.container_engine) def glob(self, path: PurePosixPath, pattern: str) -> List[PurePosixPath]: glob_pattern = path.joinpath(pattern) @@ -186,6 +264,11 @@ class DockerContainer: cwd: Optional[PathOrStr] = None, ) -> str: + if cwd is None: + # Podman does not start the a container in a specific working dir + # so we always need to specify it when making calls. + cwd = self.cwd + chdir = f"cd {cwd}" if cwd else "" env_assignments = ( " ".join(f"{shlex.quote(k)}={shlex.quote(v)}" for k, v in env.items()) @@ -271,6 +354,23 @@ class DockerContainer: # used as an EnvironmentExecutor to evaluate commands and capture output return self.call(command, env=environment, capture_output=True) + def debug_info(self) -> str: + if self.container_engine == "podman": + command = f"{self.container_engine} info --debug" + else: + command = f"{self.container_engine} info" + completed = subprocess.run( + command, + shell=True, + check=True, + cwd=self.cwd, + env=self.env, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + output = str(completed.stdout, encoding="utf8", errors="surrogateescape") + return output + def shell_quote(path: PurePath) -> str: return shlex.quote(str(path)) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 93225c21..e89a35b6 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -320,12 +320,16 @@ def build_on_docker( def build(options: Options, tmp_path: Path) -> None: # pylint: disable=unused-argument + + build_opts = options.build_options(None) try: # check docker is installed - subprocess.run(["docker", "--version"], check=True, stdout=subprocess.DEVNULL) + subprocess.run( + [build_opts.container_engine, "--version"], check=True, stdout=subprocess.DEVNULL + ) except subprocess.CalledProcessError: print( - "cibuildwheel: Docker not found. Docker is required to run Linux builds. " + f"cibuildwheel: {build_opts.container_engine} not found. An OCI exe like Docker or Podman is required to run Linux builds " "If you're building on Travis CI, add `services: [docker]` to your .travis.yml." "If you're building on Circle CI in Linux, add a `setup_remote_docker` step to your .circleci/config.yml", file=sys.stderr, @@ -355,6 +359,7 @@ def build(options: Options, tmp_path: Path) -> None: # pylint: disable=unused-a docker_image=build_step.docker_image, simulate_32_bit=build_step.platform_tag.endswith("i686"), cwd=container_project_path, + container_engine=build_opts.container_engine, ) as docker: build_on_docker( diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index 5d9744af..80ee5327 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -79,6 +79,7 @@ class BuildOptions(NamedTuple): test_extras: str build_verbosity: int build_frontend: BuildFrontend + container_engine: str @property def package_dir(self) -> Path: @@ -422,6 +423,7 @@ class Options: test_requires = self.reader.get("test-requires", sep=" ").split() test_extras = self.reader.get("test-extras", sep=",") build_verbosity_str = self.reader.get("build-verbosity") + container_engine = self.reader.get("container-engine") build_frontend: BuildFrontend if build_frontend_str == "build": @@ -520,6 +522,7 @@ class Options: manylinux_images=manylinux_images or None, musllinux_images=musllinux_images or None, build_frontend=build_frontend, + container_engine=container_engine, ) def check_for_invalid_configuration(self, identifiers: List[str]) -> None: diff --git a/cibuildwheel/resources/defaults.toml b/cibuildwheel/resources/defaults.toml index 5890c8ed..5e0bdacf 100644 --- a/cibuildwheel/resources/defaults.toml +++ b/cibuildwheel/resources/defaults.toml @@ -19,6 +19,8 @@ before-test = "" test-requires = [] test-extras = [] +container-engine = "docker" + manylinux-x86_64-image = "manylinux2014" manylinux-i686-image = "manylinux2014" manylinux-aarch64-image = "manylinux2014" diff --git a/setup.py b/setup.py index 7a9eee68..ad32d3f3 100644 --- a/setup.py +++ b/setup.py @@ -14,6 +14,7 @@ extras = { "pytest-timeout", "pytest-xdist", "build", + "toml", ], "bin": [ "click", diff --git a/unit_test/conftest.py b/unit_test/conftest.py index 2f794a21..4e22853a 100644 --- a/unit_test/conftest.py +++ b/unit_test/conftest.py @@ -16,7 +16,7 @@ def pytest_configure(config): def pytest_collection_modifyitems(config, items): if config.getoption("--run-docker"): - # --run-docker given in cli: do not skip docker tests + # --run-docker given in cli: do not skip container tests return skip_docker = pytest.mark.skip(reason="need --run-docker option to run") for item in items: diff --git a/unit_test/docker_container_test.py b/unit_test/docker_container_test.py index e9c43e77..dd317ee2 100644 --- a/unit_test/docker_container_test.py +++ b/unit_test/docker_container_test.py @@ -1,11 +1,15 @@ +import atexit +import os import platform import random import shutil import subprocess +import tempfile import textwrap from pathlib import Path, PurePath, PurePosixPath import pytest +import toml # type: ignore[import] from cibuildwheel.docker_container import DockerContainer from cibuildwheel.environment import EnvironmentAssignmentBash @@ -21,23 +25,178 @@ elif pm == "ppc64le": DEFAULT_IMAGE = "quay.io/pypa/manylinux2014_ppc64le:2020-05-17-2f8ac3b" elif pm == "s390x": DEFAULT_IMAGE = "quay.io/pypa/manylinux2014_s390x:2020-05-17-2f8ac3b" +else: + DEFAULT_IMAGE = "" + + +temp_test_dir = None + + +@atexit.register +def _cleanup_tempdir(): + """ + Cleans up any configuration written by :func:`basis_container_kwargs`. + + For podman tests, the user is not given write permissions by default in new + directories. As a workaround chown them before trying to delete them. + + It may be possible to handle this more cleanly in pytest itself, but using + atexit works well enough for now. + """ + import stat + + global temp_test_dir + if temp_test_dir is not None: + print(f"CLEANUP temp_test_dir = {temp_test_dir!r}") # type: ignore[unreachable] + for r, ds, fs in os.walk(temp_test_dir.name): + for d in ds: + dpath = os.path.join(r, d) + if not os.path.islink(dpath): + perms = os.lstat(dpath).st_mode + try: + os.chmod(dpath, stat.S_IWUSR | perms) + except Exception as ex: + print(f"issue with dpath = {dpath!r}, {ex!r}") + + for f in fs: + fpath = os.path.join(r, f) + if not os.path.islink(fpath): + perms = os.lstat(fpath).st_mode + try: + os.chmod(fpath, stat.S_IWUSR | perms) + except Exception as ex: + print(f"issue with fpath = {fpath!r}, {ex!r}") + else: + os.unlink(fpath) + try: + temp_test_dir.cleanup() + except Exception as ex: + print(f"Issue cleaning up ex = {ex!r}") + temp_test_dir = None + + +def basis_container_kwargs(): + """ + Generate keyword args that can be passed to to :class:`DockerContainer`. + + This is used with :func:`pytest.mark.parametrize` to run each test with + different configuraions of each supported containers engine. + + For docker we test the default configuration. + + For podman we test the default configuration and a configuration with VFS + (virtual file system) enabled as the storage driver. + """ + + global temp_test_dir + if temp_test_dir is None: + # Only setup the temp directory once for all tests + temp_test_dir = tempfile.TemporaryDirectory(prefix="cibw_test_") + + HAVE_DOCKER = bool(shutil.which("docker")) + HAVE_PODMAN = bool(shutil.which("podman")) + + if HAVE_DOCKER: + # Basic podman configuration + yield {"container_engine": "docker", "docker_image": DEFAULT_IMAGE} + + if HAVE_PODMAN: + # Basic podman usage + yield {"container_engine": "podman", "docker_image": DEFAULT_IMAGE} + + # VFS Podman usage (for the podman in docker use-case) + dpath = Path(temp_test_dir.name) + + # This requires that we write configuration files and point to them + # with environment variables before we run podman + # https://github.com/containers/common/blob/main/docs/containers.conf.5.md + vfs_containers_conf_data = { + "containers": { + "default_capabilities": [ + "CHOWN", + "DAC_OVERRIDE", + "FOWNER", + "FSETID", + "KILL", + "NET_BIND_SERVICE", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_CHROOT", + ] + }, + "engine": {"cgroup_manager": "cgroupfs", "events_logger": "file"}, + } + # https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md + storage_root = dpath / ".local/share/containers/vfs-storage" + run_root = dpath / ".local/share/containers/vfs-runroot" + storage_root.mkdir(parents=True, exist_ok=True) + run_root.mkdir(parents=True, exist_ok=True) + vfs_containers_storage_conf_data = { + "storage": { + "driver": "vfs", + "graphroot": str(storage_root), + "runroot": str(run_root), + "rootless_storage_path": str(storage_root), + "options": { + # "remap-user": "containers", + "aufs": {"mountopt": "rw"}, + "overlay": {"mountopt": "rw", "force_mask": "shared"}, + # "vfs": {"ignore_chown_errors": "true"}, + }, + } + } + vfs_containers_conf_fpath = dpath / "temp_vfs_containers.conf" + vfs_containers_storage_conf_fpath = dpath / "temp_vfs_containers_storage.conf" + with open(vfs_containers_conf_fpath, "w") as file: + toml.dump(vfs_containers_conf_data, file) + + with open(vfs_containers_storage_conf_fpath, "w") as file: + toml.dump(vfs_containers_storage_conf_data, file) + + oci_environ = os.environ.copy() + oci_environ.update( + { + "CONTAINERS_CONF": str(vfs_containers_conf_fpath), + "CONTAINERS_STORAGE_CONF": str(vfs_containers_storage_conf_fpath), + } + ) + + yield { + "container_engine": "podman", + "docker_image": DEFAULT_IMAGE, + "env": oci_environ, + } @pytest.mark.docker -def test_simple(): - with DockerContainer(docker_image=DEFAULT_IMAGE) as container: +@pytest.mark.parametrize("container_kwargs", basis_container_kwargs()) +def test_simple(container_kwargs): + with DockerContainer(**container_kwargs) as container: assert container.call(["echo", "hello"], capture_output=True) == "hello\n" @pytest.mark.docker -def test_no_lf(): - with DockerContainer(docker_image=DEFAULT_IMAGE) as container: +@pytest.mark.parametrize("container_kwargs", basis_container_kwargs()) +def test_no_lf(container_kwargs): + with DockerContainer(**container_kwargs) as container: assert container.call(["printf", "hello"], capture_output=True) == "hello" @pytest.mark.docker -def test_environment(): - with DockerContainer(docker_image=DEFAULT_IMAGE) as container: +@pytest.mark.parametrize("container_kwargs", basis_container_kwargs()) +def test_debug_info(container_kwargs): + container = DockerContainer(**container_kwargs) + print(container.debug_info()) + with container: + pass + + +@pytest.mark.docker +@pytest.mark.parametrize("container_kwargs", basis_container_kwargs()) +def test_environment(container_kwargs): + with DockerContainer(**container_kwargs) as container: assert ( container.call( ["sh", "-c", "echo $TEST_VAR"], env={"TEST_VAR": "1"}, capture_output=True @@ -47,40 +206,43 @@ def test_environment(): @pytest.mark.docker -def test_cwd(): - with DockerContainer( - docker_image=DEFAULT_IMAGE, cwd="/cibuildwheel/working_directory" - ) as container: +@pytest.mark.parametrize("container_kwargs", basis_container_kwargs()) +def test_cwd(container_kwargs): + with DockerContainer(cwd="/cibuildwheel/working_directory", **container_kwargs) as container: assert container.call(["pwd"], capture_output=True) == "/cibuildwheel/working_directory\n" assert container.call(["pwd"], capture_output=True, cwd="/opt") == "/opt\n" @pytest.mark.docker -def test_container_removed(): - with DockerContainer(docker_image=DEFAULT_IMAGE) as container: +@pytest.mark.parametrize("container_kwargs", basis_container_kwargs()) +def test_container_removed(container_kwargs): + with DockerContainer(**container_kwargs) as container: docker_containers_listing = subprocess.run( - "docker container ls", + f"{container.container_engine} container ls", shell=True, check=True, stdout=subprocess.PIPE, universal_newlines=True, + env=container.env, ).stdout assert container.name is not None assert container.name in docker_containers_listing old_container_name = container.name docker_containers_listing = subprocess.run( - "docker container ls", + f"{container.container_engine} container ls", shell=True, check=True, stdout=subprocess.PIPE, universal_newlines=True, + env=container.env, ).stdout assert old_container_name not in docker_containers_listing @pytest.mark.docker -def test_large_environment(): +@pytest.mark.parametrize("container_kwargs", basis_container_kwargs()) +def test_large_environment(container_kwargs): # max environment variable size is 128kB long_env_var_length = 127 * 1024 large_environment = { @@ -90,7 +252,7 @@ def test_large_environment(): "d": "0" * long_env_var_length, } - with DockerContainer(docker_image=DEFAULT_IMAGE) as container: + with DockerContainer(**container_kwargs) as container: # check the length of d assert ( container.call(["sh", "-c", "echo ${#d}"], env=large_environment, capture_output=True) @@ -99,8 +261,9 @@ def test_large_environment(): @pytest.mark.docker -def test_binary_output(): - with DockerContainer(docker_image=DEFAULT_IMAGE) as container: +@pytest.mark.parametrize("container_kwargs", basis_container_kwargs()) +def test_binary_output(container_kwargs): + with DockerContainer(**container_kwargs) as container: # note: the below embedded snippets are in python2 # check that we can pass though arbitrary binary data without erroring @@ -150,8 +313,9 @@ def test_binary_output(): @pytest.mark.docker -def test_file_operations(tmp_path: Path): - with DockerContainer(docker_image=DEFAULT_IMAGE) as container: +@pytest.mark.parametrize("container_kwargs", basis_container_kwargs()) +def test_file_operation(tmp_path: Path, container_kwargs): + with DockerContainer(**container_kwargs) as container: # test copying a file in test_binary_data = bytes(random.randrange(256) for _ in range(1000)) original_test_file = tmp_path / "test.dat" @@ -166,8 +330,9 @@ def test_file_operations(tmp_path: Path): @pytest.mark.docker -def test_dir_operations(tmp_path: Path): - with DockerContainer(docker_image=DEFAULT_IMAGE) as container: +@pytest.mark.parametrize("container_kwargs", basis_container_kwargs()) +def test_dir_operations(tmp_path: Path, container_kwargs): + with DockerContainer(**container_kwargs) as container: test_binary_data = bytes(random.randrange(256) for _ in range(1000)) original_test_file = tmp_path / "test.dat" original_test_file.write_bytes(test_binary_data) @@ -196,7 +361,8 @@ def test_dir_operations(tmp_path: Path): @pytest.mark.docker -def test_environment_executor(): - with DockerContainer(docker_image=DEFAULT_IMAGE) as container: +@pytest.mark.parametrize("container_kwargs", basis_container_kwargs()) +def test_environment_executor(container_kwargs): + with DockerContainer(**container_kwargs) as container: assignment = EnvironmentAssignmentBash("TEST=$(echo 42)") assert assignment.evaluated_value({}, container.environment_executor) == "42"