Flattened podman support commits

.github/install_latest_podman.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+__doc__="
+Based on code in: https://github.com/redhat-actions/podman-login/blob/main/.github/install_latest_podman.sh
+"
+# https://podman.io/getting-started/installation
+# shellcheck source=/dev/null
+. /etc/os-release
+echo "deb https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/ /" | sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
+curl -sSfL "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_${VERSION_ID}/Release.key" | sudo apt-key add -
+sudo apt-get update
+sudo apt-get -y upgrade
+sudo apt-get -y install podman

.github/workflows/test.yml

@@ -45,6 +45,13 @@ jobs:
       with:
         python-version: ${{ matrix.python_version }}
 
+    # Install podman on this CI instance for podman tests on linux
+    # Snippet from: https://github.com/redhat-actions/podman-login/blob/main/.github/workflows/example.yml
+    - name: Install latest podman
+      if: runner.os == 'Linux'
+      run: |
+        bash .github/install_latest_podman.sh
+
     - name: Install dependencies
       run: |
         python -m pip install ".[test]"

cibuildwheel/docker_container.py

@@ -1,4 +1,5 @@
 import io
+import os
 import json
 import platform
 import shlex
@@ -25,6 +26,18 @@ class DockerContainer:
     A bash shell is running in the remote container. When `call()` is invoked,
     the command is relayed to the remote shell, and the results are streamed
     back to cibuildwheel.
+
+    TODO:
+        - [ ] Rename to Container as this now generalizes docker and podman?
+
+    Example:
+        >>> from cibuildwheel.docker_container import *  # NOQA
+        >>> docker_image = "quay.io/pypa/manylinux2014_x86_64:2020-05-17-2f8ac3b"
+        >>> # Test the default container
+        >>> with DockerContainer(docker_image=docker_image) as self:
+        ...     self.call(["echo", "hello world"])
+        ...     self.call(["cat", "/proc/1/cgroup"])
+        ...     print(self.get_environment())
     """
 
     UTILITY_PYTHON = "/opt/python/cp38-cp38/bin/python"
@@ -34,7 +47,13 @@ class DockerContainer:
     bash_stdout: IO[bytes]
 
     def __init__(
-        self, *, docker_image: str, simulate_32_bit: bool = False, cwd: Optional[PathOrStr] = None
+        self,
+        *,
+        docker_image: str,
+        simulate_32_bit: bool = False,
+        cwd: Optional[PathOrStr] = None,
+        container_engine: str = "docker",
+        env: Optional[Dict[str, str]] = None,
     ):
         if not docker_image:
             raise ValueError("Must have a non-empty docker image to run.")
@@ -43,10 +62,12 @@ class DockerContainer:
         self.simulate_32_bit = simulate_32_bit
         self.cwd = cwd
         self.name: Optional[str] = None
+        self.container_engine = container_engine
+        self.env = env  # If specified, overwrite environment variables
 
     def __enter__(self) -> "DockerContainer":
+
         self.name = f"cibuildwheel-{uuid.uuid4()}"
-        cwd_args = ["-w", str(self.cwd)] if self.cwd else []
 
         # work-around for Travis-CI PPC64le Docker runs since 2021:
         # this avoids network splits
@@ -57,24 +78,27 @@ class DockerContainer:
             network_args = ["--network=host"]
 
         shell_args = ["linux32", "/bin/bash"] if self.simulate_32_bit else ["/bin/bash"]
+
         subprocess.run(
             [
-                "docker",
+                self.container_engine,
                 "create",
                 "--env=CIBUILDWHEEL",
                 f"--name={self.name}",
                 "--interactive",
-                "--volume=/:/host",  # ignored on CircleCI
                 *network_args,
-                *cwd_args,
+                # Z-flags is for SELinux
+                "--volume=/:/host:Z",  # ignored on CircleCI
                 self.docker_image,
                 *shell_args,
             ],
+            env=self.env,
             check=True,
         )
+
         self.process = subprocess.Popen(
             [
-                "docker",
+                self.container_engine,
                 "start",
                 "--attach",
                 "--interactive",
@@ -82,6 +106,7 @@ class DockerContainer:
             ],
             stdin=subprocess.PIPE,
             stdout=subprocess.PIPE,
+            env=self.env,
         )
 
         assert self.process.stdin and self.process.stdout
@@ -89,7 +114,13 @@ class DockerContainer:
         self.bash_stdout = self.process.stdout
 
         # run a noop command to block until the container is responding
-        self.call(["/bin/true"])
+        self.call(["/bin/true"], cwd="")
+
+        if self.cwd:
+            # Although `docker create -w` does create the working dir if it
+            # does not exist, podman does not. There does not seem to be a way
+            # to setup a workdir for a container running in podman.
+            self.call(["mkdir", "-p", str(self.cwd)], cwd="")
 
         return self
 
@@ -106,10 +137,20 @@ class DockerContainer:
         self.bash_stdin.close()
         self.bash_stdout.close()
 
+        if self.container_engine == "podman":
+            # This works around what seems to be a race condition in the podman
+            # backend. The full reason is not understood. See PR #966 for a
+            # discussion on possible causes and attempts to remove this line.
+            # For now, this seems to work "well enough".
+            self.process.wait()
+
         assert isinstance(self.name, str)
 
         subprocess.run(
-            ["docker", "rm", "--force", "-v", self.name], stdout=subprocess.DEVNULL, check=False
+            [self.container_engine, "rm", "--force", "-v", self.name],
+            stdout=subprocess.DEVNULL,
+            env=self.env,
+            check=False,
         )
         self.name = None
 
@@ -122,15 +163,16 @@ class DockerContainer:
         if from_path.is_dir():
             self.call(["mkdir", "-p", to_path])
             subprocess.run(
-                f"tar cf - . | docker exec -i {self.name} tar --no-same-owner -xC {shell_quote(to_path)} -f -",
+                f"tar cf - . | {self.container_engine} exec -i {self.name} tar --no-same-owner -xC {shell_quote(to_path)} -f -",
                 shell=True,
                 check=True,
                 cwd=from_path,
+                env=self.env,
             )
         else:
             with subprocess.Popen(
                 [
-                    "docker",
+                    "{self.container_engine}",
                     "exec",
                     "-i",
                     str(self.name),
@@ -138,6 +180,7 @@ class DockerContainer:
                     "-c",
                     f"cat > {shell_quote(to_path)}",
                 ],
+                env=self.env,
                 stdin=subprocess.PIPE,
             ) as docker:
                 docker.stdin = cast(IO[bytes], docker.stdin)
@@ -155,13 +198,48 @@ class DockerContainer:
         # note: we assume from_path is a dir
         to_path.mkdir(parents=True, exist_ok=True)
 
+        if self.container_engine == "podman":
+            # The copy out logic that works for docker does not seem to
+            # translate to podman, which seems to need the steps spelled out
+            # more explicitly.
+            command = f"{self.container_engine} exec -i {self.name} tar -cC {shell_quote(from_path)} -f /tmp/output-{self.name}.tar ."
             subprocess.run(
-            f"docker exec -i {self.name} tar -cC {shell_quote(from_path)} -f - . | tar -xf -",
+                command,
                 shell=True,
                 check=True,
                 cwd=to_path,
+                env=self.env,
             )
 
+            command = f"{self.container_engine} cp {self.name}:/tmp/output-{self.name}.tar output-{self.name}.tar"
+            subprocess.run(
+                command,
+                shell=True,
+                check=True,
+                cwd=to_path,
+                env=self.env,
+            )
+            command = f"tar -xvf output-{self.name}.tar"
+            subprocess.run(
+                command,
+                shell=True,
+                check=True,
+                cwd=to_path,
+                env=self.env,
+            )
+            os.unlink(to_path / f"output-{self.name}.tar")
+        elif self.container_engine == "docker":
+            command = f"{self.container_engine} exec -i {self.name} tar -cC {shell_quote(from_path)} -f - . | tar -xf -"
+            subprocess.run(
+                command,
+                shell=True,
+                check=True,
+                cwd=to_path,
+                env=self.env,
+            )
+        else:
+            raise KeyError(self.container_engine)
+
     def glob(self, path: PurePosixPath, pattern: str) -> List[PurePosixPath]:
         glob_pattern = path.joinpath(pattern)
 
@@ -186,6 +264,11 @@ class DockerContainer:
         cwd: Optional[PathOrStr] = None,
     ) -> str:
 
+        if cwd is None:
+            # Podman does not start the a container in a specific working dir
+            # so we always need to specify it when making calls.
+            cwd = self.cwd
+
         chdir = f"cd {cwd}" if cwd else ""
         env_assignments = (
             " ".join(f"{shlex.quote(k)}={shlex.quote(v)}" for k, v in env.items())
@@ -271,6 +354,23 @@ class DockerContainer:
         # used as an EnvironmentExecutor to evaluate commands and capture output
         return self.call(command, env=environment, capture_output=True)
 
+    def debug_info(self) -> str:
+        if self.container_engine == "podman":
+            command = f"{self.container_engine} info --debug"
+        else:
+            command = f"{self.container_engine} info"
+        completed = subprocess.run(
+            command,
+            shell=True,
+            check=True,
+            cwd=self.cwd,
+            env=self.env,
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+        )
+        output = str(completed.stdout, encoding="utf8", errors="surrogateescape")
+        return output
+
 
 def shell_quote(path: PurePath) -> str:
     return shlex.quote(str(path))

cibuildwheel/linux.py

@@ -320,12 +320,16 @@ def build_on_docker(
 
 
 def build(options: Options, tmp_path: Path) -> None:  # pylint: disable=unused-argument
+
+    build_opts = options.build_options(None)
     try:
         # check docker is installed
-        subprocess.run(["docker", "--version"], check=True, stdout=subprocess.DEVNULL)
+        subprocess.run(
+            [build_opts.container_engine, "--version"], check=True, stdout=subprocess.DEVNULL
+        )
     except subprocess.CalledProcessError:
         print(
-            "cibuildwheel: Docker not found. Docker is required to run Linux builds. "
+            f"cibuildwheel: {build_opts.container_engine} not found. An OCI exe like Docker or Podman is required to run Linux builds "
             "If you're building on Travis CI, add `services: [docker]` to your .travis.yml."
             "If you're building on Circle CI in Linux, add a `setup_remote_docker` step to your .circleci/config.yml",
             file=sys.stderr,
@@ -355,6 +359,7 @@ def build(options: Options, tmp_path: Path) -> None:  # pylint: disable=unused-a
                 docker_image=build_step.docker_image,
                 simulate_32_bit=build_step.platform_tag.endswith("i686"),
                 cwd=container_project_path,
+                container_engine=build_opts.container_engine,
             ) as docker:
 
                 build_on_docker(

cibuildwheel/options.py

@@ -79,6 +79,7 @@ class BuildOptions(NamedTuple):
     test_extras: str
     build_verbosity: int
     build_frontend: BuildFrontend
+    container_engine: str
 
     @property
     def package_dir(self) -> Path:
@@ -422,6 +423,7 @@ class Options:
             test_requires = self.reader.get("test-requires", sep=" ").split()
             test_extras = self.reader.get("test-extras", sep=",")
             build_verbosity_str = self.reader.get("build-verbosity")
+            container_engine = self.reader.get("container-engine")
 
             build_frontend: BuildFrontend
             if build_frontend_str == "build":
@@ -520,6 +522,7 @@ class Options:
                 manylinux_images=manylinux_images or None,
                 musllinux_images=musllinux_images or None,
                 build_frontend=build_frontend,
+                container_engine=container_engine,
             )
 
     def check_for_invalid_configuration(self, identifiers: List[str]) -> None:

cibuildwheel/resources/defaults.toml

@@ -19,6 +19,8 @@ before-test = ""
 test-requires = []
 test-extras = []
 
+container-engine = "docker"
+
 manylinux-x86_64-image = "manylinux2014"
 manylinux-i686-image = "manylinux2014"
 manylinux-aarch64-image = "manylinux2014"

setup.py

@@ -14,6 +14,7 @@ extras = {
         "pytest-timeout",
         "pytest-xdist",
         "build",
+        "toml",
     ],
     "bin": [
         "click",

unit_test/conftest.py

@@ -16,7 +16,7 @@ def pytest_configure(config):
 
 def pytest_collection_modifyitems(config, items):
     if config.getoption("--run-docker"):
-        # --run-docker given in cli: do not skip docker tests
+        # --run-docker given in cli: do not skip container tests
         return
     skip_docker = pytest.mark.skip(reason="need --run-docker option to run")
     for item in items:

unit_test/docker_container_test.py

@@ -1,11 +1,15 @@
+import atexit
+import os
 import platform
 import random
 import shutil
 import subprocess
+import tempfile
 import textwrap
 from pathlib import Path, PurePath, PurePosixPath
 
 import pytest
+import toml  # type: ignore[import]
 
 from cibuildwheel.docker_container import DockerContainer
 from cibuildwheel.environment import EnvironmentAssignmentBash
@@ -21,23 +25,178 @@ elif pm == "ppc64le":
     DEFAULT_IMAGE = "quay.io/pypa/manylinux2014_ppc64le:2020-05-17-2f8ac3b"
 elif pm == "s390x":
     DEFAULT_IMAGE = "quay.io/pypa/manylinux2014_s390x:2020-05-17-2f8ac3b"
+else:
+    DEFAULT_IMAGE = ""
+
+
+temp_test_dir = None
+
+
+@atexit.register
+def _cleanup_tempdir():
+    """
+    Cleans up any configuration written by :func:`basis_container_kwargs`.
+
+    For podman tests, the user is not given write permissions by default in new
+    directories. As a workaround chown them before trying to delete them.
+
+    It may be possible to handle this more cleanly in pytest itself, but using
+    atexit works well enough for now.
+    """
+    import stat
+
+    global temp_test_dir
+    if temp_test_dir is not None:
+        print(f"CLEANUP temp_test_dir = {temp_test_dir!r}")  # type: ignore[unreachable]
+        for r, ds, fs in os.walk(temp_test_dir.name):
+            for d in ds:
+                dpath = os.path.join(r, d)
+                if not os.path.islink(dpath):
+                    perms = os.lstat(dpath).st_mode
+                    try:
+                        os.chmod(dpath, stat.S_IWUSR | perms)
+                    except Exception as ex:
+                        print(f"issue with dpath = {dpath!r}, {ex!r}")
+
+            for f in fs:
+                fpath = os.path.join(r, f)
+                if not os.path.islink(fpath):
+                    perms = os.lstat(fpath).st_mode
+                    try:
+                        os.chmod(fpath, stat.S_IWUSR | perms)
+                    except Exception as ex:
+                        print(f"issue with fpath = {fpath!r}, {ex!r}")
+                    else:
+                        os.unlink(fpath)
+        try:
+            temp_test_dir.cleanup()
+        except Exception as ex:
+            print(f"Issue cleaning up ex = {ex!r}")
+    temp_test_dir = None
+
+
+def basis_container_kwargs():
+    """
+    Generate keyword args that can be passed to to :class:`DockerContainer`.
+
+    This is used with :func:`pytest.mark.parametrize` to run each test with
+    different configuraions of each supported containers engine.
+
+    For docker we test the default configuration.
+
+    For podman we test the default configuration and a configuration with VFS
+    (virtual file system) enabled as the storage driver.
+    """
+
+    global temp_test_dir
+    if temp_test_dir is None:
+        # Only setup the temp directory once for all tests
+        temp_test_dir = tempfile.TemporaryDirectory(prefix="cibw_test_")
+
+    HAVE_DOCKER = bool(shutil.which("docker"))
+    HAVE_PODMAN = bool(shutil.which("podman"))
+
+    if HAVE_DOCKER:
+        # Basic podman configuration
+        yield {"container_engine": "docker", "docker_image": DEFAULT_IMAGE}
+
+    if HAVE_PODMAN:
+        # Basic podman usage
+        yield {"container_engine": "podman", "docker_image": DEFAULT_IMAGE}
+
+        # VFS Podman usage (for the podman in docker use-case)
+        dpath = Path(temp_test_dir.name)
+
+        # This requires that we write configuration files and point to them
+        # with environment variables before we run podman
+        # https://github.com/containers/common/blob/main/docs/containers.conf.5.md
+        vfs_containers_conf_data = {
+            "containers": {
+                "default_capabilities": [
+                    "CHOWN",
+                    "DAC_OVERRIDE",
+                    "FOWNER",
+                    "FSETID",
+                    "KILL",
+                    "NET_BIND_SERVICE",
+                    "SETFCAP",
+                    "SETGID",
+                    "SETPCAP",
+                    "SETUID",
+                    "SYS_CHROOT",
+                ]
+            },
+            "engine": {"cgroup_manager": "cgroupfs", "events_logger": "file"},
+        }
+        # https://github.com/containers/storage/blob/main/docs/containers-storage.conf.5.md
+        storage_root = dpath / ".local/share/containers/vfs-storage"
+        run_root = dpath / ".local/share/containers/vfs-runroot"
+        storage_root.mkdir(parents=True, exist_ok=True)
+        run_root.mkdir(parents=True, exist_ok=True)
+        vfs_containers_storage_conf_data = {
+            "storage": {
+                "driver": "vfs",
+                "graphroot": str(storage_root),
+                "runroot": str(run_root),
+                "rootless_storage_path": str(storage_root),
+                "options": {
+                    # "remap-user": "containers",
+                    "aufs": {"mountopt": "rw"},
+                    "overlay": {"mountopt": "rw", "force_mask": "shared"},
+                    # "vfs": {"ignore_chown_errors": "true"},
+                },
+            }
+        }
+        vfs_containers_conf_fpath = dpath / "temp_vfs_containers.conf"
+        vfs_containers_storage_conf_fpath = dpath / "temp_vfs_containers_storage.conf"
+        with open(vfs_containers_conf_fpath, "w") as file:
+            toml.dump(vfs_containers_conf_data, file)
+
+        with open(vfs_containers_storage_conf_fpath, "w") as file:
+            toml.dump(vfs_containers_storage_conf_data, file)
+
+        oci_environ = os.environ.copy()
+        oci_environ.update(
+            {
+                "CONTAINERS_CONF": str(vfs_containers_conf_fpath),
+                "CONTAINERS_STORAGE_CONF": str(vfs_containers_storage_conf_fpath),
+            }
+        )
+
+        yield {
+            "container_engine": "podman",
+            "docker_image": DEFAULT_IMAGE,
+            "env": oci_environ,
+        }
 
 
 @pytest.mark.docker
-def test_simple():
+@pytest.mark.parametrize("container_kwargs", basis_container_kwargs())
-    with DockerContainer(docker_image=DEFAULT_IMAGE) as container:
+def test_simple(container_kwargs):
+    with DockerContainer(**container_kwargs) as container:
         assert container.call(["echo", "hello"], capture_output=True) == "hello\n"
 
 
 @pytest.mark.docker
-def test_no_lf():
+@pytest.mark.parametrize("container_kwargs", basis_container_kwargs())
-    with DockerContainer(docker_image=DEFAULT_IMAGE) as container:
+def test_no_lf(container_kwargs):
+    with DockerContainer(**container_kwargs) as container:
         assert container.call(["printf", "hello"], capture_output=True) == "hello"
 
 
 @pytest.mark.docker
-def test_environment():
+@pytest.mark.parametrize("container_kwargs", basis_container_kwargs())
-    with DockerContainer(docker_image=DEFAULT_IMAGE) as container:
+def test_debug_info(container_kwargs):
+    container = DockerContainer(**container_kwargs)
+    print(container.debug_info())
+    with container:
+        pass
+
+
+@pytest.mark.docker
+@pytest.mark.parametrize("container_kwargs", basis_container_kwargs())
+def test_environment(container_kwargs):
+    with DockerContainer(**container_kwargs) as container:
         assert (
             container.call(
                 ["sh", "-c", "echo $TEST_VAR"], env={"TEST_VAR": "1"}, capture_output=True
@@ -47,40 +206,43 @@ def test_environment():
 
 
 @pytest.mark.docker
-def test_cwd():
+@pytest.mark.parametrize("container_kwargs", basis_container_kwargs())
-    with DockerContainer(
+def test_cwd(container_kwargs):
-        docker_image=DEFAULT_IMAGE, cwd="/cibuildwheel/working_directory"
+    with DockerContainer(cwd="/cibuildwheel/working_directory", **container_kwargs) as container:
-    ) as container:
         assert container.call(["pwd"], capture_output=True) == "/cibuildwheel/working_directory\n"
         assert container.call(["pwd"], capture_output=True, cwd="/opt") == "/opt\n"
 
 
 @pytest.mark.docker
-def test_container_removed():
+@pytest.mark.parametrize("container_kwargs", basis_container_kwargs())
-    with DockerContainer(docker_image=DEFAULT_IMAGE) as container:
+def test_container_removed(container_kwargs):
+    with DockerContainer(**container_kwargs) as container:
         docker_containers_listing = subprocess.run(
-            "docker container ls",
+            f"{container.container_engine} container ls",
             shell=True,
             check=True,
             stdout=subprocess.PIPE,
             universal_newlines=True,
+            env=container.env,
         ).stdout
         assert container.name is not None
         assert container.name in docker_containers_listing
         old_container_name = container.name
 
     docker_containers_listing = subprocess.run(
-        "docker container ls",
+        f"{container.container_engine} container ls",
         shell=True,
         check=True,
         stdout=subprocess.PIPE,
         universal_newlines=True,
+        env=container.env,
     ).stdout
     assert old_container_name not in docker_containers_listing
 
 
 @pytest.mark.docker
-def test_large_environment():
+@pytest.mark.parametrize("container_kwargs", basis_container_kwargs())
+def test_large_environment(container_kwargs):
     # max environment variable size is 128kB
     long_env_var_length = 127 * 1024
     large_environment = {
@@ -90,7 +252,7 @@ def test_large_environment():
         "d": "0" * long_env_var_length,
     }
 
-    with DockerContainer(docker_image=DEFAULT_IMAGE) as container:
+    with DockerContainer(**container_kwargs) as container:
         # check the length of d
         assert (
             container.call(["sh", "-c", "echo ${#d}"], env=large_environment, capture_output=True)
@@ -99,8 +261,9 @@ def test_large_environment():
 
 
 @pytest.mark.docker
-def test_binary_output():
+@pytest.mark.parametrize("container_kwargs", basis_container_kwargs())
-    with DockerContainer(docker_image=DEFAULT_IMAGE) as container:
+def test_binary_output(container_kwargs):
+    with DockerContainer(**container_kwargs) as container:
         # note: the below embedded snippets are in python2
 
         # check that we can pass though arbitrary binary data without erroring
@@ -150,8 +313,9 @@ def test_binary_output():
 
 
 @pytest.mark.docker
-def test_file_operations(tmp_path: Path):
+@pytest.mark.parametrize("container_kwargs", basis_container_kwargs())
-    with DockerContainer(docker_image=DEFAULT_IMAGE) as container:
+def test_file_operation(tmp_path: Path, container_kwargs):
+    with DockerContainer(**container_kwargs) as container:
         # test copying a file in
         test_binary_data = bytes(random.randrange(256) for _ in range(1000))
         original_test_file = tmp_path / "test.dat"
@@ -166,8 +330,9 @@ def test_file_operations(tmp_path: Path):
 
 
 @pytest.mark.docker
-def test_dir_operations(tmp_path: Path):
+@pytest.mark.parametrize("container_kwargs", basis_container_kwargs())
-    with DockerContainer(docker_image=DEFAULT_IMAGE) as container:
+def test_dir_operations(tmp_path: Path, container_kwargs):
+    with DockerContainer(**container_kwargs) as container:
         test_binary_data = bytes(random.randrange(256) for _ in range(1000))
         original_test_file = tmp_path / "test.dat"
         original_test_file.write_bytes(test_binary_data)
@@ -196,7 +361,8 @@ def test_dir_operations(tmp_path: Path):
 
 
 @pytest.mark.docker
-def test_environment_executor():
+@pytest.mark.parametrize("container_kwargs", basis_container_kwargs())
-    with DockerContainer(docker_image=DEFAULT_IMAGE) as container:
+def test_environment_executor(container_kwargs):
+    with DockerContainer(**container_kwargs) as container:
         assignment = EnvironmentAssignmentBash("TEST=$(echo 42)")
         assert assignment.evaluated_value({}, container.environment_executor) == "42"