feat: use digests instead of tags for pinned images (#2915)

test/conftest.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import dataclasses
 import json
 import os
 import subprocess
@@ -12,14 +13,16 @@ from cibuildwheel.ci import detect_ci_provider
 from cibuildwheel.options import CommandLineArguments, Options
 from cibuildwheel.selector import EnableGroup
 from cibuildwheel.typing import PLATFORMS
+from cibuildwheel.util.resources import PINNED_DOCKER_IMAGES
 from cibuildwheel.venv import find_uv
 
 from . import utils
-from .utils import DEFAULT_CIBW_ENABLE, EMULATED_ARCHS, get_platform
+from .utils import DEFAULT_CIBW_ENABLE, EMULATED_ARCHS, get_enable_groups, get_platform
 
 TYPE_CHECKING = False
 if TYPE_CHECKING:
     from collections.abc import Generator
+    from pathlib import Path
 
 
 def pytest_addoption(parser: pytest.Parser) -> None:
@@ -74,7 +77,19 @@ def pytest_configure(config: pytest.Config) -> None:
         os.environ["CIBW_PLATFORM"] = flag_platform
 
 
-def docker_warmup(request: pytest.FixtureRequest) -> None:
+@dataclasses.dataclass(frozen=True)
+class DockerWarmUpImage:
+    source_name: str
+    cached_name: str
+
+
+def get_docker_warmup_images(request: pytest.FixtureRequest) -> list[DockerWarmUpImage] | None:
+    enable_groups = get_enable_groups()
+    # Check missing pre-installed interpreters are needed
+    enable_groups &= {EnableGroup.GraalPy, EnableGroup.PyPyEoL}
+    if not enable_groups:
+        return None
+
     machine = request.config.getoption("--run-emulation", default=None)
     if machine is None:
         archs = {arch.value for arch in Architecture.auto_archs("linux")}
@@ -86,7 +101,7 @@ def docker_warmup(request: pytest.FixtureRequest) -> None:
     # Only include architectures where there are missing pre-installed interpreters
     archs &= {"x86_64", "i686", "aarch64"}
     if not archs:
-        return
+        return None
 
     options = Options(
         platform="linux",
@@ -96,17 +111,28 @@ def docker_warmup(request: pytest.FixtureRequest) -> None:
     )
     build_options = options.build_options(None)
     assert build_options.manylinux_images is not None
-    assert build_options.musllinux_images is not None
-    images = [build_options.manylinux_images[arch] for arch in archs] + [
-        build_options.musllinux_images[arch] for arch in archs
-    ]
+    images: list[DockerWarmUpImage] = []
+    for arch in archs:
+        source_name = build_options.manylinux_images[arch]
+        cached_name = source_name
+        if "@sha256:" in source_name:
+            cached_name = source_name.rsplit("@sha256:", 1)[0] + ":cibw_cache_fixture"
+        images.append(DockerWarmUpImage(source_name, cached_name))
+    if images:
+        return images
+    return None
+
+
+def docker_warmup(images: list[DockerWarmUpImage], docker_warmup_config: Path) -> None:
     command = (
         "manylinux-interpreters ensure-all &&"
         "cpython3.13 -m pip download -d /tmp setuptools wheel pytest"
     )
+    config_content = PINNED_DOCKER_IMAGES.read_text()
     for image in images:
+        config_content = config_content.replace(image.source_name, image.cached_name)
         container_id = subprocess.run(
-            ["docker", "create", image, "bash", "-c", command],
+            ["docker", "create", image.source_name, "bash", "-c", command],
             text=True,
             check=True,
             stdout=subprocess.PIPE,
@@ -118,37 +144,48 @@ def docker_warmup(request: pytest.FixtureRequest) -> None:
             ).stdout.strip()
             assert exit_code == "0"
             subprocess.run(
-                ["docker", "commit", container_id, image], check=True, stdout=subprocess.DEVNULL
+                ["docker", "commit", container_id, image.cached_name],
+                check=True,
+                stdout=subprocess.DEVNULL,
             )
         finally:
             subprocess.run(["docker", "rm", container_id], check=True, stdout=subprocess.DEVNULL)
+    docker_warmup_config.write_text(config_content)
 
 
 @pytest.fixture(scope="session", autouse=True)
 def docker_warmup_fixture(
     request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory, worker_id: str
-) -> None:
+) -> Generator[None, None, None]:
     # if we're in CI testing linux, let's warm-up docker images
     if detect_ci_provider() is None or get_platform() != "linux":
-        return None
-    if request.config.getoption("--run-emulation", default=None) is not None:
+        images = None
+    elif request.config.getoption("--run-emulation", default=None) is not None:
         # emulation tests only run one test in CI, caching the image only slows down the test
-        return None
+        images = None
+    else:
+        images = get_docker_warmup_images(request)
+    if images is None:
+        yield None
+        return
 
     if worker_id == "master":
         # not executing with multiple workers
         # it might be unsafe to write to tmp_path_factory.getbasetemp().parent
-        return docker_warmup(request)
+        docker_warmup_config = tmp_path_factory.getbasetemp() / "docker_warmup.cfg"
+        docker_warmup(images, docker_warmup_config)
+    else:
+        # get the temp directory shared by all workers
+        root_tmp_dir = tmp_path_factory.getbasetemp().parent
 
-    # get the temp directory shared by all workers
-    root_tmp_dir = tmp_path_factory.getbasetemp().parent
+        docker_warmup_config = root_tmp_dir / "docker_warmup.cfg"
+        with FileLock(str(docker_warmup_config) + ".lock"):
+            if not docker_warmup_config.is_file():
+                docker_warmup(images, docker_warmup_config)
 
-    fn = root_tmp_dir / "warmup.done"
-    with FileLock(str(fn) + ".lock"):
-        if not fn.is_file():
-            docker_warmup(request)
-            fn.write_text("done")
-    return None
+    with pytest.MonkeyPatch.context() as monkeypatch:
+        monkeypatch.setenv("CIBW_INTERNAL_PINNED_DOCKER_IMAGES", str(docker_warmup_config))
+        yield None
 
 
 @pytest.fixture(params=["pip", "build"])