feat: use digests instead of tags for pinned images (#2915)
This commit is contained in:
+90
-42
@@ -9,6 +9,7 @@
|
||||
|
||||
import configparser
|
||||
import dataclasses
|
||||
from functools import cache
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
@@ -33,7 +34,7 @@ class PyPAImage(Image):
|
||||
super().__init__(manylinux_version, platforms, image_name, tag, True)
|
||||
|
||||
|
||||
images = [
|
||||
IMAGES = [
|
||||
# manylinux2014 images
|
||||
PyPAImage(
|
||||
"manylinux2014",
|
||||
@@ -88,49 +89,71 @@ images = [
|
||||
),
|
||||
]
|
||||
|
||||
config = configparser.ConfigParser()
|
||||
|
||||
for image in images:
|
||||
# get the tag name whose digest matches 'latest'
|
||||
if image.tag is not None:
|
||||
# image has been pinned, do not update
|
||||
tag_name = image.tag
|
||||
elif image.image_name.startswith("quay.io/"):
|
||||
_, _, repository_name = image.image_name.partition("/")
|
||||
response = requests.get(
|
||||
f"https://quay.io/api/v1/repository/{repository_name}?includeTags=true"
|
||||
)
|
||||
response.raise_for_status()
|
||||
repo_info = response.json()
|
||||
tags_dict = repo_info["tags"]
|
||||
|
||||
latest_tag = tags_dict.pop("latest")
|
||||
@cache
|
||||
def quay_lookup(image_name: str, tag_name: str) -> tuple[str, str]:
|
||||
_, _, repository_name = image_name.partition("/")
|
||||
if tag_name == "latest":
|
||||
url = f"https://quay.io/api/v1/repository/{repository_name}?includeTags=true"
|
||||
else:
|
||||
url = f"https://quay.io/api/v1/repository/{repository_name}/tag?specificTag={tag_name}"
|
||||
response = requests.get(url)
|
||||
response.raise_for_status()
|
||||
info = response.json()
|
||||
if tag_name == "latest":
|
||||
tags_dict = info["tags"]
|
||||
tag_info = tags_dict.pop(tag_name)
|
||||
# find the tag whose manifest matches 'latest'
|
||||
tag_name = next(
|
||||
name
|
||||
tag_name, digest = next(
|
||||
(name, info["manifest_digest"])
|
||||
for (name, info) in tags_dict.items()
|
||||
if info["manifest_digest"] == latest_tag["manifest_digest"]
|
||||
if info["manifest_digest"] == tag_info["manifest_digest"]
|
||||
)
|
||||
elif image.image_name.startswith("ghcr.io/"):
|
||||
repository = image.image_name[8:]
|
||||
response = requests.get(
|
||||
"https://ghcr.io/token", params={"scope": f"repository:{repository}:pull"}
|
||||
)
|
||||
response.raise_for_status()
|
||||
token = response.json()["token"]
|
||||
else:
|
||||
tags_list = info["tags"]
|
||||
tag_info = next(tag for tag in tags_list if tag["name"] == tag_name)
|
||||
digest = tag_info["manifest_digest"]
|
||||
|
||||
return tag_name, digest
|
||||
|
||||
|
||||
@cache
|
||||
def ghcr_lookup(image_name: str, tag_name: str) -> tuple[str, str]:
|
||||
repository = image_name[8:]
|
||||
response = requests.get(
|
||||
"https://ghcr.io/token", params={"scope": f"repository:{repository}:pull"}
|
||||
)
|
||||
response.raise_for_status()
|
||||
token = response.json()["token"]
|
||||
if tag_name == "latest":
|
||||
response = requests.get(
|
||||
f"https://ghcr.io/v2/{repository}/tags/list",
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
)
|
||||
response.raise_for_status()
|
||||
ghcr_tags = [(Version(tag), tag) for tag in response.json()["tags"] if tag != "latest"]
|
||||
info = response.json()
|
||||
ghcr_tags = [(Version(tag), tag) for tag in info["tags"] if tag != "latest"]
|
||||
ghcr_tags.sort(reverse=True)
|
||||
tag_name = ghcr_tags[0][1]
|
||||
else:
|
||||
response = requests.get(f"https://hub.docker.com/v2/repositories/{image.image_name}/tags")
|
||||
response.raise_for_status()
|
||||
tags = response.json()["results"]
|
||||
|
||||
response = requests.head(
|
||||
f"https://ghcr.io/v2/{repository}/manifests/{tag_name}",
|
||||
headers={
|
||||
"Authorization": f"Bearer {token}",
|
||||
"Accept": "application/vnd.docker.distribution.manifest.v2+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json, application/vnd.oci.artifact.manifest.v1+json",
|
||||
},
|
||||
)
|
||||
response.raise_for_status()
|
||||
digest = response.headers["Docker-Content-Digest"]
|
||||
return tag_name, digest
|
||||
|
||||
|
||||
@cache
|
||||
def dockerhub_lookup(image_name: str, tag_name: str) -> tuple[str, str]:
|
||||
response = requests.get(f"https://hub.docker.com/v2/repositories/{image_name}/tags")
|
||||
response.raise_for_status()
|
||||
tags = response.json()["results"]
|
||||
if tag_name == "latest":
|
||||
latest_tag = next(tag for tag in tags if tag["name"] == "latest")
|
||||
# i don't know what it would mean to have multiple images per tag
|
||||
assert len(latest_tag["images"]) == 1
|
||||
@@ -139,15 +162,40 @@ for image in images:
|
||||
pinned_tag = next(
|
||||
tag for tag in tags if tag != latest_tag and tag["images"][0]["digest"] == digest
|
||||
)
|
||||
tag_name = pinned_tag["name"]
|
||||
else:
|
||||
pinned_tag = next(tag for tag in tags if tag["name"] == tag_name)
|
||||
digest = pinned_tag["images"][0]["digest"]
|
||||
tag_name = pinned_tag["name"]
|
||||
return tag_name, digest
|
||||
|
||||
for platform in image.platforms:
|
||||
if not config.has_section(platform):
|
||||
config[platform] = {}
|
||||
suffix = ""
|
||||
if image.use_platform_suffix:
|
||||
suffix = f"_{platform.removeprefix('pypy_')}"
|
||||
config[platform][image.manylinux_version] = f"{image.image_name}{suffix}:{tag_name}"
|
||||
|
||||
with open(RESOURCES / "pinned_docker_images.cfg", "w") as f:
|
||||
config.write(f)
|
||||
def main() -> None:
|
||||
config = configparser.ConfigParser()
|
||||
for image in IMAGES:
|
||||
# get the tag name whose digest matches 'latest'
|
||||
# if image has been pinned, do not update
|
||||
search_tag = image.tag or "latest"
|
||||
if image.image_name.startswith("quay.io/"):
|
||||
lookup = quay_lookup
|
||||
elif image.image_name.startswith("ghcr.io/"):
|
||||
lookup = ghcr_lookup
|
||||
else:
|
||||
lookup = dockerhub_lookup
|
||||
|
||||
tag_name, digest = lookup(image.image_name, search_tag)
|
||||
for platform in image.platforms:
|
||||
if not config.has_section(platform):
|
||||
config[platform] = {}
|
||||
image_name = image.image_name
|
||||
if image.use_platform_suffix:
|
||||
image_name = f"{image_name}_{platform.removeprefix('pypy_')}"
|
||||
_, digest = lookup(image_name, tag_name)
|
||||
assert digest.startswith("sha256:")
|
||||
config[platform][image.manylinux_version] = f"{image_name}@{digest} # {tag_name}"
|
||||
|
||||
with open(RESOURCES / "pinned_docker_images.cfg", "w") as f:
|
||||
config.write(f)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
@@ -1131,6 +1131,6 @@ def _get_pinned_container_images() -> Mapping[str, Mapping[str, str]]:
|
||||
'pypy_x86_64': {'manylinux2010': '...' }
|
||||
... }
|
||||
"""
|
||||
all_pinned_images = configparser.ConfigParser()
|
||||
all_pinned_images = configparser.ConfigParser(inline_comment_prefixes="#")
|
||||
all_pinned_images.read(resources.PINNED_DOCKER_IMAGES)
|
||||
return all_pinned_images
|
||||
|
||||
@@ -1,54 +1,54 @@
|
||||
[x86_64]
|
||||
manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2026.06.21-2
|
||||
manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64:2026.06.21-2
|
||||
manylinux_2_34 = quay.io/pypa/manylinux_2_34_x86_64:2026.06.21-2
|
||||
musllinux_1_2 = quay.io/pypa/musllinux_1_2_x86_64:2026.06.21-2
|
||||
manylinux2014 = quay.io/pypa/manylinux2014_x86_64@sha256:e19f84d04229d95eff831b6d4d9a44b9ba8d25a76ee2fce0eb9d6f0785e62f02 # 2026.06.25-1
|
||||
manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64@sha256:e6cdd8b231c97f3c7b8fbc16f8009e2c19b9c5966bda945dfd1deff9ae78544c # 2026.06.25-1
|
||||
manylinux_2_34 = quay.io/pypa/manylinux_2_34_x86_64@sha256:7c2795d223e1b3b84a11e07c3c78fb3c6fd1876d3c9d8f723757042e5c6d0315 # 2026.06.25-1
|
||||
musllinux_1_2 = quay.io/pypa/musllinux_1_2_x86_64@sha256:ba5a279716d87b72a79087ef4cd7fd4b55745f39fb99d9be56bdac62f0218d88 # 2026.06.25-1
|
||||
|
||||
[i686]
|
||||
manylinux2014 = quay.io/pypa/manylinux2014_i686:2026.06.21-2
|
||||
manylinux_2_28 = quay.io/pypa/manylinux_2_28_i686:2026.06.21-2
|
||||
manylinux_2_34 = quay.io/pypa/manylinux_2_34_i686:2026.06.21-2
|
||||
musllinux_1_2 = quay.io/pypa/musllinux_1_2_i686:2026.06.21-2
|
||||
manylinux2014 = quay.io/pypa/manylinux2014_i686@sha256:ee57716c65baee033261867185f6445c689994b6f0ce6caf47725204105a0231 # 2026.06.25-1
|
||||
manylinux_2_28 = quay.io/pypa/manylinux_2_28_i686@sha256:f0c6b806dfee3a30595516fb6585f73d43770eb953c269a344762abe7047cd15 # 2026.06.25-1
|
||||
manylinux_2_34 = quay.io/pypa/manylinux_2_34_i686@sha256:a327814daa44f5e88f6ecbef233422da7a478d7513c2940801bcf611137c3f6f # 2026.06.25-1
|
||||
musllinux_1_2 = quay.io/pypa/musllinux_1_2_i686@sha256:c69ef86661fd25af356cf398abf58febca60e31b338a421e26b9f0927f434b1e # 2026.06.25-1
|
||||
|
||||
[aarch64]
|
||||
manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2026.06.21-2
|
||||
manylinux_2_28 = quay.io/pypa/manylinux_2_28_aarch64:2026.06.21-2
|
||||
manylinux_2_34 = quay.io/pypa/manylinux_2_34_aarch64:2026.06.21-2
|
||||
musllinux_1_2 = quay.io/pypa/musllinux_1_2_aarch64:2026.06.21-2
|
||||
manylinux2014 = quay.io/pypa/manylinux2014_aarch64@sha256:9c8e0985470699c10aade65c740240c5cf83e4eb96bbfc5d8fe5f91bb26e4dcf # 2026.06.25-1
|
||||
manylinux_2_28 = quay.io/pypa/manylinux_2_28_aarch64@sha256:a8b0076da75671705167cad88ff4f9970abc6b12d8cbe26e344c4d9ce959223d # 2026.06.25-1
|
||||
manylinux_2_34 = quay.io/pypa/manylinux_2_34_aarch64@sha256:b643076f14b8e0e0b2d0df3b88bb0a64e6de92956a13a51b7548741d97042150 # 2026.06.25-1
|
||||
musllinux_1_2 = quay.io/pypa/musllinux_1_2_aarch64@sha256:2070c98fd85e592cda11b1de66819f782d926debf845dac6c0bab22366050c31 # 2026.06.25-1
|
||||
|
||||
[ppc64le]
|
||||
manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2026.06.21-2
|
||||
manylinux_2_28 = quay.io/pypa/manylinux_2_28_ppc64le:2026.06.21-2
|
||||
manylinux_2_34 = quay.io/pypa/manylinux_2_34_ppc64le:2026.06.21-2
|
||||
musllinux_1_2 = quay.io/pypa/musllinux_1_2_ppc64le:2026.06.21-2
|
||||
manylinux2014 = quay.io/pypa/manylinux2014_ppc64le@sha256:d37ab4c30d4c8213a47755ba6185d67c75ffedeee1afabfa75da8cc445aba7a8 # 2026.06.25-1
|
||||
manylinux_2_28 = quay.io/pypa/manylinux_2_28_ppc64le@sha256:4c423ebdc562f7cdfd2c6fa4a7bc65a763bd3e089f8c5d0aaa2f30a33ea8f790 # 2026.06.25-1
|
||||
manylinux_2_34 = quay.io/pypa/manylinux_2_34_ppc64le@sha256:63acb306c9849ef5589beb5542cf7d6939829a8deaa8f4c88ee763652d49657e # 2026.06.25-1
|
||||
musllinux_1_2 = quay.io/pypa/musllinux_1_2_ppc64le@sha256:91494d026ea3629db0f7e5f0fb32276dfe5133f40a74503d08a18d346ae2cc79 # 2026.06.25-1
|
||||
|
||||
[s390x]
|
||||
manylinux2014 = quay.io/pypa/manylinux2014_s390x:2026.06.21-2
|
||||
manylinux_2_28 = quay.io/pypa/manylinux_2_28_s390x:2026.06.21-2
|
||||
manylinux_2_34 = quay.io/pypa/manylinux_2_34_s390x:2026.06.21-2
|
||||
musllinux_1_2 = quay.io/pypa/musllinux_1_2_s390x:2026.06.21-2
|
||||
manylinux2014 = quay.io/pypa/manylinux2014_s390x@sha256:e808e27bcb8e708f86be65cd58165f77aebeb7851c447e6510489a46123776e9 # 2026.06.25-1
|
||||
manylinux_2_28 = quay.io/pypa/manylinux_2_28_s390x@sha256:787a24d1d4822067f5e1895913790ffe3e3fdbdeb2bc8cd4a66fcbb9d135f633 # 2026.06.25-1
|
||||
manylinux_2_34 = quay.io/pypa/manylinux_2_34_s390x@sha256:9f2cf25dbc5cd28dc021adf1b25eaaac3fbd08592b2fb07775c8c2b39f5aacf2 # 2026.06.25-1
|
||||
musllinux_1_2 = quay.io/pypa/musllinux_1_2_s390x@sha256:f2b1f52a5ac11e1ef884c29e30c2179e1ad308ee1a076d4f9e93a2b617811403 # 2026.06.25-1
|
||||
|
||||
[pypy_x86_64]
|
||||
manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2026.06.21-2
|
||||
manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64:2026.06.21-2
|
||||
manylinux_2_34 = quay.io/pypa/manylinux_2_34_x86_64:2026.06.21-2
|
||||
manylinux2014 = quay.io/pypa/manylinux2014_x86_64@sha256:e19f84d04229d95eff831b6d4d9a44b9ba8d25a76ee2fce0eb9d6f0785e62f02 # 2026.06.25-1
|
||||
manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64@sha256:e6cdd8b231c97f3c7b8fbc16f8009e2c19b9c5966bda945dfd1deff9ae78544c # 2026.06.25-1
|
||||
manylinux_2_34 = quay.io/pypa/manylinux_2_34_x86_64@sha256:7c2795d223e1b3b84a11e07c3c78fb3c6fd1876d3c9d8f723757042e5c6d0315 # 2026.06.25-1
|
||||
|
||||
[pypy_i686]
|
||||
manylinux2014 = quay.io/pypa/manylinux2014_i686:2026.06.21-2
|
||||
manylinux_2_28 = quay.io/pypa/manylinux_2_28_i686:2026.06.21-2
|
||||
manylinux_2_34 = quay.io/pypa/manylinux_2_34_i686:2026.06.21-2
|
||||
manylinux2014 = quay.io/pypa/manylinux2014_i686@sha256:ee57716c65baee033261867185f6445c689994b6f0ce6caf47725204105a0231 # 2026.06.25-1
|
||||
manylinux_2_28 = quay.io/pypa/manylinux_2_28_i686@sha256:f0c6b806dfee3a30595516fb6585f73d43770eb953c269a344762abe7047cd15 # 2026.06.25-1
|
||||
manylinux_2_34 = quay.io/pypa/manylinux_2_34_i686@sha256:a327814daa44f5e88f6ecbef233422da7a478d7513c2940801bcf611137c3f6f # 2026.06.25-1
|
||||
|
||||
[pypy_aarch64]
|
||||
manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2026.06.21-2
|
||||
manylinux_2_28 = quay.io/pypa/manylinux_2_28_aarch64:2026.06.21-2
|
||||
manylinux_2_34 = quay.io/pypa/manylinux_2_34_aarch64:2026.06.21-2
|
||||
manylinux2014 = quay.io/pypa/manylinux2014_aarch64@sha256:9c8e0985470699c10aade65c740240c5cf83e4eb96bbfc5d8fe5f91bb26e4dcf # 2026.06.25-1
|
||||
manylinux_2_28 = quay.io/pypa/manylinux_2_28_aarch64@sha256:a8b0076da75671705167cad88ff4f9970abc6b12d8cbe26e344c4d9ce959223d # 2026.06.25-1
|
||||
manylinux_2_34 = quay.io/pypa/manylinux_2_34_aarch64@sha256:b643076f14b8e0e0b2d0df3b88bb0a64e6de92956a13a51b7548741d97042150 # 2026.06.25-1
|
||||
|
||||
[armv7l]
|
||||
manylinux_2_31 = quay.io/pypa/manylinux_2_31_armv7l:2026.06.21-2
|
||||
manylinux_2_35 = quay.io/pypa/manylinux_2_35_armv7l:2026.06.21-2
|
||||
musllinux_1_2 = quay.io/pypa/musllinux_1_2_armv7l:2026.06.21-2
|
||||
manylinux_2_31 = quay.io/pypa/manylinux_2_31_armv7l@sha256:e4bda68b3bbfe0dc0fd6c6b7c50528d62a8d0907917d07c16492a3bfc2356cd3 # 2026.06.25-1
|
||||
manylinux_2_35 = quay.io/pypa/manylinux_2_35_armv7l@sha256:720871eaa85e1d0ad4b13d676be610590ddfe731345d6d433a01a3ac8e3d4832 # 2026.06.25-1
|
||||
musllinux_1_2 = quay.io/pypa/musllinux_1_2_armv7l@sha256:2a03ce7ad6ceec14cc2662a860a2b84162c9d9015273e577eb2050ff2654d84a # 2026.06.25-1
|
||||
|
||||
[riscv64]
|
||||
manylinux_2_39 = quay.io/pypa/manylinux_2_39_riscv64:2026.06.21-2
|
||||
musllinux_1_2 = quay.io/pypa/musllinux_1_2_riscv64:2026.06.21-2
|
||||
manylinux_2_39 = quay.io/pypa/manylinux_2_39_riscv64@sha256:567517486f7151b48c3cad0f20ee64c7f450a05d6accbfea465a4c5c38955095 # 2026.06.25-1
|
||||
musllinux_1_2 = quay.io/pypa/musllinux_1_2_riscv64@sha256:5fadccbe13d1e522515c9323c14c86ae7e0deff6c1850b7c7fae69f5429d8f4a # 2026.06.25-1
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@ from __future__ import annotations
|
||||
__lazy_modules__ = {"tomllib"}
|
||||
|
||||
import functools
|
||||
import os
|
||||
import tomllib
|
||||
from pathlib import Path
|
||||
|
||||
@@ -18,7 +19,9 @@ FREE_THREAD_ENABLE_314: Final[Path] = PATH / "free-threaded-enable-314.xml"
|
||||
FREE_THREAD_ENABLE_315: Final[Path] = PATH / "free-threaded-enable-315.xml"
|
||||
NODEJS: Final[Path] = PATH / "nodejs.toml"
|
||||
DEFAULTS: Final[Path] = PATH / "defaults.toml"
|
||||
PINNED_DOCKER_IMAGES: Final[Path] = PATH / "pinned_docker_images.cfg"
|
||||
PINNED_DOCKER_IMAGES: Final[Path] = Path(
|
||||
os.environ.get("CIBW_INTERNAL_PINNED_DOCKER_IMAGES", str(PATH / "pinned_docker_images.cfg"))
|
||||
)
|
||||
BUILD_PLATFORMS: Final[Path] = PATH / "build-platforms.toml"
|
||||
CONSTRAINTS: Final[Path] = PATH / "constraints.txt"
|
||||
VIRTUALENV: Final[Path] = PATH / "virtualenv.toml"
|
||||
|
||||
+59
-22
@@ -1,5 +1,6 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
@@ -12,14 +13,16 @@ from cibuildwheel.ci import detect_ci_provider
|
||||
from cibuildwheel.options import CommandLineArguments, Options
|
||||
from cibuildwheel.selector import EnableGroup
|
||||
from cibuildwheel.typing import PLATFORMS
|
||||
from cibuildwheel.util.resources import PINNED_DOCKER_IMAGES
|
||||
from cibuildwheel.venv import find_uv
|
||||
|
||||
from . import utils
|
||||
from .utils import DEFAULT_CIBW_ENABLE, EMULATED_ARCHS, get_platform
|
||||
from .utils import DEFAULT_CIBW_ENABLE, EMULATED_ARCHS, get_enable_groups, get_platform
|
||||
|
||||
TYPE_CHECKING = False
|
||||
if TYPE_CHECKING:
|
||||
from collections.abc import Generator
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
def pytest_addoption(parser: pytest.Parser) -> None:
|
||||
@@ -74,7 +77,19 @@ def pytest_configure(config: pytest.Config) -> None:
|
||||
os.environ["CIBW_PLATFORM"] = flag_platform
|
||||
|
||||
|
||||
def docker_warmup(request: pytest.FixtureRequest) -> None:
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class DockerWarmUpImage:
|
||||
source_name: str
|
||||
cached_name: str
|
||||
|
||||
|
||||
def get_docker_warmup_images(request: pytest.FixtureRequest) -> list[DockerWarmUpImage] | None:
|
||||
enable_groups = get_enable_groups()
|
||||
# Check missing pre-installed interpreters are needed
|
||||
enable_groups &= {EnableGroup.GraalPy, EnableGroup.PyPyEoL}
|
||||
if not enable_groups:
|
||||
return None
|
||||
|
||||
machine = request.config.getoption("--run-emulation", default=None)
|
||||
if machine is None:
|
||||
archs = {arch.value for arch in Architecture.auto_archs("linux")}
|
||||
@@ -86,7 +101,7 @@ def docker_warmup(request: pytest.FixtureRequest) -> None:
|
||||
# Only include architectures where there are missing pre-installed interpreters
|
||||
archs &= {"x86_64", "i686", "aarch64"}
|
||||
if not archs:
|
||||
return
|
||||
return None
|
||||
|
||||
options = Options(
|
||||
platform="linux",
|
||||
@@ -96,17 +111,28 @@ def docker_warmup(request: pytest.FixtureRequest) -> None:
|
||||
)
|
||||
build_options = options.build_options(None)
|
||||
assert build_options.manylinux_images is not None
|
||||
assert build_options.musllinux_images is not None
|
||||
images = [build_options.manylinux_images[arch] for arch in archs] + [
|
||||
build_options.musllinux_images[arch] for arch in archs
|
||||
]
|
||||
images: list[DockerWarmUpImage] = []
|
||||
for arch in archs:
|
||||
source_name = build_options.manylinux_images[arch]
|
||||
cached_name = source_name
|
||||
if "@sha256:" in source_name:
|
||||
cached_name = source_name.rsplit("@sha256:", 1)[0] + ":cibw_cache_fixture"
|
||||
images.append(DockerWarmUpImage(source_name, cached_name))
|
||||
if images:
|
||||
return images
|
||||
return None
|
||||
|
||||
|
||||
def docker_warmup(images: list[DockerWarmUpImage], docker_warmup_config: Path) -> None:
|
||||
command = (
|
||||
"manylinux-interpreters ensure-all &&"
|
||||
"cpython3.13 -m pip download -d /tmp setuptools wheel pytest"
|
||||
)
|
||||
config_content = PINNED_DOCKER_IMAGES.read_text()
|
||||
for image in images:
|
||||
config_content = config_content.replace(image.source_name, image.cached_name)
|
||||
container_id = subprocess.run(
|
||||
["docker", "create", image, "bash", "-c", command],
|
||||
["docker", "create", image.source_name, "bash", "-c", command],
|
||||
text=True,
|
||||
check=True,
|
||||
stdout=subprocess.PIPE,
|
||||
@@ -118,37 +144,48 @@ def docker_warmup(request: pytest.FixtureRequest) -> None:
|
||||
).stdout.strip()
|
||||
assert exit_code == "0"
|
||||
subprocess.run(
|
||||
["docker", "commit", container_id, image], check=True, stdout=subprocess.DEVNULL
|
||||
["docker", "commit", container_id, image.cached_name],
|
||||
check=True,
|
||||
stdout=subprocess.DEVNULL,
|
||||
)
|
||||
finally:
|
||||
subprocess.run(["docker", "rm", container_id], check=True, stdout=subprocess.DEVNULL)
|
||||
docker_warmup_config.write_text(config_content)
|
||||
|
||||
|
||||
@pytest.fixture(scope="session", autouse=True)
|
||||
def docker_warmup_fixture(
|
||||
request: pytest.FixtureRequest, tmp_path_factory: pytest.TempPathFactory, worker_id: str
|
||||
) -> None:
|
||||
) -> Generator[None, None, None]:
|
||||
# if we're in CI testing linux, let's warm-up docker images
|
||||
if detect_ci_provider() is None or get_platform() != "linux":
|
||||
return None
|
||||
if request.config.getoption("--run-emulation", default=None) is not None:
|
||||
images = None
|
||||
elif request.config.getoption("--run-emulation", default=None) is not None:
|
||||
# emulation tests only run one test in CI, caching the image only slows down the test
|
||||
return None
|
||||
images = None
|
||||
else:
|
||||
images = get_docker_warmup_images(request)
|
||||
if images is None:
|
||||
yield None
|
||||
return
|
||||
|
||||
if worker_id == "master":
|
||||
# not executing with multiple workers
|
||||
# it might be unsafe to write to tmp_path_factory.getbasetemp().parent
|
||||
return docker_warmup(request)
|
||||
docker_warmup_config = tmp_path_factory.getbasetemp() / "docker_warmup.cfg"
|
||||
docker_warmup(images, docker_warmup_config)
|
||||
else:
|
||||
# get the temp directory shared by all workers
|
||||
root_tmp_dir = tmp_path_factory.getbasetemp().parent
|
||||
|
||||
# get the temp directory shared by all workers
|
||||
root_tmp_dir = tmp_path_factory.getbasetemp().parent
|
||||
docker_warmup_config = root_tmp_dir / "docker_warmup.cfg"
|
||||
with FileLock(str(docker_warmup_config) + ".lock"):
|
||||
if not docker_warmup_config.is_file():
|
||||
docker_warmup(images, docker_warmup_config)
|
||||
|
||||
fn = root_tmp_dir / "warmup.done"
|
||||
with FileLock(str(fn) + ".lock"):
|
||||
if not fn.is_file():
|
||||
docker_warmup(request)
|
||||
fn.write_text("done")
|
||||
return None
|
||||
with pytest.MonkeyPatch.context() as monkeypatch:
|
||||
monkeypatch.setenv("CIBW_INTERNAL_PINNED_DOCKER_IMAGES", str(docker_warmup_config))
|
||||
yield None
|
||||
|
||||
|
||||
@pytest.fixture(params=["pip", "build"])
|
||||
|
||||
@@ -12,7 +12,10 @@ basic_project = test_projects.new_c_project()
|
||||
|
||||
|
||||
def test_podman(
|
||||
tmp_path: Path, capfd: pytest.CaptureFixture[str], request: pytest.FixtureRequest
|
||||
tmp_path: Path,
|
||||
capfd: pytest.CaptureFixture[str],
|
||||
request: pytest.FixtureRequest,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> None:
|
||||
if utils.get_platform() != "linux":
|
||||
pytest.skip("the test is only relevant to the linux build")
|
||||
@@ -20,6 +23,9 @@ def test_podman(
|
||||
if not request.config.getoption("--run-podman"):
|
||||
pytest.skip("needs --run-podman option to run")
|
||||
|
||||
# we can't use images cached in the docker_warmup_fixture autouse fixture
|
||||
monkeypatch.delenv("CIBW_INTERNAL_PINNED_DOCKER_IMAGES", raising=False)
|
||||
|
||||
project_dir = tmp_path / "project"
|
||||
basic_project.generate(project_dir)
|
||||
|
||||
|
||||
@@ -173,19 +173,19 @@ def test_empty_selector(monkeypatch: pytest.MonkeyPatch) -> None:
|
||||
@pytest.mark.parametrize(
|
||||
("architecture", "image", "full_image"),
|
||||
[
|
||||
("x86_64", None, "quay.io/pypa/manylinux_2_28_x86_64:*"),
|
||||
("x86_64", "manylinux2014", "quay.io/pypa/manylinux2014_x86_64:*"),
|
||||
("x86_64", "manylinux_2_28", "quay.io/pypa/manylinux_2_28_x86_64:*"),
|
||||
("x86_64", "manylinux_2_34", "quay.io/pypa/manylinux_2_34_x86_64:*"),
|
||||
("x86_64", None, "quay.io/pypa/manylinux_2_28_x86_64@*"),
|
||||
("x86_64", "manylinux2014", "quay.io/pypa/manylinux2014_x86_64@*"),
|
||||
("x86_64", "manylinux_2_28", "quay.io/pypa/manylinux_2_28_x86_64@*"),
|
||||
("x86_64", "manylinux_2_34", "quay.io/pypa/manylinux_2_34_x86_64@*"),
|
||||
("x86_64", "custom_image", "custom_image"),
|
||||
("i686", None, "quay.io/pypa/manylinux_2_28_i686:*"),
|
||||
("i686", "manylinux2014", "quay.io/pypa/manylinux2014_i686:*"),
|
||||
("i686", "manylinux_2_28", "quay.io/pypa/manylinux_2_28_i686:*"),
|
||||
("i686", None, "quay.io/pypa/manylinux_2_28_i686@*"),
|
||||
("i686", "manylinux2014", "quay.io/pypa/manylinux2014_i686@*"),
|
||||
("i686", "manylinux_2_28", "quay.io/pypa/manylinux_2_28_i686@*"),
|
||||
("i686", "custom_image", "custom_image"),
|
||||
("pypy_x86_64", None, "quay.io/pypa/manylinux_2_28_x86_64:*"),
|
||||
("pypy_x86_64", "manylinux2014", "quay.io/pypa/manylinux2014_x86_64:*"),
|
||||
("pypy_x86_64", "manylinux_2_28", "quay.io/pypa/manylinux_2_28_x86_64:*"),
|
||||
("pypy_x86_64", "manylinux_2_34", "quay.io/pypa/manylinux_2_34_x86_64:*"),
|
||||
("pypy_x86_64", None, "quay.io/pypa/manylinux_2_28_x86_64@*"),
|
||||
("pypy_x86_64", "manylinux2014", "quay.io/pypa/manylinux2014_x86_64@*"),
|
||||
("pypy_x86_64", "manylinux_2_28", "quay.io/pypa/manylinux_2_28_x86_64@*"),
|
||||
("pypy_x86_64", "manylinux_2_34", "quay.io/pypa/manylinux_2_34_x86_64@*"),
|
||||
("pypy_x86_64", "custom_image", "custom_image"),
|
||||
],
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user