Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Make container-engine a build (non-global) option (#1792)

Browse Source
This commit is contained in:
Joe Rickerby
2024-05-20 02:47:10 -04:00
committed by GitHub
parent 1b354cfa63
commit 9d5f5e60b8
9 changed files with 135 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cibuildwheel/linux.py
+38 -32
View File
@@ -11,8 +11,8 @@ from typing import OrderedDict, Tuple
from ._compat.typing import assert_never
from .architecture import Architecture
from .logger import log
from .oci_container import OCIContainer
from .options import Options
from .oci_container import OCIContainer, OCIContainerEngineConfig
from .options import BuildOptions, Options
from .typing import PathOrStr
from .util import (
AlreadyBuiltWheelError,
@@ -44,6 +44,7 @@ class PythonConfiguration:
class BuildStep:
platform_configs: list[PythonConfiguration]
platform_tag: str
container_engine: OCIContainerEngineConfig
container_image: str
@@ -65,8 +66,9 @@ def get_python_configurations(
]
def container_image_for_python_configuration(config: PythonConfiguration, options: Options) -> str:
build_options = options.build_options(config.identifier)
def container_image_for_python_configuration(
config: PythonConfiguration, build_options: BuildOptions
) -> str:
# e.g
# identifier is 'cp310-manylinux_x86_64'
# platform_tag is 'manylinux_x86_64'
@@ -91,15 +93,18 @@ def get_build_steps(
Groups PythonConfigurations into BuildSteps. Each BuildStep represents a
separate container instance.
"""
steps = OrderedDict[Tuple[str, str, str], BuildStep]()
steps = OrderedDict[Tuple[str, str, str, OCIContainerEngineConfig], BuildStep]()
for config in python_configurations:
_, platform_tag = config.identifier.split("-", 1)
before_all = options.build_options(config.identifier).before_all
container_image = container_image_for_python_configuration(config, options)
build_options = options.build_options(config.identifier)
step_key = (platform_tag, container_image, before_all)
before_all = build_options.before_all
container_image = container_image_for_python_configuration(config, build_options)
container_engine = build_options.container_engine
step_key = (platform_tag, container_image, before_all, container_engine)
if step_key in steps:
steps[step_key].platform_configs.append(config)
@@ -107,6 +112,7 @@ def get_build_steps(
steps[step_key] = BuildStep(
platform_configs=[config],
platform_tag=platform_tag,
container_engine=container_engine,
container_image=container_image,
)
@@ -388,29 +394,6 @@ def build_in_container(
def build(options: Options, tmp_path: Path) -> None: # noqa: ARG001
try:
# check the container engine is installed
subprocess.run(
[options.globals.container_engine.name, "--version"],
check=True,
stdout=subprocess.DEVNULL,
)
except subprocess.CalledProcessError:
print(
unwrap(
f"""
cibuildwheel: {options.globals.container_engine} not found. An
OCI exe like Docker or Podman is required to run Linux builds.
If you're building on Travis CI, add `services: [docker]` to
your .travis.yml. If you're building on Circle CI in Linux,
add a `setup_remote_docker` step to your .circleci/config.yml.
If you're building on Cirrus CI, use `docker_builder` task.
"""
),
file=sys.stderr,
)
sys.exit(2)
python_configurations = get_python_configurations(
options.globals.build_selector, options.globals.architectures
)
@@ -425,6 +408,29 @@ def build(options: Options, tmp_path: Path) -> None: # noqa: ARG001
container_package_dir = container_project_path / abs_package_dir.relative_to(cwd)
for build_step in get_build_steps(options, python_configurations):
try:
# check the container engine is installed
subprocess.run(
[build_step.container_engine.name, "--version"],
check=True,
stdout=subprocess.DEVNULL,
)
except subprocess.CalledProcessError:
print(
unwrap(
f"""
cibuildwheel: {build_step.container_engine.name} not found. An
OCI exe like Docker or Podman is required to run Linux builds.
If you're building on Travis CI, add `services: [docker]` to
your .travis.yml. If you're building on Circle CI in Linux,
add a `setup_remote_docker` step to your .circleci/config.yml.
If you're building on Cirrus CI, use `docker_builder` task.
"""
),
file=sys.stderr,
)
sys.exit(2)
try:
ids_to_build = [x.identifier for x in build_step.platform_configs]
log.step(f"Starting container image {build_step.container_image}...")
@@ -435,7 +441,7 @@ def build(options: Options, tmp_path: Path) -> None: # noqa: ARG001
image=build_step.container_image,
enforce_32_bit=build_step.platform_tag.endswith("i686"),
cwd=container_project_path,
engine=options.globals.container_engine,
engine=build_step.container_engine,
) as container:
build_in_container(
options=options,
cibuildwheel/oci_container.py
+3 -3
View File
@@ -11,7 +11,7 @@ import sys
import typing
import uuid
from collections.abc import Mapping, Sequence
from dataclasses import dataclass
from dataclasses import dataclass, field
from pathlib import Path, PurePath, PurePosixPath
from types import TracebackType
from typing import IO, Dict, Literal
@@ -32,7 +32,7 @@ ContainerEngineName = Literal["docker", "podman"]
@dataclass(frozen=True)
class OCIContainerEngineConfig:
name: ContainerEngineName
create_args: Sequence[str] = ()
create_args: tuple[str, ...] = field(default_factory=tuple)
disable_host_mount: bool = False
@staticmethod
@@ -58,7 +58,7 @@ class OCIContainerEngineConfig:
)
return OCIContainerEngineConfig(
name=name, create_args=create_args, disable_host_mount=disable_host_mount
name=name, create_args=tuple(create_args), disable_host_mount=disable_host_mount
)
def options_summary(self) -> str | dict[str, str]:
cibuildwheel/options.py
+16 -14
View File
@@ -75,7 +75,6 @@ class GlobalOptions:
build_selector: BuildSelector
test_selector: TestSelector
architectures: set[Architecture]
container_engine: OCIContainerEngineConfig
@dataclasses.dataclass(frozen=True)
@@ -95,6 +94,7 @@ class BuildOptions:
build_verbosity: int
build_frontend: BuildFrontendConfig | None
config_settings: str
container_engine: OCIContainerEngineConfig
@property
def package_dir(self) -> Path:
@@ -545,25 +545,12 @@ class Options:
)
test_selector = TestSelector(skip_config=test_skip)
container_engine_str = self.reader.get(
"container-engine",
table_format={"item": "{k}:{v}", "sep": "; ", "quote": shlex.quote},
)
try:
container_engine = OCIContainerEngineConfig.from_config_string(container_engine_str)
except ValueError as e:
msg = f"cibuildwheel: Failed to parse container config. {e}"
print(msg, file=sys.stderr)
sys.exit(2)
return GlobalOptions(
package_dir=package_dir,
output_dir=output_dir,
build_selector=build_selector,
test_selector=test_selector,
architectures=architectures,
container_engine=container_engine,
)
def build_options(self, identifier: str | None) -> BuildOptions:
@@ -643,6 +630,8 @@ class Options:
manylinux_images: dict[str, str] = {}
musllinux_images: dict[str, str] = {}
container_engine: OCIContainerEngineConfig | None = None
if self.platform == "linux":
all_pinned_container_images = _get_pinned_container_images()
@@ -677,6 +666,18 @@ class Options:
musllinux_images[build_platform] = image
container_engine_str = self.reader.get(
"container-engine",
table_format={"item": "{k}:{v}", "sep": "; ", "quote": shlex.quote},
)
try:
container_engine = OCIContainerEngineConfig.from_config_string(container_engine_str)
except ValueError as e:
msg = f"cibuildwheel: Failed to parse container config. {e}"
print(msg, file=sys.stderr)
sys.exit(2)
return BuildOptions(
globals=self.globals,
test_command=test_command,
@@ -693,6 +694,7 @@ class Options:
musllinux_images=musllinux_images or None,
build_frontend=build_frontend,
config_settings=config_settings,
container_engine=container_engine,
)
def check_for_invalid_configuration(self, identifiers: Iterable[str]) -> None:
cibuildwheel/resources/cibuildwheel.schema.json
+6
View File
@@ -491,6 +491,9 @@
"config-settings": {
"$ref": "#/properties/config-settings"
},
"container-engine": {
"$ref": "#/properties/container-engine"
},
"dependency-versions": {
"$ref": "#/properties/dependency-versions"
},
@@ -579,6 +582,9 @@
"config-settings": {
"$ref": "#/properties/config-settings"
},
"container-engine": {
"$ref": "#/properties/container-engine"
},
"environment": {
"$ref": "#/properties/environment"
},