Inline dependency-versions syntax (#2122)

* Write docs for inline dependency-versions

* Change the `inline` keyword to `packages` for better readability

* Implement inline package constraints

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* Add schema for TOML properties

* Change the parsing of the option to parse filenames as-is

* Add a unit test for table-parsing of the option

* Remove unneeded shlex.quote on the dependency-version test

* Tidy-ups, comments, docs fixes

* Add test for empty packages option value

* Fix empty packages scenario

And, remove some optionals to reduce the problem space

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Joe Rickerby
2025-03-10 21:45:54 +00:00
committed by GitHub
co-authored by pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
parent f5e502aa88
commit 25f2d3f313
13 changed files with 375 additions and 96 deletions
+18 -1
View File
@@ -104,7 +104,24 @@ properties:
dependency-versions:
default: pinned
description: Specify how cibuildwheel controls the versions of the tools it uses
type: string
oneOf:
- enum: [pinned, latest]
- type: string
description: Path to a file containing dependency versions, or inline package specifications, starting with "packages:"
not:
enum: [pinned, latest]
- type: object
additionalProperties: false
properties:
file:
type: string
- type: object
additionalProperties: false
properties:
packages:
type: array
items:
type: string
enable:
description: Enable or disable certain builds.
oneOf:
+11 -9
View File
@@ -166,6 +166,7 @@ def build_in_container(
container: OCIContainer,
container_project_path: PurePath,
container_package_dir: PurePath,
local_tmp_dir: Path,
) -> None:
container_output_dir = PurePosixPath("/output")
@@ -199,22 +200,22 @@ def build_in_container(
for config in platform_configs:
log.build_start(config.identifier)
local_identifier_tmp_dir = local_tmp_dir / config.identifier
build_options = options.build_options(config.identifier)
build_frontend = build_options.build_frontend or BuildFrontendConfig("pip")
use_uv = build_frontend.name == "build[uv]"
pip = ["uv", "pip"] if use_uv else ["pip"]
dependency_constraint_flags: list[PathOrStr] = []
log.step("Setting up build environment...")
if build_options.dependency_constraints:
constraints_file = build_options.dependency_constraints.get_for_python_version(
config.version
)
dependency_constraint_flags: list[PathOrStr] = []
local_constraints_file = build_options.dependency_constraints.get_for_python_version(
version=config.version,
tmp_dir=local_identifier_tmp_dir,
)
if local_constraints_file:
container_constraints_file = PurePosixPath("/constraints.txt")
container.copy_into(constraints_file, container_constraints_file)
container.copy_into(local_constraints_file, container_constraints_file)
dependency_constraint_flags = ["-c", container_constraints_file]
env = container.get_environment()
@@ -426,7 +427,7 @@ def build_in_container(
log.step_end()
def build(options: Options, tmp_path: Path) -> None: # noqa: ARG001
def build(options: Options, tmp_path: Path) -> None:
python_configurations = get_python_configurations(
options.globals.build_selector, options.globals.architectures
)
@@ -480,6 +481,7 @@ def build(options: Options, tmp_path: Path) -> None: # noqa: ARG001
container=container,
container_project_path=container_project_path,
container_package_dir=container_package_dir,
local_tmp_dir=tmp_path,
)
except subprocess.CalledProcessError as error:
+8 -11
View File
@@ -421,12 +421,12 @@ def build(options: Options, tmp_path: Path) -> None:
config_is_arm64 = config.identifier.endswith("arm64")
config_is_universal2 = config.identifier.endswith("universal2")
dependency_constraint_flags: Sequence[PathOrStr] = []
if build_options.dependency_constraints:
dependency_constraint_flags = [
"-c",
build_options.dependency_constraints.get_for_python_version(config.version),
]
constraints_path = build_options.dependency_constraints.get_for_python_version(
version=config.version, tmp_dir=identifier_tmp_dir
)
dependency_constraint_flags: Sequence[PathOrStr] = (
["-c", constraints_path] if constraints_path else []
)
base_python, env = setup_python(
identifier_tmp_dir / "build",
@@ -463,12 +463,9 @@ def build(options: Options, tmp_path: Path) -> None:
build_env = env.copy()
if not use_uv:
build_env["VIRTUALENV_PIP"] = pip_version
if build_options.dependency_constraints:
constraint_path = build_options.dependency_constraints.get_for_python_version(
config.version
)
if constraints_path:
combine_constraints(
build_env, constraint_path, identifier_tmp_dir if use_uv else None
build_env, constraints_path, identifier_tmp_dir if use_uv else None
)
if build_frontend.name == "pip":
+12 -10
View File
@@ -97,7 +97,7 @@ class BuildOptions:
repair_command: str
manylinux_images: dict[str, str] | None
musllinux_images: dict[str, str] | None
dependency_constraints: DependencyConstraints | None
dependency_constraints: DependencyConstraints
test_command: str | None
before_test: str | None
test_sources: list[str]
@@ -693,7 +693,6 @@ class Options:
"config-settings", option_format=ShlexTableFormat(sep=" ", pair_sep="=")
)
dependency_versions = self.reader.get("dependency-versions")
test_command = self.reader.get("test-command", option_format=ListFormat(sep=" && "))
before_test = self.reader.get("before-test", option_format=ListFormat(sep=" && "))
test_sources = shlex.split(
@@ -739,15 +738,18 @@ class Options:
with contextlib.suppress(KeyError):
environment.add(env_var_name, self.env[env_var_name], prepend=True)
if dependency_versions == "pinned":
dependency_constraints: DependencyConstraints | None = (
DependencyConstraints.with_defaults()
dependency_versions_str = self.reader.get(
"dependency-versions",
env_plat=True,
option_format=ShlexTableFormat(sep="; ", pair_sep=":", allow_merge=False),
)
try:
dependency_constraints = DependencyConstraints.from_config_string(
dependency_versions_str
)
elif dependency_versions == "latest":
dependency_constraints = None
else:
dependency_versions_path = Path(dependency_versions)
dependency_constraints = DependencyConstraints(dependency_versions_path)
except (ValueError, OSError) as e:
msg = f"Failed to parse dependency versions. {e}"
raise errors.ConfigurationError(msg) from e
if test_extras:
test_extras = f"[{test_extras}]"
+7 -7
View File
@@ -265,12 +265,12 @@ def build(options: Options, tmp_path: Path) -> None:
built_wheel_dir.mkdir()
repaired_wheel_dir.mkdir()
dependency_constraint_flags: Sequence[PathOrStr] = []
if build_options.dependency_constraints:
constraints_path = build_options.dependency_constraints.get_for_python_version(
config.version, variant="pyodide"
)
dependency_constraint_flags = ["-c", constraints_path]
constraints_path = build_options.dependency_constraints.get_for_python_version(
version=config.version, variant="pyodide", tmp_dir=identifier_tmp_dir
)
dependency_constraint_flags: Sequence[PathOrStr] = (
["-c", constraints_path] if constraints_path else []
)
env = setup_python(
identifier_tmp_dir / "build",
@@ -319,7 +319,7 @@ def build(options: Options, tmp_path: Path) -> None:
)
build_env = env.copy()
if build_options.dependency_constraints:
if constraints_path:
combine_constraints(build_env, constraints_path, identifier_tmp_dir)
build_env["VIRTUALENV_PIP"] = pip_version
call(
@@ -233,7 +233,45 @@
"dependency-versions": {
"default": "pinned",
"description": "Specify how cibuildwheel controls the versions of the tools it uses",
"type": "string",
"oneOf": [
{
"enum": [
"pinned",
"latest"
]
},
{
"type": "string",
"description": "Path to a file containing dependency versions, or inline package specifications, starting with \"packages:\"",
"not": {
"enum": [
"pinned",
"latest"
]
}
},
{
"type": "object",
"additionalProperties": false,
"properties": {
"file": {
"type": "string"
}
}
},
{
"type": "object",
"additionalProperties": false,
"properties": {
"packages": {
"type": "array",
"items": {
"type": "string"
}
}
}
}
],
"title": "CIBW_DEPENDENCY_VERSIONS"
},
"enable": {
+89 -25
View File
@@ -1,4 +1,6 @@
import shlex
from collections.abc import Mapping, MutableMapping, Sequence
from dataclasses import dataclass, field
from pathlib import Path, PurePath
from typing import Any, Literal, Self, TypeVar
@@ -6,47 +8,109 @@ from packaging.utils import parse_wheel_filename
from . import resources
from .cmd import call
from .helpers import parse_key_value_string, unwrap
@dataclass()
class DependencyConstraints:
def __init__(self, base_file_path: Path):
assert base_file_path.exists()
self.base_file_path = base_file_path.resolve()
base_file_path: Path | None = None
packages: list[str] = field(default_factory=list)
def __post_init__(self) -> None:
if self.packages and self.base_file_path is not None:
msg = "Cannot specify both a file and packages in the dependency constraints"
raise ValueError(msg)
if self.base_file_path is not None:
if not self.base_file_path.exists():
msg = f"Dependency constraints file not found: {self.base_file_path}"
raise FileNotFoundError(msg)
self.base_file_path = self.base_file_path.resolve()
@classmethod
def with_defaults(cls) -> Self:
def pinned(cls) -> Self:
return cls(base_file_path=resources.CONSTRAINTS)
@classmethod
def latest(cls) -> Self:
return cls()
@classmethod
def from_config_string(cls, config_string: str) -> Self:
if config_string == "pinned":
return cls.pinned()
if config_string == "latest" or not config_string:
return cls.latest()
if config_string.startswith(("file:", "packages:")):
# we only do the table-style parsing if it looks like a table,
# because this option used to be only a file path. We don't want
# to break existing configurations, whose file paths might include
# special characters like ':' or ' ', which would require quoting
# if they were to be passed as a parse_key_value_string positional
# argument.
return cls.from_table_style_config_string(config_string)
return cls(base_file_path=Path(config_string))
@classmethod
def from_table_style_config_string(cls, config_string: str) -> Self:
config_dict = parse_key_value_string(config_string, kw_arg_names=["file", "packages"])
files = config_dict.get("file")
packages = config_dict.get("packages") or []
if files and packages:
msg = "Cannot specify both a file and packages in dependency-versions"
raise ValueError(msg)
if files:
if len(files) > 1:
msg = unwrap("""
Only one file can be specified in dependency-versions.
If you intended to pass only one, perhaps you need to quote the path?
""")
raise ValueError(msg)
return cls(base_file_path=Path(files[0]))
return cls(packages=packages)
def get_for_python_version(
self, version: str, *, variant: Literal["python", "pyodide"] = "python"
) -> Path:
version_parts = version.split(".")
self, *, version: str, variant: Literal["python", "pyodide"] = "python", tmp_dir: Path
) -> Path | None:
if self.packages:
constraint_file = tmp_dir / "constraints.txt"
constraint_file.write_text("\n".join(self.packages))
return constraint_file
# try to find a version-specific dependency file e.g. if
# ./constraints.txt is the base, look for ./constraints-python36.txt
specific_stem = self.base_file_path.stem + f"-{variant}{version_parts[0]}{version_parts[1]}"
specific_name = specific_stem + self.base_file_path.suffix
specific_file_path = self.base_file_path.with_name(specific_name)
if self.base_file_path is not None:
version_parts = version.split(".")
if specific_file_path.exists():
return specific_file_path
else:
return self.base_file_path
# try to find a version-specific dependency file e.g. if
# ./constraints.txt is the base, look for ./constraints-python36.txt
specific_stem = (
self.base_file_path.stem + f"-{variant}{version_parts[0]}{version_parts[1]}"
)
specific_name = specific_stem + self.base_file_path.suffix
specific_file_path = self.base_file_path.with_name(specific_name)
def __repr__(self) -> str:
return f"{self.__class__.__name__}({self.base_file_path!r})"
if specific_file_path.exists():
return specific_file_path
else:
return self.base_file_path
def __eq__(self, o: object) -> bool:
if not isinstance(o, DependencyConstraints):
return False
return self.base_file_path == o.base_file_path
return None
def options_summary(self) -> Any:
if self == DependencyConstraints.with_defaults():
if self == DependencyConstraints.pinned():
return "pinned"
else:
elif self.packages:
return {"packages": " ".join(shlex.quote(p) for p in self.packages)}
elif self.base_file_path is not None:
return self.base_file_path.name
else:
return "latest"
def get_pip_version(env: Mapping[str, str]) -> str:
+8 -10
View File
@@ -364,12 +364,13 @@ def build(options: Options, tmp_path: Path) -> None:
built_wheel_dir = identifier_tmp_dir / "built_wheel"
repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel"
dependency_constraint_flags: Sequence[PathOrStr] = []
if build_options.dependency_constraints:
dependency_constraint_flags = [
"-c",
build_options.dependency_constraints.get_for_python_version(config.version),
]
constraints_path = build_options.dependency_constraints.get_for_python_version(
version=config.version,
tmp_dir=identifier_tmp_dir,
)
dependency_constraint_flags: Sequence[PathOrStr] = (
["-c", constraints_path] if constraints_path else []
)
# install Python
base_python, env = setup_python(
@@ -411,10 +412,7 @@ def build(options: Options, tmp_path: Path) -> None:
if not use_uv:
build_env["VIRTUALENV_PIP"] = pip_version
if build_options.dependency_constraints:
constraints_path = build_options.dependency_constraints.get_for_python_version(
config.version
)
if constraints_path:
combine_constraints(build_env, constraints_path, identifier_tmp_dir)
if build_frontend.name == "pip":
+22 -4
View File
@@ -1356,9 +1356,10 @@ Options can be supplied after the name.
### `CIBW_DEPENDENCY_VERSIONS` {: #dependency-versions}
> Specify how cibuildwheel controls the versions of the tools it uses
Options: `pinned` `latest` `<your constraints file>`
> Control the versions of the tools cibuildwheel uses
Options: `pinned` `latest` `packages: SPECIFIER...` `<your constraints file>`
Default: `pinned`
@@ -1373,7 +1374,8 @@ fixes that can't wait for a new cibuildwheel release.
To control the versions of dependencies yourself, you can supply a [pip
constraints](https://pip.pypa.io/en/stable/user_guide/#constraints-files) file
here and it will be used instead.
here and it will be used instead. Alternatively, you can list constraint
specifiers inline with the `packages: SPECIFIER...` syntax.
!!! note
If you need different dependencies for each python version, provide them
@@ -1404,6 +1406,15 @@ Platform-specific environment variables are also available:<br/>
# Use your own pip constraints file
CIBW_DEPENDENCY_VERSIONS: ./constraints.txt
# Specify requirements inline
CIBW_DEPENDENCY_VERSIONS: "packages: auditwheel==6.2.0"
# Choose a specific pyodide-build version
CIBW_DEPENDENCY_VERSIONS_PYODIDE: "packages: pyodide-build==0.29.1"
# Use shell-style quoting around spaces package specifiers
CIBW_DEPENDENCY_VERSIONS: "packages: 'pip >=16.0.0, !=17'"
```
!!! tab examples "pyproject.toml"
@@ -1417,7 +1428,14 @@ Platform-specific environment variables are also available:<br/>
dependency-versions = "latest"
# Use your own pip constraints file
dependency-versions = "./constraints.txt"
dependency-versions = { file = "./constraints.txt" }
# Specify requirements inline
dependency-versions = { packages = ["auditwheel==6.2.0"] }
[tool.cibuildwheel.pyodide]
# Choose a specific pyodide-build version
dependency-versions = { packages = ["pyodide-build==0.29.1"] }
```
+20 -10
View File
@@ -91,7 +91,8 @@ def test_pinned_versions(tmp_path, python_version, build_frontend_env_nouv):
assert set(actual_wheels) == set(expected_wheels)
def test_dependency_constraints_file(tmp_path, build_frontend_env_nouv):
@pytest.mark.parametrize("method", ["inline", "file"])
def test_dependency_constraints(method, tmp_path, build_frontend_env_nouv):
if utils.platform == "linux":
pytest.skip("linux doesn't pin individual tool versions, it pins manylinux images instead")
@@ -103,15 +104,24 @@ def test_dependency_constraints_file(tmp_path, build_frontend_env_nouv):
"delocate": "0.10.3",
}
constraints_file = tmp_path / "constraints file.txt"
constraints_file.write_text(
textwrap.dedent(
"""
pip=={pip}
delocate=={delocate}
""".format(**tool_versions)
if method == "file":
constraints_file = tmp_path / "constraints file.txt"
constraints_file.write_text(
textwrap.dedent(
"""
pip=={pip}
delocate=={delocate}
""".format(**tool_versions)
)
)
)
dependency_version_option = str(constraints_file)
elif method == "inline":
dependency_version_option = "packages: " + " ".join(
f"{k}=={v}" for k, v in tool_versions.items()
)
else:
msg = f"Unknown method: {method}"
raise ValueError(msg)
build_environment = {}
@@ -126,7 +136,7 @@ def test_dependency_constraints_file(tmp_path, build_frontend_env_nouv):
project_dir,
add_env={
"CIBW_ENVIRONMENT": cibw_environment_option,
"CIBW_DEPENDENCY_VERSIONS": str(constraints_file),
"CIBW_DEPENDENCY_VERSIONS": dependency_version_option,
**build_frontend_env_nouv,
},
)
+42 -8
View File
@@ -1,21 +1,55 @@
from pathlib import Path
import pytest
from cibuildwheel.util.packaging import DependencyConstraints
def test_defaults():
dependency_constraints = DependencyConstraints.with_defaults()
def test_defaults(tmp_path: Path) -> None:
dependency_constraints = DependencyConstraints.pinned()
project_root = Path(__file__).parents[1]
resources_dir = project_root / "cibuildwheel" / "resources"
assert dependency_constraints.base_file_path
assert dependency_constraints.base_file_path.samefile(resources_dir / "constraints.txt")
assert dependency_constraints.get_for_python_version("3.99").samefile(
resources_dir / "constraints.txt"
constraints_file = dependency_constraints.get_for_python_version(
version="3.99", tmp_dir=tmp_path
)
assert dependency_constraints.get_for_python_version("3.9").samefile(
resources_dir / "constraints-python39.txt"
assert constraints_file
assert constraints_file.samefile(resources_dir / "constraints.txt")
constraints_file = dependency_constraints.get_for_python_version(
version="3.9", tmp_dir=tmp_path
)
assert dependency_constraints.get_for_python_version("3.13").samefile(
resources_dir / "constraints-python313.txt"
assert constraints_file
assert constraints_file.samefile(resources_dir / "constraints-python39.txt")
constraints_file = dependency_constraints.get_for_python_version(
version="3.13", tmp_dir=tmp_path
)
assert constraints_file
assert constraints_file.samefile(resources_dir / "constraints-python313.txt")
def test_inline_packages(tmp_path: Path) -> None:
dependency_constraints = DependencyConstraints(
base_file_path=None,
packages=["foo==1.2.3", "bar==4.5.6"],
)
constraint_file = dependency_constraints.get_for_python_version(version="x.x", tmp_dir=tmp_path)
assert constraint_file
constraints_file_contents = constraint_file.read_text()
assert constraints_file_contents == "foo==1.2.3\nbar==4.5.6"
@pytest.mark.parametrize("config_string", ["", "latest", "packages:"])
def test_empty_constraints(config_string: str) -> None:
dependency_constraints = DependencyConstraints.from_config_string(config_string)
assert not dependency_constraints.packages
assert not dependency_constraints.base_file_path
assert dependency_constraints == DependencyConstraints.latest()
+38
View File
@@ -11,6 +11,7 @@ from cibuildwheel.frontend import _split_config_settings
from cibuildwheel.options import BuildOptions, _get_pinned_container_images
from cibuildwheel.selector import BuildSelector, EnableGroup
from cibuildwheel.util import resources
from cibuildwheel.util.packaging import DependencyConstraints
# CIBW_PLATFORM is tested in main_platform_test.py
@@ -342,6 +343,43 @@ def test_before_all(before_all, platform_specific, platform, intercepted_build_a
assert build_options.before_all == (before_all or "")
@pytest.mark.parametrize(
"dependency_versions",
[None, "pinned", "latest", "FILE", "packages: pip==21.0.0"],
)
@pytest.mark.parametrize("platform_specific", [False, True])
def test_dependency_versions(
dependency_versions, platform_specific, platform, intercepted_build_args, monkeypatch, tmp_path
):
option_value = dependency_versions
if dependency_versions == "FILE":
constraints_file = tmp_path / "constraints.txt"
constraints_file.write_text("foo==1.2.3\nbar==4.5.6")
option_value = str(constraints_file)
if option_value is not None:
if platform_specific:
monkeypatch.setenv("CIBW_DEPENDENCY_VERSIONS_" + platform.upper(), option_value)
monkeypatch.setenv("CIBW_DEPENDENCY_VERSIONS", "overwritten")
else:
monkeypatch.setenv("CIBW_DEPENDENCY_VERSIONS", option_value)
main()
build_options: BuildOptions = intercepted_build_args.args[0].build_options(identifier=None)
dependency_constraints = build_options.dependency_constraints
if dependency_versions is None or dependency_versions == "pinned":
assert dependency_constraints == DependencyConstraints.pinned()
elif dependency_versions == "latest":
assert dependency_constraints == DependencyConstraints.latest()
elif dependency_versions == "FILE":
assert dependency_constraints.base_file_path
assert dependency_constraints.base_file_path.samefile(Path(option_value))
elif dependency_versions.startswith("packages:"):
assert dependency_constraints.packages == ["pip==21.0.0"]
@pytest.mark.parametrize("method", ["unset", "command_line", "env_var"])
def test_debug_traceback(monkeypatch, method, capfd):
if method == "command_line":
+61
View File
@@ -14,6 +14,8 @@ from cibuildwheel.options import (
_get_pinned_container_images,
)
from cibuildwheel.selector import EnableGroup
from cibuildwheel.util import resources
from cibuildwheel.util.packaging import DependencyConstraints
PYPROJECT_1 = """
[tool.cibuildwheel]
@@ -468,3 +470,62 @@ def test_free_threaded_support(
assert EnableGroup.CPythonFreeThreading in options.globals.build_selector.enable
else:
assert EnableGroup.CPythonFreeThreading not in options.globals.build_selector.enable
@pytest.mark.parametrize(
("toml_assignment", "base_file_path", "packages"),
[
("", resources.CONSTRAINTS, []),
("dependency-versions = 'pinned'", resources.CONSTRAINTS, []),
("dependency-versions = 'latest'", None, []),
("dependency-versions = 'constraints file.txt'", Path("constraints file.txt"), []),
(
"dependency-versions = \"file:'constraints file.txt'\"",
Path("constraints file.txt"),
[],
),
(
"dependency-versions = {file = 'constraints file.txt'}",
Path("constraints file.txt"),
[],
),
(
"dependency-versions = 'packages: foo==1.2.3 bar==4.5.6'",
None,
["foo==1.2.3", "bar==4.5.6"],
),
],
)
def test_dependency_versions_toml(
tmp_path: Path,
toml_assignment: str,
base_file_path: Path | None,
packages: list[str] | None,
monkeypatch: pytest.MonkeyPatch,
) -> None:
args = CommandLineArguments.defaults()
args.package_dir = tmp_path
(tmp_path / "constraints file.txt").write_text("")
monkeypatch.chdir(tmp_path)
pyproject_toml: Path = tmp_path / "pyproject.toml"
pyproject_toml.write_text(
textwrap.dedent(
f"""\
[tool.cibuildwheel]
{toml_assignment}
"""
)
)
options = Options(platform="linux", command_line_arguments=args, env={})
parsed_dependency_constraints = options.build_options(None).dependency_constraints
if base_file_path is None and packages is None:
assert parsed_dependency_constraints == DependencyConstraints.latest()
else:
if parsed_dependency_constraints.base_file_path and base_file_path:
assert parsed_dependency_constraints.base_file_path.samefile(base_file_path)
else:
assert parsed_dependency_constraints.base_file_path == base_file_path
assert parsed_dependency_constraints.packages == packages