Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: add inherit to override (#1730)

Browse Source 
* feat: add inherit to override

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* refactor: use dict and support prepend

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* Refactor: simplifying by splitting the responsibilities of _dig_first

* Refactor to allow merging of string settings, and preserve table cascades

* docs: add some docs for inherit

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* Apply suggestions from code review

Co-authored-by: Joe Rickerby <joerick@mac.com>

---------

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>
Co-authored-by: Joe Rickerby <joerick@mac.com>
This commit is contained in:
Henry Schreiner
2024-03-02 16:58:18 -05:00
committed by GitHub
co-authored by Joe Rickerby
parent ae2451a199
commit 8f58f71c2e
7 changed files with 567 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cibuildwheel/options.py
+150 -67
View File
@@ -5,19 +5,20 @@ import configparser
import contextlib
import dataclasses
import difflib
import enum
import functools
import shlex
import sys
import textwrap
import traceback
from collections.abc import Callable, Generator, Iterable, Iterator, Mapping, Set
from collections.abc import Callable, Generator, Iterable, Iterator, Set
from pathlib import Path
from typing import Any, Dict, List, Literal, TypedDict, Union
from typing import Any, Literal, Mapping, Sequence, TypedDict, Union # noqa: TID251
from packaging.specifiers import SpecifierSet
from ._compat import tomllib
from ._compat.typing import NotRequired
from ._compat.typing import NotRequired, assert_never
from .architecture import Architecture
from .environment import EnvironmentParseError, ParsedEnvironment, parse_environment
from .logger import log
@@ -116,13 +117,14 @@ class BuildOptions:
return self.globals.architectures
Setting = Union[Dict[str, str], List[str], str, int]
Setting = Union[Mapping[str, str], Sequence[str], str, int]
@dataclasses.dataclass(frozen=True)
class Override:
select_pattern: str
options: dict[str, Setting]
inherit: dict[str, InheritRule]
MANYLINUX_OPTIONS = {f"manylinux-{build_platform}-image" for build_platform in MANYLINUX_ARCHS}
@@ -150,28 +152,115 @@ class ConfigOptionError(KeyError):
pass
def _dig_first(*pairs: tuple[Mapping[str, Setting], str], ignore_empty: bool = False) -> Setting:
"""
Return the first dict item that matches from pairs of dicts and keys.
Will throw a KeyError if missing.
class InheritRule(enum.Enum):
NONE = enum.auto()
APPEND = enum.auto()
PREPEND = enum.auto()
_dig_first((dict1, "key1"), (dict2, "key2"), ...)
def _resolve_cascade(
*pairs: tuple[Setting | None, InheritRule],
ignore_empty: bool = False,
list_sep: str | None = None,
table_format: TableFmt | None = None,
) -> str:
"""
Given a cascade of values with inherit rules, resolve them into a single
value.
'None' values mean that the option was not set at that level, and are
ignored. If `ignore_empty` is True, empty values are ignored too.
Values start with defaults, followed by more specific rules. If rules are
NONE, the last non-null value is returned. If a rule is APPEND or PREPEND,
the value is concatenated with the previous value.
The following idiom can be used to get the first matching value:
_resolve_cascade(("value1", Inherit.NONE), ("value2", Inherit.NONE), ...)))
"""
if not pairs:
msg = "pairs cannot be empty"
raise ValueError(msg)
for dict_like, key in pairs:
if key in dict_like:
value = dict_like[key]
result: str | None = None
if ignore_empty and value == "":
continue
if table_format is not None:
merge_sep = table_format["sep"]
elif list_sep is not None:
merge_sep = list_sep
else:
merge_sep = None
return value
for value, rule in pairs:
if value is None:
continue
last_key = pairs[-1][1]
raise KeyError(last_key)
if ignore_empty and not value:
continue
value_string = _stringify_setting(value, list_sep, table_format)
result = _merge_values(
result,
value_string,
rule=rule,
merge_sep=merge_sep,
)
if result is None:
msg = "a setting should at least have a default value"
raise ValueError(msg)
return result
def _merge_values(before: str | None, after: str, rule: InheritRule, merge_sep: str | None) -> str:
if rule == InheritRule.NONE:
return after
if not before:
# if before is None, we can just return after
# if before is an empty string, we shouldn't add any separator
return after
if not after:
# if after is an empty string, we shouldn't add any separator
return before
if not merge_sep:
msg = f"Don't know how to merge {before!r} and {after!r} with {rule}"
raise ConfigOptionError(msg)
if rule == InheritRule.APPEND:
return f"{before}{merge_sep}{after}"
elif rule == InheritRule.PREPEND:
return f"{after}{merge_sep}{before}"
else:
assert_never(rule)
def _stringify_setting(
setting: Setting, list_sep: str | None, table_format: TableFmt | None
) -> str:
if isinstance(setting, Mapping):
if table_format is None:
msg = f"Error converting {setting!r} to a string: this setting doesn't accept a table"
raise ConfigOptionError(msg)
return table_format["sep"].join(
item for k, v in setting.items() for item in _inner_fmt(k, v, table_format)
)
if not isinstance(setting, str) and isinstance(setting, Sequence):
if list_sep is None:
msg = f"Error converting {setting!r} to a string: this setting doesn't accept a list"
raise ConfigOptionError(msg)
return list_sep.join(setting)
if isinstance(setting, int):
return str(setting)
return setting
class OptionsReader:
@@ -244,7 +333,16 @@ class OptionsReader:
if isinstance(select, list):
select = " ".join(select)
self.overrides.append(Override(select, config_override))
inherit = config_override.pop("inherit", {})
if not isinstance(inherit, dict) or not all(
i in {"none", "append", "prepend"} for i in inherit.values()
):
msg = "'inherit' must be a dict containing only {'none', 'append', 'prepend'} values"
raise ConfigOptionError(msg)
inherit_enum = {k: InheritRule[v.upper()] for k, v in inherit.items()}
self.overrides.append(Override(select, config_override, inherit_enum))
def _validate_global_option(self, name: str) -> None:
"""
@@ -312,8 +410,8 @@ class OptionsReader:
name: str,
*,
env_plat: bool = True,
sep: str | None = None,
table: TableFmt | None = None,
list_sep: str | None = None,
table_format: TableFmt | None = None,
ignore_empty: bool = False,
) -> str:
"""
@@ -335,41 +433,24 @@ class OptionsReader:
envvar = f"CIBW_{name.upper().replace('-', '_')}"
plat_envvar = f"{envvar}_{self.platform.upper()}"
# later overrides take precedence over earlier ones, so reverse the list
active_config_overrides = reversed(self.active_config_overrides)
# get the option from the environment, then the config file, then finally the default.
# get the option from the default, then the config file, then finally the environment.
# platform-specific options are preferred, if they're allowed.
result = _dig_first(
(self.env if env_plat else {}, plat_envvar),
(self.env, envvar),
*[(o.options, name) for o in active_config_overrides],
(self.config_platform_options, name),
(self.config_options, name),
(self.default_platform_options, name),
(self.default_options, name),
return _resolve_cascade(
(self.default_options.get(name), InheritRule.NONE),
(self.default_platform_options.get(name), InheritRule.NONE),
(self.config_options.get(name), InheritRule.NONE),
(self.config_platform_options.get(name), InheritRule.NONE),
*[
(o.options.get(name), o.inherit.get(name, InheritRule.NONE))
for o in self.active_config_overrides
],
(self.env.get(envvar), InheritRule.NONE),
(self.env.get(plat_envvar) if env_plat else None, InheritRule.NONE),
ignore_empty=ignore_empty,
list_sep=list_sep,
table_format=table_format,
)
if isinstance(result, dict):
if table is None:
msg = f"{name!r} does not accept a table"
raise ConfigOptionError(msg)
return table["sep"].join(
item for k, v in result.items() for item in _inner_fmt(k, v, table)
)
if isinstance(result, list):
if sep is None:
msg = f"{name!r} does not accept a list"
raise ConfigOptionError(msg)
return sep.join(result)
if isinstance(result, int):
return str(result)
return result
def _inner_fmt(k: str, v: Any, table: TableFmt) -> Iterator[str]:
quote_function = table.get("quote", lambda a: a)
@@ -430,9 +511,9 @@ class Options:
package_dir = args.package_dir
output_dir = args.output_dir
build_config = self.reader.get("build", env_plat=False, sep=" ") or "*"
skip_config = self.reader.get("skip", env_plat=False, sep=" ")
test_skip = self.reader.get("test-skip", env_plat=False, sep=" ")
build_config = self.reader.get("build", env_plat=False, list_sep=" ") or "*"
skip_config = self.reader.get("skip", env_plat=False, list_sep=" ")
test_skip = self.reader.get("test-skip", env_plat=False, list_sep=" ")
prerelease_pythons = args.prerelease_pythons or strtobool(
self.env.get("CIBW_PRERELEASE_PYTHONS", "0")
@@ -445,7 +526,7 @@ class Options:
)
requires_python = None if requires_python_str is None else SpecifierSet(requires_python_str)
archs_config_str = args.archs or self.reader.get("archs", sep=" ")
archs_config_str = args.archs or self.reader.get("archs", list_sep=" ")
architectures = Architecture.parse_config(archs_config_str, platform=self.platform)
# Process `--only`
@@ -464,7 +545,8 @@ class Options:
test_selector = TestSelector(skip_config=test_skip)
container_engine_str = self.reader.get(
"container-engine", table={"item": "{k}:{v}", "sep": "; ", "quote": shlex.quote}
"container-engine",
table_format={"item": "{k}:{v}", "sep": "; ", "quote": shlex.quote},
)
try:
@@ -489,29 +571,30 @@ class Options:
"""
with self.reader.identifier(identifier):
before_all = self.reader.get("before-all", sep=" && ")
before_all = self.reader.get("before-all", list_sep=" && ")
environment_config = self.reader.get(
"environment", table={"item": '{k}="{v}"', "sep": " "}
"environment", table_format={"item": '{k}="{v}"', "sep": " "}
)
environment_pass = self.reader.get("environment-pass", sep=" ").split()
before_build = self.reader.get("before-build", sep=" && ")
repair_command = self.reader.get("repair-wheel-command", sep=" && ")
environment_pass = self.reader.get("environment-pass", list_sep=" ").split()
before_build = self.reader.get("before-build", list_sep=" && ")
repair_command = self.reader.get("repair-wheel-command", list_sep=" && ")
config_settings = self.reader.get(
"config-settings", table={"item": "{k}={v}", "sep": " ", "quote": shlex.quote}
"config-settings",
table_format={"item": "{k}={v}", "sep": " ", "quote": shlex.quote},
)
dependency_versions = self.reader.get("dependency-versions")
test_command = self.reader.get("test-command", sep=" && ")
before_test = self.reader.get("before-test", sep=" && ")
test_requires = self.reader.get("test-requires", sep=" ").split()
test_extras = self.reader.get("test-extras", sep=",")
test_command = self.reader.get("test-command", list_sep=" && ")
before_test = self.reader.get("before-test", list_sep=" && ")
test_requires = self.reader.get("test-requires", list_sep=" ").split()
test_extras = self.reader.get("test-extras", list_sep=",")
build_verbosity_str = self.reader.get("build-verbosity")
build_frontend_str = self.reader.get(
"build-frontend",
env_plat=False,
table={"item": "{k}:{v}", "sep": "; ", "quote": shlex.quote},
table_format={"item": "{k}:{v}", "sep": "; ", "quote": shlex.quote},
)
build_frontend: BuildFrontendConfig | None
if not build_frontend_str or build_frontend_str == "default":