619 lines
22 KiB
Python
619 lines
22 KiB
Python
import functools
|
|
import os
|
|
import sys
|
|
import traceback
|
|
from configparser import ConfigParser
|
|
from contextlib import contextmanager
|
|
from dataclasses import asdict, dataclass
|
|
from pathlib import Path
|
|
from typing import (
|
|
Any,
|
|
Dict,
|
|
Generator,
|
|
List,
|
|
Mapping,
|
|
Optional,
|
|
Set,
|
|
Tuple,
|
|
Union,
|
|
cast,
|
|
)
|
|
|
|
if sys.version_info >= (3, 11):
|
|
import tomllib
|
|
else:
|
|
import tomli as tomllib
|
|
|
|
from packaging.specifiers import SpecifierSet
|
|
|
|
from .architecture import Architecture
|
|
from .environment import EnvironmentParseError, ParsedEnvironment, parse_environment
|
|
from .oci_container import ContainerEngine
|
|
from .projectfiles import get_requires_python_str
|
|
from .typing import PLATFORMS, Literal, PlatformName, TypedDict
|
|
from .util import (
|
|
MANYLINUX_ARCHS,
|
|
MUSLLINUX_ARCHS,
|
|
BuildFrontend,
|
|
BuildSelector,
|
|
DependencyConstraints,
|
|
TestSelector,
|
|
cached_property,
|
|
format_safe,
|
|
resources_dir,
|
|
selector_matches,
|
|
strtobool,
|
|
unwrap,
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class CommandLineArguments:
|
|
platform: Literal["auto", "linux", "macos", "windows"]
|
|
archs: Optional[str]
|
|
output_dir: Path
|
|
config_file: str
|
|
package_dir: Path
|
|
print_build_identifiers: bool
|
|
allow_empty: bool
|
|
prerelease_pythons: bool
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class GlobalOptions:
|
|
package_dir: Path
|
|
output_dir: Path
|
|
build_selector: BuildSelector
|
|
test_selector: TestSelector
|
|
architectures: Set[Architecture]
|
|
container_engine: ContainerEngine
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class BuildOptions:
|
|
globals: GlobalOptions
|
|
environment: ParsedEnvironment
|
|
before_all: str
|
|
before_build: Optional[str]
|
|
repair_command: str
|
|
manylinux_images: Optional[Dict[str, str]]
|
|
musllinux_images: Optional[Dict[str, str]]
|
|
dependency_constraints: Optional[DependencyConstraints]
|
|
test_command: Optional[str]
|
|
before_test: Optional[str]
|
|
test_requires: List[str]
|
|
test_extras: str
|
|
build_verbosity: int
|
|
build_frontend: BuildFrontend
|
|
|
|
@property
|
|
def package_dir(self) -> Path:
|
|
return self.globals.package_dir
|
|
|
|
@property
|
|
def output_dir(self) -> Path:
|
|
return self.globals.output_dir
|
|
|
|
@property
|
|
def build_selector(self) -> BuildSelector:
|
|
return self.globals.build_selector
|
|
|
|
@property
|
|
def test_selector(self) -> TestSelector:
|
|
return self.globals.test_selector
|
|
|
|
@property
|
|
def architectures(self) -> Set[Architecture]:
|
|
return self.globals.architectures
|
|
|
|
|
|
Setting = Union[Dict[str, str], List[str], str, int]
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Override:
|
|
select_pattern: str
|
|
options: Dict[str, Setting]
|
|
|
|
|
|
MANYLINUX_OPTIONS = {f"manylinux-{build_platform}-image" for build_platform in MANYLINUX_ARCHS}
|
|
MUSLLINUX_OPTIONS = {f"musllinux-{build_platform}-image" for build_platform in MUSLLINUX_ARCHS}
|
|
DISALLOWED_OPTIONS = {
|
|
"linux": {"dependency-versions"},
|
|
"macos": MANYLINUX_OPTIONS | MUSLLINUX_OPTIONS,
|
|
"windows": MANYLINUX_OPTIONS | MUSLLINUX_OPTIONS,
|
|
}
|
|
|
|
|
|
class TableFmt(TypedDict):
|
|
item: str
|
|
sep: str
|
|
|
|
|
|
class ConfigOptionError(KeyError):
|
|
pass
|
|
|
|
|
|
def _dig_first(*pairs: Tuple[Mapping[str, Setting], str], ignore_empty: bool = False) -> Setting:
|
|
"""
|
|
Return the first dict item that matches from pairs of dicts and keys.
|
|
Will throw a KeyError if missing.
|
|
|
|
_dig_first((dict1, "key1"), (dict2, "key2"), ...)
|
|
"""
|
|
if not pairs:
|
|
raise ValueError("pairs cannot be empty")
|
|
|
|
for dict_like, key in pairs:
|
|
if key in dict_like:
|
|
value = dict_like[key]
|
|
|
|
if ignore_empty and value == "":
|
|
continue
|
|
|
|
return value
|
|
|
|
last_key = pairs[-1][1]
|
|
raise KeyError(last_key)
|
|
|
|
|
|
class OptionsReader:
|
|
"""
|
|
Gets options from the environment, config or defaults, optionally scoped
|
|
by the platform.
|
|
|
|
Example:
|
|
>>> options_reader = OptionsReader(config_file, platform='macos')
|
|
>>> options_reader.get('cool-color')
|
|
|
|
This will return the value of CIBW_COOL_COLOR_MACOS if it exists,
|
|
otherwise the value of CIBW_COOL_COLOR, otherwise
|
|
'tool.cibuildwheel.macos.cool-color' or 'tool.cibuildwheel.cool-color'
|
|
from `config_file`, or from cibuildwheel/resources/defaults.toml. An
|
|
error is thrown if there are any unexpected keys or sections in
|
|
tool.cibuildwheel.
|
|
"""
|
|
|
|
def __init__(
|
|
self,
|
|
config_file_path: Optional[Path] = None,
|
|
*,
|
|
platform: PlatformName,
|
|
disallow: Optional[Dict[str, Set[str]]] = None,
|
|
) -> None:
|
|
self.platform = platform
|
|
self.disallow = disallow or {}
|
|
|
|
# Open defaults.toml, loading both global and platform sections
|
|
defaults_path = resources_dir / "defaults.toml"
|
|
self.default_options, self.default_platform_options = self._load_file(defaults_path)
|
|
|
|
# Load the project config file
|
|
config_options: Dict[str, Any] = {}
|
|
config_platform_options: Dict[str, Any] = {}
|
|
|
|
if config_file_path is not None:
|
|
config_options, config_platform_options = self._load_file(config_file_path)
|
|
|
|
# Validate project config
|
|
for option_name in config_options:
|
|
if not self._is_valid_global_option(option_name):
|
|
raise ConfigOptionError(f'Option "{option_name}" not supported in a config file')
|
|
|
|
for option_name in config_platform_options:
|
|
if not self._is_valid_platform_option(option_name):
|
|
raise ConfigOptionError(
|
|
f'Option "{option_name}" not supported in the "{self.platform}" section'
|
|
)
|
|
|
|
self.config_options = config_options
|
|
self.config_platform_options = config_platform_options
|
|
|
|
self.overrides: List[Override] = []
|
|
self.current_identifier: Optional[str] = None
|
|
|
|
config_overrides = self.config_options.get("overrides")
|
|
|
|
if config_overrides is not None:
|
|
if not isinstance(config_overrides, list):
|
|
raise ConfigOptionError('"tool.cibuildwheel.overrides" must be a list')
|
|
|
|
for config_override in config_overrides:
|
|
select = config_override.pop("select", None)
|
|
|
|
if not select:
|
|
raise ConfigOptionError('"select" must be set in an override')
|
|
|
|
if isinstance(select, list):
|
|
select = " ".join(select)
|
|
|
|
self.overrides.append(Override(select, config_override))
|
|
|
|
def _is_valid_global_option(self, name: str) -> bool:
|
|
"""
|
|
Returns True if an option with this name is allowed in the
|
|
[tool.cibuildwheel] section of a config file.
|
|
"""
|
|
allowed_option_names = self.default_options.keys() | PLATFORMS | {"overrides"}
|
|
|
|
return name in allowed_option_names
|
|
|
|
def _is_valid_platform_option(self, name: str) -> bool:
|
|
"""
|
|
Returns True if an option with this name is allowed in the
|
|
[tool.cibuildwheel.<current-platform>] section of a config file.
|
|
"""
|
|
disallowed_platform_options = self.disallow.get(self.platform, set())
|
|
if name in disallowed_platform_options:
|
|
return False
|
|
|
|
allowed_option_names = self.default_options.keys() | self.default_platform_options.keys()
|
|
|
|
return name in allowed_option_names
|
|
|
|
def _load_file(self, filename: Path) -> Tuple[Dict[str, Any], Dict[str, Any]]:
|
|
"""
|
|
Load a toml file, returns global and platform as separate dicts.
|
|
"""
|
|
with filename.open("rb") as f:
|
|
config = tomllib.load(f)
|
|
|
|
global_options = config.get("tool", {}).get("cibuildwheel", {})
|
|
platform_options = global_options.get(self.platform, {})
|
|
|
|
return global_options, platform_options
|
|
|
|
@property
|
|
def active_config_overrides(self) -> List[Override]:
|
|
if self.current_identifier is None:
|
|
return []
|
|
return [
|
|
o for o in self.overrides if selector_matches(o.select_pattern, self.current_identifier)
|
|
]
|
|
|
|
@contextmanager
|
|
def identifier(self, identifier: Optional[str]) -> Generator[None, None, None]:
|
|
self.current_identifier = identifier
|
|
try:
|
|
yield
|
|
finally:
|
|
self.current_identifier = None
|
|
|
|
def get(
|
|
self,
|
|
name: str,
|
|
*,
|
|
env_plat: bool = True,
|
|
sep: Optional[str] = None,
|
|
table: Optional[TableFmt] = None,
|
|
ignore_empty: bool = False,
|
|
) -> str:
|
|
"""
|
|
Get and return the value for the named option from environment,
|
|
configuration file, or the default. If env_plat is False, then don't
|
|
accept platform versions of the environment variable. If this is an
|
|
array it will be merged with "sep" before returning. If it is a table,
|
|
it will be formatted with "table['item']" using {k} and {v} and merged
|
|
with "table['sep']". Empty variables will not override if ignore_empty
|
|
is True.
|
|
"""
|
|
|
|
if name not in self.default_options and name not in self.default_platform_options:
|
|
raise ConfigOptionError(f"{name} must be in cibuildwheel/resources/defaults.toml file")
|
|
|
|
# Environment variable form
|
|
envvar = f"CIBW_{name.upper().replace('-', '_')}"
|
|
plat_envvar = f"{envvar}_{self.platform.upper()}"
|
|
|
|
# later overrides take precedence over earlier ones, so reverse the list
|
|
active_config_overrides = reversed(self.active_config_overrides)
|
|
|
|
# get the option from the environment, then the config file, then finally the default.
|
|
# platform-specific options are preferred, if they're allowed.
|
|
result = _dig_first(
|
|
(os.environ if env_plat else {}, plat_envvar), # type: ignore[arg-type]
|
|
(os.environ, envvar),
|
|
*[(o.options, name) for o in active_config_overrides],
|
|
(self.config_platform_options, name),
|
|
(self.config_options, name),
|
|
(self.default_platform_options, name),
|
|
(self.default_options, name),
|
|
ignore_empty=ignore_empty,
|
|
)
|
|
|
|
if isinstance(result, dict):
|
|
if table is None:
|
|
raise ConfigOptionError(f"{name} does not accept a table")
|
|
return table["sep"].join(table["item"].format(k=k, v=v) for k, v in result.items())
|
|
|
|
if isinstance(result, list):
|
|
if sep is None:
|
|
raise ConfigOptionError(f"{name} does not accept a list")
|
|
return sep.join(result)
|
|
|
|
if isinstance(result, int):
|
|
return str(result)
|
|
|
|
return result
|
|
|
|
|
|
class Options:
|
|
def __init__(self, platform: PlatformName, command_line_arguments: CommandLineArguments):
|
|
self.platform = platform
|
|
self.command_line_arguments = command_line_arguments
|
|
|
|
self.reader = OptionsReader(
|
|
self.config_file_path,
|
|
platform=platform,
|
|
disallow=DISALLOWED_OPTIONS,
|
|
)
|
|
|
|
@property
|
|
def config_file_path(self) -> Optional[Path]:
|
|
args = self.command_line_arguments
|
|
|
|
if args.config_file:
|
|
return Path(format_safe(args.config_file, package=args.package_dir))
|
|
|
|
# return pyproject.toml, if it's available
|
|
pyproject_toml_path = Path(args.package_dir) / "pyproject.toml"
|
|
if pyproject_toml_path.exists():
|
|
return pyproject_toml_path
|
|
|
|
return None
|
|
|
|
@cached_property
|
|
def package_requires_python_str(self) -> Optional[str]:
|
|
args = self.command_line_arguments
|
|
return get_requires_python_str(Path(args.package_dir))
|
|
|
|
@property
|
|
def globals(self) -> GlobalOptions:
|
|
args = self.command_line_arguments
|
|
package_dir = args.package_dir
|
|
output_dir = args.output_dir
|
|
|
|
build_config = self.reader.get("build", env_plat=False, sep=" ") or "*"
|
|
skip_config = self.reader.get("skip", env_plat=False, sep=" ")
|
|
test_skip = self.reader.get("test-skip", env_plat=False, sep=" ")
|
|
|
|
prerelease_pythons = args.prerelease_pythons or strtobool(
|
|
os.environ.get("CIBW_PRERELEASE_PYTHONS", "0")
|
|
)
|
|
|
|
# This is not supported in tool.cibuildwheel, as it comes from a standard location.
|
|
# Passing this in as an environment variable will override pyproject.toml, setup.cfg, or setup.py
|
|
requires_python_str: Optional[str] = (
|
|
os.environ.get("CIBW_PROJECT_REQUIRES_PYTHON") or self.package_requires_python_str
|
|
)
|
|
requires_python = None if requires_python_str is None else SpecifierSet(requires_python_str)
|
|
|
|
build_selector = BuildSelector(
|
|
build_config=build_config,
|
|
skip_config=skip_config,
|
|
requires_python=requires_python,
|
|
prerelease_pythons=prerelease_pythons,
|
|
)
|
|
test_selector = TestSelector(skip_config=test_skip)
|
|
|
|
archs_config_str = args.archs or self.reader.get("archs", sep=" ")
|
|
architectures = Architecture.parse_config(archs_config_str, platform=self.platform)
|
|
|
|
container_engine_str = self.reader.get("container-engine")
|
|
|
|
if container_engine_str not in ["docker", "podman"]:
|
|
msg = f"cibuildwheel: Unrecognised container_engine '{container_engine_str}', only 'docker' and 'podman' are supported"
|
|
print(msg, file=sys.stderr)
|
|
sys.exit(2)
|
|
|
|
container_engine = cast(ContainerEngine, container_engine_str)
|
|
|
|
return GlobalOptions(
|
|
package_dir=package_dir,
|
|
output_dir=output_dir,
|
|
build_selector=build_selector,
|
|
test_selector=test_selector,
|
|
architectures=architectures,
|
|
container_engine=container_engine,
|
|
)
|
|
|
|
def build_options(self, identifier: Optional[str]) -> BuildOptions:
|
|
"""
|
|
Compute BuildOptions for a single run configuration.
|
|
"""
|
|
|
|
with self.reader.identifier(identifier):
|
|
before_all = self.reader.get("before-all", sep=" && ")
|
|
|
|
build_frontend_str = self.reader.get("build-frontend", env_plat=False)
|
|
environment_config = self.reader.get(
|
|
"environment", table={"item": '{k}="{v}"', "sep": " "}
|
|
)
|
|
environment_pass = self.reader.get("environment-pass", sep=" ").split()
|
|
before_build = self.reader.get("before-build", sep=" && ")
|
|
repair_command = self.reader.get("repair-wheel-command", sep=" && ")
|
|
|
|
dependency_versions = self.reader.get("dependency-versions")
|
|
test_command = self.reader.get("test-command", sep=" && ")
|
|
before_test = self.reader.get("before-test", sep=" && ")
|
|
test_requires = self.reader.get("test-requires", sep=" ").split()
|
|
test_extras = self.reader.get("test-extras", sep=",")
|
|
build_verbosity_str = self.reader.get("build-verbosity")
|
|
|
|
build_frontend: BuildFrontend
|
|
if build_frontend_str == "build":
|
|
build_frontend = "build"
|
|
elif build_frontend_str == "pip":
|
|
build_frontend = "pip"
|
|
else:
|
|
msg = f"cibuildwheel: Unrecognised build frontend '{build_frontend_str}', only 'pip' and 'build' are supported"
|
|
print(msg, file=sys.stderr)
|
|
sys.exit(2)
|
|
|
|
try:
|
|
environment = parse_environment(environment_config)
|
|
except (EnvironmentParseError, ValueError):
|
|
print(
|
|
f'cibuildwheel: Malformed environment option "{environment_config}"',
|
|
file=sys.stderr,
|
|
)
|
|
traceback.print_exc(None, sys.stderr)
|
|
sys.exit(2)
|
|
|
|
# Pass through environment variables
|
|
if self.platform == "linux":
|
|
for env_var_name in environment_pass:
|
|
try:
|
|
environment.add(env_var_name, os.environ[env_var_name])
|
|
except KeyError:
|
|
pass
|
|
|
|
if dependency_versions == "pinned":
|
|
dependency_constraints: Optional[
|
|
DependencyConstraints
|
|
] = DependencyConstraints.with_defaults()
|
|
elif dependency_versions == "latest":
|
|
dependency_constraints = None
|
|
else:
|
|
dependency_versions_path = Path(dependency_versions)
|
|
dependency_constraints = DependencyConstraints(dependency_versions_path)
|
|
|
|
if test_extras:
|
|
test_extras = f"[{test_extras}]"
|
|
|
|
try:
|
|
build_verbosity = min(3, max(-3, int(build_verbosity_str)))
|
|
except ValueError:
|
|
build_verbosity = 0
|
|
|
|
manylinux_images: Dict[str, str] = {}
|
|
musllinux_images: Dict[str, str] = {}
|
|
if self.platform == "linux":
|
|
all_pinned_container_images = _get_pinned_container_images()
|
|
|
|
for build_platform in MANYLINUX_ARCHS:
|
|
pinned_images = all_pinned_container_images[build_platform]
|
|
|
|
config_value = self.reader.get(
|
|
f"manylinux-{build_platform}-image", ignore_empty=True
|
|
)
|
|
|
|
if not config_value:
|
|
# default to manylinux2014
|
|
image = pinned_images["manylinux2014"]
|
|
elif config_value in pinned_images:
|
|
image = pinned_images[config_value]
|
|
else:
|
|
image = config_value
|
|
|
|
manylinux_images[build_platform] = image
|
|
|
|
for build_platform in MUSLLINUX_ARCHS:
|
|
pinned_images = all_pinned_container_images[build_platform]
|
|
|
|
config_value = self.reader.get(f"musllinux-{build_platform}-image")
|
|
|
|
if not config_value:
|
|
image = pinned_images["musllinux_1_1"]
|
|
elif config_value in pinned_images:
|
|
image = pinned_images[config_value]
|
|
else:
|
|
image = config_value
|
|
|
|
musllinux_images[build_platform] = image
|
|
|
|
return BuildOptions(
|
|
globals=self.globals,
|
|
test_command=test_command,
|
|
test_requires=test_requires,
|
|
test_extras=test_extras,
|
|
before_test=before_test,
|
|
before_build=before_build,
|
|
before_all=before_all,
|
|
build_verbosity=build_verbosity,
|
|
repair_command=repair_command,
|
|
environment=environment,
|
|
dependency_constraints=dependency_constraints,
|
|
manylinux_images=manylinux_images or None,
|
|
musllinux_images=musllinux_images or None,
|
|
build_frontend=build_frontend,
|
|
)
|
|
|
|
def check_for_invalid_configuration(self, identifiers: List[str]) -> None:
|
|
if self.platform in ["macos", "windows"]:
|
|
before_all_values = {self.build_options(i).before_all for i in identifiers}
|
|
|
|
if len(before_all_values) > 1:
|
|
raise ValueError(
|
|
unwrap(
|
|
f"""
|
|
before_all cannot be set to multiple values. On macOS and Windows,
|
|
before_all is only run once, at the start of the build. before_all values
|
|
are: {before_all_values!r}
|
|
"""
|
|
)
|
|
)
|
|
|
|
def check_for_deprecated_options(self) -> None:
|
|
build_selector = self.globals.build_selector
|
|
test_selector = self.globals.test_selector
|
|
|
|
deprecated_selectors("CIBW_BUILD", build_selector.build_config, error=True)
|
|
deprecated_selectors("CIBW_SKIP", build_selector.skip_config)
|
|
deprecated_selectors("CIBW_TEST_SKIP", test_selector.skip_config)
|
|
|
|
def summary(self, identifiers: List[str]) -> str:
|
|
lines = [
|
|
f"{option_name}: {option_value!r}"
|
|
for option_name, option_value in sorted(asdict(self.globals).items())
|
|
]
|
|
|
|
build_option_defaults = self.build_options(identifier=None)
|
|
|
|
for option_name, default_value in sorted(asdict(build_option_defaults).items()):
|
|
if option_name == "globals":
|
|
continue
|
|
|
|
lines.append(f"{option_name}: {default_value!r}")
|
|
|
|
# if any identifiers have an overridden value, print that too
|
|
for identifier in identifiers:
|
|
option_value = getattr(self.build_options(identifier=identifier), option_name)
|
|
if option_value != default_value:
|
|
lines.append(f" {identifier}: {option_value!r}")
|
|
|
|
return "\n".join(lines)
|
|
|
|
|
|
def compute_options(
|
|
platform: PlatformName,
|
|
command_line_arguments: CommandLineArguments,
|
|
) -> Options:
|
|
options = Options(platform=platform, command_line_arguments=command_line_arguments)
|
|
options.check_for_deprecated_options()
|
|
return options
|
|
|
|
|
|
@functools.lru_cache(maxsize=None)
|
|
def _get_pinned_container_images() -> Mapping[str, Mapping[str, str]]:
|
|
"""
|
|
This looks like a dict of dicts, e.g.
|
|
{ 'x86_64': {'manylinux1': '...', 'manylinux2010': '...', 'manylinux2014': '...'},
|
|
'i686': {'manylinux1': '...', 'manylinux2010': '...', 'manylinux2014': '...'},
|
|
'pypy_x86_64': {'manylinux2010': '...' }
|
|
... }
|
|
"""
|
|
|
|
pinned_images_file = resources_dir / "pinned_docker_images.cfg"
|
|
all_pinned_images = ConfigParser()
|
|
all_pinned_images.read(pinned_images_file)
|
|
return all_pinned_images
|
|
|
|
|
|
def deprecated_selectors(name: str, selector: str, *, error: bool = False) -> None:
|
|
if "p2" in selector or "p35" in selector:
|
|
msg = f"cibuildwheel 2.x no longer supports Python < 3.6. Please use the 1.x series or update {name}"
|
|
print(msg, file=sys.stderr)
|
|
if error:
|
|
sys.exit(4)
|