From 8d056814912a863a5cadc034b33d23536555b8fa Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 26 Nov 2022 15:54:08 +0000 Subject: [PATCH 1/7] Improve the formatting of the preamble --- cibuildwheel/__main__.py | 5 +- cibuildwheel/architecture.py | 3 + cibuildwheel/environment.py | 5 +- cibuildwheel/options.py | 151 ++++++++++++++++++++++++---- cibuildwheel/util.py | 17 ++++ unit_test/linux_build_steps_test.py | 8 +- unit_test/options_test.py | 41 ++++---- unit_test/options_toml_test.py | 70 +++++++------ unit_test/utils.py | 21 ---- 9 files changed, 223 insertions(+), 98 deletions(-) delete mode 100644 unit_test/utils.py diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 48ba61af..afe46d2e 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -229,7 +229,7 @@ def build_in_directory(args: CommandLineArguments) -> None: ) sys.exit(2) - options = compute_options(platform=platform, command_line_arguments=args) + options = compute_options(platform=platform, command_line_arguments=args, environ=os.environ) package_dir = options.globals.package_dir package_files = {"setup.py", "setup.cfg", "pyproject.toml"} @@ -318,9 +318,10 @@ def print_preamble(platform: str, options: Options, identifiers: list[str]) -> N print(f"cibuildwheel version {cibuildwheel.__version__}\n") print("Build options:") - print(f" platform: {platform!r}") + print(f" platform: {platform}") print(textwrap.indent(options.summary(identifiers), " ")) + print() print(f"Cache folder: {CIBW_CACHE_PATH}") warnings = detect_warnings(options=options, identifiers=identifiers) diff --git a/cibuildwheel/architecture.py b/cibuildwheel/architecture.py index 90d30c58..7be0bb3a 100644 --- a/cibuildwheel/architecture.py +++ b/cibuildwheel/architecture.py @@ -43,6 +43,9 @@ class Architecture(Enum): def __lt__(self, other: Architecture) -> bool: return self.value < other.value + def __str__(self) -> str: + return self.name + @staticmethod def parse_config(config: str, platform: PlatformName) -> set[Architecture]: result = set() diff --git a/cibuildwheel/environment.py b/cibuildwheel/environment.py index 23ea9877..fd17175d 100644 --- a/cibuildwheel/environment.py +++ b/cibuildwheel/environment.py @@ -70,7 +70,7 @@ class EnvironmentAssignmentRaw: self.value = value def __repr__(self) -> str: - return f"{self.name}: {self.value}" + return f"{self.name}={self.value}" def evaluated_value(self, **_: Any) -> str: return self.value @@ -131,6 +131,9 @@ class ParsedEnvironment: def __repr__(self) -> str: return f"{self.__class__.__name__}({[repr(a) for a in self.assignments]!r})" + def options_summary(self) -> Any: + return self.assignments + def parse_environment(env_string: str) -> ParsedEnvironment: env_items = split_env_items(env_string) diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index fa313800..190978e4 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -1,13 +1,14 @@ from __future__ import annotations +import collections import configparser import contextlib import dataclasses import difflib import functools -import os import shlex import sys +import textwrap import traceback from pathlib import Path from typing import Any, Callable, Dict, Generator, Iterator, List, Mapping, Union, cast @@ -21,6 +22,7 @@ from packaging.specifiers import SpecifierSet from .architecture import Architecture from .environment import EnvironmentParseError, ParsedEnvironment, parse_environment +from .logger import log from .oci_container import ContainerEngine from .projectfiles import get_requires_python_str from .typing import PLATFORMS, Literal, NotRequired, PlatformName, TypedDict @@ -52,6 +54,20 @@ class CommandLineArguments: allow_empty: bool prerelease_pythons: bool + @staticmethod + def defaults() -> CommandLineArguments: + return CommandLineArguments( + platform="auto", + allow_empty=False, + archs=None, + only=None, + config_file="", + output_dir=Path("wheelhouse"), + package_dir=Path("."), + prerelease_pythons=False, + print_build_identifiers=False, + ) + @dataclasses.dataclass(frozen=True) class GlobalOptions: @@ -176,9 +192,11 @@ class OptionsReader: config_file_path: Path | None = None, *, platform: PlatformName, + environ: Mapping[str, str], disallow: dict[str, set[str]] | None = None, ) -> None: self.platform = platform + self.environ = environ self.disallow = disallow or {} # Open defaults.toml, loading both global and platform sections @@ -319,8 +337,8 @@ class OptionsReader: # get the option from the environment, then the config file, then finally the default. # platform-specific options are preferred, if they're allowed. result = _dig_first( - (os.environ if env_plat else {}, plat_envvar), # type: ignore[arg-type] - (os.environ, envvar), + (self.environ if env_plat else {}, plat_envvar), + (self.environ, envvar), *[(o.options, name) for o in active_config_overrides], (self.config_platform_options, name), (self.config_options, name), @@ -362,13 +380,21 @@ def _inner_fmt(k: str, v: Any, table: TableFmt) -> Iterator[str]: class Options: - def __init__(self, platform: PlatformName, command_line_arguments: CommandLineArguments): + def __init__( + self, + platform: PlatformName, + command_line_arguments: CommandLineArguments, + environ: Mapping[str, str], + read_config_file: bool = True, + ): self.platform = platform self.command_line_arguments = command_line_arguments + self.environ = environ self.reader = OptionsReader( - self.config_file_path, + self.config_file_path if read_config_file else None, platform=platform, + environ=environ, disallow=DISALLOWED_OPTIONS, ) @@ -402,13 +428,13 @@ class Options: test_skip = self.reader.get("test-skip", env_plat=False, sep=" ") prerelease_pythons = args.prerelease_pythons or strtobool( - os.environ.get("CIBW_PRERELEASE_PYTHONS", "0") + self.environ.get("CIBW_PRERELEASE_PYTHONS", "0") ) # This is not supported in tool.cibuildwheel, as it comes from a standard location. # Passing this in as an environment variable will override pyproject.toml, setup.cfg, or setup.py requires_python_str: str | None = ( - os.environ.get("CIBW_PROJECT_REQUIRES_PYTHON") or self.package_requires_python_str + self.environ.get("CIBW_PROJECT_REQUIRES_PYTHON") or self.package_requires_python_str ) requires_python = None if requires_python_str is None else SpecifierSet(requires_python_str) @@ -497,7 +523,7 @@ class Options: if self.platform == "linux": for env_var_name in environment_pass: with contextlib.suppress(KeyError): - environment.add(env_var_name, os.environ[env_var_name]) + environment.add(env_var_name, self.environ[env_var_name]) if dependency_versions == "pinned": dependency_constraints: None | ( @@ -594,37 +620,118 @@ class Options: deprecated_selectors("CIBW_SKIP", build_selector.skip_config) deprecated_selectors("CIBW_TEST_SKIP", test_selector.skip_config) - def summary(self, identifiers: list[str]) -> str: - lines = [ - f"{option_name}: {option_value!r}" - for option_name, option_value in sorted(dataclasses.asdict(self.globals).items()) - ] + @cached_property + def defaults(self) -> Options: + return Options( + platform=self.platform, + command_line_arguments=CommandLineArguments.defaults(), + environ={}, + read_config_file=False, + ) - build_option_defaults = self.build_options(identifier=None) + def summary(self, identifiers: list[str]) -> str: + lines = [] + global_option_names = sorted(f.name for f in dataclasses.fields(self.globals)) + + for option_name in global_option_names: + option_value = getattr(self.globals, option_name) + default_value = getattr(self.defaults.globals, option_name) + lines.append(self.option_summary(option_name, option_value, default_value)) + + build_options = self.build_options(identifier=None) + build_options_defaults = self.defaults.build_options(identifier=None) build_options_for_identifier = { identifier: self.build_options(identifier) for identifier in identifiers } - for option_name, default_value in sorted(dataclasses.asdict(build_option_defaults).items()): + build_option_names = sorted(f.name for f in dataclasses.fields(build_options)) + + for option_name in build_option_names: if option_name == "globals": continue - lines.append(f"{option_name}: {default_value!r}") + option_value = getattr(build_options, option_name) + default_value = getattr(build_options_defaults, option_name) + overrides = { + i: getattr(build_options_for_identifier[i], option_name) for i in identifiers + } - # if any identifiers have an overridden value, print that too - for identifier in identifiers: - option_value = getattr(build_options_for_identifier[identifier], option_name) - if option_value != default_value: - lines.append(f" {identifier}: {option_value!r}") + lines.append( + self.option_summary(option_name, option_value, default_value, overrides=overrides) + ) return "\n".join(lines) + def option_summary( + self, + option_name: str, + option_value: Any, + default_value: Any, + overrides: dict[str, Any] | None = None, + ) -> str: + value_str = self.option_summary_value(option_value) + default_value_str = self.option_summary_value(default_value) + overrides_value_strs = { + k: self.option_summary_value(v) for k, v in (overrides or {}).items() + } + # if the override value is the same as the non-overridden value, don't print it + overrides_value_strs = {k: v for k, v in overrides_value_strs.items() if v != value_str} + + has_been_set = (value_str != default_value_str) or overrides_value_strs + c = log.colors + + result = "" + result += c.faint if not has_been_set else "" + result += f"{option_name}: " + + if overrides_value_strs: + overrides_groups = collections.defaultdict(list) + for k, v in overrides_value_strs.items(): + overrides_groups[v].append(k) + + result += "\n *: " + result += self.indent_if_multiline(value_str, " ") + + for override_value_str, identifiers in overrides_groups.items(): + result += f"\n {', '.join(identifiers)}: " + result += self.indent_if_multiline(override_value_str, " ") + else: + result += self.indent_if_multiline(value_str, " ") + + result += c.end + + return result + + def indent_if_multiline(self, value: str, indent: str) -> str: + if "\n" in value: + return "\n" + textwrap.indent(value.strip(), indent) + else: + return value + + def option_summary_value(self, option_value: Any) -> str: + if hasattr(option_value, "options_summary"): + option_value = option_value.options_summary() + + if isinstance(option_value, list): + return "".join(f"{el}\n" for el in option_value) + + if isinstance(option_value, set): + return ", ".join(str(el) for el in sorted(option_value)) + + if isinstance(option_value, dict): + return "".join(f"{k}: {v}\n" for k, v in option_value.items()) + + return str(option_value) + def compute_options( platform: PlatformName, command_line_arguments: CommandLineArguments, + environ: Mapping[str, str], ) -> Options: - options = Options(platform=platform, command_line_arguments=command_line_arguments) + options = Options( + platform=platform, command_line_arguments=command_line_arguments, environ=environ + ) options.check_for_deprecated_options() return options diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 29ccda97..fa1af89b 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -270,6 +270,14 @@ class BuildSelector: return should_build and not should_skip + def options_summary(self) -> Any: + return { + "build_config": self.build_config, + "skip_config": self.skip_config, + "requires_python": str(self.requires_python), + "prerelease_pythons": self.prerelease_pythons, + } + @dataclass(frozen=True) class TestSelector: @@ -283,6 +291,9 @@ class TestSelector: should_skip = selector_matches(self.skip_config, build_id) return not should_skip + def options_summary(self) -> Any: + return {"skip_config": self.skip_config} + # Taken from https://stackoverflow.com/a/107717 class Unbuffered: @@ -356,6 +367,12 @@ class DependencyConstraints: return self.base_file_path == o.base_file_path + def options_summary(self) -> Any: + if self == DependencyConstraints.with_defaults(): + return "pinned" + else: + return self.base_file_path.name + class NonPlatformWheelError(Exception): def __init__(self) -> None: diff --git a/unit_test/linux_build_steps_test.py b/unit_test/linux_build_steps_test.py index 8423d423..4730a4a9 100644 --- a/unit_test/linux_build_steps_test.py +++ b/unit_test/linux_build_steps_test.py @@ -6,9 +6,7 @@ from pprint import pprint import cibuildwheel.linux import cibuildwheel.oci_container -from cibuildwheel.options import Options - -from .utils import get_default_command_line_arguments +from cibuildwheel.options import CommandLineArguments, Options def test_linux_container_split(tmp_path: Path, monkeypatch): @@ -16,7 +14,7 @@ def test_linux_container_split(tmp_path: Path, monkeypatch): Tests splitting linux builds by container image and before_all """ - args = get_default_command_line_arguments() + args = CommandLineArguments.defaults() args.platform = "linux" (tmp_path / "pyproject.toml").write_text( @@ -42,7 +40,7 @@ def test_linux_container_split(tmp_path: Path, monkeypatch): ) monkeypatch.chdir(tmp_path) - options = Options("linux", command_line_arguments=args) + options = Options("linux", command_line_arguments=args, environ={}) python_configurations = cibuildwheel.linux.get_python_configurations( options.globals.build_selector, options.globals.architectures diff --git a/unit_test/options_test.py b/unit_test/options_test.py index 73d558f4..d967aa40 100644 --- a/unit_test/options_test.py +++ b/unit_test/options_test.py @@ -10,13 +10,16 @@ import pytest from cibuildwheel.__main__ import get_build_identifiers from cibuildwheel.bashlex_eval import local_environment_executor from cibuildwheel.environment import parse_environment -from cibuildwheel.options import Options, _get_pinned_container_images - -from .utils import get_default_command_line_arguments +from cibuildwheel.options import ( + CommandLineArguments, + Options, + _get_pinned_container_images, +) PYPROJECT_1 = """ [tool.cibuildwheel] build = ["cp38*", "cp37*"] +skip = ["*musllinux*"] environment = {FOO="BAR"} test-command = "pyproject" @@ -39,12 +42,12 @@ def test_options_1(tmp_path, monkeypatch): with tmp_path.joinpath("pyproject.toml").open("w") as f: f.write(PYPROJECT_1) - args = get_default_command_line_arguments() + args = CommandLineArguments.defaults() args.package_dir = tmp_path monkeypatch.setattr(platform_module, "machine", lambda: "x86_64") - options = Options(platform="linux", command_line_arguments=args) + options = Options(platform="linux", command_line_arguments=args, environ={}) identifiers = get_build_identifiers( platform="linux", @@ -53,9 +56,8 @@ def test_options_1(tmp_path, monkeypatch): ) override_display = """\ -test_command: 'pyproject' - cp37-manylinux_x86_64: 'pyproject-override'""" - + *: pyproject + cp37-manylinux_x86_64, cp37-manylinux_i686: pyproject-override""" print(options.summary(identifiers)) assert override_display in options.summary(identifiers) @@ -82,13 +84,12 @@ def test_passthrough(tmp_path, monkeypatch): with tmp_path.joinpath("pyproject.toml").open("w") as f: f.write(PYPROJECT_1) - args = get_default_command_line_arguments() + args = CommandLineArguments.defaults() args.package_dir = tmp_path monkeypatch.setattr(platform_module, "machine", lambda: "x86_64") - monkeypatch.setenv("EXAMPLE_ENV", "ONE") - options = Options(platform="linux", command_line_arguments=args) + options = Options(platform="linux", command_line_arguments=args, environ={"EXAMPLE_ENV": "ONE"}) default_build_options = options.build_options(identifier=None) @@ -110,14 +111,16 @@ def test_passthrough(tmp_path, monkeypatch): ], ) def test_passthrough_evil(tmp_path, monkeypatch, env_var_value): - args = get_default_command_line_arguments() + args = CommandLineArguments.defaults() args.package_dir = tmp_path monkeypatch.setattr(platform_module, "machine", lambda: "x86_64") - monkeypatch.setenv("CIBW_ENVIRONMENT_PASS_LINUX", "ENV_VAR") - options = Options(platform="linux", command_line_arguments=args) + options = Options( + platform="linux", + command_line_arguments=args, + environ={"CIBW_ENVIRONMENT_PASS_LINUX": "ENV_VAR", "ENV_VAR": env_var_value}, + ) - monkeypatch.setenv("ENV_VAR", env_var_value) parsed_environment = options.build_options(identifier=None).environment assert parsed_environment.as_dictionary(prev_environment={}) == {"ENV_VAR": env_var_value} @@ -138,7 +141,7 @@ xfail_env_parse = pytest.mark.xfail( ], ) def test_toml_environment_evil(tmp_path, monkeypatch, env_var_value): - args = get_default_command_line_arguments() + args = CommandLineArguments.defaults() args.package_dir = tmp_path tmp_path.joinpath("pyproject.toml").write_text( @@ -150,7 +153,7 @@ def test_toml_environment_evil(tmp_path, monkeypatch, env_var_value): ) ) - options = Options(platform="linux", command_line_arguments=args) + options = Options(platform="linux", command_line_arguments=args, environ={}) parsed_environment = options.build_options(identifier=None).environment assert parsed_environment.as_dictionary(prev_environment={}) == {"EXAMPLE": env_var_value} @@ -174,7 +177,7 @@ def test_toml_environment_evil(tmp_path, monkeypatch, env_var_value): ], ) def test_toml_environment_quoting(tmp_path: Path, toml_assignment, result_value): - args = get_default_command_line_arguments() + args = CommandLineArguments.defaults() args.package_dir = tmp_path tmp_path.joinpath("pyproject.toml").write_text( @@ -186,7 +189,7 @@ def test_toml_environment_quoting(tmp_path: Path, toml_assignment, result_value) ) ) - options = Options(platform="linux", command_line_arguments=args) + options = Options(platform="linux", command_line_arguments=args, environ={}) parsed_environment = options.build_options(identifier=None).environment environment_values = parsed_environment.as_dictionary( prev_environment={**os.environ, "PARAM": "spam"}, diff --git a/unit_test/options_toml_test.py b/unit_test/options_toml_test.py index 5c01a5fe..91ef98d5 100644 --- a/unit_test/options_toml_test.py +++ b/unit_test/options_toml_test.py @@ -35,7 +35,7 @@ def test_simple_settings(tmp_path, platform, fname): config_file_path: Path = tmp_path / fname config_file_path.write_text(PYPROJECT_1) - options_reader = OptionsReader(config_file_path, platform=platform) + options_reader = OptionsReader(config_file_path, platform=platform, environ={}) assert options_reader.get("build", env_plat=False, sep=" ") == "cp39*" @@ -72,16 +72,20 @@ def test_simple_settings(tmp_path, platform, fname): def test_envvar_override(tmp_path, platform, monkeypatch): - monkeypatch.setenv("CIBW_BUILD", "cp38*") - monkeypatch.setenv("CIBW_MANYLINUX_X86_64_IMAGE", "manylinux_2_24") - monkeypatch.setenv("CIBW_TEST_COMMAND", "mytest") - monkeypatch.setenv("CIBW_TEST_REQUIRES", "docs") - monkeypatch.setenv("CIBW_TEST_REQUIRES_LINUX", "scod") - config_file_path: Path = tmp_path / "pyproject.toml" config_file_path.write_text(PYPROJECT_1) - options_reader = OptionsReader(config_file_path, platform=platform) + options_reader = OptionsReader( + config_file_path, + platform=platform, + environ={ + "CIBW_BUILD": "cp38*", + "CIBW_MANYLINUX_X86_64_IMAGE": "manylinux_2_24", + "CIBW_TEST_COMMAND": "mytest", + "CIBW_TEST_REQUIRES": "docs", + "CIBW_TEST_REQUIRES_LINUX": "scod", + }, + ) assert options_reader.get("archs", sep=" ") == "auto" @@ -104,18 +108,18 @@ def test_project_global_override_default_platform(tmp_path, platform): repair-wheel-command = "repair-project-global" """ ) - options_reader = OptionsReader(pyproject_toml, platform=platform) + options_reader = OptionsReader(pyproject_toml, platform=platform, environ={}) assert options_reader.get("repair-wheel-command") == "repair-project-global" def test_env_global_override_default_platform(tmp_path, platform, monkeypatch): - monkeypatch.setenv("CIBW_REPAIR_WHEEL_COMMAND", "repair-env-global") - options_reader = OptionsReader(platform=platform) + options_reader = OptionsReader( + platform=platform, environ={"CIBW_REPAIR_WHEEL_COMMAND": "repair-env-global"} + ) assert options_reader.get("repair-wheel-command") == "repair-env-global" def test_env_global_override_project_platform(tmp_path, platform, monkeypatch): - monkeypatch.setenv("CIBW_REPAIR_WHEEL_COMMAND", "repair-env-global") pyproject_toml = tmp_path / "pyproject.toml" pyproject_toml.write_text( """ @@ -127,7 +131,13 @@ repair-wheel-command = "repair-project-windows" repair-wheel-command = "repair-project-macos" """ ) - options_reader = OptionsReader(pyproject_toml, platform=platform) + options_reader = OptionsReader( + pyproject_toml, + platform=platform, + environ={ + "CIBW_REPAIR_WHEEL_COMMAND": "repair-env-global", + }, + ) assert options_reader.get("repair-wheel-command") == "repair-env-global" @@ -145,7 +155,7 @@ repair-wheel-command = "repair-project-macos" repair-wheel-command = "repair-project-global" """ ) - options_reader = OptionsReader(pyproject_toml, platform=platform) + options_reader = OptionsReader(pyproject_toml, platform=platform, environ={}) assert options_reader.get("repair-wheel-command") == f"repair-project-{platform}" @@ -161,7 +171,7 @@ repairs-wheel-command = "repair-project-linux" ) with pytest.raises(ConfigOptionError) as excinfo: - OptionsReader(pyproject_toml, platform="linux") + OptionsReader(pyproject_toml, platform="linux", environ={}) assert "repair-wheel-command" in str(excinfo.value) @@ -178,7 +188,7 @@ repair_wheel_command = "repair-project-linux" ) with pytest.raises(ConfigOptionError) as excinfo: - OptionsReader(pyproject_toml, platform="linux") + OptionsReader(pyproject_toml, platform="linux", environ={}) assert "repair-wheel-command" in str(excinfo.value) @@ -192,7 +202,7 @@ repair-wheel-command = "repair-project-linux" """ ) with pytest.raises(ConfigOptionError): - OptionsReader(pyproject_toml, platform="linux") + OptionsReader(pyproject_toml, platform="linux", environ={}) def test_unsupported_join(tmp_path): @@ -203,7 +213,7 @@ def test_unsupported_join(tmp_path): build = ["1", "2"] """ ) - options_reader = OptionsReader(pyproject_toml, platform="linux") + options_reader = OptionsReader(pyproject_toml, platform="linux", environ={}) assert "1, 2" == options_reader.get("build", sep=", ") with pytest.raises(ConfigOptionError): @@ -219,9 +229,9 @@ manylinux-x86_64-image = "manylinux1" """ ) disallow = {"windows": {"manylinux-x86_64-image"}} - OptionsReader(pyproject_toml, platform="linux", disallow=disallow) + OptionsReader(pyproject_toml, platform="linux", disallow=disallow, environ={}) with pytest.raises(ConfigOptionError): - OptionsReader(pyproject_toml, platform="windows", disallow=disallow) + OptionsReader(pyproject_toml, platform="windows", disallow=disallow, environ={}) def test_environment_override_empty(tmp_path, monkeypatch): @@ -234,10 +244,14 @@ manylinux-x86_64-image = "" """ ) - monkeypatch.setenv("CIBW_MANYLINUX_I686_IMAGE", "") - monkeypatch.setenv("CIBW_MANYLINUX_AARCH64_IMAGE", "manylinux1") - - options_reader = OptionsReader(pyproject_toml, platform="linux") + options_reader = OptionsReader( + pyproject_toml, + platform="linux", + environ={ + "CIBW_MANYLINUX_I686_IMAGE": "", + "CIBW_MANYLINUX_AARCH64_IMAGE": "manylinux1", + }, + ) assert options_reader.get("manylinux-x86_64-image") == "" assert options_reader.get("manylinux-i686-image") == "" @@ -306,7 +320,7 @@ def test_pyproject_2(tmp_path, platform): pyproject_toml: Path = tmp_path / "pyproject.toml" pyproject_toml.write_text(PYPROJECT_2) - options_reader = OptionsReader(config_file_path=pyproject_toml, platform=platform) + options_reader = OptionsReader(config_file_path=pyproject_toml, platform=platform, environ={}) assert options_reader.get("test-command") == "pyproject" with options_reader.identifier("random"): @@ -330,7 +344,7 @@ test-command = "pyproject-override" ) with pytest.raises(ConfigOptionError): - OptionsReader(config_file_path=pyproject_toml, platform=platform) + OptionsReader(config_file_path=pyproject_toml, platform=platform, environ={}) def test_config_settings(tmp_path): @@ -343,7 +357,7 @@ other = ["two", "three"] """ ) - options_reader = OptionsReader(config_file_path=pyproject_toml, platform="linux") + options_reader = OptionsReader(config_file_path=pyproject_toml, platform="linux", environ={}) assert ( options_reader.get("config-settings", table={"item": '{k}="{v}"', "sep": " "}) == 'example="one" other="two" other="three"' @@ -359,7 +373,7 @@ def test_pip_config_settings(tmp_path): """ ) - options_reader = OptionsReader(config_file_path=pyproject_toml, platform="linux") + options_reader = OptionsReader(config_file_path=pyproject_toml, platform="linux", environ={}) assert ( options_reader.get( "config-settings", table={"item": "--config-settings='{k}=\"{v}\"'", "sep": " "} diff --git a/unit_test/utils.py b/unit_test/utils.py deleted file mode 100644 index bf44382a..00000000 --- a/unit_test/utils.py +++ /dev/null @@ -1,21 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -from cibuildwheel.options import CommandLineArguments - - -def get_default_command_line_arguments() -> CommandLineArguments: - defaults = CommandLineArguments( - platform="auto", - allow_empty=False, - archs=None, - only=None, - config_file="", - output_dir=Path("wheelhouse"), - package_dir=Path("."), - prerelease_pythons=False, - print_build_identifiers=False, - ) - - return defaults From 76e56f08f2bba5c07d689985b46e851d1990929e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 26 Nov 2022 20:12:22 +0000 Subject: [PATCH 2/7] Use 'gray' instead of 'dim' ANSI color (github doesn't seem to support 'dim') --- cibuildwheel/logger.py | 1 + cibuildwheel/options.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index 70f22997..426ea746 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -228,6 +228,7 @@ class Colors: self.bright_red = "\033[91m" if enabled else "" self.bright_green = "\033[92m" if enabled else "" self.white = "\033[37m\033[97m" if enabled else "" + self.gray = "\033[90m" if enabled else "" self.bg_grey = "\033[48;5;235m" if enabled else "" diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index 190978e4..a0894b17 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -681,7 +681,7 @@ class Options: c = log.colors result = "" - result += c.faint if not has_been_set else "" + result += c.gray if not has_been_set else "" result += f"{option_name}: " if overrides_value_strs: From c777af1f609de62a393ee736a46c01a3ae31c68c Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 27 Nov 2022 14:11:15 +0000 Subject: [PATCH 3/7] Use an 8-bit gray instead --- cibuildwheel/logger.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index 426ea746..a95ed2e5 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -228,7 +228,7 @@ class Colors: self.bright_red = "\033[91m" if enabled else "" self.bright_green = "\033[92m" if enabled else "" self.white = "\033[37m\033[97m" if enabled else "" - self.gray = "\033[90m" if enabled else "" + self.gray = "\033[38;5;244m" if enabled else "" self.bg_grey = "\033[48;5;235m" if enabled else "" From b00547cd9763a06f95f0ce457aef02428981298f Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 4 Dec 2022 13:51:56 +0000 Subject: [PATCH 4/7] Workaround the Github forgetting styles bug --- cibuildwheel/__main__.py | 7 ++++++- cibuildwheel/options.py | 4 ++++ cibuildwheel/util.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index afe46d2e..99af21b4 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -22,9 +22,11 @@ from cibuildwheel.typing import PLATFORMS, PlatformName, assert_never from cibuildwheel.util import ( CIBW_CACHE_PATH, BuildSelector, + CIProvider, Unbuffered, chdir, detect_ci_provider, + fix_ansi_codes_for_github_actions, ) @@ -319,7 +321,10 @@ def print_preamble(platform: str, options: Options, identifiers: list[str]) -> N print("Build options:") print(f" platform: {platform}") - print(textwrap.indent(options.summary(identifiers), " ")) + options_summary = textwrap.indent(options.summary(identifiers), " ") + if detect_ci_provider() == CIProvider.github_actions: + options_summary = fix_ansi_codes_for_github_actions(options_summary) + print(options_summary) print() print(f"Cache folder: {CIBW_CACHE_PATH}") diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index a0894b17..0f2852e9 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -669,6 +669,10 @@ class Options: default_value: Any, overrides: dict[str, Any] | None = None, ) -> str: + """ + Return a summary of the option value, including any overrides, with + ANSI 'dim' color if it's the default. + """ value_str = self.option_summary_value(option_value) default_value_str = self.option_summary_value(default_value) overrides_value_strs = { diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index fa1af89b..ca83d964 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -672,3 +672,31 @@ def chdir(new_path: Path | str) -> Generator[None, None, None]: yield finally: os.chdir(cwd) + + +def fix_ansi_codes_for_github_actions(text: str) -> str: + """ + Github Actions forgets the current ANSI style on every new line. This + function repeats the current ANSI style on every new line. + """ + ansi_code_regex = re.compile(r"(\033\[[0-9;]*m)") + ansi_codes: list[str] = [] + output = "" + + for line in text.split("\n"): + # add the current ANSI codes to the beginning of the line + output += "".join(ansi_codes) + line + "\n" + + # split the line at each ANSI code + parts = ansi_code_regex.split(line) + # if there are any ANSI codes, save them + if len(parts) > 1: + # iterate over the ANSI codes in this line + for code in parts[1::2]: + if code == "\033[0m": + # reset the list of ANSI codes when the clear code is found + ansi_codes = [] + else: + ansi_codes.append(code) + + return output From ec878dc3937e5c8e630a3fe0db192cdd7734dce2 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 4 Dec 2022 19:08:47 +0000 Subject: [PATCH 5/7] Add a unit test for the workaround, and fix a bug --- cibuildwheel/util.py | 4 ++-- unit_test/utils_test.py | 36 +++++++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index ca83d964..e0b2ae12 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -683,9 +683,9 @@ def fix_ansi_codes_for_github_actions(text: str) -> str: ansi_codes: list[str] = [] output = "" - for line in text.split("\n"): + for line in text.splitlines(keepends=True): # add the current ANSI codes to the beginning of the line - output += "".join(ansi_codes) + line + "\n" + output += "".join(ansi_codes) + line # split the line at each ANSI code parts = ansi_code_regex.split(line) diff --git a/unit_test/utils_test.py b/unit_test/utils_test.py index adf21435..4628ff6d 100644 --- a/unit_test/utils_test.py +++ b/unit_test/utils_test.py @@ -1,10 +1,16 @@ from __future__ import annotations +import textwrap from pathlib import PurePath import pytest -from cibuildwheel.util import find_compatible_wheel, format_safe, prepare_command +from cibuildwheel.util import ( + find_compatible_wheel, + fix_ansi_codes_for_github_actions, + format_safe, + prepare_command, +) def test_format_safe(): @@ -90,3 +96,31 @@ def test_find_compatible_wheel_found(wheel: str, identifier: str): ) def test_find_compatible_wheel_not_found(wheel: str, identifier: str): assert find_compatible_wheel([PurePath(wheel)], identifier) is None + + +def test_fix_ansi_codes_for_github_actions(): + input = textwrap.dedent( + """ + This line is normal + \033[1mThis line is bold + This line is also bold + \033[31m this line is red and bold + This line is red and bold, too\033[0m + This line is normal again + """ + ) + + expected = textwrap.dedent( + """ + This line is normal + \033[1mThis line is bold + \033[1mThis line is also bold + \033[1m\033[31m this line is red and bold + \033[1m\033[31mThis line is red and bold, too\033[0m + This line is normal again + """ + ) + + output = fix_ansi_codes_for_github_actions(input) + + assert output == expected From 56359fba7058e0d13118c8500130f635cdbd237f Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 5 Dec 2022 19:04:35 +0000 Subject: [PATCH 6/7] Apply suggestions from code review Co-authored-by: Henry Schreiner --- cibuildwheel/options.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index 0f2852e9..6ed1b5c8 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -684,8 +684,7 @@ class Options: has_been_set = (value_str != default_value_str) or overrides_value_strs c = log.colors - result = "" - result += c.gray if not has_been_set else "" + result = c.gray if not has_been_set else "" result += f"{option_name}: " if overrides_value_strs: From 98508648510a139a196f304dc984ff04a4a96833 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 5 Dec 2022 19:18:54 +0000 Subject: [PATCH 7/7] Rename 'environ' to 'env' --- cibuildwheel/__main__.py | 2 +- cibuildwheel/options.py | 28 +++++++++++------------- unit_test/linux_build_steps_test.py | 2 +- unit_test/options_test.py | 10 ++++----- unit_test/options_toml_test.py | 34 ++++++++++++++--------------- 5 files changed, 37 insertions(+), 39 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 99af21b4..1ff447aa 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -231,7 +231,7 @@ def build_in_directory(args: CommandLineArguments) -> None: ) sys.exit(2) - options = compute_options(platform=platform, command_line_arguments=args, environ=os.environ) + options = compute_options(platform=platform, command_line_arguments=args, env=os.environ) package_dir = options.globals.package_dir package_files = {"setup.py", "setup.cfg", "pyproject.toml"} diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index 6ed1b5c8..6b7b335f 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -192,11 +192,11 @@ class OptionsReader: config_file_path: Path | None = None, *, platform: PlatformName, - environ: Mapping[str, str], + env: Mapping[str, str], disallow: dict[str, set[str]] | None = None, ) -> None: self.platform = platform - self.environ = environ + self.env = env self.disallow = disallow or {} # Open defaults.toml, loading both global and platform sections @@ -337,8 +337,8 @@ class OptionsReader: # get the option from the environment, then the config file, then finally the default. # platform-specific options are preferred, if they're allowed. result = _dig_first( - (self.environ if env_plat else {}, plat_envvar), - (self.environ, envvar), + (self.env if env_plat else {}, plat_envvar), + (self.env, envvar), *[(o.options, name) for o in active_config_overrides], (self.config_platform_options, name), (self.config_options, name), @@ -384,17 +384,17 @@ class Options: self, platform: PlatformName, command_line_arguments: CommandLineArguments, - environ: Mapping[str, str], + env: Mapping[str, str], read_config_file: bool = True, ): self.platform = platform self.command_line_arguments = command_line_arguments - self.environ = environ + self.env = env self.reader = OptionsReader( self.config_file_path if read_config_file else None, platform=platform, - environ=environ, + env=env, disallow=DISALLOWED_OPTIONS, ) @@ -428,13 +428,13 @@ class Options: test_skip = self.reader.get("test-skip", env_plat=False, sep=" ") prerelease_pythons = args.prerelease_pythons or strtobool( - self.environ.get("CIBW_PRERELEASE_PYTHONS", "0") + self.env.get("CIBW_PRERELEASE_PYTHONS", "0") ) # This is not supported in tool.cibuildwheel, as it comes from a standard location. # Passing this in as an environment variable will override pyproject.toml, setup.cfg, or setup.py requires_python_str: str | None = ( - self.environ.get("CIBW_PROJECT_REQUIRES_PYTHON") or self.package_requires_python_str + self.env.get("CIBW_PROJECT_REQUIRES_PYTHON") or self.package_requires_python_str ) requires_python = None if requires_python_str is None else SpecifierSet(requires_python_str) @@ -523,7 +523,7 @@ class Options: if self.platform == "linux": for env_var_name in environment_pass: with contextlib.suppress(KeyError): - environment.add(env_var_name, self.environ[env_var_name]) + environment.add(env_var_name, self.env[env_var_name]) if dependency_versions == "pinned": dependency_constraints: None | ( @@ -625,7 +625,7 @@ class Options: return Options( platform=self.platform, command_line_arguments=CommandLineArguments.defaults(), - environ={}, + env={}, read_config_file=False, ) @@ -730,11 +730,9 @@ class Options: def compute_options( platform: PlatformName, command_line_arguments: CommandLineArguments, - environ: Mapping[str, str], + env: Mapping[str, str], ) -> Options: - options = Options( - platform=platform, command_line_arguments=command_line_arguments, environ=environ - ) + options = Options(platform=platform, command_line_arguments=command_line_arguments, env=env) options.check_for_deprecated_options() return options diff --git a/unit_test/linux_build_steps_test.py b/unit_test/linux_build_steps_test.py index 4730a4a9..065c53c3 100644 --- a/unit_test/linux_build_steps_test.py +++ b/unit_test/linux_build_steps_test.py @@ -40,7 +40,7 @@ def test_linux_container_split(tmp_path: Path, monkeypatch): ) monkeypatch.chdir(tmp_path) - options = Options("linux", command_line_arguments=args, environ={}) + options = Options("linux", command_line_arguments=args, env={}) python_configurations = cibuildwheel.linux.get_python_configurations( options.globals.build_selector, options.globals.architectures diff --git a/unit_test/options_test.py b/unit_test/options_test.py index d967aa40..4acb8d74 100644 --- a/unit_test/options_test.py +++ b/unit_test/options_test.py @@ -47,7 +47,7 @@ def test_options_1(tmp_path, monkeypatch): monkeypatch.setattr(platform_module, "machine", lambda: "x86_64") - options = Options(platform="linux", command_line_arguments=args, environ={}) + options = Options(platform="linux", command_line_arguments=args, env={}) identifiers = get_build_identifiers( platform="linux", @@ -89,7 +89,7 @@ def test_passthrough(tmp_path, monkeypatch): monkeypatch.setattr(platform_module, "machine", lambda: "x86_64") - options = Options(platform="linux", command_line_arguments=args, environ={"EXAMPLE_ENV": "ONE"}) + options = Options(platform="linux", command_line_arguments=args, env={"EXAMPLE_ENV": "ONE"}) default_build_options = options.build_options(identifier=None) @@ -118,7 +118,7 @@ def test_passthrough_evil(tmp_path, monkeypatch, env_var_value): options = Options( platform="linux", command_line_arguments=args, - environ={"CIBW_ENVIRONMENT_PASS_LINUX": "ENV_VAR", "ENV_VAR": env_var_value}, + env={"CIBW_ENVIRONMENT_PASS_LINUX": "ENV_VAR", "ENV_VAR": env_var_value}, ) parsed_environment = options.build_options(identifier=None).environment @@ -153,7 +153,7 @@ def test_toml_environment_evil(tmp_path, monkeypatch, env_var_value): ) ) - options = Options(platform="linux", command_line_arguments=args, environ={}) + options = Options(platform="linux", command_line_arguments=args, env={}) parsed_environment = options.build_options(identifier=None).environment assert parsed_environment.as_dictionary(prev_environment={}) == {"EXAMPLE": env_var_value} @@ -189,7 +189,7 @@ def test_toml_environment_quoting(tmp_path: Path, toml_assignment, result_value) ) ) - options = Options(platform="linux", command_line_arguments=args, environ={}) + options = Options(platform="linux", command_line_arguments=args, env={}) parsed_environment = options.build_options(identifier=None).environment environment_values = parsed_environment.as_dictionary( prev_environment={**os.environ, "PARAM": "spam"}, diff --git a/unit_test/options_toml_test.py b/unit_test/options_toml_test.py index 91ef98d5..76760879 100644 --- a/unit_test/options_toml_test.py +++ b/unit_test/options_toml_test.py @@ -35,7 +35,7 @@ def test_simple_settings(tmp_path, platform, fname): config_file_path: Path = tmp_path / fname config_file_path.write_text(PYPROJECT_1) - options_reader = OptionsReader(config_file_path, platform=platform, environ={}) + options_reader = OptionsReader(config_file_path, platform=platform, env={}) assert options_reader.get("build", env_plat=False, sep=" ") == "cp39*" @@ -78,7 +78,7 @@ def test_envvar_override(tmp_path, platform, monkeypatch): options_reader = OptionsReader( config_file_path, platform=platform, - environ={ + env={ "CIBW_BUILD": "cp38*", "CIBW_MANYLINUX_X86_64_IMAGE": "manylinux_2_24", "CIBW_TEST_COMMAND": "mytest", @@ -108,13 +108,13 @@ def test_project_global_override_default_platform(tmp_path, platform): repair-wheel-command = "repair-project-global" """ ) - options_reader = OptionsReader(pyproject_toml, platform=platform, environ={}) + options_reader = OptionsReader(pyproject_toml, platform=platform, env={}) assert options_reader.get("repair-wheel-command") == "repair-project-global" def test_env_global_override_default_platform(tmp_path, platform, monkeypatch): options_reader = OptionsReader( - platform=platform, environ={"CIBW_REPAIR_WHEEL_COMMAND": "repair-env-global"} + platform=platform, env={"CIBW_REPAIR_WHEEL_COMMAND": "repair-env-global"} ) assert options_reader.get("repair-wheel-command") == "repair-env-global" @@ -134,7 +134,7 @@ repair-wheel-command = "repair-project-macos" options_reader = OptionsReader( pyproject_toml, platform=platform, - environ={ + env={ "CIBW_REPAIR_WHEEL_COMMAND": "repair-env-global", }, ) @@ -155,7 +155,7 @@ repair-wheel-command = "repair-project-macos" repair-wheel-command = "repair-project-global" """ ) - options_reader = OptionsReader(pyproject_toml, platform=platform, environ={}) + options_reader = OptionsReader(pyproject_toml, platform=platform, env={}) assert options_reader.get("repair-wheel-command") == f"repair-project-{platform}" @@ -171,7 +171,7 @@ repairs-wheel-command = "repair-project-linux" ) with pytest.raises(ConfigOptionError) as excinfo: - OptionsReader(pyproject_toml, platform="linux", environ={}) + OptionsReader(pyproject_toml, platform="linux", env={}) assert "repair-wheel-command" in str(excinfo.value) @@ -188,7 +188,7 @@ repair_wheel_command = "repair-project-linux" ) with pytest.raises(ConfigOptionError) as excinfo: - OptionsReader(pyproject_toml, platform="linux", environ={}) + OptionsReader(pyproject_toml, platform="linux", env={}) assert "repair-wheel-command" in str(excinfo.value) @@ -202,7 +202,7 @@ repair-wheel-command = "repair-project-linux" """ ) with pytest.raises(ConfigOptionError): - OptionsReader(pyproject_toml, platform="linux", environ={}) + OptionsReader(pyproject_toml, platform="linux", env={}) def test_unsupported_join(tmp_path): @@ -213,7 +213,7 @@ def test_unsupported_join(tmp_path): build = ["1", "2"] """ ) - options_reader = OptionsReader(pyproject_toml, platform="linux", environ={}) + options_reader = OptionsReader(pyproject_toml, platform="linux", env={}) assert "1, 2" == options_reader.get("build", sep=", ") with pytest.raises(ConfigOptionError): @@ -229,9 +229,9 @@ manylinux-x86_64-image = "manylinux1" """ ) disallow = {"windows": {"manylinux-x86_64-image"}} - OptionsReader(pyproject_toml, platform="linux", disallow=disallow, environ={}) + OptionsReader(pyproject_toml, platform="linux", disallow=disallow, env={}) with pytest.raises(ConfigOptionError): - OptionsReader(pyproject_toml, platform="windows", disallow=disallow, environ={}) + OptionsReader(pyproject_toml, platform="windows", disallow=disallow, env={}) def test_environment_override_empty(tmp_path, monkeypatch): @@ -247,7 +247,7 @@ manylinux-x86_64-image = "" options_reader = OptionsReader( pyproject_toml, platform="linux", - environ={ + env={ "CIBW_MANYLINUX_I686_IMAGE": "", "CIBW_MANYLINUX_AARCH64_IMAGE": "manylinux1", }, @@ -320,7 +320,7 @@ def test_pyproject_2(tmp_path, platform): pyproject_toml: Path = tmp_path / "pyproject.toml" pyproject_toml.write_text(PYPROJECT_2) - options_reader = OptionsReader(config_file_path=pyproject_toml, platform=platform, environ={}) + options_reader = OptionsReader(config_file_path=pyproject_toml, platform=platform, env={}) assert options_reader.get("test-command") == "pyproject" with options_reader.identifier("random"): @@ -344,7 +344,7 @@ test-command = "pyproject-override" ) with pytest.raises(ConfigOptionError): - OptionsReader(config_file_path=pyproject_toml, platform=platform, environ={}) + OptionsReader(config_file_path=pyproject_toml, platform=platform, env={}) def test_config_settings(tmp_path): @@ -357,7 +357,7 @@ other = ["two", "three"] """ ) - options_reader = OptionsReader(config_file_path=pyproject_toml, platform="linux", environ={}) + options_reader = OptionsReader(config_file_path=pyproject_toml, platform="linux", env={}) assert ( options_reader.get("config-settings", table={"item": '{k}="{v}"', "sep": " "}) == 'example="one" other="two" other="three"' @@ -373,7 +373,7 @@ def test_pip_config_settings(tmp_path): """ ) - options_reader = OptionsReader(config_file_path=pyproject_toml, platform="linux", environ={}) + options_reader = OptionsReader(config_file_path=pyproject_toml, platform="linux", env={}) assert ( options_reader.get( "config-settings", table={"item": "--config-settings='{k}=\"{v}\"'", "sep": " "}