From 4fe7630d9c85a0cd4a9194149139c377913ba0fa Mon Sep 17 00:00:00 2001 From: Russell Keith-Magee Date: Thu, 6 Nov 2025 05:19:41 +0800 Subject: [PATCH] feat: add configuration option for test executor arguments (#2636) * Add test-execution-args option. * Add usage of test-execution-args. * Add CI configuration to use test-execution-args. * Document the test-execution-args setting. * Simplify code using or syntax instead of inline if. Co-authored-by: Malcolm Smith * Clarified some Android-specific terminology, and added details about the default args to the test runner. * Switch to a dict-based test-execution configuration * Add tests for test-execution parsing. * Add all the files before pushing... * Add note about default Android version for testbed. * Improve description of test-execution setting. Co-authored-by: Joe Rickerby * Switch to using test-runtime. --------- Co-authored-by: Malcolm Smith Co-authored-by: Joe Rickerby --- .github/workflows/test.yml | 8 +++ README.md | 8 +-- bin/generate_schema.py | 19 +++++++ cibuildwheel/options.py | 32 ++++++++++- cibuildwheel/platforms/android.py | 14 ++++- cibuildwheel/platforms/ios.py | 25 ++++++++ .../resources/cibuildwheel.schema.json | 57 +++++++++++++++++++ cibuildwheel/resources/defaults.toml | 1 + docs/options.md | 35 ++++++++++++ docs/platforms.md | 6 +- unit_test/options_test.py | 34 +++++++++++ 11 files changed, 231 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 75e7ebf5..6b9f3c8f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -79,9 +79,13 @@ jobs: - os: macos-15 python_version: '3.13' test_select: ios + # Exercise iOS on a non-default simulator. + test_runtime: 'args: --simulator "iPhone 16e,OS=18.5"' - os: macos-15-intel python_version: '3.13' test_select: android + # Exercise Android on a non-default simulator + test_runtime: 'args: --managed minVersion' - os: macos-15 python_version: '3.13' test_select: android @@ -154,6 +158,7 @@ jobs: CIBW_ARCHS_MACOS: x86_64 universal2 arm64 CIBW_BUILD_FRONTEND: ${{ matrix.test_select && 'build' || 'build[uv]' }} CIBW_PLATFORM: ${{ matrix.test_select }} + CIBW_TEST_RUNTIME: ${{ matrix.test_runtime }} - name: Run a sample build (GitHub Action, only) uses: ./ @@ -179,6 +184,7 @@ jobs: uses: ./ env: CIBW_PLATFORM: ${{ matrix.test_select }} + CIBW_TEST_RUNTIME: ${{ matrix.test_runtime }} with: package-dir: sample_proj output-dir: wheelhouse_config_file @@ -202,6 +208,8 @@ jobs: path: wheelhouse/*.whl - name: Test cibuildwheel + env: + CIBW_TEST_RUNTIME: ${{ matrix.test_runtime }} run: | uv run --no-sync bin/run_tests.py --test-select=${{ matrix.test_select || 'native' }} ${{ (runner.os == 'Linux' && runner.arch == 'X64') && '--run-podman' || '' }} diff --git a/README.md b/README.md index 5c4eb621..f3847b88 100644 --- a/README.md +++ b/README.md @@ -59,8 +59,8 @@ Usage | | Linux | macOS | Windows | Linux ARM | macOS ARM | Windows ARM | Android | iOS | |-----------------|-------|-------|---------|-----------|-----------|-------------|---------|-----| -| GitHub Actions | ✅ | ✅ | ✅ | ✅ | ✅ | ✅² | ✅⁴ | ✅³⁵ | -| Azure Pipelines | ✅ | ✅ | ✅ | | ✅ | ✅² | ✅⁴ | ✅³⁵ | +| GitHub Actions | ✅ | ✅ | ✅ | ✅ | ✅ | ✅² | ✅⁴ | ✅³ | +| Azure Pipelines | ✅ | ✅ | ✅ | | ✅ | ✅² | ✅⁴ | ✅³ | | Travis CI | ✅ | | ✅ | ✅ | | | ✅⁴ | | | CircleCI | ✅ | ✅ | | ✅ | ✅ | | ✅⁴ | ✅³ | | Gitlab CI | ✅ | ✅ | ✅ | ✅¹ | ✅ | | ✅⁴ | ✅³ | @@ -70,7 +70,6 @@ Usage ² [Uses cross-compilation](https://cibuildwheel.pypa.io/en/stable/faq/#windows-arm64). It is not possible to test `arm64` on this CI platform.
³ Requires a macOS runner; runs tests on the simulator for the runner's architecture.
⁴ Building for Android requires the runner to be Linux x86_64, macOS ARM64 or macOS x86_64. Testing has [additional requirements](https://cibuildwheel.pypa.io/en/stable/platforms/#android).
-⁵ The `macos-15` and `macos-latest` images are [incompatible with cibuildwheel at this time](https://cibuildwheel.pypa.io/en/stable/platforms/#ios-system-requirements) when building iOS wheels.
@@ -160,12 +159,13 @@ The following diagram summarises the steps that cibuildwheel takes on each platf | | [`test-groups`](https://cibuildwheel.pypa.io/en/stable/options/#test-groups) | Specify test dependencies from your project's `dependency-groups` | | | [`test-skip`](https://cibuildwheel.pypa.io/en/stable/options/#test-skip) | Skip running tests on some builds | | | [`test-environment`](https://cibuildwheel.pypa.io/en/stable/options/#test-environment) | Set environment variables for the test environment | +| | [`test-runtime`](https://cibuildwheel.pypa.io/en/stable/options/#test-runtime) | Controls how the tests will be executed. | | **Debugging** | [`debug-keep-container`](https://cibuildwheel.pypa.io/en/stable/options/#debug-keep-container) | Keep the container after running for debugging. | | | [`debug-traceback`](https://cibuildwheel.pypa.io/en/stable/options/#debug-traceback) | Print full traceback when errors occur. | | | [`build-verbosity`](https://cibuildwheel.pypa.io/en/stable/options/#build-verbosity) | Increase/decrease the output of the build | - + These options can be specified in a pyproject.toml file, or as environment variables, see [configuration docs](https://cibuildwheel.pypa.io/en/latest/configuration/). diff --git a/bin/generate_schema.py b/bin/generate_schema.py index fe5f0918..611d19e5 100755 --- a/bin/generate_schema.py +++ b/bin/generate_schema.py @@ -224,6 +224,24 @@ properties: test-environment: description: Set environment variables for the test environment type: string_table + test-runtime: + description: Additional configuration for the test runner + oneOf: + - type: string + pattern: '^$' + - type: object + additionalProperties: false + - type: string + pattern: 'args:' + - type: object + additionalProperties: false + required: [args] + properties: + args: + type: array + items: + type: string + """ schema = yaml.safe_load(starter) @@ -304,6 +322,7 @@ items: test-sources: {"$ref": "#/$defs/inherit"} test-requires: {"$ref": "#/$defs/inherit"} test-environment: {"$ref": "#/$defs/inherit"} + test-runtime: {"$ref": "#/$defs/inherit"} """ ) diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index 22dc48d1..af946a88 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -24,7 +24,7 @@ from .projectfiles import get_requires_python_str, resolve_dependency_groups from .selector import BuildSelector, EnableGroup, TestSelector, selector_matches from .typing import PLATFORMS, PlatformName from .util import resources -from .util.helpers import format_safe, strtobool, unwrap +from .util.helpers import format_safe, parse_key_value_string, strtobool, unwrap from .util.packaging import DependencyConstraints MANYLINUX_ARCHS: Final[tuple[str, ...]] = ( @@ -92,6 +92,20 @@ class GlobalOptions: allow_empty: bool +@dataclasses.dataclass(frozen=True) +class TestRuntimeConfig: + args: Sequence[str] = () + + @classmethod + def from_config_string(cls, config_string: str) -> Self: + config_dict = parse_key_value_string(config_string, [], ["args"]) + args = config_dict.get("args") or [] + return cls(args=args) + + def options_summary(self) -> str | dict[str, str]: + return {"args": repr(self.args)} + + @dataclasses.dataclass(frozen=True, kw_only=True) class BuildOptions: globals: GlobalOptions @@ -110,6 +124,7 @@ class BuildOptions: test_extras: str test_groups: list[str] test_environment: ParsedEnvironment + test_runtime: TestRuntimeConfig build_verbosity: int build_frontend: BuildFrontendConfig config_settings: str @@ -761,6 +776,20 @@ class Options: msg = f"Malformed environment option {test_environment_config!r}" raise errors.ConfigurationError(msg) from e + test_runtime_str = self.reader.get( + "test-runtime", + env_plat=False, + option_format=ShlexTableFormat(sep="; ", pair_sep=":", allow_merge=False), + ) + if not test_runtime_str: + test_runtime = TestRuntimeConfig() + else: + try: + test_runtime = TestRuntimeConfig.from_config_string(test_runtime_str) + except ValueError as e: + msg = f"Failed to parse test runtime config. {e}" + raise errors.ConfigurationError(msg) from e + test_requires = self.reader.get( "test-requires", option_format=ListFormat(sep=" ") ).split() @@ -868,6 +897,7 @@ class Options: test_command=test_command, test_sources=test_sources, test_environment=test_environment, + test_runtime=test_runtime, test_requires=[*test_requires, *test_requirements_from_groups], test_extras=test_extras, test_groups=test_groups, diff --git a/cibuildwheel/platforms/android.py b/cibuildwheel/platforms/android.py index dca2f511..13438250 100644 --- a/cibuildwheel/platforms/android.py +++ b/cibuildwheel/platforms/android.py @@ -638,17 +638,27 @@ def test_wheel(state: BuildState, wheel: Path) -> None: ) raise errors.FatalError(msg) + # By default, run on a testbed managed emulator running the newest supported + # Android version. However, if the user specifies a --managed or --connected + # test execution argument, that argument takes precedence. + test_runtime_args = state.options.test_runtime.args + + if any(arg.startswith(("--managed", "--connected")) for arg in test_runtime_args): + emulator_args = [] + else: + emulator_args = ["--managed", "maxVersion"] + # Run the test app. call( state.python_dir / "android.py", "test", - "--managed", - "maxVersion", "--site-packages", site_packages_dir, "--cwd", cwd_dir, + *emulator_args, *(["-v"] if state.options.build_verbosity > 0 else []), + *test_runtime_args, "--", *test_args, env=state.build_env, diff --git a/cibuildwheel/platforms/ios.py b/cibuildwheel/platforms/ios.py index eba04c35..cf9ba4eb 100644 --- a/cibuildwheel/platforms/ios.py +++ b/cibuildwheel/platforms/ios.py @@ -2,6 +2,7 @@ from __future__ import annotations import dataclasses import os +import platform import shlex import shutil import subprocess @@ -653,11 +654,35 @@ def build(options: Options, tmp_path: Path) -> None: ) raise errors.FatalError(msg) + test_runtime_args = build_options.test_runtime.args + + # 2025-10: The GitHub Actions macos-15 runner has a known issue where + # the default simulator won't start due to a disk performance issue; + # see https://github.com/actions/runner-images/issues/12777 for details. + # In the meantime, if it looks like we're running on a GitHub Actions + # macos-15 runner, use a simulator that is known to work, unless the + # user explicitly specifies a simulator. + os_version, _, arch = platform.mac_ver() + if ( + "GITHUB_ACTIONS" in os.environ + and os_version.startswith("15.") + and arch == "arm64" + and not any( + arg.startswith("--simulator") for arg in test_runtime_args + ) + ): + test_runtime_args = [ + "--simulator", + "iPhone 16e,OS=18.5", + *test_runtime_args, + ] + call( "python", testbed_path, "run", *(["--verbose"] if build_options.build_verbosity > 0 else []), + *test_runtime_args, "--", *final_command, env=test_env, diff --git a/cibuildwheel/resources/cibuildwheel.schema.json b/cibuildwheel/resources/cibuildwheel.schema.json index bae1f9eb..2c52aada 100644 --- a/cibuildwheel/resources/cibuildwheel.schema.json +++ b/cibuildwheel/resources/cibuildwheel.schema.json @@ -569,6 +569,39 @@ ], "title": "CIBW_TEST_ENVIRONMENT" }, + "test-runtime": { + "description": "Additional configuration for the test runner", + "oneOf": [ + { + "type": "string", + "pattern": "^$" + }, + { + "type": "object", + "additionalProperties": false + }, + { + "type": "string", + "pattern": "args:" + }, + { + "type": "object", + "additionalProperties": false, + "required": [ + "args" + ], + "properties": { + "args": { + "type": "array", + "items": { + "type": "string" + } + } + } + } + ], + "title": "CIBW_TEST_RUNTIME" + }, "overrides": { "type": "array", "description": "An overrides array", @@ -638,6 +671,9 @@ }, "test-environment": { "$ref": "#/$defs/inherit" + }, + "test-runtime": { + "$ref": "#/$defs/inherit" } } }, @@ -748,6 +784,9 @@ }, "test-environment": { "$ref": "#/properties/test-environment" + }, + "test-runtime": { + "$ref": "#/properties/test-runtime" } } } @@ -876,6 +915,9 @@ }, "test-environment": { "$ref": "#/properties/test-environment" + }, + "test-runtime": { + "$ref": "#/properties/test-runtime" } } }, @@ -936,6 +978,9 @@ }, "test-environment": { "$ref": "#/properties/test-environment" + }, + "test-runtime": { + "$ref": "#/properties/test-runtime" } } }, @@ -1009,6 +1054,9 @@ }, "test-environment": { "$ref": "#/properties/test-environment" + }, + "test-runtime": { + "$ref": "#/properties/test-runtime" } } }, @@ -1069,6 +1117,9 @@ }, "test-environment": { "$ref": "#/properties/test-environment" + }, + "test-runtime": { + "$ref": "#/properties/test-runtime" } } }, @@ -1129,6 +1180,9 @@ }, "test-environment": { "$ref": "#/properties/test-environment" + }, + "test-runtime": { + "$ref": "#/properties/test-runtime" } } }, @@ -1189,6 +1243,9 @@ }, "test-environment": { "$ref": "#/properties/test-environment" + }, + "test-runtime": { + "$ref": "#/properties/test-runtime" } } } diff --git a/cibuildwheel/resources/defaults.toml b/cibuildwheel/resources/defaults.toml index d5c176f2..78895bf9 100644 --- a/cibuildwheel/resources/defaults.toml +++ b/cibuildwheel/resources/defaults.toml @@ -25,6 +25,7 @@ test-requires = [] test-extras = [] test-groups = [] test-environment = {} +test-runtime = {} container-engine = "docker" diff --git a/docs/options.md b/docs/options.md index 92caf5af..88a377d3 100644 --- a/docs/options.md +++ b/docs/options.md @@ -1672,6 +1672,41 @@ Platform-specific environment variables are also available:
CIBW_TEST_ENVIRONMENT: PYTHONSAFEPATH=1 ``` +### `test-runtime` {: #test-runtime toml env-var } + +> Controls how the tests will be executed. + +On desktop environments, the tests are executed on the same machine/container as the wheel was built. However on Android and iOS, the tests are run inside a virtual machine – a simulator or emulator – representing the target. + +For these embedded platforms, a testbed project is used to run the tests. The `test-runtime` setting can define an `args` key that defines additional arguments that will be used when starting the testbed project. + +Platform-specific environment variables are also available:
+`CIBW_TEST_RUNTIME_ANDROID` |`CIBW_TEST_RUNTIME_IOS` + +#### Examples + +!!! tab examples "pyproject.toml" + + ```toml + [tool.cibuildwheel.ios] + # Run the tests on an iPhone 16e simulator running iOS 18.5. + test-runtime = { args = ["--simulator='iPhone 16e,OS=18.5'"] } + + [tool.cibuildwheel.android] + # Run the Android tests on the minimum supported Android version. + test-runtime = { args = ["--managed", "minVersion"] } + ``` + +!!! tab examples "Environment variables" + + ```yaml + # Run the tests on an iPhone 16e simulator running iOS 18.5. + CIBW_TEST_RUNTIME_IOS: "args: --simulator='iPhone 16e,OS=18.5'" + + # Run the Android tests on the minimum supported Android version. + CIBW_TEST_RUNTIME_ANDROID: "args: --managed minVersion" + ``` + ## Debugging diff --git a/docs/platforms.md b/docs/platforms.md index 1cb931eb..dee82249 100644 --- a/docs/platforms.md +++ b/docs/platforms.md @@ -233,6 +233,8 @@ machine – for example, if you're building on an ARM64 machine, then you can te ARM64 wheel. Wheels of other architectures can still be built, but testing will automatically be skipped. +Any arguments specified using [`test-runtime`](options.md#test-runtime) will be passed as arguments to the Python script that starts the [testbed project](https://github.com/python/cpython/blob/main/Android/README.md#testing). cibuildwheel will automatically start the testbed project with `--site-packages` and `--cwd` arguments matching your test environment, as well as enabling verbose output with `-v` if [`build-verbosity`](options.md#build-verbosity) is enabled. The most common additional arguments to use will be `--managed minVersion` or `--managed maxVersion`, specifying the use of a managed Android emulator with the minimum or maximum supported Android version; or `--connected `, specifying the use of an existing booted Android emulator or device. By default, the testbed project will run with `--managed maxVersion`. + Running an emulator requires the build machine to either be bare-metal or support nested virtualization. CI platforms known to meet this requirement are: @@ -320,4 +322,6 @@ If tests have been configured, the test suite will be executed on the simulator The iOS test environment can't support running shell scripts, so the [`test-command`](options.md#test-command) value must be specified as if it were a command line being passed to `python -m ...`. -The test process uses the same testbed used by CPython itself to run the CPython test suite. It is an Xcode project that has been configured to have a single Xcode "XCUnit" test - the result of which reports the success or failure of running `python -m `. +The test process uses the [same testbed used by CPython itself](https://github.com/python/cpython/tree/main/Apple/iOS#testing-python-on-ios) to run the CPython test suite. It is an Xcode project that has been configured to have a single Xcode "XCUnit" test - the result of which reports the success or failure of running `python -m `. + +Any arguments specified using [`test-runtime`](options.md#test-runtime) will be passed as arguments to the Python script that starts the testbed project. The testbed project will be started with `-v` enabling verbose output if [`build-verbosity`](options.md#build-verbosity) is enabled; the most common additional argument to use will be `--simulator`, which allows the specification of a specific device or iOS version for the test simulator. By default, the testbed project will attempt to find an "SE class" simulator (i.e., an iPhone SE, iPhone 16e, or similar), running the newest iOS version available. diff --git a/unit_test/options_test.py b/unit_test/options_test.py index 52f2f1ad..b9847ed5 100644 --- a/unit_test/options_test.py +++ b/unit_test/options_test.py @@ -2,6 +2,7 @@ import os import platform as platform_module import textwrap import unittest.mock +from collections.abc import Sequence from pathlib import Path from typing import Literal @@ -626,6 +627,39 @@ def test_get_build_frontend_extra_flags_warning( mock_warning.assert_called_once() +@pytest.mark.parametrize( + ("definition", "expected_args"), + [ + ("", ()), + ('test-runtime = ""', ()), + ("test-runtime = {}", ()), + ('test-runtime = {args = ""}', []), + ('test-runtime = "args: --simulator foo"', ["--simulator", "foo"]), + ('test-runtime = {args = ["--simulator", "foo"]}', ["--simulator", "foo"]), + ], +) +def test_test_runtime_handling( + tmp_path: Path, definition: str, expected_args: Sequence[str] | None +) -> None: + args = CommandLineArguments.defaults() + args.package_dir = tmp_path + + pyproject_toml: Path = tmp_path / "pyproject.toml" + pyproject_toml.write_text( + textwrap.dedent( + f"""\ + [tool.cibuildwheel] + {definition} + """ + ) + ) + + options = Options(platform="ios", command_line_arguments=args, env={}) + + local = options.build_options("cp313-ios_13_0_arm64_iphoneos") + assert local.test_runtime.args == expected_args + + @pytest.mark.parametrize( ("definition", "expected"), [