feat: add configuration option for test executor arguments (#2636)
* Add test-execution-args option. * Add usage of test-execution-args. * Add CI configuration to use test-execution-args. * Document the test-execution-args setting. * Simplify code using or syntax instead of inline if. Co-authored-by: Malcolm Smith <smith@chaquo.com> * Clarified some Android-specific terminology, and added details about the default args to the test runner. * Switch to a dict-based test-execution configuration * Add tests for test-execution parsing. * Add all the files before pushing... * Add note about default Android version for testbed. * Improve description of test-execution setting. Co-authored-by: Joe Rickerby <joerick@mac.com> * Switch to using test-runtime. --------- Co-authored-by: Malcolm Smith <smith@chaquo.com> Co-authored-by: Joe Rickerby <joerick@mac.com>
This commit is contained in:
Russell Keith-Magee
co-authored by
Malcolm Smith
Joe Rickerby
Malcolm Smith
Joe Rickerby
parent
c53e541c2d
commit
4fe7630d9c
@@ -79,9 +79,13 @@ jobs:
|
||||
- os: macos-15
|
||||
python_version: '3.13'
|
||||
test_select: ios
|
||||
# Exercise iOS on a non-default simulator.
|
||||
test_runtime: 'args: --simulator "iPhone 16e,OS=18.5"'
|
||||
- os: macos-15-intel
|
||||
python_version: '3.13'
|
||||
test_select: android
|
||||
# Exercise Android on a non-default simulator
|
||||
test_runtime: 'args: --managed minVersion'
|
||||
- os: macos-15
|
||||
python_version: '3.13'
|
||||
test_select: android
|
||||
@@ -154,6 +158,7 @@ jobs:
|
||||
CIBW_ARCHS_MACOS: x86_64 universal2 arm64
|
||||
CIBW_BUILD_FRONTEND: ${{ matrix.test_select && 'build' || 'build[uv]' }}
|
||||
CIBW_PLATFORM: ${{ matrix.test_select }}
|
||||
CIBW_TEST_RUNTIME: ${{ matrix.test_runtime }}
|
||||
|
||||
- name: Run a sample build (GitHub Action, only)
|
||||
uses: ./
|
||||
@@ -179,6 +184,7 @@ jobs:
|
||||
uses: ./
|
||||
env:
|
||||
CIBW_PLATFORM: ${{ matrix.test_select }}
|
||||
CIBW_TEST_RUNTIME: ${{ matrix.test_runtime }}
|
||||
with:
|
||||
package-dir: sample_proj
|
||||
output-dir: wheelhouse_config_file
|
||||
@@ -202,6 +208,8 @@ jobs:
|
||||
path: wheelhouse/*.whl
|
||||
|
||||
- name: Test cibuildwheel
|
||||
env:
|
||||
CIBW_TEST_RUNTIME: ${{ matrix.test_runtime }}
|
||||
run: |
|
||||
uv run --no-sync bin/run_tests.py --test-select=${{ matrix.test_select || 'native' }} ${{ (runner.os == 'Linux' && runner.arch == 'X64') && '--run-podman' || '' }}
|
||||
|
||||
|
||||
@@ -59,8 +59,8 @@ Usage
|
||||
|
||||
| | Linux | macOS | Windows | Linux ARM | macOS ARM | Windows ARM | Android | iOS |
|
||||
|-----------------|-------|-------|---------|-----------|-----------|-------------|---------|-----|
|
||||
| GitHub Actions | ✅ | ✅ | ✅ | ✅ | ✅ | ✅² | ✅⁴ | ✅³⁵ |
|
||||
| Azure Pipelines | ✅ | ✅ | ✅ | | ✅ | ✅² | ✅⁴ | ✅³⁵ |
|
||||
| GitHub Actions | ✅ | ✅ | ✅ | ✅ | ✅ | ✅² | ✅⁴ | ✅³ |
|
||||
| Azure Pipelines | ✅ | ✅ | ✅ | | ✅ | ✅² | ✅⁴ | ✅³ |
|
||||
| Travis CI | ✅ | | ✅ | ✅ | | | ✅⁴ | |
|
||||
| CircleCI | ✅ | ✅ | | ✅ | ✅ | | ✅⁴ | ✅³ |
|
||||
| Gitlab CI | ✅ | ✅ | ✅ | ✅¹ | ✅ | | ✅⁴ | ✅³ |
|
||||
@@ -70,7 +70,6 @@ Usage
|
||||
<sup>² [Uses cross-compilation](https://cibuildwheel.pypa.io/en/stable/faq/#windows-arm64). It is not possible to test `arm64` on this CI platform.</sup><br>
|
||||
<sup>³ Requires a macOS runner; runs tests on the simulator for the runner's architecture. </sup><br>
|
||||
<sup>⁴ Building for Android requires the runner to be Linux x86_64, macOS ARM64 or macOS x86_64. Testing has [additional requirements](https://cibuildwheel.pypa.io/en/stable/platforms/#android).</sup><br>
|
||||
<sup>⁵ The `macos-15` and `macos-latest` images are [incompatible with cibuildwheel at this time](https://cibuildwheel.pypa.io/en/stable/platforms/#ios-system-requirements) when building iOS wheels.</sup><br>
|
||||
|
||||
<!--intro-end-->
|
||||
|
||||
@@ -160,12 +159,13 @@ The following diagram summarises the steps that cibuildwheel takes on each platf
|
||||
| | [`test-groups`](https://cibuildwheel.pypa.io/en/stable/options/#test-groups) | Specify test dependencies from your project's `dependency-groups` |
|
||||
| | [`test-skip`](https://cibuildwheel.pypa.io/en/stable/options/#test-skip) | Skip running tests on some builds |
|
||||
| | [`test-environment`](https://cibuildwheel.pypa.io/en/stable/options/#test-environment) | Set environment variables for the test environment |
|
||||
| | [`test-runtime`](https://cibuildwheel.pypa.io/en/stable/options/#test-runtime) | Controls how the tests will be executed. |
|
||||
| **Debugging** | [`debug-keep-container`](https://cibuildwheel.pypa.io/en/stable/options/#debug-keep-container) | Keep the container after running for debugging. |
|
||||
| | [`debug-traceback`](https://cibuildwheel.pypa.io/en/stable/options/#debug-traceback) | Print full traceback when errors occur. |
|
||||
| | [`build-verbosity`](https://cibuildwheel.pypa.io/en/stable/options/#build-verbosity) | Increase/decrease the output of the build |
|
||||
|
||||
|
||||
<!--[[[end]]] (sum: FxE3nIgFiY) -->
|
||||
<!--[[[end]]] (sum: dbfwOkj/k/) -->
|
||||
|
||||
These options can be specified in a pyproject.toml file, or as environment variables, see [configuration docs](https://cibuildwheel.pypa.io/en/latest/configuration/).
|
||||
|
||||
|
||||
@@ -224,6 +224,24 @@ properties:
|
||||
test-environment:
|
||||
description: Set environment variables for the test environment
|
||||
type: string_table
|
||||
test-runtime:
|
||||
description: Additional configuration for the test runner
|
||||
oneOf:
|
||||
- type: string
|
||||
pattern: '^$'
|
||||
- type: object
|
||||
additionalProperties: false
|
||||
- type: string
|
||||
pattern: 'args:'
|
||||
- type: object
|
||||
additionalProperties: false
|
||||
required: [args]
|
||||
properties:
|
||||
args:
|
||||
type: array
|
||||
items:
|
||||
type: string
|
||||
|
||||
"""
|
||||
|
||||
schema = yaml.safe_load(starter)
|
||||
@@ -304,6 +322,7 @@ items:
|
||||
test-sources: {"$ref": "#/$defs/inherit"}
|
||||
test-requires: {"$ref": "#/$defs/inherit"}
|
||||
test-environment: {"$ref": "#/$defs/inherit"}
|
||||
test-runtime: {"$ref": "#/$defs/inherit"}
|
||||
"""
|
||||
)
|
||||
|
||||
|
||||
+31
-1
@@ -24,7 +24,7 @@ from .projectfiles import get_requires_python_str, resolve_dependency_groups
|
||||
from .selector import BuildSelector, EnableGroup, TestSelector, selector_matches
|
||||
from .typing import PLATFORMS, PlatformName
|
||||
from .util import resources
|
||||
from .util.helpers import format_safe, strtobool, unwrap
|
||||
from .util.helpers import format_safe, parse_key_value_string, strtobool, unwrap
|
||||
from .util.packaging import DependencyConstraints
|
||||
|
||||
MANYLINUX_ARCHS: Final[tuple[str, ...]] = (
|
||||
@@ -92,6 +92,20 @@ class GlobalOptions:
|
||||
allow_empty: bool
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True)
|
||||
class TestRuntimeConfig:
|
||||
args: Sequence[str] = ()
|
||||
|
||||
@classmethod
|
||||
def from_config_string(cls, config_string: str) -> Self:
|
||||
config_dict = parse_key_value_string(config_string, [], ["args"])
|
||||
args = config_dict.get("args") or []
|
||||
return cls(args=args)
|
||||
|
||||
def options_summary(self) -> str | dict[str, str]:
|
||||
return {"args": repr(self.args)}
|
||||
|
||||
|
||||
@dataclasses.dataclass(frozen=True, kw_only=True)
|
||||
class BuildOptions:
|
||||
globals: GlobalOptions
|
||||
@@ -110,6 +124,7 @@ class BuildOptions:
|
||||
test_extras: str
|
||||
test_groups: list[str]
|
||||
test_environment: ParsedEnvironment
|
||||
test_runtime: TestRuntimeConfig
|
||||
build_verbosity: int
|
||||
build_frontend: BuildFrontendConfig
|
||||
config_settings: str
|
||||
@@ -761,6 +776,20 @@ class Options:
|
||||
msg = f"Malformed environment option {test_environment_config!r}"
|
||||
raise errors.ConfigurationError(msg) from e
|
||||
|
||||
test_runtime_str = self.reader.get(
|
||||
"test-runtime",
|
||||
env_plat=False,
|
||||
option_format=ShlexTableFormat(sep="; ", pair_sep=":", allow_merge=False),
|
||||
)
|
||||
if not test_runtime_str:
|
||||
test_runtime = TestRuntimeConfig()
|
||||
else:
|
||||
try:
|
||||
test_runtime = TestRuntimeConfig.from_config_string(test_runtime_str)
|
||||
except ValueError as e:
|
||||
msg = f"Failed to parse test runtime config. {e}"
|
||||
raise errors.ConfigurationError(msg) from e
|
||||
|
||||
test_requires = self.reader.get(
|
||||
"test-requires", option_format=ListFormat(sep=" ")
|
||||
).split()
|
||||
@@ -868,6 +897,7 @@ class Options:
|
||||
test_command=test_command,
|
||||
test_sources=test_sources,
|
||||
test_environment=test_environment,
|
||||
test_runtime=test_runtime,
|
||||
test_requires=[*test_requires, *test_requirements_from_groups],
|
||||
test_extras=test_extras,
|
||||
test_groups=test_groups,
|
||||
|
||||
@@ -638,17 +638,27 @@ def test_wheel(state: BuildState, wheel: Path) -> None:
|
||||
)
|
||||
raise errors.FatalError(msg)
|
||||
|
||||
# By default, run on a testbed managed emulator running the newest supported
|
||||
# Android version. However, if the user specifies a --managed or --connected
|
||||
# test execution argument, that argument takes precedence.
|
||||
test_runtime_args = state.options.test_runtime.args
|
||||
|
||||
if any(arg.startswith(("--managed", "--connected")) for arg in test_runtime_args):
|
||||
emulator_args = []
|
||||
else:
|
||||
emulator_args = ["--managed", "maxVersion"]
|
||||
|
||||
# Run the test app.
|
||||
call(
|
||||
state.python_dir / "android.py",
|
||||
"test",
|
||||
"--managed",
|
||||
"maxVersion",
|
||||
"--site-packages",
|
||||
site_packages_dir,
|
||||
"--cwd",
|
||||
cwd_dir,
|
||||
*emulator_args,
|
||||
*(["-v"] if state.options.build_verbosity > 0 else []),
|
||||
*test_runtime_args,
|
||||
"--",
|
||||
*test_args,
|
||||
env=state.build_env,
|
||||
|
||||
@@ -2,6 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import dataclasses
|
||||
import os
|
||||
import platform
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
@@ -653,11 +654,35 @@ def build(options: Options, tmp_path: Path) -> None:
|
||||
)
|
||||
raise errors.FatalError(msg)
|
||||
|
||||
test_runtime_args = build_options.test_runtime.args
|
||||
|
||||
# 2025-10: The GitHub Actions macos-15 runner has a known issue where
|
||||
# the default simulator won't start due to a disk performance issue;
|
||||
# see https://github.com/actions/runner-images/issues/12777 for details.
|
||||
# In the meantime, if it looks like we're running on a GitHub Actions
|
||||
# macos-15 runner, use a simulator that is known to work, unless the
|
||||
# user explicitly specifies a simulator.
|
||||
os_version, _, arch = platform.mac_ver()
|
||||
if (
|
||||
"GITHUB_ACTIONS" in os.environ
|
||||
and os_version.startswith("15.")
|
||||
and arch == "arm64"
|
||||
and not any(
|
||||
arg.startswith("--simulator") for arg in test_runtime_args
|
||||
)
|
||||
):
|
||||
test_runtime_args = [
|
||||
"--simulator",
|
||||
"iPhone 16e,OS=18.5",
|
||||
*test_runtime_args,
|
||||
]
|
||||
|
||||
call(
|
||||
"python",
|
||||
testbed_path,
|
||||
"run",
|
||||
*(["--verbose"] if build_options.build_verbosity > 0 else []),
|
||||
*test_runtime_args,
|
||||
"--",
|
||||
*final_command,
|
||||
env=test_env,
|
||||
|
||||
@@ -569,6 +569,39 @@
|
||||
],
|
||||
"title": "CIBW_TEST_ENVIRONMENT"
|
||||
},
|
||||
"test-runtime": {
|
||||
"description": "Additional configuration for the test runner",
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "string",
|
||||
"pattern": "^$"
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "string",
|
||||
"pattern": "args:"
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"required": [
|
||||
"args"
|
||||
],
|
||||
"properties": {
|
||||
"args": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"type": "string"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
],
|
||||
"title": "CIBW_TEST_RUNTIME"
|
||||
},
|
||||
"overrides": {
|
||||
"type": "array",
|
||||
"description": "An overrides array",
|
||||
@@ -638,6 +671,9 @@
|
||||
},
|
||||
"test-environment": {
|
||||
"$ref": "#/$defs/inherit"
|
||||
},
|
||||
"test-runtime": {
|
||||
"$ref": "#/$defs/inherit"
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -748,6 +784,9 @@
|
||||
},
|
||||
"test-environment": {
|
||||
"$ref": "#/properties/test-environment"
|
||||
},
|
||||
"test-runtime": {
|
||||
"$ref": "#/properties/test-runtime"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -876,6 +915,9 @@
|
||||
},
|
||||
"test-environment": {
|
||||
"$ref": "#/properties/test-environment"
|
||||
},
|
||||
"test-runtime": {
|
||||
"$ref": "#/properties/test-runtime"
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -936,6 +978,9 @@
|
||||
},
|
||||
"test-environment": {
|
||||
"$ref": "#/properties/test-environment"
|
||||
},
|
||||
"test-runtime": {
|
||||
"$ref": "#/properties/test-runtime"
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -1009,6 +1054,9 @@
|
||||
},
|
||||
"test-environment": {
|
||||
"$ref": "#/properties/test-environment"
|
||||
},
|
||||
"test-runtime": {
|
||||
"$ref": "#/properties/test-runtime"
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -1069,6 +1117,9 @@
|
||||
},
|
||||
"test-environment": {
|
||||
"$ref": "#/properties/test-environment"
|
||||
},
|
||||
"test-runtime": {
|
||||
"$ref": "#/properties/test-runtime"
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -1129,6 +1180,9 @@
|
||||
},
|
||||
"test-environment": {
|
||||
"$ref": "#/properties/test-environment"
|
||||
},
|
||||
"test-runtime": {
|
||||
"$ref": "#/properties/test-runtime"
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -1189,6 +1243,9 @@
|
||||
},
|
||||
"test-environment": {
|
||||
"$ref": "#/properties/test-environment"
|
||||
},
|
||||
"test-runtime": {
|
||||
"$ref": "#/properties/test-runtime"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,6 +25,7 @@ test-requires = []
|
||||
test-extras = []
|
||||
test-groups = []
|
||||
test-environment = {}
|
||||
test-runtime = {}
|
||||
|
||||
container-engine = "docker"
|
||||
|
||||
|
||||
@@ -1672,6 +1672,41 @@ Platform-specific environment variables are also available:<br/>
|
||||
CIBW_TEST_ENVIRONMENT: PYTHONSAFEPATH=1
|
||||
```
|
||||
|
||||
### `test-runtime` {: #test-runtime toml env-var }
|
||||
|
||||
> Controls how the tests will be executed.
|
||||
|
||||
On desktop environments, the tests are executed on the same machine/container as the wheel was built. However on Android and iOS, the tests are run inside a virtual machine – a simulator or emulator – representing the target.
|
||||
|
||||
For these embedded platforms, a testbed project is used to run the tests. The `test-runtime` setting can define an `args` key that defines additional arguments that will be used when starting the testbed project.
|
||||
|
||||
Platform-specific environment variables are also available:<br/>
|
||||
`CIBW_TEST_RUNTIME_ANDROID` |`CIBW_TEST_RUNTIME_IOS`
|
||||
|
||||
#### Examples
|
||||
|
||||
!!! tab examples "pyproject.toml"
|
||||
|
||||
```toml
|
||||
[tool.cibuildwheel.ios]
|
||||
# Run the tests on an iPhone 16e simulator running iOS 18.5.
|
||||
test-runtime = { args = ["--simulator='iPhone 16e,OS=18.5'"] }
|
||||
|
||||
[tool.cibuildwheel.android]
|
||||
# Run the Android tests on the minimum supported Android version.
|
||||
test-runtime = { args = ["--managed", "minVersion"] }
|
||||
```
|
||||
|
||||
!!! tab examples "Environment variables"
|
||||
|
||||
```yaml
|
||||
# Run the tests on an iPhone 16e simulator running iOS 18.5.
|
||||
CIBW_TEST_RUNTIME_IOS: "args: --simulator='iPhone 16e,OS=18.5'"
|
||||
|
||||
# Run the Android tests on the minimum supported Android version.
|
||||
CIBW_TEST_RUNTIME_ANDROID: "args: --managed minVersion"
|
||||
```
|
||||
|
||||
|
||||
## Debugging
|
||||
|
||||
|
||||
+5
-1
@@ -233,6 +233,8 @@ machine – for example, if you're building on an ARM64 machine, then you can te
|
||||
ARM64 wheel. Wheels of other architectures can still be built, but testing will
|
||||
automatically be skipped.
|
||||
|
||||
Any arguments specified using [`test-runtime`](options.md#test-runtime) will be passed as arguments to the Python script that starts the [testbed project](https://github.com/python/cpython/blob/main/Android/README.md#testing). cibuildwheel will automatically start the testbed project with `--site-packages` and `--cwd` arguments matching your test environment, as well as enabling verbose output with `-v` if [`build-verbosity`](options.md#build-verbosity) is enabled. The most common additional arguments to use will be `--managed minVersion` or `--managed maxVersion`, specifying the use of a managed Android emulator with the minimum or maximum supported Android version; or `--connected <serial>`, specifying the use of an existing booted Android emulator or device. By default, the testbed project will run with `--managed maxVersion`.
|
||||
|
||||
Running an emulator requires the build machine to either be bare-metal or support
|
||||
nested virtualization. CI platforms known to meet this requirement are:
|
||||
|
||||
@@ -320,4 +322,6 @@ If tests have been configured, the test suite will be executed on the simulator
|
||||
|
||||
The iOS test environment can't support running shell scripts, so the [`test-command`](options.md#test-command) value must be specified as if it were a command line being passed to `python -m ...`.
|
||||
|
||||
The test process uses the same testbed used by CPython itself to run the CPython test suite. It is an Xcode project that has been configured to have a single Xcode "XCUnit" test - the result of which reports the success or failure of running `python -m <test-command>`.
|
||||
The test process uses the [same testbed used by CPython itself](https://github.com/python/cpython/tree/main/Apple/iOS#testing-python-on-ios) to run the CPython test suite. It is an Xcode project that has been configured to have a single Xcode "XCUnit" test - the result of which reports the success or failure of running `python -m <test-command>`.
|
||||
|
||||
Any arguments specified using [`test-runtime`](options.md#test-runtime) will be passed as arguments to the Python script that starts the testbed project. The testbed project will be started with `-v` enabling verbose output if [`build-verbosity`](options.md#build-verbosity) is enabled; the most common additional argument to use will be `--simulator`, which allows the specification of a specific device or iOS version for the test simulator. By default, the testbed project will attempt to find an "SE class" simulator (i.e., an iPhone SE, iPhone 16e, or similar), running the newest iOS version available.
|
||||
|
||||
@@ -2,6 +2,7 @@ import os
|
||||
import platform as platform_module
|
||||
import textwrap
|
||||
import unittest.mock
|
||||
from collections.abc import Sequence
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
@@ -626,6 +627,39 @@ def test_get_build_frontend_extra_flags_warning(
|
||||
mock_warning.assert_called_once()
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("definition", "expected_args"),
|
||||
[
|
||||
("", ()),
|
||||
('test-runtime = ""', ()),
|
||||
("test-runtime = {}", ()),
|
||||
('test-runtime = {args = ""}', []),
|
||||
('test-runtime = "args: --simulator foo"', ["--simulator", "foo"]),
|
||||
('test-runtime = {args = ["--simulator", "foo"]}', ["--simulator", "foo"]),
|
||||
],
|
||||
)
|
||||
def test_test_runtime_handling(
|
||||
tmp_path: Path, definition: str, expected_args: Sequence[str] | None
|
||||
) -> None:
|
||||
args = CommandLineArguments.defaults()
|
||||
args.package_dir = tmp_path
|
||||
|
||||
pyproject_toml: Path = tmp_path / "pyproject.toml"
|
||||
pyproject_toml.write_text(
|
||||
textwrap.dedent(
|
||||
f"""\
|
||||
[tool.cibuildwheel]
|
||||
{definition}
|
||||
"""
|
||||
)
|
||||
)
|
||||
|
||||
options = Options(platform="ios", command_line_arguments=args, env={})
|
||||
|
||||
local = options.build_options("cp313-ios_13_0_arm64_iphoneos")
|
||||
assert local.test_runtime.args == expected_args
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("definition", "expected"),
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user