From eec3efa84e89571a9507345da107db8bf394564f Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 5 Jun 2026 08:09:16 -0400 Subject: [PATCH] ci: improve Azure test reliability (#2890) * ci: improve Azure test reliability Azure has intermittent failures, often the macOS framework-Python `posix_spawn: Undefined error: 0` spawn race under load, plus transient download flakiness. Address both: - Retry integration tests via pytest-rerunfailures (already required): add `--reruns=2 --reruns-delay=5` to the serial and non-serial runs so a single transient flake self-heals instead of failing the whole job. - Cap Azure at `--num-processes 3` to reduce concurrent process spawning (the macOS runner was using 4 xdist workers), lowering the spawn-race probability and peak resource pressure. - Cache downloads across runs: relocate CIBW_CACHE_PATH to a stable path and add a Cache@2 task (interpreter downloads + per-worker test pip caches live under it), cutting the network-download flake surface. - Bump the Azure host interpreter to 3.13. Assisted-by: ClaudeCode:claude-opus-4.8 * ci: stabilize iOS tests, bound rerun cost An iOS run on Azure (build 9043) hung for the full 40-min pytest timeout on the second build config of test_ios_platforms, then reran twice (reruns=2), blowing past the 180-min job cap with no useful signal. The hang was a stuck simulator/xcodebuild on the second config, after the first config had run. - Shut down running simulators before each parametrized config via a `clean_ios_simulators` fixture, so a simulator left booted/wedged by the previous config isn't reused. - Drop iOS reruns from 2 to 1. One retry still covers the documented "fails the first time" simulator-boot flake, while halving the worst-case retry cost (3x40=120min -> 2x40=80min) on a hang. Assisted-by: ClaudeCode:claude-opus-4.8 * ci: don't limit num processes Signed-off-by: Henry Schreiner * ci: rerun OCI unit tests that pull from Docker Hub test_local_image et al. occasionally fail on a transient Docker Hub anonymous-pull blip ("unauthorized: authentication required") when pulling debian:trixie-slim. The integration runs already self-heal via pytest-rerunfailures, but the unit run had no reruns, so a single flake failed the whole job. Mark the three network-dependent OCI tests (test_local_image, test_enter_error, test_multiarch_image) flaky so they retry instead of blanket-rerunning the suite. Assisted-by: ClaudeCode:claude-opus-4.8 * Drop iOS clean step Co-authored-by: Henry Schreiner * ci: scope integration reruns to test_0_basic::test Drop the blanket --reruns from the serial and non-serial integration runs and instead mark test_0_basic.py::test as flaky, so only the known flaky basic test is retried. Assisted-by: ClaudeCode:claude-opus-4.8 --------- Signed-off-by: Henry Schreiner --- azure-pipelines.yml | 39 +++++++++++++++++++++++---------- test/test_0_basic.py | 1 + test/test_ios.py | 4 ++-- unit_test/oci_container_test.py | 3 +++ 4 files changed, 33 insertions(+), 14 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index accd67ee..0f2621d8 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -19,39 +19,54 @@ jobs: - job: tests strategy: matrix: - linux_311: + linux_313: imageName: "ubuntu-latest" - pythonVersion: "3.11" + pythonVersion: "3.13" testSelect: "native" - android_311: + android_313: imageName: "ubuntu-latest" - pythonVersion: "3.11" + pythonVersion: "3.13" testSelect: "android" - macos_311: + macos_313: imageName: "macos-latest" - pythonVersion: "3.11" + pythonVersion: "3.13" testSelect: "native" - ios_311: + ios_313: imageName: "macos-14" # See https://github.com/actions/runner-images/issues/12777 - pythonVersion: "3.11" + pythonVersion: "3.13" testSelect: "ios" - android_macos_311: + android_macos_313: imageName: "macos-latest" - pythonVersion: "3.11" + pythonVersion: "3.13" testSelect: "android" - windows_311: + windows_313: imageName: "windows-latest" - pythonVersion: "3.11" + pythonVersion: "3.13" testSelect: "native" timeoutInMinutes: 180 pool: vmImage: $(imageName) + variables: + # Relocate cibuildwheel's download cache (interpreters) and the per-worker + # test pip caches (which live under it) to a stable, cacheable path. + CIBW_CACHE_PATH: $(Pipeline.Workspace)/.cibw-cache steps: - task: UsePythonVersion@0 inputs: versionSpec: $(pythonVersion) + - task: Cache@2 + displayName: 'Cache cibuildwheel downloads' + inputs: + # build-platforms.toml carries the interpreter pins; bump the version + # prefix to force a cold cache. restoreKeys lets a stale cache still seed + # a new run (cibuildwheel re-downloads only what's missing). + key: 'cibw-cache | v1 | "$(imageName)" | "$(testSelect)" | cibuildwheel/resources/build-platforms.toml' + restoreKeys: | + cibw-cache | v1 | "$(imageName)" | "$(testSelect)" + path: $(CIBW_CACHE_PATH) + - task: JavaToolInstaller@0 condition: and(eq(variables['testSelect'], 'android'), eq(variables['Agent.OS'], 'Linux')) inputs: diff --git a/test/test_0_basic.py b/test/test_0_basic.py index 8c1d9b75..08c44905 100644 --- a/test/test_0_basic.py +++ b/test/test_0_basic.py @@ -35,6 +35,7 @@ def test_dummy_serial() -> None: """ +@pytest.mark.flaky(reruns=2, reruns_delay=5) def test( tmp_path: Path, build_frontend_env: dict[str, str], capfd: pytest.CaptureFixture[str] ) -> None: diff --git a/test/test_ios.py b/test/test_ios.py index d36437d9..4ee4b3e1 100644 --- a/test/test_ios.py +++ b/test/test_ios.py @@ -48,7 +48,7 @@ def skip_if_ios_testing_not_supported() -> None: # which is guaranteed to run single-process. # This can also fail the first time sometimes. @pytest.mark.serial -@pytest.mark.flaky(reruns=2) +@pytest.mark.flaky(reruns=1) @pytest.mark.parametrize( "build_config", [ @@ -173,7 +173,7 @@ def test_ios_testing_with_placeholder(tmp_path: Path, capfd: pytest.CaptureFixtu @pytest.mark.serial -@pytest.mark.flaky(reruns=2) +@pytest.mark.flaky(reruns=1) def test_ios_test_command_short_circuit(tmp_path: Path, capfd: pytest.CaptureFixture[str]) -> None: skip_if_ios_testing_not_supported() diff --git a/unit_test/oci_container_test.py b/unit_test/oci_container_test.py index 76f03111..8b94187c 100644 --- a/unit_test/oci_container_test.py +++ b/unit_test/oci_container_test.py @@ -546,6 +546,7 @@ def test_disable_host_mount( container.call(["cat", host_mount_path], capture_output=True) +@pytest.mark.flaky(reruns=2, reruns_delay=5) @pytest.mark.parametrize("platform", list(OCIPlatform)) def test_local_image( container_engine: OCIContainerEngineConfig, platform: OCIPlatform, tmp_path: Path @@ -587,6 +588,7 @@ def test_local_image( assert container._get_platform_args() == expected_platform_args +@pytest.mark.flaky(reruns=2, reruns_delay=5) def test_enter_error(container_engine: OCIContainerEngineConfig, tmp_path: Path) -> None: remote_image = "debian:trixie-slim" platform = DEFAULT_OCI_PLATFORM @@ -745,6 +747,7 @@ def test_exit_kills_process_on_shutdown_timeout(monkeypatch: pytest.MonkeyPatch) assert bash_stdout.closed +@pytest.mark.flaky(reruns=2, reruns_delay=5) @pytest.mark.parametrize("platform", list(OCIPlatform)) def test_multiarch_image(container_engine: OCIContainerEngineConfig, platform: OCIPlatform) -> None: if detect_ci_provider() == CIProvider.travis_ci and DEFAULT_OCI_PLATFORM not in {