ci: rework tests to break out ios/android tests (#2519)

* ci: rework tests to break out ios/android tests

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* Apply suggestions from code review

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Apply suggestions from code review

* ci: fix check for wheels

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* Update .github/workflows/test.yml

* tests: pass via flag instead of envvar

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* tests: apply to serial/parallel

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* tests: join together marks

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* tests: join together marks

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* ci: break up azure job

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* ci: use matrix for azure pipelines

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* Try installing Java on macos as well

* Revert "Try installing Java on macos as well"

This reverts commit aedf0f5106d2467938705b85180a0b249d49b99d.

* Try to get the most basic android test running first

* Rename platform to test_select, move headers in run_tests

Maybe it's a personal thing, but the header location was really making
me have to think more than I needed to look at that file!

* Only run docker unit tests on machines that are testing linux

* Update the flag name in azure-pipelines too

* Update azure-pipelines.yml

---------

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: Matthieu Darbois <mayeut@users.noreply.github.com>
Co-authored-by: Joe Rickerby <joerick@mac.com>

.github/workflows/test.yml

@@ -47,19 +47,44 @@ jobs:
       run: pipx run --python "${{ steps.python.outputs.python-path }}" nox -s pylint -- --output-format=github
 
   test:
-    name: Test on ${{ matrix.os }} (${{ matrix.python_version }})
+    name: Test on ${{ matrix.os }} (${{ matrix.python_version }}) ${{ matrix.test_select }}
     needs: lint
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, windows-11-arm, macos-13, macos-15]
-        python_version: ['3.13']
         include:
+        # Min Python
         - os: ubuntu-latest
           python_version: '3.11'
+        # Max Python
         - os: ubuntu-latest
           python_version: '3.14'
+        - os: ubuntu-latest
+          python_version: '3.13'
+          test_select: android
+        - os: ubuntu-24.04-arm
+          python_version: '3.13'
+        - os: windows-latest
+          python_version: '3.13'
+        - os: windows-11-arm
+          python_version: '3.13'
+        - os: macos-13
+          python_version: '3.13'
+        - os: macos-15
+          python_version: '3.13'
+        - os: macos-13
+          python_version: '3.13'
+          test_select: ios
+        - os: macos-15
+          python_version: '3.13'
+          test_select: ios
+        - os: macos-13
+          python_version: '3.13'
+          test_select: android
+        - os: macos-15
+          python_version: '3.13'
+          test_select: android
     timeout-minutes: 180
     steps:
     - uses: actions/checkout@v5
@@ -129,10 +154,12 @@ jobs:
         output-dir: wheelhouse
       env:
         CIBW_ARCHS_MACOS: x86_64 universal2 arm64
-        CIBW_BUILD_FRONTEND: 'build[uv]'
+        CIBW_BUILD_FRONTEND: ${{ matrix.test_select && 'build' || 'build[uv]' }}
+        CIBW_PLATFORM: ${{ matrix.test_select }}
 
     - name: Run a sample build (GitHub Action, only)
       uses: ./
+      if: matrix.test_select == ''
       with:
         package-dir: sample_proj
         output-dir: wheelhouse_only
@@ -152,6 +179,8 @@ jobs:
 
     - name: Run a sample build (GitHub Action, config-file)
       uses: ./
+      env:
+        CIBW_PLATFORM: ${{ matrix.test_select }}
       with:
         package-dir: sample_proj
         output-dir: wheelhouse_config_file
@@ -161,9 +190,14 @@ jobs:
       shell: bash
       run: |
         test $(find wheelhouse -name '*.whl' | wc -l) -ge 1
-        test $(find wheelhouse_only -name '*.whl' | wc -l) -eq 1
         test $(find wheelhouse_config_file -name '*.whl' | wc -l) -eq 1
 
+    - name: Check Action artifacts (native build only)
+      if: matrix.test_select == ''
+      shell: bash
+      run: |
+        test $(find wheelhouse_only -name '*.whl' | wc -l) -eq 1
+
     - uses: actions/upload-artifact@v4
       with:
         name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }}
@@ -171,7 +205,7 @@ jobs:
 
     - name: Test cibuildwheel
       run: |
-        uv run --no-sync bin/run_tests.py ${{ (runner.os == 'Linux' && runner.arch == 'X64') && '--run-podman' || '' }}
+        uv run --no-sync bin/run_tests.py --test-select=${{ matrix.test_select || 'native' }} ${{ (runner.os == 'Linux' && runner.arch == 'X64') && '--run-podman' || '' }}
 
   emulated-archs:
     name: Get qemu emulated architectures

azure-pipelines.yml

@@ -16,62 +16,61 @@ pr:
       - noxfile.py
 
 jobs:
-- job: linux_311
+- job: tests
+  strategy:
+    matrix:
+      linux_311:
+        imageName: "ubuntu-latest"
+        pythonVersion: "3.11"
+        testSelect: "native"
+      android_311:
+        imageName: "ubuntu-latest"
+        pythonVersion: "3.11"
+        testSelect: "android"
+      macos_311:
+        imageName: "macos-latest"
+        pythonVersion: "3.11"
+        testSelect: "native"
+      ios_311:
+        imageName: "macos-latest"
+        pythonVersion: "3.11"
+        testSelect: "ios"
+      android_macos_311:
+        imageName: "macos-latest"
+        pythonVersion: "3.11"
+        testSelect: "android"
+      windows_311:
+        imageName: "windows-latest"
+        pythonVersion: "3.11"
+        testSelect: "native"
   timeoutInMinutes: 180
-  pool: {vmImage: 'ubuntu-latest'}
+  pool:
+    vmImage: $(imageName)
+
   steps:
     - task: UsePythonVersion@0
       inputs:
-        versionSpec: '3.11'
+        versionSpec: $(pythonVersion)
+
     - task: JavaToolInstaller@0
+      condition: and(eq(variables['testSelect'], 'android'), eq(variables['Agent.OS'], 'Linux'))
       inputs:
         versionSpec: '17'
         jdkArchitectureOption: 'x64'
         jdkSourceOption: 'PreInstalled'
-    - bash: |
-        docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all
-        python -m pip install -U pip
-        python -m pip install -e. --group test
-        if [ "$(Build.SourceBranch)" = "refs/heads/main" ]; then
-          echo "INFO: Exporting CIBW_ENABLE=all for main branch test run."
-          export CIBW_ENABLE=all
-        else
-          echo "INFO: CIBW_ENABLE not set for this branch ($(Build.SourceBranch))."
-        fi
-        python ./bin/run_tests.py
 
-- job: macos_311
+    - bash: docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all
-  pool: {vmImage: 'macOS-latest'}
+      condition: and(eq(variables['imageName'], 'ubuntu-latest'), eq(variables['testSelect'], 'native'))
-  timeoutInMinutes: 120
+      displayName: 'Install binfmt on Linux'
-  steps:
+
-    - task: UsePythonVersion@0
+    - bash: python -m pip install -U pip && python -m pip install -e. --group test
-      inputs:
+      displayName: 'Update pip and install cibuildwheel --group test'
-        versionSpec: '3.11'
+
-    - bash: |
+    - bash: echo "##vso[task.setvariable variable=CIBW_ENABLE;]all"
-        python -m pip install -U pip
+      condition: eq(variables['Build.SourceBranch'], 'refs/heads/main')
-        python -m pip install -e. --group test
+      displayName: Set CIBW_ENABLE to all (main branch)
-        if [ "$(Build.SourceBranch)" = "refs/heads/main" ]; then
-          echo "INFO: Exporting CIBW_ENABLE=all for main branch test run."
-          export CIBW_ENABLE=all
-        else
-          echo "INFO: CIBW_ENABLE not set for this branch ($(Build.SourceBranch))."
-        fi
-        python ./bin/run_tests.py
 
-- job: windows_311
-  pool: {vmImage: 'windows-latest'}
-  timeoutInMinutes: 180
-  steps:
-    - task: UsePythonVersion@0
-      inputs:
-        versionSpec: '3.11'
     - bash: |
-        python -m pip install -U pip
+        echo "CIBW_ENABLE = $CIBW_ENABLE"
-        python -m pip install -e. --group test
+        python ./bin/run_tests.py --test-select $(testSelect)
-        if [ "$(Build.SourceBranch)" = "refs/heads/main" ]; then
+      displayName: 'Run tests'
-          echo "INFO: Exporting CIBW_ENABLE=all for main branch test run."
-          export CIBW_ENABLE=all
-        else
-          echo "INFO: CIBW_ENABLE not set for this branch ($(Build.SourceBranch))."
-        fi
-        python ./bin/run_tests.py

bin/run_tests.py

@@ -27,6 +27,12 @@ if __name__ == "__main__":
         default=default_cpu_count,
         help="number of processes to use for testing",
     )
+    parser.add_argument(
+        "--test-select",
+        choices={"all", "native", "android", "ios", "pyodide"},
+        default="all",
+        help="Either 'native' or 'android'/'ios'/'pyodide'",
+    )
     args = parser.parse_args()
 
     # move cwd to the project root
@@ -50,20 +56,37 @@ if __name__ == "__main__":
         )
 
     # unit tests
+    print(
+        "\n\n================================== UNIT TESTS ==================================",
+        flush=True,
+    )
     unit_test_args = [sys.executable, "-m", "pytest", "unit_test"]
 
-    if sys.platform.startswith("linux") and os.environ.get("CIBW_PLATFORM", "linux") == "linux":
+    if (
+        sys.platform.startswith("linux")
+        and os.environ.get("CIBW_PLATFORM", "linux") == "linux"
+        and args.test_select in ["all", "native"]
+    ):
         # run the docker unit tests only on Linux
         unit_test_args += ["--run-docker"]
 
         if args.run_podman:
             unit_test_args += ["--run-podman"]
 
+    subprocess.run(unit_test_args, check=True)
+
     print(
-        "\n\n================================== UNIT TESTS ==================================",
+        "\n\n=========================== SERIAL INTEGRATION TESTS ===========================",
         flush=True,
     )
-    subprocess.run(unit_test_args, check=True)
+
+    match args.test_select:
+        case "all":
+            marks = []
+        case "native":
+            marks = ["not pyodide", "not android", "not ios"]
+        case mark:
+            marks = [f"{mark}"]
 
     # Run the serial integration tests without multiple processes
     serial_integration_test_args = [
@@ -71,7 +94,7 @@ if __name__ == "__main__":
         "-m",
         "pytest",
         "-m",
-        "serial",
+        f"{' and '.join(['serial', *marks])}",
         "-x",
         "--durations",
         "0",
@@ -79,19 +102,19 @@ if __name__ == "__main__":
         "test",
         "-vv",
     ]
-    print(
+
-        "\n\n=========================== SERIAL INTEGRATION TESTS ===========================",
-        flush=True,
-    )
     subprocess.run(serial_integration_test_args, check=True)
 
-    # Non-serial integration tests
+    print(
+        "\n\n========================= NON-SERIAL INTEGRATION TESTS =========================",
+        flush=True,
+    )
     integration_test_args = [
         sys.executable,
         "-m",
         "pytest",
         "-m",
-        "not serial",
+        f"{' and '.join(['not serial', *marks])}",
         f"--numprocesses={args.num_processes}",
         "-x",
         "--durations",
@@ -104,8 +127,4 @@ if __name__ == "__main__":
     if sys.platform.startswith("linux") and args.run_podman:
         integration_test_args += ["--run-podman"]
 
-    print(
-        "\n\n========================= NON-SERIAL INTEGRATION TESTS =========================",
-        flush=True,
-    )
     subprocess.run(integration_test_args, check=True)

test/test_android.py

@@ -90,8 +90,17 @@ def test_android_home(tmp_path, capfd):
     assert "ANDROID_HOME environment variable is not set" in capfd.readouterr().err
 
 
-# Can fail to setup
+# the first build can fail to setup - mark as flaky, and serial to make sure it runs first
+@pytest.mark.serial
 @pytest.mark.flaky(reruns=2)
+def test_expected_wheels(tmp_path):
+    new_c_project().generate(tmp_path)
+    wheels = cibuildwheel_run(tmp_path, add_env={"CIBW_PLATFORM": "android"})
+    assert wheels == expected_wheels(
+        "spam", "0.1.0", platform="android", machine_arch=native_arch.android_abi
+    )
+
+
 def test_frontend_good(tmp_path):
     new_c_project().generate(tmp_path)
     wheels = cibuildwheel_run(
@@ -112,14 +121,6 @@ def test_frontend_bad(frontend, tmp_path, capfd):
     assert "Android requires the build frontend to be 'build'" in capfd.readouterr().err
 
 
-def test_expected_wheels(tmp_path):
-    new_c_project().generate(tmp_path)
-    wheels = cibuildwheel_run(tmp_path, add_env={"CIBW_PLATFORM": "android"})
-    assert wheels == expected_wheels(
-        "spam", "0.1.0", platform="android", machine_arch=native_arch.android_abi
-    )
-
-
 @needs_emulator
 def test_archs(tmp_path, capfd):
     new_c_project().generate(tmp_path)