ci: rework tests to break out ios/android tests (#2519)

* ci: rework tests to break out ios/android tests Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * Apply suggestions from code review Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> * Apply suggestions from code review * ci: fix check for wheels Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * Update .github/workflows/test.yml * tests: pass via flag instead of envvar Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * tests: apply to serial/parallel Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * tests: join together marks Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * tests: join together marks Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * ci: break up azure job Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * ci: use matrix for azure pipelines Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * Try installing Java on macos as well * Revert "Try installing Java on macos as well" This reverts commit aedf0f5106d2467938705b85180a0b249d49b99d. * Try to get the most basic android test running first * Rename platform to test_select, move headers in run_tests Maybe it's a personal thing, but the header location was really making me have to think more than I needed to look at that file! * Only run docker unit tests on machines that are testing linux * Update the flag name in azure-pipelines too * Update azure-pipelines.yml --------- Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: Matthieu Darbois <mayeut@users.noreply.github.com> Co-authored-by: Joe Rickerby <joerick@mac.com>
2025-08-13 08:52:27 -04:00
parent 63bdd4f043
commit cade3864de
4 changed files with 129 additions and 76 deletions
@@ -47,19 +47,44 @@ jobs:
      run: pipx run --python "${{ steps.python.outputs.python-path }}" nox -s pylint -- --output-format=github

  test:
-    name: Test on ${{ matrix.os }} (${{ matrix.python_version }})
+    name: Test on ${{ matrix.os }} (${{ matrix.python_version }}) ${{ matrix.test_select }}
    needs: lint
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
-        os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, windows-11-arm, macos-13, macos-15]
-        python_version: ['3.13']
        include:
+        # Min Python
        - os: ubuntu-latest
          python_version: '3.11'
+        # Max Python
        - os: ubuntu-latest
          python_version: '3.14'
+        - os: ubuntu-latest
+          python_version: '3.13'
+          test_select: android
+        - os: ubuntu-24.04-arm
+          python_version: '3.13'
+        - os: windows-latest
+          python_version: '3.13'
+        - os: windows-11-arm
+          python_version: '3.13'
+        - os: macos-13
+          python_version: '3.13'
+        - os: macos-15
+          python_version: '3.13'
+        - os: macos-13
+          python_version: '3.13'
+          test_select: ios
+        - os: macos-15
+          python_version: '3.13'
+          test_select: ios
+        - os: macos-13
+          python_version: '3.13'
+          test_select: android
+        - os: macos-15
+          python_version: '3.13'
+          test_select: android
    timeout-minutes: 180
    steps:
    - uses: actions/checkout@v5
@@ -129,10 +154,12 @@ jobs:
        output-dir: wheelhouse
      env:
        CIBW_ARCHS_MACOS: x86_64 universal2 arm64
-        CIBW_BUILD_FRONTEND: 'build[uv]'
+        CIBW_BUILD_FRONTEND: ${{ matrix.test_select && 'build' || 'build[uv]' }}
+        CIBW_PLATFORM: ${{ matrix.test_select }}

    - name: Run a sample build (GitHub Action, only)
      uses: ./
+      if: matrix.test_select == ''
      with:
        package-dir: sample_proj
        output-dir: wheelhouse_only
@@ -152,6 +179,8 @@ jobs:

    - name: Run a sample build (GitHub Action, config-file)
      uses: ./
+      env:
+        CIBW_PLATFORM: ${{ matrix.test_select }}
      with:
        package-dir: sample_proj
        output-dir: wheelhouse_config_file
@@ -161,9 +190,14 @@ jobs:
      shell: bash
      run: |
        test $(find wheelhouse -name '*.whl' | wc -l) -ge 1
-        test $(find wheelhouse_only -name '*.whl' | wc -l) -eq 1
        test $(find wheelhouse_config_file -name '*.whl' | wc -l) -eq 1

+    - name: Check Action artifacts (native build only)
+      if: matrix.test_select == ''
+      shell: bash
+      run: |
+        test $(find wheelhouse_only -name '*.whl' | wc -l) -eq 1
+
    - uses: actions/upload-artifact@v4
      with:
        name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }}
@@ -171,7 +205,7 @@ jobs:

    - name: Test cibuildwheel
      run: |
-        uv run --no-sync bin/run_tests.py ${{ (runner.os == 'Linux' && runner.arch == 'X64') && '--run-podman' || '' }}
+        uv run --no-sync bin/run_tests.py --test-select=${{ matrix.test_select || 'native' }} ${{ (runner.os == 'Linux' && runner.arch == 'X64') && '--run-podman' || '' }}

  emulated-archs:
    name: Get qemu emulated architectures
@@ -16,62 +16,61 @@ pr:
      - noxfile.py

 jobs:
- job: linux_311
+- job: tests
+  strategy:
+    matrix:
+      linux_311:
+        imageName: "ubuntu-latest"
+        pythonVersion: "3.11"
+        testSelect: "native"
+      android_311:
+        imageName: "ubuntu-latest"
+        pythonVersion: "3.11"
+        testSelect: "android"
+      macos_311:
+        imageName: "macos-latest"
+        pythonVersion: "3.11"
+        testSelect: "native"
+      ios_311:
+        imageName: "macos-latest"
+        pythonVersion: "3.11"
+        testSelect: "ios"
+      android_macos_311:
+        imageName: "macos-latest"
+        pythonVersion: "3.11"
+        testSelect: "android"
+      windows_311:
+        imageName: "windows-latest"
+        pythonVersion: "3.11"
+        testSelect: "native"
  timeoutInMinutes: 180
-  pool: {vmImage: 'ubuntu-latest'}
+  pool:
+    vmImage: $(imageName)
+
  steps:
    - task: UsePythonVersion@0
      inputs:
-        versionSpec: '3.11'
+        versionSpec: $(pythonVersion)
+
    - task: JavaToolInstaller@0
+      condition: and(eq(variables['testSelect'], 'android'), eq(variables['Agent.OS'], 'Linux'))
      inputs:
        versionSpec: '17'
        jdkArchitectureOption: 'x64'
        jdkSourceOption: 'PreInstalled'
-    - bash: |
-        docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all
-        python -m pip install -U pip
-        python -m pip install -e. --group test
-        if [ "$(Build.SourceBranch)" = "refs/heads/main" ]; then
-          echo "INFO: Exporting CIBW_ENABLE=all for main branch test run."
-          export CIBW_ENABLE=all
-        else
-          echo "INFO: CIBW_ENABLE not set for this branch ($(Build.SourceBranch))."
-        fi
-        python ./bin/run_tests.py

- job: macos_311
-  pool: {vmImage: 'macOS-latest'}
-  timeoutInMinutes: 120
-  steps:
-    - task: UsePythonVersion@0
-      inputs:
-        versionSpec: '3.11'
-    - bash: |
-        python -m pip install -U pip
-        python -m pip install -e. --group test
-        if [ "$(Build.SourceBranch)" = "refs/heads/main" ]; then
-          echo "INFO: Exporting CIBW_ENABLE=all for main branch test run."
-          export CIBW_ENABLE=all
-        else
-          echo "INFO: CIBW_ENABLE not set for this branch ($(Build.SourceBranch))."
-        fi
-        python ./bin/run_tests.py
+    - bash: docker run --rm --privileged docker.io/tonistiigi/binfmt:latest --install all
+      condition: and(eq(variables['imageName'], 'ubuntu-latest'), eq(variables['testSelect'], 'native'))
+      displayName: 'Install binfmt on Linux'
+
+    - bash: python -m pip install -U pip && python -m pip install -e. --group test
+      displayName: 'Update pip and install cibuildwheel --group test'
+
+    - bash: echo "##vso[task.setvariable variable=CIBW_ENABLE;]all"
+      condition: eq(variables['Build.SourceBranch'], 'refs/heads/main')
+      displayName: Set CIBW_ENABLE to all (main branch)

- job: windows_311
-  pool: {vmImage: 'windows-latest'}
-  timeoutInMinutes: 180
-  steps:
-    - task: UsePythonVersion@0
-      inputs:
-        versionSpec: '3.11'
    - bash: |
-        python -m pip install -U pip
-        python -m pip install -e. --group test
-        if [ "$(Build.SourceBranch)" = "refs/heads/main" ]; then
-          echo "INFO: Exporting CIBW_ENABLE=all for main branch test run."
-          export CIBW_ENABLE=all
-        else
-          echo "INFO: CIBW_ENABLE not set for this branch ($(Build.SourceBranch))."
-        fi
-        python ./bin/run_tests.py
+        echo "CIBW_ENABLE = $CIBW_ENABLE"
+        python ./bin/run_tests.py --test-select $(testSelect)
+      displayName: 'Run tests'
@@ -27,6 +27,12 @@ if __name__ == "__main__":
        default=default_cpu_count,
        help="number of processes to use for testing",
    )
+    parser.add_argument(
+        "--test-select",
+        choices={"all", "native", "android", "ios", "pyodide"},
+        default="all",
+        help="Either 'native' or 'android'/'ios'/'pyodide'",
+    )
    args = parser.parse_args()

    # move cwd to the project root
@@ -50,20 +56,37 @@ if __name__ == "__main__":
        )

    # unit tests
+    print(
+        "\n\n================================== UNIT TESTS ==================================",
+        flush=True,
+    )
    unit_test_args = [sys.executable, "-m", "pytest", "unit_test"]

-    if sys.platform.startswith("linux") and os.environ.get("CIBW_PLATFORM", "linux") == "linux":
+    if (
+        sys.platform.startswith("linux")
+        and os.environ.get("CIBW_PLATFORM", "linux") == "linux"
+        and args.test_select in ["all", "native"]
+    ):
        # run the docker unit tests only on Linux
        unit_test_args += ["--run-docker"]

        if args.run_podman:
            unit_test_args += ["--run-podman"]

+    subprocess.run(unit_test_args, check=True)
+
    print(
-        "\n\n================================== UNIT TESTS ==================================",
+        "\n\n=========================== SERIAL INTEGRATION TESTS ===========================",
        flush=True,
    )
-    subprocess.run(unit_test_args, check=True)
+
+    match args.test_select:
+        case "all":
+            marks = []
+        case "native":
+            marks = ["not pyodide", "not android", "not ios"]
+        case mark:
+            marks = [f"{mark}"]

    # Run the serial integration tests without multiple processes
    serial_integration_test_args = [
@@ -71,7 +94,7 @@ if __name__ == "__main__":
        "-m",
        "pytest",
        "-m",
-        "serial",
+        f"{' and '.join(['serial', *marks])}",
        "-x",
        "--durations",
        "0",
@@ -79,19 +102,19 @@ if __name__ == "__main__":
        "test",
        "-vv",
    ]
-    print(
-        "\n\n=========================== SERIAL INTEGRATION TESTS ===========================",
-        flush=True,
-    )
+
    subprocess.run(serial_integration_test_args, check=True)

-    # Non-serial integration tests
+    print(
+        "\n\n========================= NON-SERIAL INTEGRATION TESTS =========================",
+        flush=True,
+    )
    integration_test_args = [
        sys.executable,
        "-m",
        "pytest",
        "-m",
-        "not serial",
+        f"{' and '.join(['not serial', *marks])}",
        f"--numprocesses={args.num_processes}",
        "-x",
        "--durations",
@@ -104,8 +127,4 @@ if __name__ == "__main__":
    if sys.platform.startswith("linux") and args.run_podman:
        integration_test_args += ["--run-podman"]

-    print(
-        "\n\n========================= NON-SERIAL INTEGRATION TESTS =========================",
-        flush=True,
-    )
    subprocess.run(integration_test_args, check=True)
@@ -90,8 +90,17 @@ def test_android_home(tmp_path, capfd):
    assert "ANDROID_HOME environment variable is not set" in capfd.readouterr().err


-# Can fail to setup
+# the first build can fail to setup - mark as flaky, and serial to make sure it runs first
+@pytest.mark.serial
@pytest.mark.flaky(reruns=2)
+def test_expected_wheels(tmp_path):
+    new_c_project().generate(tmp_path)
+    wheels = cibuildwheel_run(tmp_path, add_env={"CIBW_PLATFORM": "android"})
+    assert wheels == expected_wheels(
+        "spam", "0.1.0", platform="android", machine_arch=native_arch.android_abi
+    )
+
+
 def test_frontend_good(tmp_path):
    new_c_project().generate(tmp_path)
    wheels = cibuildwheel_run(
@@ -112,14 +121,6 @@ def test_frontend_bad(frontend, tmp_path, capfd):
    assert "Android requires the build frontend to be 'build'" in capfd.readouterr().err


-def test_expected_wheels(tmp_path):
-    new_c_project().generate(tmp_path)
-    wheels = cibuildwheel_run(tmp_path, add_env={"CIBW_PLATFORM": "android"})
-    assert wheels == expected_wheels(
-        "spam", "0.1.0", platform="android", machine_arch=native_arch.android_abi
-    )
-
-
@needs_emulator
 def test_archs(tmp_path, capfd):
    new_c_project().generate(tmp_path)