Debug Cirrus CI testing (#2414)

* Skip Cirrus CI iOS testing

* Remove support tick in readme

* Debugging cirrus windows failure

* Use a separate file to avoid error messages from vsdevcmd getting into the output

* cirrus: TEMP target just the test i care about

* Revert "cirrus: TEMP target just the test i care about"

This reverts commit 7ed4417970930ab9e0e1cb1f17497d406310d6ff.

* Remove debug code

* Fix windows issues with multiline -c Python commands

README.md

@@ -63,7 +63,7 @@ Usage
 | Travis CI       | ✅    |       | ✅      | ✅        |           |             |     |
 | CircleCI        | ✅    | ✅    |         | ✅        | ✅        |             | ✅³  |
 | Gitlab CI       | ✅    | ✅    | ✅      | ✅¹       | ✅        |             | ✅³  |
-| Cirrus CI       | ✅    | ✅    | ✅      | ✅        | ✅        |             | ✅³  |
+| Cirrus CI       | ✅    | ✅    | ✅      | ✅        | ✅        |             |      |
 
 <sup>¹ [Requires emulation](https://cibuildwheel.pypa.io/en/stable/faq/#emulation), distributed separately. Other services may also support Linux ARM through emulation or third-party build hosts, but these are not tested in our CI.</sup><br>
 <sup>² [Uses cross-compilation](https://cibuildwheel.pypa.io/en/stable/faq/#windows-arm64). It is not possible to test `arm64` on this CI platform.</sup><br>

cibuildwheel/platforms/windows.py

@@ -336,37 +336,36 @@ def setup_python(
         # variables. Adapted from
         # https://github.com/microsoft/vswhere/wiki/Start-Developer-Command-Prompt
         # Remove when https://github.com/oracle/graalpython/issues/492 is fixed.
-        vcpath = subprocess.check_output(
+        vcpath = call(
-            [
+            Path(os.environ["PROGRAMFILES(X86)"])
-                Path(os.environ["PROGRAMFILES(X86)"])
+            / "Microsoft Visual Studio"
-                / "Microsoft Visual Studio"
+            / "Installer"
-                / "Installer"
+            / "vswhere.exe",
-                / "vswhere.exe",
+            "-products",
-                "-products",
+            "*",
-                "*",
+            "-latest",
-                "-latest",
+            "-property",
-                "-property",
+            "installationPath",
-                "installationPath",
+            capture_stdout=True,
-            ],
-            text=True,
         ).strip()
         log.notice(f"Discovering Visual Studio for GraalPy at {vcpath}")
-        vcvars = subprocess.check_output(
+        vcvars_file = tmp / "vcvars.json"
-            [
+        call(
-                f"{vcpath}\\Common7\\Tools\\vsdevcmd.bat",
+            f"{vcpath}\\Common7\\Tools\\vsdevcmd.bat",
-                "-no_logo",
+            "-no_logo",
-                "-arch=amd64",
+            "-arch=amd64",
-                "-host_arch=amd64",
+            "-host_arch=amd64",
-                "&&",
+            "&&",
-                "python",
+            "python",
-                "-c",
+            "-c",
-                "import os, json, sys; json.dump(dict(os.environ), sys.stdout);",
+            # this command needs to be one line for Windows reasons
-            ],
+            "import sys, json, pathlib, os; pathlib.Path(sys.argv[1]).write_text(json.dumps(dict(os.environ)))",
-            shell=True,
+            vcvars_file,
-            text=True,
             env=env,
         )
-        env.update(json.loads(vcvars))
+        with open(vcvars_file, encoding="utf-8") as f:
+            vcvars = json.load(f)
+        env.update(vcvars)
 
     return base_python, env
 

test/test_ios.py

@@ -8,6 +8,8 @@ import textwrap
 
 import pytest
 
+from cibuildwheel.ci import CIProvider, detect_ci_provider
+
 from . import test_projects, utils
 
 basic_project_files = {
@@ -23,6 +25,19 @@ class TestPlatform(TestCase):
 }
 
 
+def skip_if_ios_testing_not_supported() -> None:
+    """Skip the test if iOS testing is not supported on this machine."""
+    if utils.get_platform() != "macos":
+        pytest.skip("this test can only run on macOS")
+    if utils.get_xcode_version() < (13, 0):
+        pytest.skip("this test only works with Xcode 13.0 or greater")
+    if detect_ci_provider() == CIProvider.cirrus_ci:
+        pytest.skip(
+            "iOS testing not currently supported on Cirrus CI due to a failure "
+            "to start the simulator."
+        )
+
+
 # iOS tests shouldn't be run in parallel, because they're dependent on calling
 # Xcode, and starting a simulator. These are both multi-threaded operations, and
 # it's easy to overload the CI machine if there are multiple test processes
@@ -39,10 +54,7 @@ class TestPlatform(TestCase):
     ],
 )
 def test_ios_platforms(tmp_path, build_config, monkeypatch, capfd):
-    if utils.get_platform() != "macos":
+    skip_if_ios_testing_not_supported()
-        pytest.skip("this test can only run on macOS")
-    if utils.get_xcode_version() < (13, 0):
-        pytest.skip("this test only works with Xcode 13.0 or greater")
 
     # Create a temporary "bin" directory, symlink a tool that we know eixsts
     # (/usr/bin/true) into that location under a name that should be unique,
@@ -94,10 +106,7 @@ def test_ios_platforms(tmp_path, build_config, monkeypatch, capfd):
 @pytest.mark.serial
 def test_no_test_sources(tmp_path, capfd):
     """Build will provide a helpful error if pytest is run and test-sources is not defined."""
-    if utils.get_platform() != "macos":
+    skip_if_ios_testing_not_supported()
-        pytest.skip("this test can only run on macOS")
-    if utils.get_xcode_version() < (13, 0):
-        pytest.skip("this test only works with Xcode 13.0 or greater")
 
     project_dir = tmp_path / "project"
     basic_project = test_projects.new_c_project()
@@ -126,10 +135,7 @@ def test_no_test_sources(tmp_path, capfd):
 
 def test_ios_testing_with_placeholder(tmp_path, capfd):
     """Build will run tests with the {project} placeholder."""
-    if utils.get_platform() != "macos":
+    skip_if_ios_testing_not_supported()
-        pytest.skip("this test can only run on macOS")
-    if utils.get_xcode_version() < (13, 0):
-        pytest.skip("this test only works with Xcode 13.0 or greater")
 
     project_dir = tmp_path / "project"
     basic_project = test_projects.new_c_project()
@@ -155,10 +161,7 @@ def test_ios_testing_with_placeholder(tmp_path, capfd):
 
 def test_missing_xbuild_tool(tmp_path, capfd):
     """Build will fail if xbuild-tools references a non-existent tool."""
-    if utils.get_platform() != "macos":
+    skip_if_ios_testing_not_supported()
-        pytest.skip("this test can only run on macOS")
-    if utils.get_xcode_version() < (13, 0):
-        pytest.skip("this test only works with Xcode 13.0 or greater")
 
     project_dir = tmp_path / "project"
     basic_project = test_projects.new_c_project()
@@ -183,10 +186,7 @@ def test_missing_xbuild_tool(tmp_path, capfd):
 
 def test_no_xbuild_tool_definition(tmp_path, capfd):
     """Build will succeed with a warning if there is no xbuild-tools definition."""
-    if utils.get_platform() != "macos":
+    skip_if_ios_testing_not_supported()
-        pytest.skip("this test can only run on macOS")
-    if utils.get_xcode_version() < (13, 0):
-        pytest.skip("this test only works with Xcode 13.0 or greater")
 
     project_dir = tmp_path / "project"
     basic_project = test_projects.new_c_project()
@@ -220,10 +220,7 @@ def test_no_xbuild_tool_definition(tmp_path, capfd):
 
 def test_empty_xbuild_tool_definition(tmp_path, capfd):
     """Build will succeed with no warning if there is an empty xbuild-tools definition."""
-    if utils.get_platform() != "macos":
+    skip_if_ios_testing_not_supported()
-        pytest.skip("this test can only run on macOS")
-    if utils.get_xcode_version() < (13, 0):
-        pytest.skip("this test only works with Xcode 13.0 or greater")
 
     project_dir = tmp_path / "project"
     basic_project = test_projects.new_c_project()
@@ -255,10 +252,7 @@ def test_empty_xbuild_tool_definition(tmp_path, capfd):
 @pytest.mark.serial
 def test_ios_test_command_without_python_dash_m(tmp_path, capfd):
     """pytest should be able to run without python -m, but it should warn."""
-    if utils.get_platform() != "macos":
+    skip_if_ios_testing_not_supported()
-        pytest.skip("this test can only run on macOS")
-    if utils.get_xcode_version() < (13, 0):
-        pytest.skip("this test only works with Xcode 13.0 or greater")
 
     project_dir = tmp_path / "project"
 
@@ -296,10 +290,7 @@ def test_ios_test_command_without_python_dash_m(tmp_path, capfd):
 
 def test_ios_test_command_invalid(tmp_path, capfd):
     """Test command should raise an error if it's clearly invalid."""
-    if utils.get_platform() != "macos":
+    skip_if_ios_testing_not_supported()
-        pytest.skip("this test can only run on macOS")
-    if utils.get_xcode_version() < (13, 0):
-        pytest.skip("this test only works with Xcode 13.0 or greater")
 
     project_dir = tmp_path / "project"
     basic_project = test_projects.new_c_project()