Don't attempt to test CPython 3.8 wheels on Apple Silicon. (#1171)

I've added a warning for users, so they know that it doesn't get tested.
See discussion in #1169 for the reason why.

Fixes #1168, fixes #1169

cibuildwheel/macos.py

@@ -465,6 +465,23 @@ def build(options: Options, tmp_path: Path) -> None:
                         # skip this test
                         continue
 
+                    if testing_arch == "arm64" and config.identifier.startswith("cp38-"):
+                        log.warning(
+                            unwrap(
+                                """
+                                While cibuildwheel can build CPython 3.8 universal2/arm64 wheels, we
+                                cannot test the arm64 part of them, even when running on an Apple
+                                Silicon machine. This is because we use the x86_64 installer of
+                                CPython 3.8. See the discussion in
+                                https://github.com/pypa/cibuildwheel/pull/1169 for the details. To
+                                silence this warning, set `CIBW_TEST_SKIP: cp38-macosx_*:arm64`.
+                                """
+                            )
+                        )
+
+                        # skip this test
+                        continue
+
                     log.step(
                         "Testing wheel..."
                         if testing_arch == machine_arch

test/test_macos_archs.py

@@ -133,3 +133,36 @@ def test_universal2_testing(tmp_path, capfd, skip_arm64_test):
     expected_wheels = [w for w in ALL_MACOS_WHEELS if "cp39" in w and "universal2" in w]
 
     assert set(actual_wheels) == set(expected_wheels)
+
+
+def test_cp38_arm64_testing(tmp_path, capfd):
+    if utils.platform != "macos":
+        pytest.skip("this test is only relevant to macos")
+    if get_xcode_version() < (12, 2):
+        pytest.skip("this test only works with Xcode 12.2 or greater")
+    if platform.machine() != "arm64":
+        pytest.skip("this test only works on arm64")
+
+    project_dir = tmp_path / "project"
+    basic_project.generate(project_dir)
+
+    actual_wheels = utils.cibuildwheel_run(
+        project_dir,
+        add_env={
+            "CIBW_BUILD": "cp38-*",
+            "CIBW_TEST_COMMAND": '''python -c "import platform; print('running tests on ' + platform.machine())"''',
+            "CIBW_ARCHS": "x86_64,universal2,arm64",
+        },
+    )
+
+    captured = capfd.readouterr()
+
+    assert "running tests on x86_64" in captured.out
+    assert "running tests on arm64" not in captured.out
+
+    warning_message = "While cibuildwheel can build CPython 3.8 universal2/arm64 wheels, we cannot test the arm64 part of them"
+    assert warning_message in captured.err
+
+    expected_wheels = [w for w in ALL_MACOS_WHEELS if "cp38" in w]
+
+    assert set(actual_wheels) == set(expected_wheels)

test/utils.py

@@ -154,8 +154,8 @@ def expected_wheels(
             python_abi_tags += ["pp37-pypy37_pp73", "pp38-pypy38_pp73", "pp39-pypy39_pp73"]
 
         if platform == "macos" and machine_arch == "arm64":
-            # currently, arm64 macs are only supported by cp39, cp310 & cp311
-            python_abi_tags = ["cp39-cp39", "cp310-cp310", "cp311-cp311"]
+            # arm64 macs are only supported by cp38+
+            python_abi_tags = ["cp38-cp38", "cp39-cp39", "cp310-cp310", "cp311-cp311"]
 
     wheels = []
 
@@ -193,7 +193,7 @@ def expected_wheels(
                 platform_tags = ["win32", "win_amd64"]
 
         elif platform == "macos":
-            if python_abi_tag == "cp39-cp39" and machine_arch == "arm64":
+            if machine_arch == "arm64":
                 arm64_macosx_deployment_target = _get_arm64_macosx_deployment_target(
                     macosx_deployment_target
                 )