fix: change to the iOS testing option semantics (#2363)

* Change to the iOS testing option semantics

Don't assume the presence of `python -m` in the test command. Less magic
and allows more option reuse between platforms.

* Update schema

* Add a test for this warning

* Don't try to execute a test-command when it doesn't look like a module

* Update docs/options.md

Co-authored-by: Malcolm Smith <smith@chaquo.com>

* Only allow invalid test command if the first part is 'pytest'

* Responses to code review from @freakboy3742

* Fixes post-merge

---------

Co-authored-by: Malcolm Smith <smith@chaquo.com>

cibuildwheel/platforms/ios.py

@@ -32,7 +32,7 @@ from ..util.file import (
     download,
     move_file,
 )
-from ..util.helpers import prepare_command
+from ..util.helpers import prepare_command, unwrap_preserving_paragraphs
 from ..util.packaging import (
     combine_constraints,
     find_compatible_wheel,
@@ -593,6 +593,43 @@ def build(options: Options, tmp_path: Path) -> None:
                     )
 
                     log.step("Running test suite...")
+
+                    test_command_parts = shlex.split(build_options.test_command)
+                    if test_command_parts[0:2] != ["python", "-m"]:
+                        first_part = test_command_parts[0]
+                        if first_part == "pytest":
+                            # pytest works exactly the same as a module, so we
+                            # can just run it as a module.
+                            log.warning(
+                                unwrap_preserving_paragraphs(f"""
+                                    iOS tests configured with a test command which doesn't start
+                                    with 'python -m'. iOS tests must execute python modules - other
+                                    entrypoints are not supported.
+
+                                    cibuildwheel will try to execute it as if it started with
+                                    'python -m'. If this works, all you need to do is add that to
+                                    your test command.
+
+                                    Test command: {build_options.test_command!r}
+                                """)
+                            )
+                        else:
+                            msg = unwrap_preserving_paragraphs(
+                                f"""
+                                    iOS tests configured with a test command which doesn't start
+                                    with 'python -m'. iOS tests must execute python modules - other
+                                    entrypoints are not supported.
+
+                                    Test command: {build_options.test_command!r}
+                                """
+                            )
+                            raise errors.FatalError(msg)
+                    else:
+                        # the testbed run command actually doesn't want the
+                        # python -m prefix - it's implicit, so we remove it
+                        # here.
+                        test_command_parts = test_command_parts[2:]
+
                     try:
                         call(
                             "python",
@@ -600,7 +637,7 @@ def build(options: Options, tmp_path: Path) -> None:
                             "run",
                             *(["--verbose"] if build_options.build_verbosity > 0 else []),
                             "--",
-                            *(shlex.split(build_options.test_command)),
+                            *test_command_parts,
                             env=build_env,
                         )
                         failed = False

cibuildwheel/util/helpers.py

@@ -76,6 +76,21 @@ def unwrap(text: str) -> str:
     return re.sub(r"\s+", " ", text)
 
 
+def unwrap_preserving_paragraphs(text: str) -> str:
+    """
+    Unwraps multi-line text to a single line, but preserves paragraphs
+    """
+    # remove initial line indent
+    text = textwrap.dedent(text)
+    # remove leading/trailing whitespace
+    text = text.strip()
+
+    paragraphs = text.split("\n\n")
+    # remove consecutive whitespace
+    paragraphs = [re.sub(r"\s+", " ", paragraph) for paragraph in paragraphs]
+    return "\n\n".join(paragraphs)
+
+
 def parse_key_value_string(
     key_value_string: str,
     positional_arg_names: Sequence[str] | None = None,

docs/options.md

@@ -1253,7 +1253,7 @@ run your test suite.
 
 On all platforms other than iOS, the command is run in a shell, so you can write things like `cmd1 && cmd2`.
 
-On iOS, the value of the `CIBW_TEST_COMMAND` setting is interpreted as the arguments to pass to `python -m` - that is, a Python module name, followed by arguments that will be assigned to `sys.argv`. Shell commands cannot be used.
+On iOS, the value of the `CIBW_TEST_COMMAND` setting must follow the format `python -m MODULE [ARGS...]` - where MODULE is a Python module name, followed by arguments that will be assigned to `sys.argv`. Other commands cannot be used.
 
 Platform-specific environment variables are also available:<br/>
 `CIBW_TEST_COMMAND_MACOS` | `CIBW_TEST_COMMAND_WINDOWS` | `CIBW_TEST_COMMAND_LINUX` | `CIBW_TEST_COMMAND_IOS` | `CIBW_TEST_COMMAND_PYODIDE`
@@ -1273,6 +1273,10 @@ Platform-specific environment variables are also available:<br/>
     CIBW_TEST_COMMAND: >
       pytest ./tests &&
       python ./test.py
+
+    # run tests on ios
+    CIBW_TEST_SOURCES_IOS: tests
+    CIBW_TEST_COMMAND_IOS: python -m pytest ./tests
     ```
 
 !!! tab examples "pyproject.toml"
@@ -1290,6 +1294,11 @@ Platform-specific environment variables are also available:<br/>
       "pytest ./tests",
       "python ./test.py",
     ]
+
+    # run tests on ios
+    [tool.cibuildwheel.ios]
+    test-sources = ["tests"]
+    test-command = "python -m pytest ./tests"
     ```
 
     In configuration files, you can use an array, and the items will be joined with `&&`.

test/test_ios.py

@@ -4,6 +4,7 @@ import os
 import platform
 import shutil
 import subprocess
+import textwrap
 
 import pytest
 
@@ -73,28 +74,17 @@ def test_ios_platforms(tmp_path, build_config, monkeypatch, capfd):
             "CIBW_BUILD": "cp313-*",
             "CIBW_XBUILD_TOOLS": "does-exist",
             "CIBW_TEST_SOURCES": "tests",
-            "CIBW_TEST_COMMAND": "unittest discover tests test_platform.py",
+            "CIBW_TEST_COMMAND": "python -m unittest discover tests test_platform.py",
             "CIBW_BUILD_VERBOSITY": "1",
             **build_config,
         },
     )
 
     # The expected wheels were produced.
-    ios_version = os.getenv("IPHONEOS_DEPLOYMENT_TARGET", "13.0").replace(".", "_")
+    expected_wheels = utils.expected_wheels(
-    platform_machine = platform.machine()
+        "spam", "0.1.0", platform="ios", python_abi_tags=["cp313-cp313"]
-
+    )
-    if platform_machine == "x86_64":
+    assert set(actual_wheels) == set(expected_wheels)
-        expected_wheels = {
-            f"spam-0.1.0-cp313-cp313-ios_{ios_version}_x86_64_iphonesimulator.whl",
-        }
-
-    elif platform_machine == "arm64":
-        expected_wheels = {
-            f"spam-0.1.0-cp313-cp313-ios_{ios_version}_arm64_iphoneos.whl",
-            f"spam-0.1.0-cp313-cp313-ios_{ios_version}_arm64_iphonesimulator.whl",
-        }
-
-    assert set(actual_wheels) == expected_wheels
 
     # The user was notified that the cross-build tool was found.
     captured = capfd.readouterr()
@@ -119,7 +109,7 @@ def test_no_test_sources(tmp_path, capfd):
             add_env={
                 "CIBW_PLATFORM": "ios",
                 "CIBW_BUILD": "cp313-*",
-                "CIBW_TEST_COMMAND": "tests",
+                "CIBW_TEST_COMMAND": "python -m tests",
             },
         )
 
@@ -146,7 +136,7 @@ def test_missing_xbuild_tool(tmp_path, capfd):
             add_env={
                 "CIBW_PLATFORM": "ios",
                 "CIBW_BUILD": "cp313-*",
-                "CIBW_TEST_COMMAND": "tests",
+                "CIBW_TEST_COMMAND": "python -m tests",
                 "CIBW_XBUILD_TOOLS": "does-not-exist",
             },
         )
@@ -180,21 +170,13 @@ def test_no_xbuild_tool_definition(tmp_path, capfd):
     )
 
     # The expected wheels were produced.
-    ios_version = os.getenv("IPHONEOS_DEPLOYMENT_TARGET", "13.0").replace(".", "_")
+    expected_wheels = utils.expected_wheels(
-    platform_machine = platform.machine()
+        "spam",
-
+        "0.1.0",
-    if platform_machine == "x86_64":
+        platform="ios",
-        expected_wheels = {
+        python_abi_tags=["cp313-cp313"],
-            f"spam-0.1.0-cp313-cp313-ios_{ios_version}_x86_64_iphonesimulator.whl",
+    )
-        }
+    assert set(actual_wheels) == set(expected_wheels)
-
-    elif platform_machine == "arm64":
-        expected_wheels = {
-            f"spam-0.1.0-cp313-cp313-ios_{ios_version}_arm64_iphoneos.whl",
-            f"spam-0.1.0-cp313-cp313-ios_{ios_version}_arm64_iphonesimulator.whl",
-        }
-
-    assert set(actual_wheels) == expected_wheels
 
     # The user was notified that there was no cross-build tool definition.
     captured = capfd.readouterr()
@@ -225,23 +207,79 @@ def test_empty_xbuild_tool_definition(tmp_path, capfd):
         },
     )
 
-    # The expected wheels were produced.
+    expected_wheels = utils.expected_wheels(
-    ios_version = os.getenv("IPHONEOS_DEPLOYMENT_TARGET", "13.0").replace(".", "_")
+        "spam", "0.1.0", platform="ios", python_abi_tags=["cp313-cp313"]
-    platform_machine = platform.machine()
+    )
-
+    assert set(actual_wheels) == set(expected_wheels)
-    if platform_machine == "x86_64":
-        expected_wheels = {
-            f"spam-0.1.0-cp313-cp313-ios_{ios_version}_x86_64_iphonesimulator.whl",
-        }
-
-    elif platform_machine == "arm64":
-        expected_wheels = {
-            f"spam-0.1.0-cp313-cp313-ios_{ios_version}_arm64_iphoneos.whl",
-            f"spam-0.1.0-cp313-cp313-ios_{ios_version}_arm64_iphonesimulator.whl",
-        }
-
-    assert set(actual_wheels) == expected_wheels
 
     # The warnings about cross-build notifications were silenced.
     captured = capfd.readouterr()
     assert "Your project configuration does not define any cross-build tools." not in captured.err
+
+
+@pytest.mark.serial
+def test_ios_test_command_without_python_dash_m(tmp_path, capfd):
+    """pytest should be able to run without python -m, but it should warn."""
+    if utils.get_platform() != "macos":
+        pytest.skip("this test can only run on macOS")
+    if utils.get_xcode_version() < (13, 0):
+        pytest.skip("this test only works with Xcode 13.0 or greater")
+
+    project_dir = tmp_path / "project"
+
+    project = test_projects.new_c_project()
+    project.files["tests/__init__.py"] = ""
+    project.files["tests/test_spam.py"] = textwrap.dedent("""
+        import spam
+        def test_spam():
+            assert spam.filter("spam") == 0
+            assert spam.filter("ham") != 0
+    """)
+    project.generate(project_dir)
+
+    actual_wheels = utils.cibuildwheel_run(
+        project_dir,
+        add_env={
+            "CIBW_PLATFORM": "ios",
+            "CIBW_BUILD": "cp313-*",
+            "CIBW_TEST_COMMAND": "pytest ./tests",
+            "CIBW_TEST_SOURCES": "tests",
+            "CIBW_TEST_REQUIRES": "pytest",
+            "CIBW_XBUILD_TOOLS": "",
+        },
+    )
+
+    expected_wheels = utils.expected_wheels(
+        "spam", "0.1.0", platform="ios", python_abi_tags=["cp313-cp313"]
+    )
+    assert set(actual_wheels) == set(expected_wheels)
+
+    out, err = capfd.readouterr()
+
+    assert "iOS tests configured with a test command which doesn't start with 'python -m'" in err
+
+
+def test_ios_test_command_invalid(tmp_path, capfd):
+    """Test command should raise an error if it's clearly invalid."""
+    if utils.get_platform() != "macos":
+        pytest.skip("this test can only run on macOS")
+    if utils.get_xcode_version() < (13, 0):
+        pytest.skip("this test only works with Xcode 13.0 or greater")
+
+    project_dir = tmp_path / "project"
+    basic_project = test_projects.new_c_project()
+    basic_project.files["./my_test_script.sh"] = "echo hello"
+    basic_project.generate(project_dir)
+
+    with pytest.raises(subprocess.CalledProcessError):
+        utils.cibuildwheel_run(
+            project_dir,
+            add_env={
+                "CIBW_PLATFORM": "ios",
+                "CIBW_TEST_COMMAND": "./my_test_script.sh",
+                "CIBW_TEST_SOURCES": "./my_test_script.sh",
+                "CIBW_XBUILD_TOOLS": "",
+            },
+        )
+    out, err = capfd.readouterr()
+    assert "iOS tests configured with a test command which doesn't start with 'python -m'" in err

test/utils.py

@@ -167,8 +167,10 @@ def expected_wheels(
     package_version: str,
     manylinux_versions: list[str] | None = None,
     musllinux_versions: list[str] | None = None,
-    macosx_deployment_target: str = "10.9",
+    macosx_deployment_target: str | None = None,
+    iphoneos_deployment_target: str | None = None,
     machine_arch: str | None = None,
+    platform: str | None = None,
     python_abi_tags: list[str] | None = None,
     include_universal2: bool = False,
     single_python: bool = False,
@@ -177,14 +179,22 @@ def expected_wheels(
     """
     Returns the expected wheels from a run of cibuildwheel.
     """
+    platform = platform or get_platform()
+
     if machine_arch is None:
         machine_arch = pm.machine()
-        if get_platform() == "linux":
+        if platform == "linux":
             machine_arch = arch_name_for_linux(machine_arch)
 
+    if macosx_deployment_target is None:
+        macosx_deployment_target = os.environ.get("MACOSX_DEPLOYMENT_TARGET", "10.9")
+
+    if iphoneos_deployment_target is None:
+        iphoneos_deployment_target = os.environ.get("IPHONEOS_DEPLOYMENT_TARGET", "13.0")
+
     architectures = [machine_arch]
     if not single_arch:
-        if get_platform() == "linux":
+        if platform == "linux":
             if machine_arch == "x86_64":
                 architectures.append("i686")
             elif (
@@ -193,22 +203,24 @@ def expected_wheels(
                 and _AARCH64_CAN_RUN_ARMV7
             ):
                 architectures.append("armv7l")
-        elif get_platform() == "windows" and machine_arch == "AMD64":
+        elif platform == "windows" and machine_arch == "AMD64":
             architectures.append("x86")
 
     return [
         wheel
         for architecture in architectures
         for wheel in _expected_wheels(
-            package_name,
+            package_name=package_name,
-            package_version,
+            package_version=package_version,
-            architecture,
+            machine_arch=architecture,
-            manylinux_versions,
+            manylinux_versions=manylinux_versions,
-            musllinux_versions,
+            musllinux_versions=musllinux_versions,
-            macosx_deployment_target,
+            macosx_deployment_target=macosx_deployment_target,
-            python_abi_tags,
+            iphoneos_deployment_target=iphoneos_deployment_target,
-            include_universal2,
+            platform=platform,
-            single_python,
+            python_abi_tags=python_abi_tags,
+            include_universal2=include_universal2,
+            single_python=single_python,
         )
     ]
 
@@ -220,6 +232,8 @@ def _expected_wheels(
     manylinux_versions: list[str] | None,
     musllinux_versions: list[str] | None,
     macosx_deployment_target: str,
+    iphoneos_deployment_target: str,
+    platform: str,
     python_abi_tags: list[str] | None,
     include_universal2: bool,
     single_python: bool,
@@ -227,8 +241,6 @@ def _expected_wheels(
     """
     Returns a list of expected wheels from a run of cibuildwheel.
     """
-    platform = get_platform()
-
     # per PEP 425 (https://www.python.org/dev/peps/pep-0425/), wheel files shall have name of the form
     # {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl
     # {python tag} and {abi tag} are closely related to the python interpreter used to build the wheel
@@ -249,7 +261,9 @@ def _expected_wheels(
 
     if platform == "pyodide" and python_abi_tags is None:
         python_abi_tags = ["cp312-cp312"]
-    if python_abi_tags is None:
+    elif platform == "ios" and python_abi_tags is None:
+        python_abi_tags = ["cp313-cp313"]
+    elif python_abi_tags is None:
         python_abi_tags = [
             "cp38-cp38",
             "cp39-cp39",
@@ -353,6 +367,20 @@ def _expected_wheels(
             if include_universal2:
                 platform_tags.append(f"macosx_{min_macosx.replace('.', '_')}_universal2")
 
+        elif platform == "ios":
+            if machine_arch == "x86_64":
+                platform_tags = [
+                    f"ios_{iphoneos_deployment_target.replace('.', '_')}_x86_64_iphonesimulator"
+                ]
+            elif machine_arch == "arm64":
+                platform_tags = [
+                    f"ios_{iphoneos_deployment_target.replace('.', '_')}_arm64_iphoneos",
+                    f"ios_{iphoneos_deployment_target.replace('.', '_')}_arm64_iphonesimulator",
+                ]
+            else:
+                msg = f"Unsupported architecture {machine_arch!r} for iOS"
+                raise Exception(msg)
+
         elif platform == "pyodide":
             platform_tags = ["pyodide_2024_0_wasm32"]