Update Android test-command handling (#2590)

* Update Android test-command handling

* Update to Python 3.13.8

* Update documentation and tests

* Deal with `sysconfig.get_config_var("exec_prefix")` changing in Python 3.14, and add cross venv tests

* Allow test commands starting with `python3`

* test-command cleanups

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

test/test_android.py

@@ -32,22 +32,25 @@ if (platform.system(), platform.machine()) not in [
         allow_module_level=True,
     )
 
-# Detect CI services which have the Android SDK pre-installed.
-ci_supports_build = (
-    ("CIRRUS_CI" in os.environ and platform.system() == "Darwin")
-    or "GITHUB_ACTIONS" in os.environ
-    or "TF_BUILD" in os.environ  # Azure Pipelines
-)
+# Azure Pipelines does not set the CI variable.
+ci = any(key in os.environ for key in ["CI", "TF_BUILD"])
 
 if "ANDROID_HOME" not in os.environ:
     msg = "ANDROID_HOME environment variable is not set"
-    if ci_supports_build:
+
+    # Fail if we're on a CI service which is supposed to have the Android SDK
+    # pre-installed; otherwise skip the module.
+    if (
+        ("CIRRUS_CI" in os.environ and platform.system() == "Darwin")
+        or "GITHUB_ACTIONS" in os.environ
+        or "TF_BUILD" in os.environ
+    ):
         pytest.fail(msg)
     else:
         pytest.skip(msg, allow_module_level=True)
 
 # Many CI services don't support running the Android emulator: see platforms.md.
-ci_supports_emulator = "GITHUB_ACTIONS" in os.environ and platform.system() == "Linux"
+supports_emulator = (not ci) or ("GITHUB_ACTIONS" in os.environ and platform.system() == "Linux")
 
 
 def needs_emulator(test):
@@ -55,7 +58,7 @@ def needs_emulator(test):
     # application ID, so these tests must be run serially.
     test = pytest.mark.serial(test)
 
-    if ci_supports_build and not ci_supports_emulator:
+    if not supports_emulator:
         test = pytest.mark.skip("This CI platform doesn't support the emulator")(test)
     return test
 
@@ -92,12 +95,24 @@ def test_android_home(tmp_path, capfd):
     assert "ANDROID_HOME environment variable is not set" in capfd.readouterr().err
 
 
-# the first build can fail to setup - mark as flaky, and serial to make sure it runs first
+# android-env.sh may need to install the NDK, and it isn't safe to do that multiple
+# times in parallel. So make sure there's at least one test which gets as far as doing
+# a build, which is marked as serial so it will run before the parallel tests, but isn't
+# marked as needs_emulator so it will run on all CI platforms.
 @pytest.mark.serial
-@pytest.mark.flaky(reruns=2)
-def test_expected_wheels(tmp_path):
-    new_c_project().generate(tmp_path)
-    wheels = cibuildwheel_run(tmp_path, add_env={"CIBW_PLATFORM": "android"})
+def test_expected_wheels(tmp_path, spam_env):
+    # Since this test covers all Python versions, check the cross venv.
+    test_module = "_cross_venv_test_android"
+    project = new_c_project(setup_py_add=f"import {test_module}")
+    project.files[f"{test_module}.py"] = (Path(__file__).parent / f"{test_module}.py").read_text()
+    project.generate(tmp_path)
+
+    # Build wheels for all Python versions on the current architecture.
+    del spam_env["CIBW_BUILD"]
+    if not supports_emulator:
+        del spam_env["CIBW_TEST_COMMAND"]
+
+    wheels = cibuildwheel_run(tmp_path, add_env=spam_env)
     assert wheels == expected_wheels(
         "spam", "0.1.0", platform="android", machine_arch=native_arch.android_abi
     )
@@ -222,12 +237,20 @@ def spam_env(tmp_path):
             print("Spam test passed")
         """
     )
+    project.files["test_empty.py"] = dedent(
+        """\
+        def test_empty():
+            pass
+        """
+    )
+
     project.generate(tmp_path)
 
     return {
         **cp313_env,
-        "CIBW_TEST_SOURCES": "test_spam.py",
+        "CIBW_TEST_SOURCES": "test_spam.py test_empty.py",
         "CIBW_TEST_REQUIRES": "pytest==8.3.5",
+        "CIBW_TEST_COMMAND": "python -m pytest",
     }
 
 
@@ -235,7 +258,8 @@ def spam_env(tmp_path):
 @pytest.mark.parametrize(
     ("command", "expected_output"),
     [
-        ("python -c 'import test_spam; test_spam.test_spam()'", "Spam test passed"),
+        ("python3 -c 'import test_spam; test_spam.test_spam()'", "Spam test passed"),
+        ("python -m pytest", "=== 2 passed in "),
         ("python -m pytest test_spam.py", "=== 1 passed in "),
         ("pytest test_spam.py", "=== 1 passed in "),
     ],
@@ -252,27 +276,25 @@ def test_test_command_good(command, expected_output, tmp_path, spam_env, capfd):
         ) in stderr
 
 
+BAD_FORMAT_ERROR = (
+    "Test command '{}' is not supported on Android. "
+    "Command must begin with 'python' or 'python3', and contain '-m' or '-c'."
+)
+BAD_PLACEHOLDER_ERROR = (
+    "Test command '{}' with a '{{project}}' or '{{package}}' placeholder "
+    "is not supported on Android"
+)
+
+
 @needs_emulator
 @pytest.mark.parametrize(
     ("command", "expected_output"),
     [
-        # Build-time failure: unrecognized command
-        (
-            "./test_spam.py",
-            "Test command './test_spam.py' is not supported on Android. "
-            "Supported commands are 'python -m' and 'python -c'.",
-        ),
-        # Build-time failure: unrecognized placeholder
-        (
-            "pytest {project}",
-            "Test command 'pytest {project}' with a '{project}' or '{package}' "
-            "placeholder is not supported on Android",
-        ),
-        (
-            "pytest {package}",
-            "Test command 'pytest {package}' with a '{project}' or '{package}' "
-            "placeholder is not supported on Android",
-        ),
+        # Build-time failure
+        ("./test_spam.py", BAD_FORMAT_ERROR.format("./test_spam.py")),
+        ("python test_spam.py", BAD_FORMAT_ERROR.format("python test_spam.py")),
+        ("pytest {project}", BAD_PLACEHOLDER_ERROR.format("pytest {project}")),
+        ("pytest {package}", BAD_PLACEHOLDER_ERROR.format("pytest {package}")),
         # Runtime failure
         ("pytest test_ham.py", "not found: test_ham.py"),
     ],
@@ -283,6 +305,29 @@ def test_test_command_bad(command, expected_output, tmp_path, spam_env, capfd):
     assert expected_output in capfd.readouterr().err
 
 
+@needs_emulator
+@pytest.mark.parametrize(
+    ("options", "expected"),
+    [
+        ("", 0),
+        ("-E", 1),
+    ],
+)
+def test_test_command_python_options(options, expected, tmp_path, capfd):
+    project = new_c_project()
+    project.generate(tmp_path)
+
+    command = 'import sys; print(f"{sys.flags.ignore_environment=}")'
+    cibuildwheel_run(
+        tmp_path,
+        add_env={
+            **cp313_env,
+            "CIBW_TEST_COMMAND": f"python {options} -c '{command}'",
+        },
+    )
+    assert f"sys.flags.ignore_environment={expected}" in capfd.readouterr().out
+
+
 @needs_emulator
 def test_package_subdir(tmp_path, spam_env, capfd):
     spam_paths = list(tmp_path.iterdir())
@@ -291,17 +336,11 @@ def test_package_subdir(tmp_path, spam_env, capfd):
     for path in spam_paths:
         path.rename(package_dir / path.name)
 
-    test_filename = "package/" + spam_env["CIBW_TEST_SOURCES"]
-    cibuildwheel_run(
-        tmp_path,
-        package_dir,
-        add_env={
-            **spam_env,
-            "CIBW_TEST_SOURCES": test_filename,
-            "CIBW_TEST_COMMAND": f"python -m pytest {test_filename}",
-        },
+    spam_env["CIBW_TEST_SOURCES"] = " ".join(
+        f"package/{path}" for path in spam_env["CIBW_TEST_SOURCES"].split()
     )
-    assert "=== 1 passed in " in capfd.readouterr().out
+    cibuildwheel_run(tmp_path, package_dir, add_env=spam_env)
+    assert "=== 2 passed in " in capfd.readouterr().out
 
 
 @needs_emulator