Merge pull request #1336 from pypa/test-cwd-error-message

Add test failure with descriptive error message when running tests without a {project} placeholder

cibuildwheel/linux.py

@@ -21,6 +21,7 @@ from .util import (
     prepare_command,
     read_python_configs,
     split_config_settings,
+    test_fail_cwd_file,
     unwrap,
 )
 
@@ -306,9 +307,11 @@ def build_in_container(
             # set up a virtual environment to install and test from, to make sure
             # there are no dependencies that were pulled in at build time.
             container.call(["pip", "install", "virtualenv", *dependency_constraint_flags], env=env)
-            venv_dir = (
+
-                PurePath(container.call(["mktemp", "-d"], capture_output=True).strip()) / "venv"
+            testing_temp_dir = PurePosixPath(
+                container.call(["mktemp", "-d"], capture_output=True).strip()
             )
+            venv_dir = testing_temp_dir / "venv"
 
             container.call(["python", "-m", "virtualenv", "--no-download", venv_dir], env=env)
 
@@ -345,10 +348,14 @@ def build_in_container(
                 project=container_project_path,
                 package=container_package_dir,
             )
-            container.call(["sh", "-c", test_command_prepared], cwd="/root", env=virtualenv_env)
+            test_cwd = testing_temp_dir / "test_cwd"
+            container.call(["mkdir", "-p", test_cwd])
+            container.copy_into(test_fail_cwd_file, test_cwd / "test_fail.py")
+
+            container.call(["sh", "-c", test_command_prepared], cwd=test_cwd, env=virtualenv_env)
 
             # clean up test environment
-            container.call(["rm", "-rf", venv_dir])
+            container.call(["rm", "-rf", testing_temp_dir])
 
         # move repaired wheels to output
         if compatible_wheel is None:

cibuildwheel/macos.py

@@ -36,6 +36,7 @@ from .util import (
     read_python_configs,
     shell,
     split_config_settings,
+    test_fail_cwd_file,
     unwrap,
     virtualenv,
 )
@@ -563,7 +564,7 @@ def build(options: Options, tmp_path: Path) -> None:
                             "pip", "install", *build_options.test_requires, env=virtualenv_env
                         )
 
-                    # run the tests from $HOME, with an absolute path in the command
+                    # run the tests from a temp dir, with an absolute path in the command
                     # (this ensures that Python runs the tests against the installed wheel
                     # and not the repo code)
                     test_command_prepared = prepare_command(
@@ -571,9 +572,12 @@ def build(options: Options, tmp_path: Path) -> None:
                         project=Path(".").resolve(),
                         package=build_options.package_dir.resolve(),
                     )
-                    shell_with_arch(
+
-                        test_command_prepared, cwd=os.environ["HOME"], env=virtualenv_env
+                    test_cwd = identifier_tmp_dir / "test_cwd"
-                    )
+                    test_cwd.mkdir(exist_ok=True)
+                    (test_cwd / "test_fail.py").write_text(test_fail_cwd_file.read_text())
+
+                    shell_with_arch(test_command_prepared, cwd=test_cwd, env=virtualenv_env)
 
             # we're all done here; move it to output (overwrite existing)
             if compatible_wheel is None:

cibuildwheel/resources/testing_temp_dir_file.py

@@ -0,0 +1,17 @@
+# this file is copied to the testing cwd, to raise the below error message if
+# pytest/unittest is run from there.
+
+import unittest
+
+
+class TestStringMethods(unittest.TestCase):
+    def test_fail(self):
+        self.fail(
+            "cibuildwheel executes tests from a different working directory to "
+            "your project. This ensures only your wheel is imported, preventing "
+            "Python from accessing files that haven't been packaged into the "
+            "wheel. Please specify a path to your tests when invoking pytest "
+            "using the {project} placeholder, e.g. `pytest {project}` or "
+            "`pytest {project}/tests`. cibuildwheel will replace {project} with "
+            "the path to your project."
+        )

cibuildwheel/util.py

@@ -67,6 +67,8 @@ resources_dir: Final[Path] = Path(__file__).parent / "resources"
 
 install_certifi_script: Final[Path] = resources_dir / "install_certifi.py"
 
+test_fail_cwd_file: Final[Path] = resources_dir / "testing_temp_dir_file.py"
+
 BuildFrontend = Literal["pip", "build"]
 
 MANYLINUX_ARCHS: Final[tuple[str, ...]] = (

cibuildwheel/windows.py

@@ -35,6 +35,7 @@ from .util import (
     read_python_configs,
     shell,
     split_config_settings,
+    test_fail_cwd_file,
     unwrap,
     virtualenv,
 )
@@ -542,7 +543,7 @@ def build(options: Options, tmp_path: Path) -> None:
                 if build_options.test_requires:
                     call("pip", "install", *build_options.test_requires, env=virtualenv_env)
 
-                # run the tests from c:\, with an absolute path in the command
+                # run the tests from a temp dir, with an absolute path in the command
                 # (this ensures that Python runs the tests against the installed wheel
                 # and not the repo code)
                 test_command_prepared = prepare_command(
@@ -550,7 +551,11 @@ def build(options: Options, tmp_path: Path) -> None:
                     project=Path(".").resolve(),
                     package=options.globals.package_dir.resolve(),
                 )
-                shell(test_command_prepared, cwd="c:\\", env=virtualenv_env)
+                test_cwd = identifier_tmp_dir / "test_cwd"
+                test_cwd.mkdir()
+                (test_cwd / "test_fail.py").write_text(test_fail_cwd_file.read_text())
+
+                shell(test_command_prepared, cwd=test_cwd, env=virtualenv_env)
 
             # we're all done here; move it to output (remove if already exists)
             if compatible_wheel is None:

test/test_testing.py

@@ -3,6 +3,7 @@ from __future__ import annotations
 import os
 import subprocess
 import textwrap
+from pathlib import Path
 
 import pytest
 
@@ -151,3 +152,37 @@ def test_failing_test(tmp_path):
         )
 
     assert len(os.listdir(output_dir)) == 0
+
+
+@pytest.mark.parametrize("test_runner", ["pytest", "unittest"])
+def test_bare_pytest_invocation(
+    tmp_path: Path, capfd: pytest.CaptureFixture[str], test_runner: str
+):
+    """Check that if a user runs pytest in the the test cwd, it raises a helpful error"""
+    project_dir = tmp_path / "project"
+    output_dir = tmp_path / "output"
+    project_with_a_test.generate(project_dir)
+
+    with pytest.raises(subprocess.CalledProcessError):
+        utils.cibuildwheel_run(
+            project_dir,
+            output_dir=output_dir,
+            add_env={
+                "CIBW_TEST_REQUIRES": "pytest" if test_runner == "pytest" else "",
+                "CIBW_TEST_COMMAND": (
+                    "python -m pytest" if test_runner == "pytest" else "python -m unittest"
+                ),
+                # Skip CPython 3.8 on macOS arm64, see comment above in
+                # 'test_failing_test'
+                "CIBW_SKIP": "cp38-macosx_arm64",
+            },
+        )
+
+    assert len(os.listdir(output_dir)) == 0
+
+    captured = capfd.readouterr()
+
+    assert (
+        "Please specify a path to your tests when invoking pytest using the {project} placeholder"
+        in captured.out + captured.err
+    )