Add TEST_ENVIRONMENT option and set PYTHONSAFEPATH=1 in the test env (#2388)

* Add TEST_ENVIRONMENT option and set PYTHONSAFEPATH=1 in the test env

Closes https://github.com/pypa/cibuildwheel/issues/2358

* Add toml docs examples

* Add TOML examples

* Use double-quotes for windows

bin/generate_schema.py

@@ -221,6 +221,9 @@ properties:
   test-skip:
     description: Skip running tests on some builds.
     type: string_array
+  test-environment:
+    description: Set environment variables for the test environment
+    type: string_table
 """
 
 schema = yaml.safe_load(starter)
@@ -300,6 +303,7 @@ items:
         test-extras: {"$ref": "#/$defs/inherit"}
         test-sources: {"$ref": "#/$defs/inherit"}
         test-requires: {"$ref": "#/$defs/inherit"}
+        test-environment: {"$ref": "#/$defs/inherit"}
 """
 )
 

cibuildwheel/options.py

@@ -107,6 +107,7 @@ class BuildOptions:
     test_requires: list[str]
     test_extras: str
     test_groups: list[str]
+    test_environment: ParsedEnvironment
     build_verbosity: int
     build_frontend: BuildFrontendConfig | None
     config_settings: str
@@ -739,6 +740,15 @@ class Options:
                     "test-sources", option_format=ListFormat(sep=" ", quote=shlex.quote)
                 )
             )
+            test_environment_config = self.reader.get(
+                "test-environment", option_format=EnvironmentFormat()
+            )
+            try:
+                test_environment = parse_environment(test_environment_config)
+            except (EnvironmentParseError, ValueError) as e:
+                msg = f"Malformed environment option {test_environment_config!r}"
+                raise errors.ConfigurationError(msg) from e
+
             test_requires = self.reader.get(
                 "test-requires", option_format=ListFormat(sep=" ")
             ).split()
@@ -844,6 +854,7 @@ class Options:
                 globals=self.globals,
                 test_command=test_command,
                 test_sources=test_sources,
+                test_environment=test_environment,
                 test_requires=[*test_requires, *test_requirements_from_groups],
                 test_extras=test_extras,
                 test_groups=test_groups,

cibuildwheel/platforms/ios.py

@@ -536,13 +536,17 @@ def build(options: Options, tmp_path: Path) -> None:
                 elif config.arch != os.uname().machine:
                     log.step("Skipping tests on non-native simulator architecture")
                 else:
+                    test_env = build_options.test_environment.as_dictionary(
+                        prev_environment=build_env
+                    )
+
                     if build_options.before_test:
                         before_test_prepared = prepare_command(
                             build_options.before_test,
                             project=".",
                             package=build_options.package_dir,
                         )
-                        shell(before_test_prepared, env=env)
+                        shell(before_test_prepared, env=test_env)
 
                     log.step("Setting up test harness...")
                     # Clone the testbed project into the build directory
@@ -552,7 +556,7 @@ def build(options: Options, tmp_path: Path) -> None:
                         target_install_path / "testbed",
                         "clone",
                         testbed_path,
-                        env=build_env,
+                        env=test_env,
                     )
 
                     if not build_options.test_sources:
@@ -574,7 +578,7 @@ def build(options: Options, tmp_path: Path) -> None:
                     # the test requirements. Use the --platform tag to force
                     # the installation of iOS wheels; this requires the use of
                     # --only-binary=:all:
-                    ios_version = build_env["IPHONEOS_DEPLOYMENT_TARGET"]
+                    ios_version = test_env["IPHONEOS_DEPLOYMENT_TARGET"]
                     platform_tag = f"ios_{ios_version.replace('.', '_')}_{config.arch}_{config.sdk}"
 
                     call(
@@ -589,7 +593,7 @@ def build(options: Options, tmp_path: Path) -> None:
                         testbed_path / "iOSTestbed" / "app_packages",
                         f"{test_wheel}{build_options.test_extras}",
                         *build_options.test_requires,
-                        env=build_env,
+                        env=test_env,
                     )
 
                     log.step("Running test suite...")
@@ -638,7 +642,7 @@ def build(options: Options, tmp_path: Path) -> None:
                             *(["--verbose"] if build_options.build_verbosity > 0 else []),
                             "--",
                             *test_command_parts,
-                            env=build_env,
+                            env=test_env,
                         )
                         failed = False
                     except subprocess.CalledProcessError:

cibuildwheel/platforms/linux.py

@@ -361,6 +361,10 @@ def build_in_container(
             virtualenv_env = env.copy()
             virtualenv_env["PATH"] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}"
             virtualenv_env["VIRTUAL_ENV"] = str(venv_dir)
+            virtualenv_env["PYTHONSAFEPATH"] = "1"
+            virtualenv_env = build_options.test_environment.as_dictionary(
+                prev_environment=virtualenv_env
+            )
 
             if build_options.before_test:
                 before_test_prepared = prepare_command(

cibuildwheel/platforms/macos.py

@@ -648,6 +648,13 @@ def build(options: Options, tmp_path: Path) -> None:
 
                     virtualenv_env["MACOSX_DEPLOYMENT_TARGET"] = get_test_macosx_deployment_target()
 
+                    # see https://github.com/pypa/cibuildwheel/issues/2358 for discussion
+                    virtualenv_env["PYTHONSAFEPATH"] = "1"
+
+                    virtualenv_env = build_options.test_environment.as_dictionary(
+                        prev_environment=virtualenv_env
+                    )
+
                     # check that we are using the Python from the virtual environment
                     call_with_arch("which", "python", env=virtualenv_env)
 

cibuildwheel/platforms/pyodide.py

@@ -380,6 +380,12 @@ def build(options: Options, tmp_path: Path) -> None:
                 )
                 virtualenv_env["VIRTUAL_ENV"] = str(venv_dir)
 
+                virtualenv_env["PYTHONSAFEPATH"] = "1"
+
+                virtualenv_env = build_options.test_environment.as_dictionary(
+                    prev_environment=virtualenv_env
+                )
+
                 # check that we are using the Python from the virtual environment
                 call("which", "python", env=virtualenv_env)
 

cibuildwheel/platforms/windows.py

@@ -558,6 +558,13 @@ def build(options: Options, tmp_path: Path) -> None:
                     pip_version=pip_version,
                 )
 
+                # see https://github.com/pypa/cibuildwheel/issues/2358 for discussion
+                virtualenv_env["PYTHONSAFEPATH"] = "1"
+
+                virtualenv_env = build_options.test_environment.as_dictionary(
+                    prev_environment=virtualenv_env
+                )
+
                 # check that we are using the Python from the virtual environment
                 call("where", "python", env=virtualenv_env)
 

cibuildwheel/resources/cibuildwheel.schema.json

@@ -544,6 +544,24 @@
       ],
       "title": "CIBW_TEST_SKIP"
     },
+    "test-environment": {
+      "description": "Set environment variables for the test environment",
+      "oneOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "object",
+          "additionalProperties": false,
+          "patternProperties": {
+            ".+": {
+              "type": "string"
+            }
+          }
+        }
+      ],
+      "title": "CIBW_TEST_ENVIRONMENT"
+    },
     "overrides": {
       "type": "array",
       "description": "An overrides array",
@@ -610,6 +628,9 @@
               },
               "test-requires": {
                 "$ref": "#/$defs/inherit"
+              },
+              "test-environment": {
+                "$ref": "#/$defs/inherit"
               }
             }
           },
@@ -714,6 +735,9 @@
           },
           "test-requires": {
             "$ref": "#/properties/test-requires"
+          },
+          "test-environment": {
+            "$ref": "#/properties/test-environment"
           }
         }
       }
@@ -836,6 +860,9 @@
         },
         "test-requires": {
           "$ref": "#/properties/test-requires"
+        },
+        "test-environment": {
+          "$ref": "#/properties/test-environment"
         }
       }
     },
@@ -890,6 +917,9 @@
         },
         "test-requires": {
           "$ref": "#/properties/test-requires"
+        },
+        "test-environment": {
+          "$ref": "#/properties/test-environment"
         }
       }
     },
@@ -957,6 +987,9 @@
         },
         "test-requires": {
           "$ref": "#/properties/test-requires"
+        },
+        "test-environment": {
+          "$ref": "#/properties/test-environment"
         }
       }
     },
@@ -1011,6 +1044,9 @@
         },
         "test-requires": {
           "$ref": "#/properties/test-requires"
+        },
+        "test-environment": {
+          "$ref": "#/properties/test-environment"
         }
       }
     },
@@ -1065,6 +1101,9 @@
         },
         "test-requires": {
           "$ref": "#/properties/test-requires"
+        },
+        "test-environment": {
+          "$ref": "#/properties/test-environment"
         }
       }
     }

cibuildwheel/resources/defaults.toml

@@ -24,6 +24,7 @@ test-sources = []
 test-requires = []
 test-extras = []
 test-groups = []
+test-environment = {}
 
 container-engine = "docker"
 

docs/options.md

@@ -1539,6 +1539,42 @@ This option is not supported in the overrides section in `pyproject.toml`.
     test-skip = "*-macosx_arm64 *-macosx_universal2:arm64"
     ```
 
+### `CIBW_TEST_ENVIRONMENT` {: #test-environment}
+
+> Set environment variables for the test environment
+
+A space-separated list of environment variables to set in the test environment.
+
+The syntax is the same as for [`CIBW_ENVIRONMENT`](#environment).
+
+cibuildwheel sets the variable [`PYTHONSAFEPATH`](https://docs.python.org/3/using/cmdline.html#envvar-PYTHONSAFEPATH) to avoid picking up in-tree dependencies when running the tests - we want to test the installed wheel, not the in-tree version. However, this can be overridden by setting `PYTHONSAFEPATH` to an empty string.
+
+Platform-specific environment variables are also available:<br/>
+`CIBW_TEST_ENVIRONMENT_MACOS` | `CIBW_TEST_ENVIRONMENT_WINDOWS` | `CIBW_TEST_ENVIRONMENT_LINUX` | `CIBW_TEST_ENVIRONMENT_IOS` | `CIBW_TEST_ENVIRONMENT_PYODIDE`
+
+#### Examples
+
+!!! tab examples "Environment variables"
+
+    ```yaml
+    # Set the environment variable MY_ENV_VAR to "my_value" in the test environment
+    CIBW_TEST_ENVIRONMENT: MY_ENV_VAR=my_value
+
+    # Unset PYTHONSAFEPATH in the test environment
+    CIBW_TEST_ENVIRONMENT: PYTHONSAFEPATH=
+    ```
+
+!!! tab examples "pyproject.toml"
+
+    ```toml
+    [tool.cibuildwheel]
+    # Set the environment variable MY_ENV_VAR to "my_value" in the test environment
+    test-environment = { MY_ENV_VAR="my_value" }
+
+    # Unset PYTHONSAFEPATH in the test environment
+    test-environment = { PYTHONSAFEPATH="" }
+    ```
+
 ## Debugging
 
 ### `CIBW_DEBUG_KEEP_CONTAINER`

test/test_testing.py

@@ -66,6 +66,12 @@ class TestSpam(TestCase):
         bits = struct.calcsize("P") * 8
         if bits == 32:
             self.assertIn(platform.machine(), ["i686", "armv7l","armv8l", "wasm32"])
+
+    def test_pythonsafepath(self):
+        # check that the PYTHONSAFEPATH environment variable is set
+        value = os.environ.get("PYTHONSAFEPATH")
+        self.assertIsNotNone(value)
+        self.assertNotEqual(value, "")
 '''
 
 
@@ -229,3 +235,19 @@ def test_test_sources(tmp_path):
     # also check that we got the right wheels
     expected_wheels = utils.expected_wheels("spam", "0.1.0")
     assert set(actual_wheels) == set(expected_wheels)
+
+
+def test_test_environment(tmp_path):
+    project_dir = tmp_path / "project"
+    test_projects.new_c_project().generate(project_dir)
+
+    actual_wheels = utils.cibuildwheel_run(
+        project_dir,
+        add_env={
+            "CIBW_TEST_ENVIRONMENT": "MYVAR=somevalue PYTHONSAFEPATH=",
+            "CIBW_TEST_COMMAND": "python -c \"import os; assert os.environ.get('MYVAR') == 'somevalue'; assert os.environ.get('PYTHONSAFEPATH') == ''\"",
+        },
+    )
+    # also check that we got the right wheels
+    expected_wheels = utils.expected_wheels("spam", "0.1.0")
+    assert set(actual_wheels) == set(expected_wheels)