feat: CIBW_TEST_SKIP (#537)

* feat: CIBW_TEST_SKIP

* refactor: adjust a bit and add docs

* fix: typo in name

* docs: Update docs/options.md

Co-authored-by: Joe Rickerby <joerick@mac.com>

* docs: add README line from markdown

Co-authored-by: Joe Rickerby <joerick@mac.com>

README.md

@@ -112,6 +112,7 @@ Options
 |   | [`CIBW_BEFORE_TEST`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-test)  | Execute a shell command before testing each wheel |
 |   | [`CIBW_TEST_REQUIRES`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-requires)  | Install Python dependencies before running the tests |
 |   | [`CIBW_TEST_EXTRAS`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-extras)  | Install your wheel for testing using extras_require |
+|   | [`CIBW_TEST_SKIP`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-skip)  | Skip running tests on some builds |
 | **Other** | [`CIBW_BUILD_VERBOSITY`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-verbosity)  | Increase/decrease the output of pip wheel |
 
 

cibuildwheel/__main__.py

@@ -18,6 +18,7 @@ from cibuildwheel.util import (
     BuildOptions,
     BuildSelector,
     DependencyConstraints,
+    TestSelector,
     Unbuffered,
     detect_ci_provider,
     resources_dir,
@@ -140,6 +141,7 @@ def main() -> None:
         assert_never(platform)
 
     build_config, skip_config = os.environ.get('CIBW_BUILD', '*'), os.environ.get('CIBW_SKIP', '')
+    test_skip = os.environ.get('CIBW_TEST_SKIP', '')
     environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform, default='')
     before_all = get_option_from_environment('CIBW_BEFORE_ALL', platform=platform, default='')
     before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform)
@@ -151,7 +153,8 @@ def main() -> None:
     test_extras = get_option_from_environment('CIBW_TEST_EXTRAS', platform=platform, default='')
     build_verbosity_str = get_option_from_environment('CIBW_BUILD_VERBOSITY', platform=platform, default='')
 
-    build_selector = BuildSelector(build_config, skip_config)
+    build_selector = BuildSelector(build_config=build_config, skip_config=skip_config)
+    test_selector = TestSelector(skip_config=test_skip)
 
     try:
         environment = parse_environment(environment_config)
@@ -236,6 +239,7 @@ def main() -> None:
         before_all=before_all,
         build_verbosity=build_verbosity,
         build_selector=build_selector,
+        test_selector=test_selector,
         repair_command=repair_command,
         environment=environment,
         dependency_constraints=dependency_constraints,

cibuildwheel/linux.py

@@ -183,7 +183,7 @@ def build(options: BuildOptions) -> None:
 
                     repaired_wheels = docker.glob(repaired_wheel_dir, '*.whl')
 
-                    if options.test_command:
+                    if options.test_command and options.test_selector(config.identifier):
                         log.step('Testing wheel...')
 
                         # set up a virtual environment to install and test from, to make sure

cibuildwheel/macos.py

@@ -252,7 +252,7 @@ def build(options: BuildOptions) -> None:
 
             repaired_wheel = next(repaired_wheel_dir.glob('*.whl'))
 
-            if options.test_command:
+            if options.test_command and options.test_selector(config.identifier):
                 log.step('Testing wheel...')
                 # set up a virtual environment to install and test from, to make sure
                 # there are no dependencies that were pulled in at build time.

cibuildwheel/util.py

@@ -57,8 +57,13 @@ def read_python_configs(config: PlatformName) -> List[Dict[str, str]]:
     return results
 
 
-class BuildSelector:
+class IdentifierSelector:
-    def __init__(self, build_config: str, skip_config: str):
+    """
+    This class holds a set of build/skip patterns. You call an instance with a
+    build identifier, and it returns True if that identifier should be
+    included.
+    """
+    def __init__(self, *, build_config: str, skip_config: str):
         self.build_patterns = build_config.split()
         self.skip_patterns = skip_config.split()
 
@@ -72,9 +77,18 @@ class BuildSelector:
 
     def __repr__(self) -> str:
         if not self.skip_patterns:
-            return f'BuildSelector({" ".join(self.build_patterns)!r})'
+            return f'{self.__class__.__name__}({" ".join(self.build_patterns)!r})'
         else:
-            return f'BuildSelector({" ".join(self.build_patterns)!r} - {" ".join(self.skip_patterns)!r})'
+            return f'{self.__class__.__name__}({" ".join(self.build_patterns)!r} - {" ".join(self.skip_patterns)!r})'
+
+
+class BuildSelector(IdentifierSelector):
+    pass
+
+
+class TestSelector(IdentifierSelector):
+    def __init__(self, *, skip_config: str):
+        super().__init__(build_config="*", skip_config=skip_config)
 
 
 # Taken from https://stackoverflow.com/a/107717
@@ -217,6 +231,7 @@ class BuildOptions(NamedTuple):
     manylinux_images: Optional[Dict[str, str]]
     dependency_constraints: Optional[DependencyConstraints]
     test_command: Optional[str]
+    test_selector: TestSelector
     before_test: Optional[str]
     test_requires: List[str]
     test_extras: str

cibuildwheel/windows.py

@@ -277,7 +277,7 @@ def build(options: BuildOptions) -> None:
 
             repaired_wheel = next(repaired_wheel_dir.glob('*.whl'))
 
-            if options.test_command:
+            if options.test_command and options.test_selector(config.identifier):
                 log.step('Testing wheel...')
                 # set up a virtual environment to install and test from, to make sure
                 # there are no dependencies that were pulled in at build time.

docs/options.md

@@ -518,6 +518,18 @@ Platform-specific variants also available:<br/>
 CIBW_TEST_EXTRAS: test,qt
 ```
 
+### `CIBW_TEST_SKIP` {: #test-skip}
+> Skip running tests on some builds
+
+This will skip testing on any identifiers that match the given skip patterns (see [`CIBW_SKIP`](#build-skip)). This can be used to mask out tests for wheels that have missing dependencies upstream that are slow or hard to build, or to mask up slow tests on emulated architectures.
+
+#### Examples
+
+```yaml
+# Will avoid testing on emulated architectures
+CIBW_TEST_SKIP: "*-manylinux_{aarch64,ppc64le,s390x}"
+```
+
 
 ## Other