feat: support test-groups (#2063)

* feat: support test-groups

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* refactor: address review comments

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* tests: add a integration test

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* Apply suggestions from code review

Co-authored-by: Matthieu Darbois <mayeut@users.noreply.github.com>

* fix: better error messages based on feedback

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

---------

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>
Co-authored-by: Matthieu Darbois <mayeut@users.noreply.github.com>

.pre-commit-config.yaml

@@ -29,6 +29,7 @@ repos:
     args: ["--python-version=3.8"]
     additional_dependencies: &mypy-dependencies
       - bracex
+      - dependency-groups>=1.2
       - nox
       - orjson
       - packaging

bin/generate_schema.py

@@ -164,6 +164,9 @@ properties:
   test-extras:
     description: Install your wheel for testing using `extras_require`
     type: string_array
+  test-groups:
+    description: Install extra groups when testing
+    type: string_array
   test-requires:
     description: Install Python dependencies before running the tests
     type: string_array

cibuildwheel/options.py

@@ -22,7 +22,7 @@ from .architecture import Architecture
 from .environment import EnvironmentParseError, ParsedEnvironment, parse_environment
 from .logger import log
 from .oci_container import OCIContainerEngineConfig
-from .projectfiles import get_requires_python_str
+from .projectfiles import get_requires_python_str, resolve_dependency_groups
 from .typing import PLATFORMS, PlatformName
 from .util import (
     MANYLINUX_ARCHS,
@@ -92,6 +92,7 @@ class BuildOptions:
     before_test: str | None
     test_requires: list[str]
     test_extras: str
+    test_groups: list[str]
     build_verbosity: int
     build_frontend: BuildFrontendConfig | None
     config_settings: str
@@ -550,6 +551,8 @@ class OptionsReader:
 
 
 class Options:
+    pyproject_toml: dict[str, Any] | None
+
     def __init__(
         self,
         platform: PlatformName,
@@ -568,6 +571,13 @@ class Options:
             disallow=DISALLOWED_OPTIONS,
         )
 
+        self.package_dir = Path(command_line_arguments.package_dir)
+        try:
+            with self.package_dir.joinpath("pyproject.toml").open("rb") as f:
+                self.pyproject_toml = tomllib.load(f)
+        except FileNotFoundError:
+            self.pyproject_toml = None
+
     @property
     def config_file_path(self) -> Path | None:
         args = self.command_line_arguments
@@ -584,8 +594,7 @@ class Options:
 
     @functools.cached_property
     def package_requires_python_str(self) -> str | None:
-        args = self.command_line_arguments
+        return get_requires_python_str(self.package_dir, self.pyproject_toml)
-        return get_requires_python_str(Path(args.package_dir))
 
     @property
     def globals(self) -> GlobalOptions:
@@ -672,6 +681,11 @@ class Options:
                 "test-requires", option_format=ListFormat(sep=" ")
             ).split()
             test_extras = self.reader.get("test-extras", option_format=ListFormat(sep=","))
+            test_groups_str = self.reader.get("test-groups", option_format=ListFormat(sep=" "))
+            test_groups = [x for x in test_groups_str.split() if x]
+            test_requirements_from_groups = resolve_dependency_groups(
+                self.pyproject_toml, *test_groups
+            )
             build_verbosity_str = self.reader.get("build-verbosity")
 
             build_frontend_str = self.reader.get(
@@ -771,8 +785,9 @@ class Options:
             return BuildOptions(
                 globals=self.globals,
                 test_command=test_command,
-                test_requires=test_requires,
+                test_requires=[*test_requires, *test_requirements_from_groups],
                 test_extras=test_extras,
+                test_groups=test_groups,
                 before_test=before_test,
                 before_build=before_build,
                 before_all=before_all,

cibuildwheel/projectfiles.py

@@ -4,8 +4,9 @@ import ast
 import configparser
 import contextlib
 from pathlib import Path
+from typing import Any
 
-from ._compat import tomllib
+import dependency_groups
 
 
 def get_parent(node: ast.AST | None, depth: int = 1) -> ast.AST | None:
@@ -84,15 +85,12 @@ def setup_py_python_requires(content: str) -> str | None:
         return None
 
 
-def get_requires_python_str(package_dir: Path) -> str | None:
+def get_requires_python_str(package_dir: Path, pyproject_toml: dict[str, Any] | None) -> str | None:
     """Return the python requires string from the most canonical source available, or None"""
 
     # Read in from pyproject.toml:project.requires-python
-    with contextlib.suppress(FileNotFoundError):
+    with contextlib.suppress(KeyError, IndexError, TypeError):
-        with (package_dir / "pyproject.toml").open("rb") as f1:
+        return str((pyproject_toml or {})["project"]["requires-python"])
-            info = tomllib.load(f1)
-        with contextlib.suppress(KeyError, IndexError, TypeError):
-            return str(info["project"]["requires-python"])
 
     # Read in from setup.cfg:options.python_requires
     config = configparser.ConfigParser()
@@ -106,3 +104,26 @@ def get_requires_python_str(package_dir: Path) -> str | None:
         return setup_py_python_requires(f2.read())
 
     return None
+
+
+def resolve_dependency_groups(
+    pyproject_toml: dict[str, Any] | None, *groups: str
+) -> tuple[str, ...]:
+    """
+    Get the packages in dependency-groups for a package.
+    """
+
+    if not groups:
+        return ()
+
+    if pyproject_toml is None:
+        msg = f"Didn't find a pyproject.toml, so can't read [dependency-groups] {groups!r} from it!"
+        raise FileNotFoundError(msg)
+
+    try:
+        dependency_groups_toml = pyproject_toml["dependency-groups"]
+    except KeyError:
+        msg = f"Didn't find [dependency-groups] in pyproject.toml, which is needed to resolve {groups!r}."
+        raise KeyError(msg) from None
+
+    return dependency_groups.resolve(dependency_groups_toml, *groups)

cibuildwheel/resources/cibuildwheel.schema.json

@@ -397,6 +397,21 @@
       ],
       "title": "CIBW_TEST_EXTRAS"
     },
+    "test-groups": {
+      "description": "Install extra groups when testing",
+      "oneOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "array",
+          "items": {
+            "type": "string"
+          }
+        }
+      ],
+      "title": "CIBW_TEST_GROUPS"
+    },
     "test-requires": {
       "description": "Install Python dependencies before running the tests",
       "oneOf": [
@@ -571,6 +586,9 @@
           "test-extras": {
             "$ref": "#/properties/test-extras"
           },
+          "test-groups": {
+            "$ref": "#/properties/test-groups"
+          },
           "test-requires": {
             "$ref": "#/properties/test-requires"
           }
@@ -675,6 +693,9 @@
         "test-extras": {
           "$ref": "#/properties/test-extras"
         },
+        "test-groups": {
+          "$ref": "#/properties/test-groups"
+        },
         "test-requires": {
           "$ref": "#/properties/test-requires"
         }
@@ -720,6 +741,9 @@
         "test-extras": {
           "$ref": "#/properties/test-extras"
         },
+        "test-groups": {
+          "$ref": "#/properties/test-groups"
+        },
         "test-requires": {
           "$ref": "#/properties/test-requires"
         }
@@ -778,6 +802,9 @@
         "test-extras": {
           "$ref": "#/properties/test-extras"
         },
+        "test-groups": {
+          "$ref": "#/properties/test-groups"
+        },
         "test-requires": {
           "$ref": "#/properties/test-requires"
         }
@@ -823,6 +850,9 @@
         "test-extras": {
           "$ref": "#/properties/test-extras"
         },
+        "test-groups": {
+          "$ref": "#/properties/test-groups"
+        },
         "test-requires": {
           "$ref": "#/properties/test-requires"
         }

cibuildwheel/resources/defaults.toml

@@ -20,6 +20,7 @@ test-command = ""
 before-test = ""
 test-requires = []
 test-extras = []
+test-groups = []
 
 container-engine = "docker"
 

docs/options.md

@@ -1604,6 +1604,40 @@ Platform-specific environment variables are also available:<br/>
 
     In configuration files, you can use an inline array, and the items will be joined with a comma.
 
+
+### `CIBW_TEST_GROUPS` {: #test-groups}
+> Specify test dependencies from your project's `dependency-groups`
+
+List of
+[dependency-groups](https://peps.python.org/pep-0735)
+that should be included when installing the wheel prior to running the
+tests. This can be used to avoid having to redefine test dependencies in
+`CIBW_TEST_REQUIRES` if they are already defined in `pyproject.toml`.
+
+Platform-specific environment variables are also available:<br/>
+`CIBW_TEST_GROUPS_MACOS` | `CIBW_TEST_GROUPS_WINDOWS` | `CIBW_TEST_GROUPS_LINUX` | `CIBW_TEST_GROUPS_PYODIDE`
+
+#### Examples
+
+!!! tab examples "Environment variables"
+
+    ```yaml
+    # Will cause the wheel to be installed with these groups of dependencies
+    CIBW_TEST_GROUPS: "test qt"
+    ```
+
+    Separate multiple items with a space.
+
+!!! tab examples "pyproject.toml"
+
+    ```toml
+    [tool.cibuildwheel]
+    # Will cause the wheel to be installed with these groups of dependencies
+    test-groups = ["test", "qt"]
+    ```
+
+    In configuration files, you can use an inline array, and the items will be joined with a space.
+
 ### `CIBW_TEST_SKIP` {: #test-skip}
 > Skip running tests on some builds
 

pyproject.toml

@@ -43,6 +43,7 @@ dependencies = [
     "bashlex!=0.13",
     "bracex",
     "certifi",
+    "dependency-groups>=1.2",
     "filelock",
     "packaging>=20.9",
     "platformdirs",

test/test_testing.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import inspect
 import os
 import subprocess
 import textwrap
@@ -114,6 +115,38 @@ def test_extras_require(tmp_path):
     assert set(actual_wheels) == set(expected_wheels)
 
 
+def test_dependency_groups(tmp_path):
+    group_project = project_with_a_test.copy()
+    group_project.files["pyproject.toml"] = inspect.cleandoc("""
+        [build-system]
+        requires = ["setuptools"]
+        build-backend = "setuptools.build_meta"
+
+        [dependency-groups]
+        dev = ["pytest"]
+        """)
+
+    project_dir = tmp_path / "project"
+    group_project.generate(project_dir)
+
+    # build and test the wheels
+    actual_wheels = utils.cibuildwheel_run(
+        project_dir,
+        add_env={
+            "CIBW_TEST_GROUPS": "dev",
+            # the 'false ||' bit is to ensure this command runs in a shell on
+            # mac/linux.
+            "CIBW_TEST_COMMAND": f"false || {utils.invoke_pytest()} {{project}}/test",
+            "CIBW_TEST_COMMAND_WINDOWS": "COLOR 00 || pytest {project}/test",
+        },
+        single_python=True,
+    )
+
+    # also check that we got the right wheels
+    expected_wheels = utils.expected_wheels("spam", "0.1.0", single_python=True)
+    assert set(actual_wheels) == set(expected_wheels)
+
+
 project_with_a_failing_test = test_projects.new_c_project()
 project_with_a_failing_test.files["test/spam_test.py"] = r"""
 from unittest import TestCase

unit_test/options_toml_test.py

@@ -22,6 +22,7 @@ environment = {THING = "OTHER", FOO="BAR"}
 test-command = "pyproject"
 test-requires = "something"
 test-extras = ["one", "two"]
+test-groups = ["three", "four"]
 
 manylinux-x86_64-image = "manylinux1"
 
@@ -60,6 +61,7 @@ def test_simple_settings(tmp_path, platform, fname):
         == 'THING="OTHER" FOO="BAR"'
     )
     assert options_reader.get("test-extras", option_format=ListFormat(",")) == "one,two"
+    assert options_reader.get("test-groups", option_format=ListFormat(" ")) == "three four"
 
     assert options_reader.get("manylinux-x86_64-image") == "manylinux1"
     assert options_reader.get("manylinux-i686-image") == "manylinux2014"
@@ -85,7 +87,9 @@ def test_envvar_override(tmp_path, platform):
             "CIBW_MANYLINUX_X86_64_IMAGE": "manylinux_2_24",
             "CIBW_TEST_COMMAND": "mytest",
             "CIBW_TEST_REQUIRES": "docs",
+            "CIBW_TEST_GROUPS": "mgroup two",
             "CIBW_TEST_REQUIRES_LINUX": "scod",
+            "CIBW_TEST_GROUPS_LINUX": "lgroup",
         },
     )
 
@@ -99,6 +103,10 @@ def test_envvar_override(tmp_path, platform):
         options_reader.get("test-requires", option_format=ListFormat(" "))
         == {"windows": "docs", "macos": "docs", "linux": "scod"}[platform]
     )
+    assert (
+        options_reader.get("test-groups", option_format=ListFormat(" "))
+        == {"windows": "mgroup two", "macos": "mgroup two", "linux": "lgroup"}[platform]
+    )
     assert options_reader.get("test-command") == "mytest"
 
 

unit_test/projectfiles_test.py

@@ -2,7 +2,14 @@ from __future__ import annotations
 
 from textwrap import dedent
 
-from cibuildwheel.projectfiles import get_requires_python_str, setup_py_python_requires
+import pytest
+
+from cibuildwheel._compat import tomllib
+from cibuildwheel.projectfiles import (
+    get_requires_python_str,
+    resolve_dependency_groups,
+    setup_py_python_requires,
+)
 
 
 def test_read_setup_py_simple(tmp_path):
@@ -23,7 +30,7 @@ def test_read_setup_py_simple(tmp_path):
         )
 
     assert setup_py_python_requires(tmp_path.joinpath("setup.py").read_text()) == "1.23"
-    assert get_requires_python_str(tmp_path) == "1.23"
+    assert get_requires_python_str(tmp_path, {}) == "1.23"
 
 
 def test_read_setup_py_if_main(tmp_path):
@@ -45,7 +52,7 @@ def test_read_setup_py_if_main(tmp_path):
         )
 
     assert setup_py_python_requires(tmp_path.joinpath("setup.py").read_text()) == "1.23"
-    assert get_requires_python_str(tmp_path) == "1.23"
+    assert get_requires_python_str(tmp_path, {}) == "1.23"
 
 
 def test_read_setup_py_if_main_reversed(tmp_path):
@@ -67,7 +74,7 @@ def test_read_setup_py_if_main_reversed(tmp_path):
         )
 
     assert setup_py_python_requires(tmp_path.joinpath("setup.py").read_text()) == "1.23"
-    assert get_requires_python_str(tmp_path) == "1.23"
+    assert get_requires_python_str(tmp_path, {}) == "1.23"
 
 
 def test_read_setup_py_if_invalid(tmp_path):
@@ -89,7 +96,7 @@ def test_read_setup_py_if_invalid(tmp_path):
         )
 
     assert not setup_py_python_requires(tmp_path.joinpath("setup.py").read_text())
-    assert not get_requires_python_str(tmp_path)
+    assert not get_requires_python_str(tmp_path, {})
 
 
 def test_read_setup_py_full(tmp_path):
@@ -115,7 +122,7 @@ def test_read_setup_py_full(tmp_path):
     assert (
         setup_py_python_requires(tmp_path.joinpath("setup.py").read_text(encoding="utf8")) == "1.24"
     )
-    assert get_requires_python_str(tmp_path) == "1.24"
+    assert get_requires_python_str(tmp_path, {}) == "1.24"
 
 
 def test_read_setup_py_assign(tmp_path):
@@ -138,7 +145,7 @@ def test_read_setup_py_assign(tmp_path):
         )
 
     assert setup_py_python_requires(tmp_path.joinpath("setup.py").read_text()) is None
-    assert get_requires_python_str(tmp_path) is None
+    assert get_requires_python_str(tmp_path, {}) is None
 
 
 def test_read_setup_py_None(tmp_path):
@@ -161,7 +168,7 @@ def test_read_setup_py_None(tmp_path):
         )
 
     assert setup_py_python_requires(tmp_path.joinpath("setup.py").read_text()) is None
-    assert get_requires_python_str(tmp_path) is None
+    assert get_requires_python_str(tmp_path, {}) is None
 
 
 def test_read_setup_py_empty(tmp_path):
@@ -183,7 +190,7 @@ def test_read_setup_py_empty(tmp_path):
         )
 
     assert setup_py_python_requires(tmp_path.joinpath("setup.py").read_text()) is None
-    assert get_requires_python_str(tmp_path) is None
+    assert get_requires_python_str(tmp_path, {}) is None
 
 
 def test_read_setup_cfg(tmp_path):
@@ -199,7 +206,7 @@ def test_read_setup_cfg(tmp_path):
             )
         )
 
-    assert get_requires_python_str(tmp_path) == "1.234"
+    assert get_requires_python_str(tmp_path, {}) == "1.234"
 
 
 def test_read_setup_cfg_empty(tmp_path):
@@ -215,7 +222,7 @@ def test_read_setup_cfg_empty(tmp_path):
             )
         )
 
-    assert get_requires_python_str(tmp_path) is None
+    assert get_requires_python_str(tmp_path, {}) is None
 
 
 def test_read_pyproject_toml(tmp_path):
@@ -231,8 +238,10 @@ def test_read_pyproject_toml(tmp_path):
                 """
             )
         )
+    with open(tmp_path / "pyproject.toml", "rb") as f:
+        pyproject_toml = tomllib.load(f)
 
-    assert get_requires_python_str(tmp_path) == "1.654"
+    assert get_requires_python_str(tmp_path, pyproject_toml) == "1.654"
 
 
 def test_read_pyproject_toml_empty(tmp_path):
@@ -245,5 +254,25 @@ def test_read_pyproject_toml_empty(tmp_path):
                 """
             )
         )
+    with open(tmp_path / "pyproject.toml", "rb") as f:
+        pyproject_toml = tomllib.load(f)
 
-    assert get_requires_python_str(tmp_path) is None
+    assert get_requires_python_str(tmp_path, pyproject_toml) is None
+
+
+def test_read_dep_groups():
+    pyproject_toml = {"dependency-groups": {"group1": ["pkg1", "pkg2"], "group2": ["pkg3"]}}
+    assert resolve_dependency_groups(pyproject_toml) == ()
+    assert resolve_dependency_groups(pyproject_toml, "group1") == ("pkg1", "pkg2")
+    assert resolve_dependency_groups(pyproject_toml, "group2") == ("pkg3",)
+    assert resolve_dependency_groups(pyproject_toml, "group1", "group2") == ("pkg1", "pkg2", "pkg3")
+
+
+def test_dep_group_no_file_error():
+    with pytest.raises(FileNotFoundError, match="pyproject.toml"):
+        resolve_dependency_groups(None, "test")
+
+
+def test_dep_group_no_section_error():
+    with pytest.raises(KeyError, match="pyproject.toml"):
+        resolve_dependency_groups({}, "test")