refactor: pull out dependency_constraint (#2347)

* refactor: pull out dependency_constraint

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* refactor: address feedback

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* ci: better parallel

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

---------

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

azure-pipelines.yml

@@ -7,7 +7,7 @@ pr:
 jobs:
 - job: linux_311
   timeoutInMinutes: 120
-  pool: {vmImage: 'Ubuntu-20.04'}
+  pool: {vmImage: 'Ubuntu-22.04'}
   steps:
     - task: UsePythonVersion@0
       inputs:
@@ -27,7 +27,7 @@ jobs:
     - bash: |
         python -m pip install dependency-groups
         python -m dependency_groups test | xargs python -m pip install -e.
-        python ./bin/run_tests.py --num-processes 2
+        python ./bin/run_tests.py
 
 - job: windows_311
   pool: {vmImage: 'windows-2019'}

bin/run_tests.py

@@ -8,7 +8,11 @@ import sys
 from pathlib import Path
 
 if __name__ == "__main__":
-    default_cpu_count = os.cpu_count() or 2
+    if sys.version_info < (3, 13):
+        default_cpu_count = os.cpu_count() or 2
+    else:
+        default_cpu_count = os.process_cpu_count() or 2
+
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "--run-podman", action="store_true", default=False, help="run podman tests (linux only)"

cibuildwheel/platforms/ios.py

@@ -24,7 +24,6 @@ from ..frontend import (
 from ..logger import log
 from ..options import Options
 from ..selector import BuildSelector
-from ..typing import PathOrStr
 from ..util import resources
 from ..util.cmd import call, shell
 from ..util.file import (
@@ -39,7 +38,7 @@ from ..util.packaging import (
     find_compatible_wheel,
     get_pip_version,
 )
-from ..venv import virtualenv
+from ..venv import constraint_flags, virtualenv
 from .macos import install_cpython as install_build_cpython
 
 
@@ -151,7 +150,7 @@ def cross_virtualenv(
     multiarch: str,
     build_python: Path,
     venv_path: Path,
-    dependency_constraint_flags: Sequence[PathOrStr],
+    dependency_constraint: Path | None,
     xbuild_tools: Sequence[str] | None,
 ) -> dict[str, str]:
     """Create a cross-compilation virtual environment.
@@ -178,8 +177,8 @@ def cross_virtualenv(
     :param build_python: The path to the python binary for the build platform
     :param venv_path: The path where the cross virtual environment should be
         created.
-    :param dependency_constraint_flags: Any flags that should be used when
+    :param dependency_constraint: A path to a constraint file that should be
-        constraining dependencies in the environment.
+        used when constraining dependencies in the environment.
     :param xbuild_tools: A list of executable names (without paths) that are
         on the path, but must be preserved in the cross environment.
     """
@@ -188,7 +187,7 @@ def cross_virtualenv(
         py_version,
         build_python,
         venv_path,
-        dependency_constraint_flags,
+        dependency_constraint,
         use_uv=False,
     )
 
@@ -279,7 +278,7 @@ def setup_python(
     tmp: Path,
     *,
     python_configuration: PythonConfiguration,
-    dependency_constraint_flags: Sequence[PathOrStr],
+    dependency_constraint: Path | None,
     environment: ParsedEnvironment,
     build_frontend: BuildFrontendName,
     xbuild_tools: Sequence[str] | None,
@@ -334,7 +333,7 @@ def setup_python(
         multiarch=python_configuration.multiarch,
         build_python=build_python,
         venv_path=venv_path,
-        dependency_constraint_flags=dependency_constraint_flags,
+        dependency_constraint=dependency_constraint,
         xbuild_tools=xbuild_tools,
     )
     venv_bin_path = venv_path / "bin"
@@ -351,7 +350,7 @@ def setup_python(
         "install",
         "--upgrade",
         "pip",
-        *dependency_constraint_flags,
+        *constraint_flags(dependency_constraint),
         env=env,
         cwd=venv_path,
     )
@@ -397,7 +396,7 @@ def setup_python(
             "install",
             "--upgrade",
             "build[virtualenv]",
-            *dependency_constraint_flags,
+            *constraint_flags(dependency_constraint),
             env=env,
         )
     else:
@@ -453,14 +452,11 @@ def build(options: Options, tmp_path: Path) -> None:
             constraints_path = build_options.dependency_constraints.get_for_python_version(
                 version=config.version, tmp_dir=identifier_tmp_dir
             )
-            dependency_constraint_flags: Sequence[PathOrStr] = (
-                ["-c", constraints_path] if constraints_path else []
-            )
 
             target_install_path, env = setup_python(
                 identifier_tmp_dir / "build",
                 python_configuration=config,
-                dependency_constraint_flags=dependency_constraint_flags,
+                dependency_constraint=constraints_path,
                 environment=build_options.environment,
                 build_frontend=build_frontend.name,
                 xbuild_tools=build_options.xbuild_tools,

cibuildwheel/platforms/macos.py

@@ -7,7 +7,7 @@ import shutil
 import subprocess
 import sys
 import typing
-from collections.abc import Sequence, Set
+from collections.abc import Set
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Literal, assert_never
@@ -23,7 +23,6 @@ from ..frontend import BuildFrontendConfig, BuildFrontendName, get_build_fronten
 from ..logger import log
 from ..options import Options
 from ..selector import BuildSelector
-from ..typing import PathOrStr
 from ..util import resources
 from ..util.cmd import call, shell
 from ..util.file import (
@@ -34,7 +33,7 @@ from ..util.file import (
 )
 from ..util.helpers import prepare_command, unwrap
 from ..util.packaging import combine_constraints, find_compatible_wheel, get_pip_version
-from ..venv import find_uv, virtualenv
+from ..venv import constraint_flags, find_uv, virtualenv
 
 
 @functools.cache
@@ -195,7 +194,7 @@ def install_pypy(tmp: Path, url: str) -> Path:
 def setup_python(
     tmp: Path,
     python_configuration: PythonConfiguration,
-    dependency_constraint_flags: Sequence[PathOrStr],
+    dependency_constraint: Path | None,
     environment: ParsedEnvironment,
     build_frontend: BuildFrontendName,
 ) -> tuple[Path, dict[str, str]]:
@@ -226,7 +225,7 @@ def setup_python(
         python_configuration.version,
         base_python,
         venv_path,
-        dependency_constraint_flags,
+        dependency_constraint,
         use_uv=use_uv,
     )
     venv_bin_path = venv_path / "bin"
@@ -256,7 +255,7 @@ def setup_python(
             "install",
             "--upgrade",
             "pip",
-            *dependency_constraint_flags,
+            *constraint_flags(dependency_constraint),
             env=env,
             cwd=venv_path,
         )
@@ -351,7 +350,7 @@ def setup_python(
             "install",
             "--upgrade",
             "delocate",
-            *dependency_constraint_flags,
+            *constraint_flags(dependency_constraint),
             env=env,
         )
     elif build_frontend == "build":
@@ -361,7 +360,7 @@ def setup_python(
             "--upgrade",
             "delocate",
             "build[virtualenv]",
-            *dependency_constraint_flags,
+            *constraint_flags(dependency_constraint),
             env=env,
         )
     elif build_frontend == "build[uv]":
@@ -373,7 +372,7 @@ def setup_python(
             "--upgrade",
             "delocate",
             "build[virtualenv, uv]",
-            *dependency_constraint_flags,
+            *constraint_flags(dependency_constraint),
             env=env,
         )
     else:
@@ -427,14 +426,11 @@ def build(options: Options, tmp_path: Path) -> None:
             constraints_path = build_options.dependency_constraints.get_for_python_version(
                 version=config.version, tmp_dir=identifier_tmp_dir
             )
-            dependency_constraint_flags: Sequence[PathOrStr] = (
-                ["-c", constraints_path] if constraints_path else []
-            )
 
             base_python, env = setup_python(
                 identifier_tmp_dir / "build",
                 config,
-                dependency_constraint_flags,
+                constraints_path,
                 build_options.environment,
                 build_frontend.name,
             )
@@ -621,7 +617,13 @@ def build(options: Options, tmp_path: Path) -> None:
                     # set up a virtual environment to install and test from, to make sure
                     # there are no dependencies that were pulled in at build time.
                     if not use_uv:
-                        call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env)
+                        call(
+                            "pip",
+                            "install",
+                            "virtualenv",
+                            *constraint_flags(constraints_path),
+                            env=env,
+                        )
 
                     venv_dir = identifier_tmp_dir / f"venv-test-{testing_arch}"
 

cibuildwheel/platforms/pyodide.py

@@ -3,7 +3,7 @@ import os
 import shutil
 import sys
 import tomllib
-from collections.abc import Sequence, Set
+from collections.abc import Set
 from dataclasses import dataclass
 from pathlib import Path
 from tempfile import TemporaryDirectory
@@ -18,7 +18,6 @@ from ..frontend import BuildFrontendConfig, get_build_frontend_extra_flags
 from ..logger import log
 from ..options import Options
 from ..selector import BuildSelector
-from ..typing import PathOrStr
 from ..util import resources
 from ..util.cmd import call, shell
 from ..util.file import (
@@ -31,7 +30,7 @@ from ..util.file import (
 )
 from ..util.helpers import prepare_command
 from ..util.packaging import combine_constraints, find_compatible_wheel, get_pip_version
-from ..venv import virtualenv
+from ..venv import constraint_flags, virtualenv
 
 IS_WIN: Final[bool] = sys.platform.startswith("win")
 
@@ -145,14 +144,14 @@ def get_base_python(identifier: str) -> Path:
 def setup_python(
     tmp: Path,
     python_configuration: PythonConfiguration,
-    dependency_constraint_flags: Sequence[PathOrStr],
+    constraints_path: Path | None,
     environment: ParsedEnvironment,
 ) -> dict[str, str]:
     base_python = get_base_python(python_configuration.identifier)
 
     log.step("Setting up build environment...")
     venv_path = tmp / "venv"
-    env = virtualenv(python_configuration.version, base_python, venv_path, [], use_uv=False)
+    env = virtualenv(python_configuration.version, base_python, venv_path, None, use_uv=False)
     venv_bin_path = venv_path / "bin"
     assert venv_bin_path.exists()
     env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
@@ -166,7 +165,7 @@ def setup_python(
         "install",
         "--upgrade",
         "pip",
-        *dependency_constraint_flags,
+        *constraint_flags(constraints_path),
         env=env,
         cwd=venv_path,
     )
@@ -198,7 +197,7 @@ def setup_python(
         "auditwheel-emscripten",
         "build[virtualenv]",
         "pyodide-build",
-        *dependency_constraint_flags,
+        *constraint_flags(constraints_path),
         env=env,
     )
 
@@ -269,14 +268,11 @@ def build(options: Options, tmp_path: Path) -> None:
             constraints_path = build_options.dependency_constraints.get_for_python_version(
                 version=config.version, variant="pyodide", tmp_dir=identifier_tmp_dir
             )
-            dependency_constraint_flags: Sequence[PathOrStr] = (
-                ["-c", constraints_path] if constraints_path else []
-            )
 
             env = setup_python(
                 identifier_tmp_dir / "build",
                 config,
-                dependency_constraint_flags,
+                constraints_path,
                 build_options.environment,
             )
             pip_version = get_pip_version(env)

cibuildwheel/platforms/windows.py

@@ -3,7 +3,7 @@ import platform as platform_module
 import shutil
 import subprocess
 import textwrap
-from collections.abc import MutableMapping, Sequence, Set
+from collections.abc import MutableMapping, Set
 from dataclasses import dataclass
 from functools import cache
 from pathlib import Path
@@ -18,13 +18,12 @@ from ..frontend import BuildFrontendConfig, BuildFrontendName, get_build_fronten
 from ..logger import log
 from ..options import Options
 from ..selector import BuildSelector
-from ..typing import PathOrStr
 from ..util import resources
 from ..util.cmd import call, shell
 from ..util.file import CIBW_CACHE_PATH, copy_test_sources, download, extract_zip, move_file
 from ..util.helpers import prepare_command, unwrap
 from ..util.packaging import combine_constraints, find_compatible_wheel, get_pip_version
-from ..venv import find_uv, virtualenv
+from ..venv import constraint_flags, find_uv, virtualenv
 
 
 def get_nuget_args(
@@ -219,7 +218,7 @@ def can_use_uv(python_configuration: PythonConfiguration) -> bool:
 def setup_python(
     tmp: Path,
     python_configuration: PythonConfiguration,
-    dependency_constraint_flags: Sequence[PathOrStr],
+    dependency_constraint: Path | None,
     environment: ParsedEnvironment,
     build_frontend: BuildFrontendName,
 ) -> tuple[Path, dict[str, str]]:
@@ -257,7 +256,7 @@ def setup_python(
         python_configuration.version,
         base_python,
         venv_path,
-        dependency_constraint_flags,
+        dependency_constraint,
         use_uv=use_uv,
     )
 
@@ -276,7 +275,7 @@ def setup_python(
             "install",
             "--upgrade",
             "pip",
-            *dependency_constraint_flags,
+            *constraint_flags(dependency_constraint),
             env=env,
             cwd=venv_path,
         )
@@ -310,7 +309,7 @@ def setup_python(
             "install",
             "--upgrade",
             "build[virtualenv]",
-            *dependency_constraint_flags,
+            *constraint_flags(dependency_constraint),
             env=env,
         )
     elif build_frontend == "build[uv]":
@@ -321,7 +320,7 @@ def setup_python(
             "install",
             "--upgrade",
             "build[virtualenv]",
-            *dependency_constraint_flags,
+            *constraint_flags(dependency_constraint),
             env=env,
         )
 
@@ -370,15 +369,12 @@ def build(options: Options, tmp_path: Path) -> None:
                 version=config.version,
                 tmp_dir=identifier_tmp_dir,
             )
-            dependency_constraint_flags: Sequence[PathOrStr] = (
-                ["-c", constraints_path] if constraints_path else []
-            )
 
             # install Python
             base_python, env = setup_python(
                 identifier_tmp_dir / "build",
                 config,
-                dependency_constraint_flags,
+                constraints_path,
                 build_options.environment,
                 build_frontend.name,
             )
@@ -491,7 +487,9 @@ def build(options: Options, tmp_path: Path) -> None:
                 # set up a virtual environment to install and test from, to make sure
                 # there are no dependencies that were pulled in at build time.
                 if not use_uv:
-                    call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env)
+                    call(
+                        "pip", "install", "virtualenv", *constraint_flags(constraints_path), env=env
+                    )
 
                 venv_dir = identifier_tmp_dir / "venv-test"
 

cibuildwheel/venv.py

@@ -12,7 +12,6 @@ from filelock import FileLock
 from packaging.requirements import InvalidRequirement, Requirement
 from packaging.version import Version
 
-from .typing import PathOrStr
 from .util import resources
 from .util.cmd import call
 from .util.file import CIBW_CACHE_PATH, download
@@ -36,8 +35,18 @@ def _ensure_virtualenv(version: str) -> Path:
     return path
 
 
+def constraint_flags(
+    dependency_constraint: Path | None,
+) -> Sequence[str]:
+    """
+    Returns the flags to pass to pip for the given dependency constraint.
+    """
+
+    return ["-c", dependency_constraint.as_uri()] if dependency_constraint else []
+
+
 def _parse_pip_constraint_for_virtualenv(
-    dependency_constraint_flags: Sequence[PathOrStr],
+    constraint_path: Path | None,
 ) -> str:
     """
     Parses the constraints file referenced by `dependency_constraint_flags` and returns a dict where
@@ -48,10 +57,7 @@ def _parse_pip_constraint_for_virtualenv(
     If it can't get an exact version, the real constraint will be handled by the
     {macos|windows}.setup_python function.
     """
-    assert len(dependency_constraint_flags) in {0, 2}
+    if constraint_path:
-    if len(dependency_constraint_flags) == 2:
-        assert dependency_constraint_flags[0] == "-c"
-        constraint_path = Path(dependency_constraint_flags[1])
         assert constraint_path.exists()
         with constraint_path.open(encoding="utf-8") as constraint_file:
             for line_ in constraint_file:
@@ -84,7 +90,7 @@ def virtualenv(
     version: str,
     python: Path,
     venv_path: Path,
-    dependency_constraint_flags: Sequence[PathOrStr],
+    dependency_constraint: Path | None,
     *,
     use_uv: bool,
 ) -> dict[str, str]:
@@ -103,7 +109,7 @@ def virtualenv(
         call("uv", "venv", venv_path, "--python", python)
     else:
         virtualenv_app = _ensure_virtualenv(version)
-        pip_constraint = _parse_pip_constraint_for_virtualenv(dependency_constraint_flags)
+        pip_constraint = _parse_pip_constraint_for_virtualenv(dependency_constraint)
         additional_flags = [f"--pip={pip_constraint}", "--no-setuptools", "--no-wheel"]
 
         # Using symlinks to pre-installed seed packages is really the fastest way to get a virtual