Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: pull out dependency_constraint (#2347)

Browse Source 
* refactor: pull out dependency_constraint

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* refactor: address feedback

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* ci: better parallel

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

---------

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>
This commit is contained in:
Henry Schreiner
2025-04-09 09:58:57 -04:00
committed by GitHub
parent 8e2cc78913
commit 93314a8d65
7 changed files with 65 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
azure-pipelines.yml
+2 -2
View File
@@ -7,7 +7,7 @@ pr:
jobs:
- job: linux_311
timeoutInMinutes: 120
pool: {vmImage: 'Ubuntu-20.04'}
pool: {vmImage: 'Ubuntu-22.04'}
steps:
- task: UsePythonVersion@0
inputs:
@@ -27,7 +27,7 @@ jobs:
- bash: |
python -m pip install dependency-groups
python -m dependency_groups test | xargs python -m pip install -e.
python ./bin/run_tests.py --num-processes 2
python ./bin/run_tests.py
- job: windows_311
pool: {vmImage: 'windows-2019'}
bin/run_tests.py
+5 -1
View File
@@ -8,7 +8,11 @@ import sys
from pathlib import Path
if __name__ == "__main__":
default_cpu_count = os.cpu_count() or 2
if sys.version_info < (3, 13):
default_cpu_count = os.cpu_count() or 2
else:
default_cpu_count = os.process_cpu_count() or 2
parser = argparse.ArgumentParser()
parser.add_argument(
"--run-podman", action="store_true", default=False, help="run podman tests (linux only)"
cibuildwheel/platforms/ios.py
+10 -14
View File
@@ -24,7 +24,6 @@ from ..frontend import (
from ..logger import log
from ..options import Options
from ..selector import BuildSelector
from ..typing import PathOrStr
from ..util import resources
from ..util.cmd import call, shell
from ..util.file import (
@@ -39,7 +38,7 @@ from ..util.packaging import (
find_compatible_wheel,
get_pip_version,
)
from ..venv import virtualenv
from ..venv import constraint_flags, virtualenv
from .macos import install_cpython as install_build_cpython
@@ -151,7 +150,7 @@ def cross_virtualenv(
multiarch: str,
build_python: Path,
venv_path: Path,
dependency_constraint_flags: Sequence[PathOrStr],
dependency_constraint: Path | None,
xbuild_tools: Sequence[str] | None,
) -> dict[str, str]:
"""Create a cross-compilation virtual environment.
@@ -178,8 +177,8 @@ def cross_virtualenv(
:param build_python: The path to the python binary for the build platform
:param venv_path: The path where the cross virtual environment should be
created.
:param dependency_constraint_flags: Any flags that should be used when
constraining dependencies in the environment.
:param dependency_constraint: A path to a constraint file that should be
used when constraining dependencies in the environment.
:param xbuild_tools: A list of executable names (without paths) that are
on the path, but must be preserved in the cross environment.
"""
@@ -188,7 +187,7 @@ def cross_virtualenv(
py_version,
build_python,
venv_path,
dependency_constraint_flags,
dependency_constraint,
use_uv=False,
)
@@ -279,7 +278,7 @@ def setup_python(
tmp: Path,
*,
python_configuration: PythonConfiguration,
dependency_constraint_flags: Sequence[PathOrStr],
dependency_constraint: Path | None,
environment: ParsedEnvironment,
build_frontend: BuildFrontendName,
xbuild_tools: Sequence[str] | None,
@@ -334,7 +333,7 @@ def setup_python(
multiarch=python_configuration.multiarch,
build_python=build_python,
venv_path=venv_path,
dependency_constraint_flags=dependency_constraint_flags,
dependency_constraint=dependency_constraint,
xbuild_tools=xbuild_tools,
)
venv_bin_path = venv_path / "bin"
@@ -351,7 +350,7 @@ def setup_python(
"install",
"--upgrade",
"pip",
*dependency_constraint_flags,
*constraint_flags(dependency_constraint),
env=env,
cwd=venv_path,
)
@@ -397,7 +396,7 @@ def setup_python(
"install",
"--upgrade",
"build[virtualenv]",
*dependency_constraint_flags,
*constraint_flags(dependency_constraint),
env=env,
)
else:
@@ -453,14 +452,11 @@ def build(options: Options, tmp_path: Path) -> None:
constraints_path = build_options.dependency_constraints.get_for_python_version(
version=config.version, tmp_dir=identifier_tmp_dir
)
dependency_constraint_flags: Sequence[PathOrStr] = (
["-c", constraints_path] if constraints_path else []
)
target_install_path, env = setup_python(
identifier_tmp_dir / "build",
python_configuration=config,
dependency_constraint_flags=dependency_constraint_flags,
dependency_constraint=constraints_path,
environment=build_options.environment,
build_frontend=build_frontend.name,
xbuild_tools=build_options.xbuild_tools,
cibuildwheel/platforms/macos.py
+16 -14
View File
@@ -7,7 +7,7 @@ import shutil
import subprocess
import sys
import typing
from collections.abc import Sequence, Set
from collections.abc import Set
from dataclasses import dataclass
from pathlib import Path
from typing import Literal, assert_never
@@ -23,7 +23,6 @@ from ..frontend import BuildFrontendConfig, BuildFrontendName, get_build_fronten
from ..logger import log
from ..options import Options
from ..selector import BuildSelector
from ..typing import PathOrStr
from ..util import resources
from ..util.cmd import call, shell
from ..util.file import (
@@ -34,7 +33,7 @@ from ..util.file import (
)
from ..util.helpers import prepare_command, unwrap
from ..util.packaging import combine_constraints, find_compatible_wheel, get_pip_version
from ..venv import find_uv, virtualenv
from ..venv import constraint_flags, find_uv, virtualenv
@functools.cache
@@ -195,7 +194,7 @@ def install_pypy(tmp: Path, url: str) -> Path:
def setup_python(
tmp: Path,
python_configuration: PythonConfiguration,
dependency_constraint_flags: Sequence[PathOrStr],
dependency_constraint: Path | None,
environment: ParsedEnvironment,
build_frontend: BuildFrontendName,
) -> tuple[Path, dict[str, str]]:
@@ -226,7 +225,7 @@ def setup_python(
python_configuration.version,
base_python,
venv_path,
dependency_constraint_flags,
dependency_constraint,
use_uv=use_uv,
)
venv_bin_path = venv_path / "bin"
@@ -256,7 +255,7 @@ def setup_python(
"install",
"--upgrade",
"pip",
*dependency_constraint_flags,
*constraint_flags(dependency_constraint),
env=env,
cwd=venv_path,
)
@@ -351,7 +350,7 @@ def setup_python(
"install",
"--upgrade",
"delocate",
*dependency_constraint_flags,
*constraint_flags(dependency_constraint),
env=env,
)
elif build_frontend == "build":
@@ -361,7 +360,7 @@ def setup_python(
"--upgrade",
"delocate",
"build[virtualenv]",
*dependency_constraint_flags,
*constraint_flags(dependency_constraint),
env=env,
)
elif build_frontend == "build[uv]":
@@ -373,7 +372,7 @@ def setup_python(
"--upgrade",
"delocate",
"build[virtualenv, uv]",
*dependency_constraint_flags,
*constraint_flags(dependency_constraint),
env=env,
)
else:
@@ -427,14 +426,11 @@ def build(options: Options, tmp_path: Path) -> None:
constraints_path = build_options.dependency_constraints.get_for_python_version(
version=config.version, tmp_dir=identifier_tmp_dir
)
dependency_constraint_flags: Sequence[PathOrStr] = (
["-c", constraints_path] if constraints_path else []
)
base_python, env = setup_python(
identifier_tmp_dir / "build",
config,
dependency_constraint_flags,
constraints_path,
build_options.environment,
build_frontend.name,
)
@@ -621,7 +617,13 @@ def build(options: Options, tmp_path: Path) -> None:
# set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
if not use_uv:
call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env)
call(
"pip",
"install",
"virtualenv",
*constraint_flags(constraints_path),
env=env,
)
venv_dir = identifier_tmp_dir / f"venv-test-{testing_arch}"
cibuildwheel/platforms/pyodide.py
+7 -11
View File
@@ -3,7 +3,7 @@ import os
import shutil
import sys
import tomllib
from collections.abc import Sequence, Set
from collections.abc import Set
from dataclasses import dataclass
from pathlib import Path
from tempfile import TemporaryDirectory
@@ -18,7 +18,6 @@ from ..frontend import BuildFrontendConfig, get_build_frontend_extra_flags
from ..logger import log
from ..options import Options
from ..selector import BuildSelector
from ..typing import PathOrStr
from ..util import resources
from ..util.cmd import call, shell
from ..util.file import (
@@ -31,7 +30,7 @@ from ..util.file import (
)
from ..util.helpers import prepare_command
from ..util.packaging import combine_constraints, find_compatible_wheel, get_pip_version
from ..venv import virtualenv
from ..venv import constraint_flags, virtualenv
IS_WIN: Final[bool] = sys.platform.startswith("win")
@@ -145,14 +144,14 @@ def get_base_python(identifier: str) -> Path:
def setup_python(
tmp: Path,
python_configuration: PythonConfiguration,
dependency_constraint_flags: Sequence[PathOrStr],
constraints_path: Path | None,
environment: ParsedEnvironment,
) -> dict[str, str]:
base_python = get_base_python(python_configuration.identifier)
log.step("Setting up build environment...")
venv_path = tmp / "venv"
env = virtualenv(python_configuration.version, base_python, venv_path, [], use_uv=False)
env = virtualenv(python_configuration.version, base_python, venv_path, None, use_uv=False)
venv_bin_path = venv_path / "bin"
assert venv_bin_path.exists()
env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
@@ -166,7 +165,7 @@ def setup_python(
"install",
"--upgrade",
"pip",
*dependency_constraint_flags,
*constraint_flags(constraints_path),
env=env,
cwd=venv_path,
)
@@ -198,7 +197,7 @@ def setup_python(
"auditwheel-emscripten",
"build[virtualenv]",
"pyodide-build",
*dependency_constraint_flags,
*constraint_flags(constraints_path),
env=env,
)
@@ -269,14 +268,11 @@ def build(options: Options, tmp_path: Path) -> None:
constraints_path = build_options.dependency_constraints.get_for_python_version(
version=config.version, variant="pyodide", tmp_dir=identifier_tmp_dir
)
dependency_constraint_flags: Sequence[PathOrStr] = (
["-c", constraints_path] if constraints_path else []
)
env = setup_python(
identifier_tmp_dir / "build",
config,
dependency_constraint_flags,
constraints_path,
build_options.environment,
)
pip_version = get_pip_version(env)
cibuildwheel/platforms/windows.py
+11 -13
View File
@@ -3,7 +3,7 @@ import platform as platform_module
import shutil
import subprocess
import textwrap
from collections.abc import MutableMapping, Sequence, Set
from collections.abc import MutableMapping, Set
from dataclasses import dataclass
from functools import cache
from pathlib import Path
@@ -18,13 +18,12 @@ from ..frontend import BuildFrontendConfig, BuildFrontendName, get_build_fronten
from ..logger import log
from ..options import Options
from ..selector import BuildSelector
from ..typing import PathOrStr
from ..util import resources
from ..util.cmd import call, shell
from ..util.file import CIBW_CACHE_PATH, copy_test_sources, download, extract_zip, move_file
from ..util.helpers import prepare_command, unwrap
from ..util.packaging import combine_constraints, find_compatible_wheel, get_pip_version
from ..venv import find_uv, virtualenv
from ..venv import constraint_flags, find_uv, virtualenv
def get_nuget_args(
@@ -219,7 +218,7 @@ def can_use_uv(python_configuration: PythonConfiguration) -> bool:
def setup_python(
tmp: Path,
python_configuration: PythonConfiguration,
dependency_constraint_flags: Sequence[PathOrStr],
dependency_constraint: Path | None,
environment: ParsedEnvironment,
build_frontend: BuildFrontendName,
) -> tuple[Path, dict[str, str]]:
@@ -257,7 +256,7 @@ def setup_python(
python_configuration.version,
base_python,
venv_path,
dependency_constraint_flags,
dependency_constraint,
use_uv=use_uv,
)
@@ -276,7 +275,7 @@ def setup_python(
"install",
"--upgrade",
"pip",
*dependency_constraint_flags,
*constraint_flags(dependency_constraint),
env=env,
cwd=venv_path,
)
@@ -310,7 +309,7 @@ def setup_python(
"install",
"--upgrade",
"build[virtualenv]",
*dependency_constraint_flags,
*constraint_flags(dependency_constraint),
env=env,
)
elif build_frontend == "build[uv]":
@@ -321,7 +320,7 @@ def setup_python(
"install",
"--upgrade",
"build[virtualenv]",
*dependency_constraint_flags,
*constraint_flags(dependency_constraint),
env=env,
)
@@ -370,15 +369,12 @@ def build(options: Options, tmp_path: Path) -> None:
version=config.version,
tmp_dir=identifier_tmp_dir,
)
dependency_constraint_flags: Sequence[PathOrStr] = (
["-c", constraints_path] if constraints_path else []
)
# install Python
base_python, env = setup_python(
identifier_tmp_dir / "build",
config,
dependency_constraint_flags,
constraints_path,
build_options.environment,
build_frontend.name,
)
@@ -491,7 +487,9 @@ def build(options: Options, tmp_path: Path) -> None:
# set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
if not use_uv:
call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env)
call(
"pip", "install", "virtualenv", *constraint_flags(constraints_path), env=env
)
venv_dir = identifier_tmp_dir / "venv-test"
cibuildwheel/venv.py
+14 -8
View File
@@ -12,7 +12,6 @@ from filelock import FileLock
from packaging.requirements import InvalidRequirement, Requirement
from packaging.version import Version
from .typing import PathOrStr
from .util import resources
from .util.cmd import call
from .util.file import CIBW_CACHE_PATH, download
@@ -36,8 +35,18 @@ def _ensure_virtualenv(version: str) -> Path:
return path
def constraint_flags(
dependency_constraint: Path | None,
) -> Sequence[str]:
"""
Returns the flags to pass to pip for the given dependency constraint.
"""
return ["-c", dependency_constraint.as_uri()] if dependency_constraint else []
def _parse_pip_constraint_for_virtualenv(
dependency_constraint_flags: Sequence[PathOrStr],
constraint_path: Path | None,
) -> str:
"""
Parses the constraints file referenced by `dependency_constraint_flags` and returns a dict where
@@ -48,10 +57,7 @@ def _parse_pip_constraint_for_virtualenv(
If it can't get an exact version, the real constraint will be handled by the
{macos|windows}.setup_python function.
"""
assert len(dependency_constraint_flags) in {0, 2}
if len(dependency_constraint_flags) == 2:
assert dependency_constraint_flags[0] == "-c"
constraint_path = Path(dependency_constraint_flags[1])
if constraint_path:
assert constraint_path.exists()
with constraint_path.open(encoding="utf-8") as constraint_file:
for line_ in constraint_file:
@@ -84,7 +90,7 @@ def virtualenv(
version: str,
python: Path,
venv_path: Path,
dependency_constraint_flags: Sequence[PathOrStr],
dependency_constraint: Path | None,
*,
use_uv: bool,
) -> dict[str, str]:
@@ -103,7 +109,7 @@ def virtualenv(
call("uv", "venv", venv_path, "--python", python)
else:
virtualenv_app = _ensure_virtualenv(version)
pip_constraint = _parse_pip_constraint_for_virtualenv(dependency_constraint_flags)
pip_constraint = _parse_pip_constraint_for_virtualenv(dependency_constraint)
additional_flags = [f"--pip={pip_constraint}", "--no-setuptools", "--no-wheel"]
# Using symlinks to pre-installed seed packages is really the fastest way to get a virtual