From 384c8d5c82400a9e01f6e54ae9ade3d3ef4cfa39 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sun, 9 Jun 2024 15:45:31 -0400 Subject: [PATCH] feat: build[uv] (#1856) --- .github/workflows/test.yml | 7 +- .pre-commit-config.yaml | 1 + bin/generate_schema.py | 8 +- cibuildwheel/linux.py | 55 ++++-- cibuildwheel/macos.py | 152 ++++++++++++----- cibuildwheel/pyodide.py | 2 +- .../resources/cibuildwheel.schema.json | 10 +- cibuildwheel/util.py | 158 ++++++++++++------ cibuildwheel/windows.py | 121 ++++++++------ docs/options.md | 31 +++- pyproject.toml | 2 + test/conftest.py | 13 +- test/test_dependency_versions.py | 8 +- test/test_macos_archs.py | 11 +- 14 files changed, 399 insertions(+), 180 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 31c6a4b0..ff15ec4b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -49,6 +49,8 @@ jobs: with: python-version: ${{ matrix.python_version }} + - uses: yezz123/setup-uv@v4 + # Install podman on this CI instance for podman tests on linux # Snippet from: https://github.com/redhat-actions/podman-login/blob/main/.github/workflows/example.yml - name: Install latest podman @@ -69,7 +71,7 @@ jobs: - name: Install dependencies run: | - python -m pip install ".[test]" + uv pip install --system ".[test]" - name: Generate a sample project run: | @@ -82,6 +84,7 @@ jobs: output-dir: wheelhouse env: CIBW_ARCHS_MACOS: x86_64 universal2 arm64 + CIBW_BUILD_FRONTEND: 'build[uv]' CIBW_FREE_THREADED_SUPPORT: 1 CIBW_PRERELEASE_PYTHONS: 1 @@ -161,7 +164,7 @@ jobs: with: python-version: "3.x" - name: Install dependencies - run: python -m pip install ".[test]" + run: python -m pip install ".[test,uv]" - name: Set up QEMU id: qemu diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fb3ac473..38899cef 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,6 +40,7 @@ repos: - types-jinja2 - types-pyyaml - types-requests + - uv - validate-pyproject - id: mypy name: mypy 3.12 diff --git a/bin/generate_schema.py b/bin/generate_schema.py index a1ac23c3..dee1aad3 100755 --- a/bin/generate_schema.py +++ b/bin/generate_schema.py @@ -48,19 +48,21 @@ properties: type: string_array build-frontend: default: default - description: Set the tool to use to build, either "pip" (default for now) or "build" + description: Set the tool to use to build, either "pip" (default for now), "build", or "build[uv]" oneOf: - - enum: [pip, build, default] + - enum: [pip, build, "build[uv]", default] - type: string pattern: '^pip; ?args:' - type: string pattern: '^build; ?args:' + - type: string + pattern: '^build\[uv\]; ?args:' - type: object additionalProperties: false required: [name] properties: name: - enum: [pip, build] + enum: [pip, build, "build[uv]"] args: type: array items: diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index c33c0de4..8e885cbc 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -196,6 +196,8 @@ def build_in_container( log.build_start(config.identifier) build_options = options.build_options(config.identifier) build_frontend = build_options.build_frontend or BuildFrontendConfig("pip") + use_uv = build_frontend.name == "build[uv]" and Version(config.version) >= Version("3.8") + pip = ["uv", "pip"] if use_uv else ["pip"] dependency_constraint_flags: list[PathOrStr] = [] @@ -229,13 +231,22 @@ def build_in_container( ) sys.exit(1) - which_pip = container.call(["which", "pip"], env=env, capture_output=True).strip() - if PurePosixPath(which_pip) != python_bin / "pip": - print( - "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", - file=sys.stderr, - ) - sys.exit(1) + if use_uv: + which_uv = container.call(["which", "uv"], env=env, capture_output=True).strip() + if not which_uv: + print( + "cibuildwheel: uv not found on PATH. You must use a supported manylinux or musllinux environment with uv.", + file=sys.stderr, + ) + sys.exit(1) + else: + which_pip = container.call(["which", "pip"], env=env, capture_output=True).strip() + if PurePosixPath(which_pip) != python_bin / "pip": + print( + "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", + file=sys.stderr, + ) + sys.exit(1) compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) if compatible_wheel: @@ -279,10 +290,12 @@ def build_in_container( ], env=env, ) - elif build_frontend.name == "build": + elif build_frontend.name == "build" or build_frontend.name == "build[uv]": if not 0 <= build_options.build_verbosity < 2: msg = f"build_verbosity {build_options.build_verbosity} is not supported for build frontend. Ignoring." log.warning(msg) + if use_uv: + extra_flags += ["--installer=uv"] container.call( [ "python", @@ -327,26 +340,32 @@ def build_in_container( # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. - container.call(["pip", "install", "virtualenv", *dependency_constraint_flags], env=env) + if not use_uv: + container.call( + ["pip", "install", "virtualenv", *dependency_constraint_flags], env=env + ) testing_temp_dir = PurePosixPath( container.call(["mktemp", "-d"], capture_output=True).strip() ) venv_dir = testing_temp_dir / "venv" - # Use embedded dependencies from virtualenv to ensure determinism - venv_args = ["--no-periodic-update", "--pip=embed"] - # In Python<3.12, setuptools & wheel are installed as well - if Version(config.version) < Version("3.12"): - venv_args.extend(("--setuptools=embed", "--wheel=embed")) - container.call(["python", "-m", "virtualenv", *venv_args, venv_dir], env=env) + if use_uv: + container.call(["uv", "venv", venv_dir], env=env) + else: + # Use embedded dependencies from virtualenv to ensure determinism + venv_args = ["--no-periodic-update", "--pip=embed"] + # In Python<3.12, setuptools & wheel are installed as well + if Version(config.version) < Version("3.12"): + venv_args.extend(("--setuptools=embed", "--wheel=embed")) + container.call(["python", "-m", "virtualenv", *venv_args, venv_dir], env=env) virtualenv_env = env.copy() virtualenv_env["PATH"] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}" virtualenv_env["VIRTUAL_ENV"] = str(venv_dir) # TODO remove me once virtualenv provides pip>=24.1b1 - if config.version == "3.13": + if config.version == "3.13" and not use_uv: container.call(["pip", "install", "pip>=24.1b1"], env=virtualenv_env) if build_options.before_test: @@ -365,13 +384,13 @@ def build_in_container( # Let's just pick the first one. wheel_to_test = repaired_wheels[0] container.call( - ["pip", "install", str(wheel_to_test) + build_options.test_extras], + [*pip, "install", str(wheel_to_test) + build_options.test_extras], env=virtualenv_env, ) # Install any requirements to run the tests if build_options.test_requires: - container.call(["pip", "install", *build_options.test_requires], env=virtualenv_env) + container.call([*pip, "install", *build_options.test_requires], env=virtualenv_env) # Run the tests from a different directory test_command_prepared = prepare_command( diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 2906b584..890ca89e 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -30,9 +30,11 @@ from .util import ( BuildSelector, NonPlatformWheelError, call, + combine_constraints, detect_ci_provider, download, find_compatible_wheel, + find_uv, free_thread_enable_313, get_build_verbosity_extra_flags, get_pip_version, @@ -48,6 +50,7 @@ from .util import ( ) +@functools.lru_cache(maxsize=None) def get_macos_version() -> tuple[int, int]: """ Returns the macOS major/minor version, as a tuple, e.g. (10, 15) or (11, 0) @@ -59,9 +62,29 @@ def get_macos_version() -> tuple[int, int]: """ version_str, _, _ = platform.mac_ver() version = tuple(map(int, version_str.split(".")[:2])) + if (10, 15) < version < (11, 0): + # When built against an older macOS SDK, Python will report macOS 10.16 + # instead of the real version. + version_str = call( + sys.executable, + "-sS", + "-c", + "import platform; print(platform.mac_ver()[0])", + env={"SYSTEM_VERSION_COMPAT": "0"}, + capture_stdout=True, + ) + version = tuple(map(int, version_str.split(".")[:2])) return typing.cast(Tuple[int, int], version) +@functools.lru_cache(maxsize=None) +def get_test_macosx_deployment_target() -> str: + version = get_macos_version() + if version >= (11, 0): + return f"{version[0]}.0" + return f"{version[0]}.{version[1]}" + + def get_macos_sdks() -> list[str]: output = call("xcodebuild", "-showsdks", capture_stdout=True) return [m.group(1) for m in re.finditer(r"-sdk (macosx\S+)", output)] @@ -179,6 +202,14 @@ def setup_python( environment: ParsedEnvironment, build_frontend: BuildFrontendName, ) -> dict[str, str]: + if build_frontend == "build[uv]" and Version(python_configuration.version) < Version("3.8"): + build_frontend = "build" + + uv_path = find_uv() + use_uv = build_frontend == "build[uv]" and Version(python_configuration.version) >= Version( + "3.8" + ) + tmp.mkdir() implementation_id = python_configuration.identifier.split("-")[0] log.step(f"Installing Python {implementation_id}...") @@ -200,7 +231,11 @@ def setup_python( log.step("Setting up build environment...") venv_path = tmp / "venv" env = virtualenv( - python_configuration.version, base_python, venv_path, dependency_constraint_flags + python_configuration.version, + base_python, + venv_path, + dependency_constraint_flags, + use_uv=use_uv, ) venv_bin_path = venv_path / "bin" assert venv_bin_path.exists() @@ -217,32 +252,38 @@ def setup_python( # upgrade pip to the version matching our constraints # if necessary, reinstall it to ensure that it's available on PATH as 'pip' - call( - "python", - "-m", - "pip", - "install", - "--upgrade", - "pip", - *dependency_constraint_flags, - env=env, - cwd=venv_path, - ) + if build_frontend == "build[uv]": + assert uv_path is not None + pip = [str(uv_path), "pip"] + else: + pip = ["python", "-m", "pip"] + + if not use_uv: + call( + *pip, + "install", + "--upgrade", + "pip", + *dependency_constraint_flags, + env=env, + cwd=venv_path, + ) # Apply our environment after pip is ready env = environment.as_dictionary(prev_environment=env) # check what pip version we're on - assert (venv_bin_path / "pip").exists() - call("which", "pip", env=env) - call("pip", "--version", env=env) - which_pip = call("which", "pip", env=env, capture_stdout=True).strip() - if which_pip != str(venv_bin_path / "pip"): - print( - "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", - file=sys.stderr, - ) - sys.exit(1) + if not use_uv: + assert (venv_bin_path / "pip").exists() + call("which", "pip", env=env) + call("pip", "--version", env=env) + which_pip = call("which", "pip", env=env, capture_stdout=True).strip() + if which_pip != str(venv_bin_path / "pip"): + print( + "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", + file=sys.stderr, + ) + sys.exit(1) # check what Python version we're on call("which", "python", env=env) @@ -338,6 +379,18 @@ def setup_python( *dependency_constraint_flags, env=env, ) + elif build_frontend == "build[uv]": + assert uv_path is not None + call( + uv_path, + "pip", + "install", + "--upgrade", + "delocate", + "build[virtualenv, uv]", + *dependency_constraint_flags, + env=env, + ) else: assert_never(build_frontend) @@ -370,6 +423,14 @@ def build(options: Options, tmp_path: Path) -> None: for config in python_configurations: build_options = options.build_options(config.identifier) build_frontend = build_options.build_frontend or BuildFrontendConfig("pip") + use_uv = build_frontend.name == "build[uv]" and Version(config.version) >= Version( + "3.8" + ) + uv_path = find_uv() + if use_uv and uv_path is None: + msg = "uv not found" + raise AssertionError(msg) + pip = ["pip"] if not use_uv else [str(uv_path), "pip"] log.build_start(config.identifier) identifier_tmp_dir = tmp_path / config.identifier @@ -394,7 +455,8 @@ def build(options: Options, tmp_path: Path) -> None: build_options.environment, build_frontend.name, ) - pip_version = get_pip_version(env) + if not use_uv: + pip_version = get_pip_version(env) compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) if compatible_wheel: @@ -420,15 +482,14 @@ def build(options: Options, tmp_path: Path) -> None: extra_flags += build_frontend.args build_env = env.copy() - build_env["VIRTUALENV_PIP"] = pip_version + if not use_uv: + build_env["VIRTUALENV_PIP"] = pip_version if build_options.dependency_constraints: constraint_path = build_options.dependency_constraints.get_for_python_version( config.version ) - user_constraints = build_env.get("PIP_CONSTRAINT") - our_constraints = constraint_path.as_uri() - build_env["PIP_CONSTRAINT"] = " ".join( - c for c in [user_constraints, our_constraints] if c + combine_constraints( + build_env, constraint_path, identifier_tmp_dir if use_uv else None ) if build_frontend.name == "pip": @@ -446,10 +507,12 @@ def build(options: Options, tmp_path: Path) -> None: *extra_flags, env=build_env, ) - elif build_frontend.name == "build": + elif build_frontend.name == "build" or build_frontend.name == "build[uv]": if not 0 <= build_options.build_verbosity < 2: msg = f"build_verbosity {build_options.build_verbosity} is not supported for build frontend. Ignoring." log.warning(msg) + if use_uv: + extra_flags.append("--installer=uv") call( "python", "-m", @@ -579,15 +642,18 @@ def build(options: Options, tmp_path: Path) -> None: # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. - call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env) + if not use_uv: + call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env) venv_dir = identifier_tmp_dir / f"venv-test-{testing_arch}" arch_prefix = [] + uv_arch_args = [] if testing_arch != machine_arch: if machine_arch == "arm64" and testing_arch == "x86_64": # rosetta2 will provide the emulation with just the arch prefix. arch_prefix = ["arch", "-x86_64"] + uv_arch_args = ["--python-platform", "x86_64-apple-darwin"] else: msg = f"don't know how to emulate {testing_arch} on {machine_arch}" raise RuntimeError(msg) @@ -596,14 +662,20 @@ def build(options: Options, tmp_path: Path) -> None: call_with_arch = functools.partial(call, *arch_prefix) shell_with_arch = functools.partial(call, *arch_prefix, "/bin/sh", "-c") - # Use pip version from the initial env to ensure determinism - venv_args = ["--no-periodic-update", f"--pip={pip_version}"] - # In Python<3.12, setuptools & wheel are installed as well, use virtualenv embedded ones - if Version(config.version) < Version("3.12"): - venv_args.extend(("--setuptools=embed", "--wheel=embed")) - call_with_arch("python", "-m", "virtualenv", *venv_args, venv_dir, env=env) + if use_uv: + pip_install = functools.partial(call, *pip, "install", *uv_arch_args) + call("uv", "venv", venv_dir, "--python=python", env=env) + else: + pip_install = functools.partial(call_with_arch, *pip, "install") + # Use pip version from the initial env to ensure determinism + venv_args = ["--no-periodic-update", f"--pip={pip_version}"] + # In Python<3.12, setuptools & wheel are installed as well, use virtualenv embedded ones + if Version(config.version) < Version("3.12"): + venv_args.extend(("--setuptools=embed", "--wheel=embed")) + call_with_arch("python", "-m", "virtualenv", *venv_args, venv_dir, env=env) virtualenv_env = env.copy() + virtualenv_env["MACOSX_DEPLOYMENT_TARGET"] = get_test_macosx_deployment_target() virtualenv_env["PATH"] = os.pathsep.join( [ str(venv_dir / "bin"), @@ -641,18 +713,14 @@ def build(options: Options, tmp_path: Path) -> None: else: virtualenv_env_install_wheel = virtualenv_env - call_with_arch( - "pip", - "install", + pip_install( f"{repaired_wheel}{build_options.test_extras}", env=virtualenv_env_install_wheel, ) # test the wheel if build_options.test_requires: - call_with_arch( - "pip", - "install", + pip_install( *build_options.test_requires, env=virtualenv_env_install_wheel, ) diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index e9e75a5f..780e43ce 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -114,7 +114,7 @@ def setup_python( log.step("Setting up build environment...") venv_path = tmp / "venv" - env = virtualenv(python_configuration.version, base_python, venv_path, []) + env = virtualenv(python_configuration.version, base_python, venv_path, [], use_uv=False) venv_bin_path = venv_path / "bin" assert venv_bin_path.exists() env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" diff --git a/cibuildwheel/resources/cibuildwheel.schema.json b/cibuildwheel/resources/cibuildwheel.schema.json index 088b5d54..976751a5 100644 --- a/cibuildwheel/resources/cibuildwheel.schema.json +++ b/cibuildwheel/resources/cibuildwheel.schema.json @@ -96,12 +96,13 @@ }, "build-frontend": { "default": "default", - "description": "Set the tool to use to build, either \"pip\" (default for now) or \"build\"", + "description": "Set the tool to use to build, either \"pip\" (default for now), \"build\", or \"build[uv]\"", "oneOf": [ { "enum": [ "pip", "build", + "build[uv]", "default" ] }, @@ -113,6 +114,10 @@ "type": "string", "pattern": "^build; ?args:" }, + { + "type": "string", + "pattern": "^build\\[uv\\]; ?args:" + }, { "type": "object", "additionalProperties": false, @@ -123,7 +128,8 @@ "name": { "enum": [ "pip", - "build" + "build", + "build[uv]" ] }, "args": { diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index a2b853c2..ce6e60a5 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -16,7 +16,7 @@ import time import typing import urllib.request from collections import defaultdict -from collections.abc import Generator, Iterable, Mapping, Sequence +from collections.abc import Generator, Iterable, Mapping, MutableMapping, Sequence from dataclasses import dataclass from enum import Enum from functools import cached_property, lru_cache @@ -40,20 +40,22 @@ from .architecture import Architecture from .typing import PathOrStr, PlatformName __all__ = [ - "resources_dir", "MANYLINUX_ARCHS", - "call", - "shell", - "find_compatible_wheel", - "format_safe", - "prepare_command", - "get_build_verbosity_extra_flags", - "read_python_configs", - "selector_matches", - "strtobool", "cached_property", + "call", "chdir", + "combine_constraints", + "find_compatible_wheel", + "find_uv", + "format_safe", + "get_build_verbosity_extra_flags", + "prepare_command", + "read_python_configs", + "resources_dir", + "selector_matches", + "shell", "split_config_settings", + "strtobool", ] resources_dir: Final[Path] = Path(__file__).parent / "resources" @@ -202,7 +204,9 @@ def get_build_verbosity_extra_flags(level: int) -> list[str]: return [] -def split_config_settings(config_settings: str, frontend: Literal["pip", "build"]) -> list[str]: +def split_config_settings( + config_settings: str, frontend: Literal["pip", "build", "build[uv]"] +) -> list[str]: config_settings_list = shlex.split(config_settings) s = "s" if frontend == "pip" else "" return [f"--config-setting{s}={setting}" for setting in config_settings_list] @@ -431,7 +435,7 @@ class DependencyConstraints: return self.base_file_path.name -BuildFrontendName = Literal["pip", "build"] +BuildFrontendName = Literal["pip", "build", "build[uv]"] @dataclass(frozen=True) @@ -443,8 +447,8 @@ class BuildFrontendConfig: def from_config_string(config_string: str) -> BuildFrontendConfig: config_dict = parse_key_value_string(config_string, ["name"], ["args"]) name = " ".join(config_dict["name"]) - if name not in {"pip", "build"}: - msg = f"Unrecognised build frontend {name}, only 'pip' and 'build' are supported" + if name not in {"pip", "build", "build[uv]"}: + msg = f"Unrecognised build frontend {name}, only 'pip', 'build', and 'build[uv]' are supported" raise ValueError(msg) name = typing.cast(BuildFrontendName, name) @@ -689,45 +693,60 @@ def _parse_constraints_for_virtualenv( def virtualenv( - version: str, python: Path, venv_path: Path, dependency_constraint_flags: Sequence[PathOrStr] + version: str, + python: Path, + venv_path: Path, + dependency_constraint_flags: Sequence[PathOrStr], + *, + use_uv: bool, ) -> dict[str, str]: + """ + Create a virtual environment. If `use_uv` is True, + dependency_constraint_flags are ignored since nothing is installed in the + venv. Otherwise, pip is installed, and setuptools + wheel if Python < 3.12. + """ assert python.exists() - virtualenv_app = _ensure_virtualenv(version) - allowed_seed_packages = ["pip", "setuptools", "wheel"] - constraints = _parse_constraints_for_virtualenv( - allowed_seed_packages, dependency_constraint_flags - ) - additional_flags: list[str] = [] - for package in allowed_seed_packages: - if package in constraints: - additional_flags.append(f"--{package}={constraints[package]}") - else: - additional_flags.append(f"--no-{package}") - # Using symlinks to pre-installed seed packages is really the fastest way to get a virtual - # environment. The initial cost is a bit higher but reusing is much faster. - # Windows does not always allow symlinks so just disabling for now. - # Requires pip>=19.3 so disabling for "embed" because this means we don't know what's the - # version of pip that will end-up installed. - # c.f. https://virtualenv.pypa.io/en/latest/cli_interface.html#section-seeder - if ( - not IS_WIN - and constraints["pip"] != "embed" - and Version(constraints["pip"]) >= Version("19.3") - ): - additional_flags.append("--symlink-app-data") + if use_uv: + call("uv", "venv", venv_path, "--python", python) + else: + virtualenv_app = _ensure_virtualenv(version) + allowed_seed_packages = ["pip", "setuptools", "wheel"] + constraints = _parse_constraints_for_virtualenv( + allowed_seed_packages, dependency_constraint_flags + ) + additional_flags: list[str] = [] + for package in allowed_seed_packages: + if package in constraints: + additional_flags.append(f"--{package}={constraints[package]}") + else: + additional_flags.append(f"--no-{package}") + + # Using symlinks to pre-installed seed packages is really the fastest way to get a virtual + # environment. The initial cost is a bit higher but reusing is much faster. + # Windows does not always allow symlinks so just disabling for now. + # Requires pip>=19.3 so disabling for "embed" because this means we don't know what's the + # version of pip that will end-up installed. + # c.f. https://virtualenv.pypa.io/en/latest/cli_interface.html#section-seeder + if ( + not IS_WIN + and constraints["pip"] != "embed" + and Version(constraints["pip"]) >= Version("19.3") + ): + additional_flags.append("--symlink-app-data") + + call( + sys.executable, + "-sS", # just the stdlib, https://github.com/pypa/virtualenv/issues/2133#issuecomment-1003710125 + virtualenv_app, + "--activators=", + "--no-periodic-update", + *additional_flags, + "--python", + python, + venv_path, + ) - call( - sys.executable, - "-sS", # just the stdlib, https://github.com/pypa/virtualenv/issues/2133#issuecomment-1003710125 - virtualenv_app, - "--activators=", - "--no-periodic-update", - *additional_flags, - "--python", - python, - venv_path, - ) paths = [str(venv_path), str(venv_path / "Scripts")] if IS_WIN else [str(venv_path / "bin")] env = os.environ.copy() env["PATH"] = os.pathsep.join([*paths, env["PATH"]]) @@ -873,3 +892,42 @@ def parse_key_value_string( result[field_name] += values return dict(result) + + +def find_uv() -> Path | None: + # Prefer uv in our environment + with contextlib.suppress(ImportError, FileNotFoundError): + # pylint: disable-next=import-outside-toplevel + from uv import find_uv_bin + + return Path(find_uv_bin()) + + uv_on_path = shutil.which("uv") + return Path(uv_on_path) if uv_on_path else None + + +def combine_constraints( + env: MutableMapping[str, str], /, constraints_path: Path, tmp_dir: Path | None +) -> None: + """ + This will workaround a bug in pip<=21.1.1 or uv<=0.2.0 if a tmp_dir is given. + If set to None, this will use the modern URI method. + """ + + if tmp_dir: + if " " in str(constraints_path): + assert " " not in str(tmp_dir) + tmp_file = tmp_dir / "constraints.txt" + tmp_file.write_bytes(constraints_path.read_bytes()) + constraints_path = tmp_file + our_constraints = str(constraints_path) + else: + our_constraints = ( + constraints_path.as_uri() if " " in str(constraints_path) else str(constraints_path) + ) + + user_constraints = env.get("PIP_CONSTRAINT") + + env["UV_CONSTRAINT"] = env["PIP_CONSTRAINT"] = " ".join( + c for c in [our_constraints, user_constraints] if c + ) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index f5daa28c..263e8cf9 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -28,9 +28,11 @@ from .util import ( BuildSelector, NonPlatformWheelError, call, + combine_constraints, download, extract_zip, find_compatible_wheel, + find_uv, get_build_verbosity_extra_flags, get_pip_version, move_file, @@ -244,10 +246,19 @@ def setup_python( raise ValueError(msg) assert base_python.exists() + use_uv = build_frontend == "build[uv]" and Version(python_configuration.version) >= Version( + "3.8" + ) + uv_path = find_uv() + log.step("Setting up build environment...") venv_path = tmp / "venv" env = virtualenv( - python_configuration.version, base_python, venv_path, dependency_constraint_flags + python_configuration.version, + base_python, + venv_path, + dependency_constraint_flags, + use_uv=use_uv, ) # set up environment variables for run_with_env @@ -257,17 +268,18 @@ def setup_python( # upgrade pip to the version matching our constraints # if necessary, reinstall it to ensure that it's available on PATH as 'pip.exe' - call( - "python", - "-m", - "pip", - "install", - "--upgrade", - "pip", - *dependency_constraint_flags, - env=env, - cwd=venv_path, - ) + if not use_uv: + call( + "python", + "-m", + "pip", + "install", + "--upgrade", + "pip", + *dependency_constraint_flags, + env=env, + cwd=venv_path, + ) # update env with results from CIBW_ENVIRONMENT env = environment.as_dictionary(prev_environment=env) @@ -285,16 +297,17 @@ def setup_python( sys.exit(1) # check what pip version we're on - assert (venv_path / "Scripts" / "pip.exe").exists() - where_pip = call("where", "pip", env=env, capture_stdout=True).splitlines()[0].strip() - if where_pip.strip() != str(venv_path / "Scripts" / "pip.exe"): - print( - "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", - file=sys.stderr, - ) - sys.exit(1) + if not use_uv: + assert (venv_path / "Scripts" / "pip.exe").exists() + where_pip = call("where", "pip", env=env, capture_stdout=True).splitlines()[0].strip() + if where_pip.strip() != str(venv_path / "Scripts" / "pip.exe"): + print( + "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", + file=sys.stderr, + ) + sys.exit(1) - call("pip", "--version", env=env) + call("pip", "--version", env=env) log.step("Installing build tools...") if build_frontend == "build": @@ -306,6 +319,17 @@ def setup_python( *dependency_constraint_flags, env=env, ) + elif build_frontend == "build[uv]": + assert uv_path is not None + call( + uv_path, + "pip", + "install", + "--upgrade", + "build[virtualenv]", + *dependency_constraint_flags, + env=env, + ) if python_libs_base: # Set up the environment for various backends to enable cross-compilation @@ -340,6 +364,9 @@ def build(options: Options, tmp_path: Path) -> None: for config in python_configurations: build_options = options.build_options(config.identifier) build_frontend = build_options.build_frontend or BuildFrontendConfig("pip") + use_uv = build_frontend.name == "build[uv]" and Version(config.version) >= Version( + "3.8" + ) log.build_start(config.identifier) identifier_tmp_dir = tmp_path / config.identifier @@ -362,7 +389,8 @@ def build(options: Options, tmp_path: Path) -> None: build_options.environment, build_frontend.name, ) - pip_version = get_pip_version(env) + if not use_uv: + pip_version = get_pip_version(env) compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) if compatible_wheel: @@ -391,26 +419,14 @@ def build(options: Options, tmp_path: Path) -> None: extra_flags += build_frontend.args build_env = env.copy() - build_env["VIRTUALENV_PIP"] = pip_version + if not use_uv: + build_env["VIRTUALENV_PIP"] = pip_version + if build_options.dependency_constraints: constraints_path = build_options.dependency_constraints.get_for_python_version( config.version ) - # Bug in pip <= 21.1.3 - we can't have a space in the - # constraints file, and pip doesn't support drive letters - # in uhi. After probably pip 21.2, we can use uri. For - # now, use a temporary file. - if " " in str(constraints_path): - assert " " not in str(identifier_tmp_dir) - tmp_file = identifier_tmp_dir / "constraints.txt" - tmp_file.write_bytes(constraints_path.read_bytes()) - constraints_path = tmp_file - - our_constraints = str(constraints_path) - user_constraints = build_env.get("PIP_CONSTRAINT") - build_env["PIP_CONSTRAINT"] = " ".join( - c for c in [our_constraints, user_constraints] if c - ) + combine_constraints(build_env, constraints_path, identifier_tmp_dir) if build_frontend.name == "pip": extra_flags += get_build_verbosity_extra_flags(build_options.build_verbosity) @@ -427,10 +443,12 @@ def build(options: Options, tmp_path: Path) -> None: *extra_flags, env=build_env, ) - elif build_frontend.name == "build": + elif build_frontend.name == "build" or build_frontend.name == "build[uv]": if not 0 <= build_options.build_verbosity < 2: msg = f"build_verbosity {build_options.build_verbosity} is not supported for build frontend. Ignoring." log.warning(msg) + if use_uv: + extra_flags.append("--installer=uv") call( "python", "-m", @@ -484,15 +502,20 @@ def build(options: Options, tmp_path: Path) -> None: log.step("Testing wheel...") # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. - call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env) + if not use_uv: + call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env) + venv_dir = identifier_tmp_dir / "venv-test" - # Use pip version from the initial env to ensure determinism - venv_args = ["--no-periodic-update", f"--pip={pip_version}"] - # In Python<3.12, setuptools & wheel are installed as well, use virtualenv embedded ones - if Version(config.version) < Version("3.12"): - venv_args.extend(("--setuptools=embed", "--wheel=embed")) - call("python", "-m", "virtualenv", *venv_args, venv_dir, env=env) + if use_uv: + call("uv", "venv", venv_dir, "--python=python", env=env) + else: + # Use pip version from the initial env to ensure determinism + venv_args = ["--no-periodic-update", f"--pip={pip_version}"] + # In Python<3.12, setuptools & wheel are installed as well, use virtualenv embedded ones + if Version(config.version) < Version("3.12"): + venv_args.extend(("--setuptools=embed", "--wheel=embed")) + call("python", "-m", "virtualenv", *venv_args, venv_dir, env=env) virtualenv_env = env.copy() virtualenv_env["PATH"] = os.pathsep.join( @@ -514,9 +537,11 @@ def build(options: Options, tmp_path: Path) -> None: ) shell(before_test_prepared, env=virtualenv_env) + pip = ["uv", "pip"] if use_uv else ["pip"] + # install the wheel call( - "pip", + *pip, "install", str(repaired_wheel) + build_options.test_extras, env=virtualenv_env, @@ -524,7 +549,7 @@ def build(options: Options, tmp_path: Path) -> None: # test the wheel if build_options.test_requires: - call("pip", "install", *build_options.test_requires, env=virtualenv_env) + call(*pip, "install", *build_options.test_requires, env=virtualenv_env) # run the tests from a temp dir, with an absolute path in the command # (this ensures that Python runs the tests against the installed wheel diff --git a/docs/options.md b/docs/options.md index 8eaa6094..b322317a 100644 --- a/docs/options.md +++ b/docs/options.md @@ -624,7 +624,7 @@ This option can also be set using the [command-line option](#command-line) `--pr ## Build customization ### `CIBW_BUILD_FRONTEND` {: #build-frontend} -> Set the tool to use to build, either "pip" (default for now) or "build" +> Set the tool to use to build, either "pip" (default for now), "build", or "build[uv]" Options: @@ -636,17 +636,30 @@ Default: `pip` Choose which build frontend to use. Can either be "pip", which will run `python -m pip wheel`, or "build", which will run `python -m build --wheel`. +You can also use "build[uv]", which will use an external [uv][] everywhere +possible, both through `--installer=uv` passed to build, as well as when making +all build and test environments. This will generally speed up cibuildwheel. +Make sure you have an external uv on Windows and macOS, either by +pre-installing it, or installing cibuildwheel with the uv extra, +`cibuildwheel[uv]`. `uv` will not be used for Python 3.6 or Python 3.7. You +cannot use uv currently on Windows for ARM or for musllinux on s390x as +binaries are not provided by uv. Legacy dependencies like setuptools on Python +< 3.12 and pip are not installed if using uv. + +Pyodide ignores this setting, as only "build" is supported. + You can specify extra arguments to pass to `pip wheel` or `build` using the optional `args` option. !!! tip - Until v2.0.0, [pip] was the only way to build wheels, and is still the + Until v2.0.0, [pip][] was the only way to build wheels, and is still the default. However, we expect that at some point in the future, cibuildwheel - will change the default to [build], in line with the PyPA's recommendation. + will change the default to [build][], in line with the PyPA's recommendation. If you want to try `build` before this, you can use this option. [pip]: https://pip.pypa.io/en/stable/cli/pip_wheel/ [build]: https://github.com/pypa/build/ +[uv]: https://github.com/astral-sh/uv #### Examples @@ -661,6 +674,12 @@ optional `args` option. # supply an extra argument to 'pip wheel' CIBW_BUILD_FRONTEND: "pip; args: --no-build-isolation" + + # Use uv and build + CIBW_BUILD_FRONTEND: "build[uv]" + + # Use uv and build with an argument + CIBW_BUILD_FRONTEND: "build[uv]; args: --no-isolation" ``` !!! tab examples "pyproject.toml" @@ -675,6 +694,12 @@ optional `args` option. # supply an extra argument to 'pip wheel' build-frontend = { name = "pip", args = ["--no-build-isolation"] } + + # Use uv and build + build-frontend = "build[uv]" + + # Use uv and build with an argument + build-frontend = { name = "build[uv]", args = ["--no-isolation"] } ``` ### `CIBW_CONFIG_SETTINGS` {: #config-settings} diff --git a/pyproject.toml b/pyproject.toml index 23f8c8c4..22557937 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -78,6 +78,7 @@ test = [ "tomli_w", "validate-pyproject", ] +uv = ["uv"] [project.scripts] cibuildwheel = "cibuildwheel.__main__:main" @@ -176,6 +177,7 @@ messages_control.disable = [ "wrong-import-position", "unused-argument", # Handled by Ruff "broad-exception-raised", # Could be improved eventually + "consider-using-in", # MyPy can't narrow "in" ] [tool.ruff] diff --git a/test/conftest.py b/test/conftest.py index 08b0ee11..3fbf1d67 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -6,7 +6,7 @@ from typing import Generator import pytest -from cibuildwheel.util import detect_ci_provider +from cibuildwheel.util import detect_ci_provider, find_uv from .utils import EMULATED_ARCHS, platform @@ -31,12 +31,21 @@ def pytest_addoption(parser) -> None: @pytest.fixture( params=[{"CIBW_BUILD_FRONTEND": "pip"}, {"CIBW_BUILD_FRONTEND": "build"}], ids=["pip", "build"] ) -def build_frontend_env(request) -> dict[str, str]: +def build_frontend_env_nouv(request) -> dict[str, str]: if platform == "pyodide": pytest.skip("Can't use pip as build frontend for pyodide platform") + return request.param # type: ignore[no-any-return] +@pytest.fixture() +def build_frontend_env(build_frontend_env_nouv: dict[str, str]) -> dict[str, str]: + if build_frontend_env_nouv["CIBW_BUILD_FRONTEND"] == "build" and find_uv() is not None: + return {"CIBW_BUILD_FRONTEND": "build[uv]"} + + return build_frontend_env_nouv + + @pytest.fixture() def docker_cleanup() -> Generator[None, None, None]: def get_images() -> set[str]: diff --git a/test/test_dependency_versions.py b/test/test_dependency_versions.py index 17edd10f..defefc74 100644 --- a/test/test_dependency_versions.py +++ b/test/test_dependency_versions.py @@ -53,7 +53,7 @@ def get_versions_from_constraint_file(constraint_file): @pytest.mark.parametrize("python_version", ["3.6", "3.8", "3.10"]) -def test_pinned_versions(tmp_path, python_version, build_frontend_env): +def test_pinned_versions(tmp_path, python_version, build_frontend_env_nouv): if utils.platform == "linux": pytest.skip("linux doesn't pin individual tool versions, it pins manylinux images instead") if python_version == "3.6" and utils.platform == "macos" and platform.machine() == "arm64": @@ -79,7 +79,7 @@ def test_pinned_versions(tmp_path, python_version, build_frontend_env): add_env={ "CIBW_BUILD": build_pattern, "CIBW_ENVIRONMENT": cibw_environment_option, - **build_frontend_env, + **build_frontend_env_nouv, }, ) @@ -93,7 +93,7 @@ def test_pinned_versions(tmp_path, python_version, build_frontend_env): assert set(actual_wheels) == set(expected_wheels) -def test_dependency_constraints_file(tmp_path, build_frontend_env): +def test_dependency_constraints_file(tmp_path, build_frontend_env_nouv): if utils.platform == "linux": pytest.skip("linux doesn't pin individual tool versions, it pins manylinux images instead") @@ -130,7 +130,7 @@ def test_dependency_constraints_file(tmp_path, build_frontend_env): "CIBW_ENVIRONMENT": cibw_environment_option, "CIBW_DEPENDENCY_VERSIONS": str(constraints_file), "CIBW_SKIP": "cp36-*", - **build_frontend_env, + **build_frontend_env_nouv, }, ) diff --git a/test/test_macos_archs.py b/test/test_macos_archs.py index 31e57405..53d90d19 100644 --- a/test/test_macos_archs.py +++ b/test/test_macos_archs.py @@ -172,7 +172,7 @@ def test_universal2_testing_on_x86_64(tmp_path, capfd, skip_arm64_test): assert set(actual_wheels) == set(expected_wheels) -def test_universal2_testing_on_arm64(tmp_path, capfd): +def test_universal2_testing_on_arm64(build_frontend_env, tmp_path, capfd): # cibuildwheel should test the universal2 wheel on both x86_64 and arm64, when run on arm64 if utils.platform != "macos": pytest.skip("this test is only relevant to macos") @@ -187,15 +187,16 @@ def test_universal2_testing_on_arm64(tmp_path, capfd): add_env={ "CIBW_ARCHS": "universal2", # check that a native dependency is correctly installed, once per each testing arch - "CIBW_TEST_REQUIRES": "numpy", - "CIBW_TEST_COMMAND": '''python -c "import numpy, platform; print(f'running tests on {platform.machine()} with numpy {numpy.__version__}')"''', + "CIBW_TEST_REQUIRES": "--only-binary :all: pillow>=10.3", # pillow>=10.3 provides wheels for macOS 10.10, not 10.9 + "CIBW_TEST_COMMAND": '''python -c "import PIL, platform; print(f'running tests on {platform.machine()} with pillow {PIL.__version__}')"''', + **build_frontend_env, }, single_python=True, ) captured = capfd.readouterr() - assert "running tests on arm64" in captured.out - assert "running tests on x86_64" in captured.out + assert "running tests on arm64 with pillow" in captured.out + assert "running tests on x86_64 with pillow" in captured.out python_tag = "cp{}{}".format(*utils.SINGLE_PYTHON_VERSION) expected_wheels = [w for w in ALL_MACOS_WHEELS if python_tag in w and "universal2" in w]