From b7130867638989b7f4a2921ca893c7e457bb8551 Mon Sep 17 00:00:00 2001 From: Matthieu Darbois Date: Mon, 27 May 2024 23:04:25 +0200 Subject: [PATCH] chore: better determinism for the test virtual environment (#1838) * chore: better determinism for the test virtual environment * review: apply suggestions --- cibuildwheel/linux.py | 9 ++++++++- cibuildwheel/macos.py | 17 +++++++++-------- cibuildwheel/windows.py | 16 ++++++++-------- 3 files changed, 25 insertions(+), 17 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 1fc68612..b245d1cd 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -8,6 +8,8 @@ from dataclasses import dataclass from pathlib import Path, PurePath, PurePosixPath from typing import OrderedDict, Tuple +from packaging.version import Version + from ._compat.typing import assert_never from .architecture import Architecture from .logger import log @@ -332,7 +334,12 @@ def build_in_container( ) venv_dir = testing_temp_dir / "venv" - container.call(["python", "-m", "virtualenv", "--no-download", venv_dir], env=env) + # Use embedded dependencies from virtualenv to ensure determinism + venv_args = ["--no-periodic-update", "--pip=embed"] + # In Python<3.12, setuptools & wheel are installed as well + if Version(config.version) < Version("3.12"): + venv_args.extend(("--setuptools=embed", "--wheel=embed")) + container.call(["python", "-m", "virtualenv", *venv_args, venv_dir], env=env) virtualenv_env = env.copy() virtualenv_env["PATH"] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}" diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 3df23961..c54b83b4 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -15,6 +15,7 @@ from pathlib import Path from typing import Literal, Tuple from filelock import FileLock +from packaging.version import Version from ._compat.typing import assert_never from .architecture import Architecture @@ -359,6 +360,7 @@ def build(options: Options, tmp_path: Path) -> None: build_options.environment, build_frontend.name, ) + pip_version = get_pip_version(env) compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) if compatible_wheel: @@ -384,7 +386,7 @@ def build(options: Options, tmp_path: Path) -> None: extra_flags += build_frontend.args build_env = env.copy() - build_env["VIRTUALENV_PIP"] = get_pip_version(env) + build_env["VIRTUALENV_PIP"] = pip_version if build_options.dependency_constraints: constraint_path = build_options.dependency_constraints.get_for_python_version( config.version @@ -560,9 +562,12 @@ def build(options: Options, tmp_path: Path) -> None: call_with_arch = functools.partial(call, *arch_prefix) shell_with_arch = functools.partial(call, *arch_prefix, "/bin/sh", "-c") - # Use --no-download to ensure determinism by using seed libraries - # built into virtualenv - call_with_arch("python", "-m", "virtualenv", "--no-download", venv_dir, env=env) + # Use pip version from the initial env to ensure determinism + venv_args = ["--no-periodic-update", f"--pip={pip_version}"] + # In Python<3.12, setuptools & wheel are installed as well, use virtualenv embedded ones + if Version(config.version) < Version("3.12"): + venv_args.extend(("--setuptools=embed", "--wheel=embed")) + call_with_arch("python", "-m", "virtualenv", *venv_args, venv_dir, env=env) virtualenv_env = env.copy() virtualenv_env["PATH"] = os.pathsep.join( @@ -575,10 +580,6 @@ def build(options: Options, tmp_path: Path) -> None: # check that we are using the Python from the virtual environment call_with_arch("which", "python", env=virtualenv_env) - # TODO remove me once virtualenv provides pip>=24.1b1 - if config.version == "3.13": - call("python", "-m", "pip", "install", "pip>=24.1b1", env=virtualenv_env) - if build_options.before_test: before_test_prepared = prepare_command( build_options.before_test, diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index f2ddc012..c233a735 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -387,6 +387,7 @@ def build(options: Options, tmp_path: Path) -> None: build_options.environment, build_frontend.name, ) + pip_version = get_pip_version(env) compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) if compatible_wheel: @@ -415,7 +416,7 @@ def build(options: Options, tmp_path: Path) -> None: extra_flags += build_frontend.args build_env = env.copy() - build_env["VIRTUALENV_PIP"] = get_pip_version(env) + build_env["VIRTUALENV_PIP"] = pip_version if build_options.dependency_constraints: constraints_path = build_options.dependency_constraints.get_for_python_version( config.version @@ -511,9 +512,12 @@ def build(options: Options, tmp_path: Path) -> None: call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env) venv_dir = identifier_tmp_dir / "venv-test" - # Use --no-download to ensure determinism by using seed libraries - # built into virtualenv - call("python", "-m", "virtualenv", "--no-download", venv_dir, env=env) + # Use pip version from the initial env to ensure determinism + venv_args = ["--no-periodic-update", f"--pip={pip_version}"] + # In Python<3.12, setuptools & wheel are installed as well, use virtualenv embedded ones + if Version(config.version) < Version("3.12"): + venv_args.extend(("--setuptools=embed", "--wheel=embed")) + call("python", "-m", "virtualenv", *venv_args, venv_dir, env=env) virtualenv_env = env.copy() virtualenv_env["PATH"] = os.pathsep.join( @@ -526,10 +530,6 @@ def build(options: Options, tmp_path: Path) -> None: # check that we are using the Python from the virtual environment call("where", "python", env=virtualenv_env) - # TODO remove me once virtualenv provides pip>=24.1b1 - if config.version.startswith("3.13."): - call("python", "-m", "pip", "install", "--pre", "-U", "pip", env=virtualenv_env) - if build_options.before_test: before_test_prepared = prepare_command( build_options.before_test,