feature: allow local runs on windows/macOS (#974)

* feature: allow local runs on windows/macOS

Cache python installations to a user cache folder using platformdirs.
The build environment is now a virtual environment to allow proper isolation.
Allows to run tests in parallel.

Co-authored-by: Joe Rickerby <joerick@mac.com>

README.md

@@ -59,8 +59,6 @@ Usage
 
 <sup>¹ [Requires emulation](https://cibuildwheel.readthedocs.io/en/stable/faq/#emulation), distributed separately. Other services may also support Linux ARM through emulation or third-party build hosts, but these are not tested in our CI.</sup><br>
 
-`cibuildwheel` is not intended to run on your development machine. Because it uses system Python from Python.org on macOS and Windows, it will try to install packages globally - not what you expect from a build tool! Instead, isolated CI services like those mentioned above are ideal. For Linux builds, it uses manylinux docker images, so those can be done locally for testing in a pinch.
-
 <!--intro-end-->
 
 Example setup

bin/run_tests.py

@@ -16,15 +16,13 @@ if __name__ == "__main__":
         unit_test_args += ["--run-docker"]
     subprocess.run(unit_test_args, check=True)
 
-    xdist_test_args = ["-n", "2"] if sys.platform.startswith("linux") else []
-
     # run the integration tests
     subprocess.run(
         [
             sys.executable,
             "-m",
             "pytest",
-            *xdist_test_args,
+            "--numprocesses=2",
             "-x",
             "--durations",
             "0",

bin/update_virtualenv.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+
+from __future__ import annotations
+
+import difflib
+import logging
+import subprocess
+from pathlib import Path
+from typing import NamedTuple
+
+import click
+import rich
+import tomli
+from packaging.version import InvalidVersion, Version
+from rich.logging import RichHandler
+from rich.syntax import Syntax
+
+from cibuildwheel.typing import Final
+
+log = logging.getLogger("cibw")
+
+# Looking up the dir instead of using utils.resources_dir
+# since we want to write to it.
+DIR: Final[Path] = Path(__file__).parent.parent.resolve()
+RESOURCES_DIR: Final[Path] = DIR / "cibuildwheel/resources"
+
+GET_VIRTUALENV_GITHUB: Final[str] = "https://github.com/pypa/get-virtualenv"
+GET_VIRTUALENV_URL_TEMPLATE: Final[
+    str
+] = f"{GET_VIRTUALENV_GITHUB}/blob/{{version}}/public/virtualenv.pyz?raw=true"
+
+
+class VersionTuple(NamedTuple):
+    version: Version
+    version_string: str
+
+
+def git_ls_remote_versions(url) -> list[VersionTuple]:
+    versions: list[VersionTuple] = []
+    tags = subprocess.run(
+        ["git", "ls-remote", "--tags", url], check=True, text=True, capture_output=True
+    ).stdout.splitlines()
+    for tag in tags:
+        _, ref = tag.split()
+        assert ref.startswith("refs/tags/")
+        version_string = ref[10:]
+        try:
+            version = Version(version_string)
+            if version.is_devrelease:
+                log.info(f"Ignoring development release '{version}'")
+                continue
+            if version.is_prerelease:
+                log.info(f"Ignoring pre-release '{version}'")
+                continue
+            versions.append(VersionTuple(version, version_string))
+        except InvalidVersion:
+            log.warning(f"Ignoring ref '{ref}'")
+    versions.sort(reverse=True)
+    return versions
+
+
+@click.command()
+@click.option("--force", is_flag=True)
+@click.option(
+    "--level", default="INFO", type=click.Choice(["WARNING", "INFO", "DEBUG"], case_sensitive=False)
+)
+def update_virtualenv(force: bool, level: str) -> None:
+
+    logging.basicConfig(
+        level="INFO",
+        format="%(message)s",
+        datefmt="[%X]",
+        handlers=[RichHandler(rich_tracebacks=True, markup=True)],
+    )
+    log.setLevel(level)
+
+    toml_file_path = RESOURCES_DIR / "virtualenv.toml"
+
+    original_toml = toml_file_path.read_text()
+    with toml_file_path.open("rb") as f:
+        loaded_file = tomli.load(f)
+    version = str(loaded_file["version"])
+    versions = git_ls_remote_versions(GET_VIRTUALENV_GITHUB)
+    if versions[0].version > Version(version):
+        version = versions[0].version_string
+
+    result_toml = (
+        f'version = "{version}"\n'
+        f'url = "{GET_VIRTUALENV_URL_TEMPLATE.format(version=version)}"\n'
+    )
+
+    rich.print()  # spacer
+
+    if original_toml == result_toml:
+        rich.print("[green]Check complete, virtualenv version unchanged.")
+        return
+
+    rich.print("virtualenv version updated.")
+    rich.print("Changes:")
+    rich.print()
+
+    toml_relpath = toml_file_path.relative_to(DIR).as_posix()
+    diff_lines = difflib.unified_diff(
+        original_toml.splitlines(keepends=True),
+        result_toml.splitlines(keepends=True),
+        fromfile=toml_relpath,
+        tofile=toml_relpath,
+    )
+    rich.print(Syntax("".join(diff_lines), "diff", theme="ansi_light"))
+    rich.print()
+
+    if force:
+        toml_file_path.write_text(result_toml)
+        rich.print("[green]TOML file updated.")
+    else:
+        rich.print("[yellow]File left unchanged. Use --force flag to update.")
+
+
+if __name__ == "__main__":
+    update_virtualenv()

cibuildwheel/__main__.py

@@ -1,7 +1,10 @@
 import argparse
 import os
+import shutil
 import sys
 import textwrap
+from pathlib import Path
+from tempfile import mkdtemp
 from typing import List, Set, Union
 
 import cibuildwheel
@@ -10,9 +13,15 @@ import cibuildwheel.macos
 import cibuildwheel.util
 import cibuildwheel.windows
 from cibuildwheel.architecture import Architecture, allowed_architectures_check
+from cibuildwheel.logger import log
 from cibuildwheel.options import CommandLineArguments, Options, compute_options
 from cibuildwheel.typing import PLATFORMS, PlatformName, assert_never
-from cibuildwheel.util import BuildSelector, Unbuffered, detect_ci_provider
+from cibuildwheel.util import (
+    CIBW_CACHE_PATH,
+    BuildSelector,
+    Unbuffered,
+    detect_ci_provider,
+)
 
 
 def main() -> None:
@@ -32,12 +41,11 @@ def main() -> None:
         choices=["auto", "linux", "macos", "windows"],
         default=os.environ.get("CIBW_PLATFORM", "auto"),
         help="""
-            Platform to build for. For "linux" you need docker running, on Mac
-            or Linux. For "macos", you need a Mac machine, and note that this
-            script is going to automatically install MacPython on your system,
-            so don't run on your development machine. For "windows", you need to
-            run in Windows, and it will build and test for all versions of
-            Python. Default: auto.
+            Platform to build for. Use this option to override the
+            auto-detected platform or to run cibuildwheel on your development
+            machine. Specifying "macos" or "windows" only works on that
+            operating system, but "linux" works on all three, as long as
+            Docker is installed. Default: auto.
         """,
     )
 
@@ -165,6 +173,9 @@ def main() -> None:
     # Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print'
     sys.stdout = Unbuffered(sys.stdout)  # type: ignore[assignment]
 
+    # create the cache dir before it gets printed & builds performed
+    CIBW_CACHE_PATH.mkdir(parents=True, exist_ok=True)
+
     print_preamble(platform=platform, options=options, identifiers=identifiers)
 
     try:
@@ -187,17 +198,23 @@ def main() -> None:
     if not output_dir.exists():
         output_dir.mkdir(parents=True)
 
-    with cibuildwheel.util.print_new_wheels(
-        "\n{n} wheels produced in {m:.0f} minutes:", output_dir
-    ):
-        if platform == "linux":
-            cibuildwheel.linux.build(options)
-        elif platform == "windows":
-            cibuildwheel.windows.build(options)
-        elif platform == "macos":
-            cibuildwheel.macos.build(options)
-        else:
-            assert_never(platform)
+    tmp_path = Path(mkdtemp(prefix="cibw-run-")).resolve(strict=True)
+    try:
+        with cibuildwheel.util.print_new_wheels(
+            "\n{n} wheels produced in {m:.0f} minutes:", output_dir
+        ):
+            if platform == "linux":
+                cibuildwheel.linux.build(options, tmp_path)
+            elif platform == "windows":
+                cibuildwheel.windows.build(options, tmp_path)
+            elif platform == "macos":
+                cibuildwheel.macos.build(options, tmp_path)
+            else:
+                assert_never(platform)
+    finally:
+        shutil.rmtree(tmp_path, ignore_errors=sys.platform.startswith("win"))
+        if tmp_path.exists():
+            log.warning(f"Can't delete temporary folder '{str(tmp_path)}'")
 
 
 def print_preamble(platform: str, options: Options, identifiers: List[str]) -> None:
@@ -218,6 +235,8 @@ def print_preamble(platform: str, options: Options, identifiers: List[str]) -> N
     print(f"  platform: {platform!r}")
     print(textwrap.indent(options.summary(identifiers), "  "))
 
+    print(f"Cache folder: {CIBW_CACHE_PATH}")
+
     warnings = detect_warnings(platform=platform, options=options, identifiers=identifiers)
     if warnings:
         print("\nWarnings:")

cibuildwheel/linux.py

@@ -303,7 +303,7 @@ def build_on_docker(
     log.step_end()
 
 
-def build(options: Options) -> None:
+def build(options: Options, tmp_path: Path) -> None:
     try:
         # check docker is installed
         subprocess.run(["docker", "--version"], check=True, stdout=subprocess.DEVNULL)

cibuildwheel/macos.py

@@ -4,20 +4,23 @@ import re
 import shutil
 import subprocess
 import sys
-import tempfile
 from pathlib import Path
 from typing import Any, Dict, List, NamedTuple, Sequence, Set, Tuple, cast
 
+from filelock import FileLock
+
 from .architecture import Architecture
 from .environment import ParsedEnvironment
 from .logger import log
 from .options import Options
 from .typing import Literal, PathOrStr, assert_never
 from .util import (
+    CIBW_CACHE_PATH,
     BuildFrontend,
     BuildSelector,
     NonPlatformWheelError,
     call,
+    detect_ci_provider,
     download,
     get_build_verbosity_extra_flags,
     get_pip_version,
@@ -26,6 +29,7 @@ from .util import (
     read_python_configs,
     shell,
     unwrap,
+    virtualenv,
 )
 
 
@@ -73,98 +77,75 @@ def get_python_configurations(
     return [c for c in python_configurations if build_selector(c.identifier)]
 
 
-SYMLINKS_DIR = Path("/tmp/cibw_bin")
+def install_cpython(tmp: Path, version: str, url: str) -> Path:
+    installation_path = Path(f"/Library/Frameworks/Python.framework/Versions/{version}")
+    with FileLock(CIBW_CACHE_PATH / f"cpython{version}.lock"):
+        installed_system_packages = call("pkgutil", "--pkgs", capture_stdout=True).splitlines()
+        # if this version of python isn't installed, get it from python.org and install
+        python_package_identifier = f"org.python.Python.PythonFramework-{version}"
+        if python_package_identifier not in installed_system_packages:
+            if detect_ci_provider() is None:
+                # if running locally, we don't want to install CPython with sudo
+                # let the user know & provide a link to the installer
+                print(
+                    f"Error: CPython {version} is not installed.\n"
+                    "cibuildwheel will not perform system-wide installs when running outside of CI.\n"
+                    f"To build locally, install CPython {version} on this machine, or, disable this version of Python using CIBW_SKIP=cp{version.replace('.', '')}-macosx_*\n"
+                    f"\nDownload link: {url}",
+                    file=sys.stderr,
+                )
+                raise SystemExit(1)
+            pkg_path = tmp / "Python.pkg"
+            # download the pkg
+            download(url, pkg_path)
+            # install
+            call("sudo", "installer", "-pkg", pkg_path, "-target", "/")
+            pkg_path.unlink()
+            env = os.environ.copy()
+            env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
+            call(installation_path / "bin" / "python3", install_certifi_script, env=env)
+
+    return installation_path / "bin" / "python3"
 
 
-def make_symlinks(installation_bin_path: Path, python_executable: str, pip_executable: str) -> None:
-    assert (installation_bin_path / python_executable).exists()
-
-    # Python bin folders on Mac don't symlink `python3` to `python`, and neither
-    # does PyPy for `pypy` or `pypy3`, so we do that so `python` and `pip` always
-    # point to the active configuration.
-    if SYMLINKS_DIR.exists():
-        shutil.rmtree(SYMLINKS_DIR)
-    SYMLINKS_DIR.mkdir(parents=True)
-
-    (SYMLINKS_DIR / "python").symlink_to(installation_bin_path / python_executable)
-    (SYMLINKS_DIR / "python-config").symlink_to(
-        installation_bin_path / (python_executable + "-config")
-    )
-    (SYMLINKS_DIR / "pip").symlink_to(installation_bin_path / pip_executable)
-
-
-def install_cpython(version: str, url: str) -> Path:
-    installed_system_packages = call("pkgutil", "--pkgs", capture_stdout=True).splitlines()
-
-    # if this version of python isn't installed, get it from python.org and install
-    python_package_identifier = f"org.python.Python.PythonFramework-{version}"
-    python_executable = "python3"
-    installation_bin_path = Path(f"/Library/Frameworks/Python.framework/Versions/{version}/bin")
-
-    if python_package_identifier not in installed_system_packages:
-        # download the pkg
-        download(url, Path("/tmp/Python.pkg"))
-        # install
-        call("sudo", "installer", "-pkg", "/tmp/Python.pkg", "-target", "/")
-        env = os.environ.copy()
-        env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
-        call(str(installation_bin_path / python_executable), str(install_certifi_script), env=env)
-
-    pip_executable = "pip3"
-    make_symlinks(installation_bin_path, python_executable, pip_executable)
-
-    return installation_bin_path
-
-
-def install_pypy(version: str, url: str) -> Path:
+def install_pypy(tmp: Path, version: str, url: str) -> Path:
     pypy_tar_bz2 = url.rsplit("/", 1)[-1]
     extension = ".tar.bz2"
     assert pypy_tar_bz2.endswith(extension)
-    pypy_base_filename = pypy_tar_bz2[: -len(extension)]
-    installation_path = Path("/tmp") / pypy_base_filename
-    if not installation_path.exists():
-        downloaded_tar_bz2 = Path("/tmp") / pypy_tar_bz2
-        download(url, downloaded_tar_bz2)
-        call("tar", "-C", "/tmp", "-xf", downloaded_tar_bz2)
-
-    installation_bin_path = installation_path / "bin"
-    python_executable = "pypy3"
-    pip_executable = "pip3"
-    make_symlinks(installation_bin_path, python_executable, pip_executable)
-
-    return installation_bin_path
+    installation_path = CIBW_CACHE_PATH / pypy_tar_bz2[: -len(extension)]
+    with FileLock(str(installation_path) + ".lock"):
+        if not installation_path.exists():
+            downloaded_tar_bz2 = tmp / pypy_tar_bz2
+            download(url, downloaded_tar_bz2)
+            installation_path.parent.mkdir(parents=True, exist_ok=True)
+            call("tar", "-C", installation_path.parent, "-xf", downloaded_tar_bz2)
+            downloaded_tar_bz2.unlink()
+    return installation_path / "bin" / "pypy3"
 
 
 def setup_python(
+    tmp: Path,
     python_configuration: PythonConfiguration,
     dependency_constraint_flags: Sequence[PathOrStr],
     environment: ParsedEnvironment,
     build_frontend: BuildFrontend,
 ) -> Dict[str, str]:
-
+    tmp.mkdir()
     implementation_id = python_configuration.identifier.split("-")[0]
     log.step(f"Installing Python {implementation_id}...")
-
     if implementation_id.startswith("cp"):
-        installation_bin_path = install_cpython(
-            python_configuration.version, python_configuration.url
-        )
+        base_python = install_cpython(tmp, python_configuration.version, python_configuration.url)
     elif implementation_id.startswith("pp"):
-        installation_bin_path = install_pypy(python_configuration.version, python_configuration.url)
+        base_python = install_pypy(tmp, python_configuration.version, python_configuration.url)
     else:
         raise ValueError("Unknown Python implementation")
+    assert base_python.exists()
 
     log.step("Setting up build environment...")
-
-    env = os.environ.copy()
-    env["PATH"] = os.pathsep.join(
-        [
-            str(SYMLINKS_DIR),
-            str(installation_bin_path),
-            env["PATH"],
-        ]
-    )
-
+    venv_path = tmp / "venv"
+    env = virtualenv(base_python, venv_path, dependency_constraint_flags)
+    venv_bin_path = venv_path / "bin"
+    assert venv_bin_path.exists()
     # Fix issue with site.py setting the wrong `sys.prefix`, `sys.exec_prefix`,
     # `sys.path`, ... for PyPy: https://foss.heptapod.net/pypy/pypy/issues/3175
     # Also fix an issue with the shebang of installed scripts inside the
@@ -176,13 +157,6 @@ def setup_python(
     # we version pip ourselves, so we don't care about pip version checking
     env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
 
-    # Install pip
-
-    requires_reinstall = not (installation_bin_path / "pip").exists()
-    if requires_reinstall:
-        # maybe pip isn't installed at all. ensurepip resolves that.
-        call("python", "-m", "ensurepip", env=env, cwd="/tmp")
-
     # upgrade pip to the version matching our constraints
     # if necessary, reinstall it to ensure that it's available on PATH as 'pip'
     call(
@@ -190,22 +164,22 @@ def setup_python(
         "-m",
         "pip",
         "install",
-        "--force-reinstall" if requires_reinstall else "--upgrade",
+        "--upgrade",
         "pip",
         *dependency_constraint_flags,
         env=env,
-        cwd="/tmp",
+        cwd=venv_path,
     )
 
     # Apply our environment after pip is ready
     env = environment.as_dictionary(prev_environment=env)
 
     # check what pip version we're on
-    assert (installation_bin_path / "pip").exists()
+    assert (venv_bin_path / "pip").exists()
     call("which", "pip", env=env)
     call("pip", "--version", env=env)
     which_pip = call("which", "pip", env=env, capture_stdout=True).strip()
-    if which_pip != "/tmp/cibw_bin/pip":
+    if which_pip != str(venv_bin_path / "pip"):
         print(
             "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.",
             file=sys.stderr,
@@ -216,7 +190,7 @@ def setup_python(
     call("which", "python", env=env)
     call("python", "--version", env=env)
     which_python = call("which", "python", env=env, capture_stdout=True).strip()
-    if which_python != "/tmp/cibw_bin/python":
+    if which_python != str(venv_bin_path / "python"):
         print(
             "cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.",
             file=sys.stderr,
@@ -295,11 +269,7 @@ def setup_python(
     return env
 
 
-def build(options: Options) -> None:
-    temp_dir = Path(tempfile.mkdtemp(prefix="cibuildwheel"))
-    built_wheel_dir = temp_dir / "built_wheel"
-    repaired_wheel_dir = temp_dir / "repaired_wheel"
-
+def build(options: Options, tmp_path: Path) -> None:
     python_configurations = get_python_configurations(
         options.globals.build_selector, options.globals.architectures
     )
@@ -321,6 +291,11 @@ def build(options: Options) -> None:
             build_options = options.build_options(config.identifier)
             log.build_start(config.identifier)
 
+            identifier_tmp_dir = tmp_path / config.identifier
+            identifier_tmp_dir.mkdir()
+            built_wheel_dir = identifier_tmp_dir / "built_wheel"
+            repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel"
+
             config_is_arm64 = config.identifier.endswith("arm64")
             config_is_universal2 = config.identifier.endswith("universal2")
 
@@ -332,6 +307,7 @@ def build(options: Options) -> None:
                 ]
 
             env = setup_python(
+                identifier_tmp_dir / "build",
                 config,
                 dependency_constraint_flags,
                 build_options.environment,
@@ -346,9 +322,7 @@ def build(options: Options) -> None:
                 shell(before_build_prepared, env=env)
 
             log.step("Building wheel...")
-            if built_wheel_dir.exists():
-                shutil.rmtree(built_wheel_dir)
-            built_wheel_dir.mkdir(parents=True)
+            built_wheel_dir.mkdir()
 
             verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity)
 
@@ -390,9 +364,7 @@ def build(options: Options) -> None:
 
             built_wheel = next(built_wheel_dir.glob("*.whl"))
 
-            if repaired_wheel_dir.exists():
-                shutil.rmtree(repaired_wheel_dir)
-            repaired_wheel_dir.mkdir(parents=True)
+            repaired_wheel_dir.mkdir()
 
             if built_wheel.name.endswith("none-any.whl"):
                 raise NonPlatformWheelError()
@@ -479,7 +451,8 @@ def build(options: Options) -> None:
                     # set up a virtual environment to install and test from, to make sure
                     # there are no dependencies that were pulled in at build time.
                     call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env)
-                    venv_dir = Path(tempfile.mkdtemp())
+
+                    venv_dir = identifier_tmp_dir / "venv-test"
 
                     arch_prefix = []
                     if testing_arch != machine_arch:
@@ -548,11 +521,12 @@ def build(options: Options) -> None:
                         test_command_prepared, cwd=os.environ["HOME"], env=virtualenv_env
                     )
 
-                    # clean up
-                    shutil.rmtree(venv_dir)
-
             # we're all done here; move it to output (overwrite existing)
             shutil.move(str(repaired_wheel), build_options.output_dir)
+
+            # clean up
+            shutil.rmtree(identifier_tmp_dir)
+
             log.build_end()
     except subprocess.CalledProcessError as error:
         log.step_end_with_error(

cibuildwheel/resources/virtualenv.toml

@@ -0,0 +1,2 @@
+version = "20.13.0"
+url = "https://github.com/pypa/get-virtualenv/blob/20.13.0/public/virtualenv.pyz?raw=true"

cibuildwheel/util.py

@@ -11,6 +11,7 @@ import textwrap
 import time
 import urllib.request
 from enum import Enum
+from functools import lru_cache
 from pathlib import Path
 from time import sleep
 from typing import (
@@ -21,6 +22,7 @@ from typing import (
     List,
     NamedTuple,
     Optional,
+    Sequence,
     TextIO,
     cast,
     overload,
@@ -29,10 +31,13 @@ from typing import (
 import bracex
 import certifi
 import tomli
+from filelock import FileLock
+from packaging.requirements import InvalidRequirement, Requirement
 from packaging.specifiers import SpecifierSet
 from packaging.version import Version
+from platformdirs import user_cache_path
 
-from .typing import Literal, PathOrStr, PlatformName
+from cibuildwheel.typing import Literal, PathOrStr, PlatformName
 
 resources_dir = Path(__file__).parent / "resources"
 
@@ -59,6 +64,9 @@ MUSLLINUX_ARCHS = (
     "s390x",
 )
 
+DEFAULT_CIBW_CACHE_PATH = user_cache_path(appname="cibuildwheel", appauthor="pypa")
+CIBW_CACHE_PATH = Path(os.environ.get("CIBW_CACHE_PATH", DEFAULT_CIBW_CACHE_PATH)).resolve()
+
 IS_WIN = sys.platform.startswith("win")
 
 
@@ -451,3 +459,104 @@ def get_pip_version(env: Dict[str, str]) -> str:
         if version.startswith("pip==")
     )
     return pip_version
+
+
+@lru_cache(maxsize=None)
+def _ensure_virtualenv() -> Path:
+    input_file = resources_dir / "virtualenv.toml"
+    with input_file.open("rb") as f:
+        loaded_file = tomli.load(f)
+    version = str(loaded_file["version"])
+    url = str(loaded_file["url"])
+    path = CIBW_CACHE_PATH / f"virtualenv-{version}.pyz"
+    with FileLock(str(path) + ".lock"):
+        if not path.exists():
+            download(url, path)
+    return path
+
+
+def _parse_constraints_for_virtualenv(
+    dependency_constraint_flags: Sequence[PathOrStr],
+) -> Dict[str, str]:
+    """
+    Parses the constraints file referenced by `dependency_constraint_flags` and returns a dict where
+    the key is the package name, and the value is the constraint version.
+    If a package version cannot be found, its value is "embed" meaning that virtualenv will install
+    its bundled version, already available locally.
+    The function does not try to be too smart and just handles basic constraints.
+    If it can't get an exact version, the real constraint will be handled by the
+    {macos|windows}.setup_python function.
+    """
+    assert len(dependency_constraint_flags) in {0, 2}
+    packages = ["pip", "setuptools", "wheel"]
+    constraints_dict = {package: "embed" for package in packages}
+    if len(dependency_constraint_flags) == 2:
+        assert dependency_constraint_flags[0] == "-c"
+        constraint_path = Path(dependency_constraint_flags[1])
+        assert constraint_path.exists()
+        with constraint_path.open() as constraint_file:
+            for line in constraint_file:
+                line = line.strip()
+                if len(line) == 0:
+                    continue
+                if line.startswith("#"):
+                    continue
+                try:
+                    requirement = Requirement(line)
+                    package = requirement.name
+                    if (
+                        package not in packages
+                        or requirement.url is not None
+                        or requirement.marker is not None
+                        or len(requirement.extras) != 0
+                        or len(requirement.specifier) != 1
+                    ):
+                        continue
+                    specifier = next(iter(requirement.specifier))
+                    if specifier.operator != "==":
+                        continue
+                    constraints_dict[package] = specifier.version
+                except InvalidRequirement:
+                    continue
+    return constraints_dict
+
+
+def virtualenv(
+    python: Path, venv_path: Path, dependency_constraint_flags: Sequence[PathOrStr]
+) -> Dict[str, str]:
+    assert python.exists()
+    virtualenv_app = _ensure_virtualenv()
+    constraints = _parse_constraints_for_virtualenv(dependency_constraint_flags)
+    additional_flags = [f"--{package}={version}" for package, version in constraints.items()]
+
+    # Using symlinks to pre-installed seed packages is really the fastest way to get a virtual
+    # environment. The initial cost is a bit higher but reusing is much faster.
+    # Windows does not always allow symlinks so just disabling for now.
+    # Requires pip>=19.3 so disabling for "embed" because this means we don't know what's the
+    # version of pip that will end-up installed.
+    # c.f. https://virtualenv.pypa.io/en/latest/cli_interface.html#section-seeder
+    if (
+        not IS_WIN
+        and constraints["pip"] != "embed"
+        and Version(constraints["pip"]) >= Version("19.3")
+    ):
+        additional_flags.append("--symlink-app-data")
+
+    call(
+        sys.executable,
+        "-sS",  # just the stdlib, https://github.com/pypa/virtualenv/issues/2133#issuecomment-1003710125
+        virtualenv_app,
+        "--activators=",
+        "--no-periodic-update",
+        *additional_flags,
+        "--python",
+        python,
+        venv_path,
+    )
+    if IS_WIN:
+        paths = [str(venv_path), str(venv_path / "Scripts")]
+    else:
+        paths = [str(venv_path / "bin")]
+    env = os.environ.copy()
+    env["PATH"] = os.pathsep.join(paths + [env["PATH"]])
+    return env

cibuildwheel/windows.py

@@ -2,11 +2,12 @@ import os
 import shutil
 import subprocess
 import sys
-import tempfile
+from functools import lru_cache
 from pathlib import Path
 from typing import Dict, List, NamedTuple, Optional, Sequence, Set
 from zipfile import ZipFile
 
+from filelock import FileLock
 from packaging.version import Version
 
 from .architecture import Architecture
@@ -15,6 +16,7 @@ from .logger import log
 from .options import Options
 from .typing import PathOrStr, assert_never
 from .util import (
+    CIBW_CACHE_PATH,
     BuildFrontend,
     BuildSelector,
     NonPlatformWheelError,
@@ -25,12 +27,11 @@ from .util import (
     prepare_command,
     read_python_configs,
     shell,
+    virtualenv,
 )
 
-CIBW_INSTALL_PATH = Path("C:\\cibw")
 
-
-def get_nuget_args(version: str, arch: str) -> List[str]:
+def get_nuget_args(version: str, arch: str, output_directory: Path) -> List[str]:
     platform_suffix = {"32": "x86", "64": "", "ARM64": "arm64"}
     python_name = "python" + platform_suffix[arch]
     return [
@@ -40,7 +41,7 @@ def get_nuget_args(version: str, arch: str) -> List[str]:
         "-FallbackSource",
         "https://api.nuget.org/v3/index.json",
         "-OutputDirectory",
-        str(CIBW_INSTALL_PATH / "python"),
+        str(output_directory),
     ]
 
 
@@ -77,82 +78,70 @@ def extract_zip(zip_src: Path, dest: Path) -> None:
         zip_.extractall(dest)
 
 
-def install_cpython(version: str, arch: str, nuget: Path) -> Path:
-    nuget_args = get_nuget_args(version, arch)
-    installation_path = Path(nuget_args[-1]) / (nuget_args[0] + "." + version) / "tools"
-    call(nuget, "install", *nuget_args)
-    # "python3" is not included in the vanilla nuget package,
-    # though it can be present if modified (like on Azure).
-    if not (installation_path / "python3.exe").exists():
-        (installation_path / "python3.exe").symlink_to(installation_path / "python.exe")
-    return installation_path
+@lru_cache(maxsize=None)
+def _ensure_nuget() -> Path:
+    nuget = CIBW_CACHE_PATH / "nuget.exe"
+    with FileLock(str(nuget) + ".lock"):
+        if not nuget.exists():
+            download("https://dist.nuget.org/win-x86-commandline/latest/nuget.exe", nuget)
+    return nuget
 
 
-def install_pypy(version: str, arch: str, url: str) -> Path:
+def install_cpython(version: str, arch: str) -> Path:
+    base_output_dir = CIBW_CACHE_PATH / "nuget-cpython"
+    nuget_args = get_nuget_args(version, arch, base_output_dir)
+    installation_path = base_output_dir / (nuget_args[0] + "." + version) / "tools"
+    with FileLock(str(base_output_dir) + f"-{version}-{arch}.lock"):
+        if not installation_path.exists():
+            nuget = _ensure_nuget()
+            call(nuget, "install", *nuget_args)
+    return installation_path / "python.exe"
+
+
+def install_pypy(tmp: Path, arch: str, url: str) -> Path:
     assert arch == "64" and "win64" in url
     # Inside the PyPy zip file is a directory with the same name
     zip_filename = url.rsplit("/", 1)[-1]
     extension = ".zip"
     assert zip_filename.endswith(extension)
-    installation_path = CIBW_INSTALL_PATH / zip_filename[: -len(extension)]
-    if not installation_path.exists():
-        pypy_zip = CIBW_INSTALL_PATH / zip_filename
-        download(url, pypy_zip)
-        # Extract to the parent directory because the zip file still contains a directory
-        extract_zip(pypy_zip, installation_path.parent)
-    return installation_path
+    installation_path = CIBW_CACHE_PATH / zip_filename[: -len(extension)]
+    with FileLock(str(installation_path) + ".lock"):
+        if not installation_path.exists():
+            pypy_zip = tmp / zip_filename
+            download(url, pypy_zip)
+            # Extract to the parent directory because the zip file still contains a directory
+            extract_zip(pypy_zip, installation_path.parent)
+    return installation_path / "python.exe"
 
 
 def setup_python(
+    tmp: Path,
     python_configuration: PythonConfiguration,
     dependency_constraint_flags: Sequence[PathOrStr],
     environment: ParsedEnvironment,
     build_frontend: BuildFrontend,
 ) -> Dict[str, str]:
-
-    nuget = CIBW_INSTALL_PATH / "nuget.exe"
-    if not nuget.exists():
-        log.step("Downloading nuget...")
-        download("https://dist.nuget.org/win-x86-commandline/latest/nuget.exe", nuget)
-
+    tmp.mkdir()
     implementation_id = python_configuration.identifier.split("-")[0]
     log.step(f"Installing Python {implementation_id}...")
-
     if implementation_id.startswith("cp"):
-        installation_path = install_cpython(
-            python_configuration.version, python_configuration.arch, nuget
-        )
+        base_python = install_cpython(python_configuration.version, python_configuration.arch)
     elif implementation_id.startswith("pp"):
         assert python_configuration.url is not None
-        installation_path = install_pypy(
-            python_configuration.version, python_configuration.arch, python_configuration.url
-        )
+        base_python = install_pypy(tmp, python_configuration.arch, python_configuration.url)
     else:
         raise ValueError("Unknown Python implementation")
-
-    assert (installation_path / "python.exe").exists()
+    assert base_python.exists()
 
     log.step("Setting up build environment...")
+    venv_path = tmp / "venv"
+    env = virtualenv(base_python, venv_path, dependency_constraint_flags)
 
-    # set up PATH and environment variables for run_with_env
-    env = os.environ.copy()
+    # set up environment variables for run_with_env
     env["PYTHON_VERSION"] = python_configuration.version
     env["PYTHON_ARCH"] = python_configuration.arch
-    env["PATH"] = os.pathsep.join(
-        [str(installation_path), str(installation_path / "Scripts"), env["PATH"]]
-    )
     env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
 
-    log.step("Installing build tools...")
-
-    # Install pip
-
-    requires_reinstall = not (installation_path / "Scripts" / "pip.exe").exists()
-
-    if requires_reinstall:
-        # maybe pip isn't installed at all. ensurepip resolves that.
-        call("python", "-m", "ensurepip", env=env, cwd=CIBW_INSTALL_PATH)
-
     # pip older than 21.3 builds executables such as pip.exe for x64 platform.
     # The first re-install of pip updates pip module but builds pip.exe using
     # the old pip which still generates x64 executable. But the second
@@ -170,7 +159,7 @@ def setup_python(
             "pip",
             *dependency_constraint_flags,
             env=env,
-            cwd=CIBW_INSTALL_PATH,
+            cwd=venv_path,
         )
 
     # upgrade pip to the version matching our constraints
@@ -180,11 +169,11 @@ def setup_python(
         "-m",
         "pip",
         "install",
-        "--force-reinstall" if requires_reinstall else "--upgrade",
+        "--upgrade",
         "pip",
         *dependency_constraint_flags,
         env=env,
-        cwd=CIBW_INSTALL_PATH,
+        cwd=venv_path,
     )
 
     # update env with results from CIBW_ENVIRONMENT
@@ -195,7 +184,7 @@ def setup_python(
     call("python", "--version", env=env)
     call("python", "-c", "\"import struct; print(struct.calcsize('P') * 8)\"", env=env)
     where_python = call("where", "python", env=env, capture_stdout=True).splitlines()[0].strip()
-    if where_python != str(installation_path / "python.exe"):
+    if where_python != str(venv_path / "Scripts" / "python.exe"):
         print(
             "cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.",
             file=sys.stderr,
@@ -203,9 +192,9 @@ def setup_python(
         sys.exit(1)
 
     # check what pip version we're on
-    assert (installation_path / "Scripts" / "pip.exe").exists()
+    assert (venv_path / "Scripts" / "pip.exe").exists()
     where_pip = call("where", "pip", env=env, capture_stdout=True).splitlines()[0].strip()
-    if where_pip.strip() != str(installation_path / "Scripts" / "pip.exe"):
+    if where_pip.strip() != str(venv_path / "Scripts" / "pip.exe"):
         print(
             "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.",
             file=sys.stderr,
@@ -214,6 +203,7 @@ def setup_python(
 
     call("pip", "--version", env=env)
 
+    log.step("Installing build tools...")
     if build_frontend == "pip":
         call(
             "pip",
@@ -239,11 +229,7 @@ def setup_python(
     return env
 
 
-def build(options: Options) -> None:
-    temp_dir = Path(tempfile.mkdtemp(prefix="cibuildwheel"))
-    built_wheel_dir = temp_dir / "built_wheel"
-    repaired_wheel_dir = temp_dir / "repaired_wheel"
-
+def build(options: Options, tmp_path: Path) -> None:
     python_configurations = get_python_configurations(
         options.globals.build_selector, options.globals.architectures
     )
@@ -264,6 +250,11 @@ def build(options: Options) -> None:
             build_options = options.build_options(config.identifier)
             log.build_start(config.identifier)
 
+            identifier_tmp_dir = tmp_path / config.identifier
+            identifier_tmp_dir.mkdir()
+            built_wheel_dir = identifier_tmp_dir / "built_wheel"
+            repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel"
+
             dependency_constraint_flags: Sequence[PathOrStr] = []
             if build_options.dependency_constraints:
                 dependency_constraint_flags = [
@@ -273,6 +264,7 @@ def build(options: Options) -> None:
 
             # install Python
             env = setup_python(
+                identifier_tmp_dir / "build",
                 config,
                 dependency_constraint_flags,
                 build_options.environment,
@@ -288,9 +280,7 @@ def build(options: Options) -> None:
                 shell(before_build_prepared, env=env)
 
             log.step("Building wheel...")
-            if built_wheel_dir.exists():
-                shutil.rmtree(built_wheel_dir)
-            built_wheel_dir.mkdir(parents=True)
+            built_wheel_dir.mkdir()
 
             verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity)
 
@@ -320,12 +310,10 @@ def build(options: Options) -> None:
                     # in uhi.  After probably pip 21.2, we can use uri. For
                     # now, use a temporary file.
                     if " " in str(constraints_path):
-                        tmp_file = tempfile.NamedTemporaryFile(
-                            "w", suffix="constraints.txt", delete=False, dir=CIBW_INSTALL_PATH
-                        )
-                        with tmp_file as new_constraints_file, open(constraints_path) as f:
-                            new_constraints_file.write(f.read())
-                            constraints_path = Path(new_constraints_file.name)
+                        assert " " not in str(identifier_tmp_dir)
+                        tmp_file = identifier_tmp_dir / "constraints.txt"
+                        tmp_file.write_bytes(constraints_path.read_bytes())
+                        constraints_path = tmp_file
 
                     build_env["PIP_CONSTRAINT"] = str(constraints_path)
                     build_env["VIRTUALENV_PIP"] = get_pip_version(env)
@@ -345,9 +333,7 @@ def build(options: Options) -> None:
             built_wheel = next(built_wheel_dir.glob("*.whl"))
 
             # repair the wheel
-            if repaired_wheel_dir.exists():
-                shutil.rmtree(repaired_wheel_dir)
-            repaired_wheel_dir.mkdir(parents=True)
+            repaired_wheel_dir.mkdir()
 
             if built_wheel.name.endswith("none-any.whl"):
                 raise NonPlatformWheelError()
@@ -368,7 +354,7 @@ def build(options: Options) -> None:
                 # set up a virtual environment to install and test from, to make sure
                 # there are no dependencies that were pulled in at build time.
                 call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env)
-                venv_dir = Path(tempfile.mkdtemp())
+                venv_dir = identifier_tmp_dir / "venv-test"
 
                 # Use --no-download to ensure determinism by using seed libraries
                 # built into virtualenv
@@ -415,13 +401,14 @@ def build(options: Options) -> None:
                 )
                 shell(test_command_prepared, cwd="c:\\", env=virtualenv_env)
 
-                # clean up
-                # (we ignore errors because occasionally Windows fails to unlink a file and we
-                # don't want to abort a build because of that)
-                shutil.rmtree(venv_dir, ignore_errors=True)
-
             # we're all done here; move it to output (remove if already exists)
             shutil.move(str(repaired_wheel), build_options.output_dir)
+
+            # clean up
+            # (we ignore errors because occasionally Windows fails to unlink a file and we
+            # don't want to abort a build because of that)
+            shutil.rmtree(identifier_tmp_dir, ignore_errors=True)
+
             log.build_end()
     except subprocess.CalledProcessError as error:
         log.step_end_with_error(

docs/options.md

@@ -174,14 +174,13 @@ Default: `auto`
 
 `auto` will auto-detect platform using environment variables, such as `TRAVIS_OS_NAME`/`APPVEYOR`/`CIRCLECI`.
 
-- For `linux`, you need Docker running, on macOS or Linux.
-- For `macos`, you need a Mac machine. Note that cibuildwheel is going to install MacPython on your system, so you probably don't want to run this on your development machine.
-- For `windows`, you need to run in Windows. cibuildwheel will install required versions of Python to `C:\cibw\python` using NuGet.
+- For `linux`, you need Docker running, on Linux, macOS, or Windows.
+- For `macos` and `windows`, you need to be running on the respective system, with a working compiler toolchain installed - Xcode Command Line tools for macOS, and MSVC for Windows.
 
 This option can also be set using the [command-line option](#command-line) `--platform`. This option is not available in the `pyproject.toml` config.
 
 !!! tip
-    If you have Docker installed, you can locally debug your cibuildwheel Linux config, instead of pushing to CI to test every change. For example:
+    You can use this option to locally debug your cibuildwheel config, instead of pushing to CI to test every change. For example:
 
     ```bash
     export CIBW_BUILD='cp37-*'
@@ -189,6 +188,7 @@ This option can also be set using the [command-line option](#command-line) `--pl
     cibuildwheel --platform linux .
     ```
 
+    This is even more convenient if you store your cibuildwheel config in [`pyproject.toml`](#configuration-file).
 
 ### `CIBW_BUILD`, `CIBW_SKIP` {: #build-skip}
 

docs/setup.md

@@ -5,31 +5,49 @@ title: 'Setup'
 # Run cibuildwheel locally (optional) {: #local}
 
 Before getting to CI setup, it can be convenient to test cibuildwheel
-locally to quickly iterate and track down issues. If you've got
-[Docker](https://www.docker.com/products/docker-desktop) installed on
-your development machine, you can run a Linux build without even
-touching CI.
-
-!!! tip
-    You can run the Linux build on any platform. Even Windows can run
-    Linux containers these days, but there are a few  hoops to jump
-    through. Check [this document](https://docs.microsoft.com/en-us/virtualization/windowscontainers/quick-start/quick-start-windows-10-linux)
-    for more info.
-
-This is convenient as it's quicker to iterate, and because the builds are
-happening in manylinux Docker containers, they're perfectly reproducible.
+locally to quickly iterate and track down issues without even touching CI.
 
 Install cibuildwheel and run a build like this:
 
-```sh
-pip install cibuildwheel
-cibuildwheel --platform linux
-```
+!!! tab "Linux"
 
-Or, using [pipx](https://github.com/pypa/pipx):
-```sh
-pipx run cibuildwheel --platform linux
-```
+    Using [pipx](https://github.com/pypa/pipx):
+    ```sh
+    pipx run cibuildwheel --platform linux
+    ```
+
+    Or,
+    ```sh
+    pip install cibuildwheel
+    cibuildwheel --platform linux
+    ```
+
+!!! tab "macOS"
+
+    Using [pipx](https://github.com/pypa/pipx):
+    ```sh
+    pipx run cibuildwheel --platform macos
+    ```
+
+    Or,
+    ```sh
+    pip install cibuildwheel
+    cibuildwheel --platform macos
+    ```
+
+
+!!! tab "Windows"
+
+    Using [pipx](https://github.com/pypa/pipx):
+    ```bat
+    pipx run cibuildwheel --platform windows
+    ```
+
+    Or,
+    ```bat
+    pip install cibuildwheel
+    cibuildwheel --platform windows
+    ```
 
 You should see the builds taking place. You can experiment with options using environment variables or pyproject.toml.
 
@@ -71,6 +89,53 @@ You should see the builds taking place. You can experiment with options using en
     cibuildwheel --platform linux
     ```
 
+## Linux builds
+
+If you've got [Docker](https://www.docker.com/products/docker-desktop) installed on
+your development machine, you can run a Linux build.
+
+!!! tip
+    You can run the Linux build on any platform. Even Windows can run
+    Linux containers these days, but there are a few  hoops to jump
+    through. Check [this document](https://docs.microsoft.com/en-us/virtualization/windowscontainers/quick-start/quick-start-windows-10-linux)
+    for more info.
+
+Because the builds are happening in manylinux Docker containers,
+they're perfectly reproducible.
+
+The only side effect to your system will be docker images being pulled.
+
+## macOS / Windows builds
+
+Pre-requisite: you need to have native build tools installed.
+
+Because the builds are happening without full isolation, there might be some
+differences compared to CI builds (Xcode version, Visual Studio version,
+OS version, local files, ...) that might prevent you from finding an issue only
+seen in CI.
+
+In order to speed-up builds, cibuildwheel will cache the tools it needs to be
+reused for future builds. The folder used for caching is system/user dependent and is
+reported in the printed preamble of each run (e.g. "Cache folder: /Users/Matt/Library/Caches/cibuildwheel").
+
+You can override the cache folder using the ``CIBW_CACHE_PATH`` environment variable.
+
+!!! warning
+    cibuildwheel uses official python.org macOS installers for CPython but
+    those can only be installed globally.
+
+    In order not to mess with your system, cibuildwheel won't install those if they are
+    missing. Instead, it will error out with a message to let you install the missing
+    CPython:
+
+    ```console
+    Error: CPython 3.6 is not installed.
+    cibuildwheel will not perform system-wide installs when running outside of CI.
+    To build locally, install CPython 3.6 on this machine, or, disable this version of Python using CIBW_SKIP=cp36-macosx_*
+
+    Download link: https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg
+    ```
+
 
 # Configure a CI service
 

noxfile.py

@@ -56,11 +56,12 @@ def update_constraints(session: nox.Session) -> None:
 @nox.session
 def update_pins(session: nox.Session) -> None:
     """
-    Update the python and docker pins version inplace.
+    Update the python, docker and virtualenv pins version inplace.
     """
     session.install("-e", ".[bin]")
     session.run("python", "bin/update_pythons.py", "--force")
     session.run("python", "bin/update_docker.py")
+    session.run("python", "bin/update_virtualenv.py", "--force")
 
 
 @nox.session

setup.cfg

@@ -34,7 +34,9 @@ install_requires =
     bashlex!=0.13
     bracex
     certifi
+    filelock
     packaging
+    platformdirs
     tomli
     dataclasses;python_version < '3.7'
     typing-extensions>=3.10.0.0;python_version < '3.8'

setup.py

@@ -11,7 +11,7 @@ extras = {
         "jinja2",
         "pytest>=6",
         "pytest-timeout",
-        "pytest-xdist; sys_platform == 'linux'",
+        "pytest-xdist",
     ],
     "bin": [
         "click",