feat: support pypa build (#521)

* feat: support pypa build

fix: update `cibuildwheel.linux.troubleshoot` to know about build

Workaround issue with PyPy venv module by installing build[virtualenv]

fix: test_dependency_constraints_file when using build

fix: test/test_pep518.py

fix: use `strtobool` to parse `CIBW_PYPA_BUILD`

fix: update `cibuildwheel.linux.troubleshoot` to know about `python -m pip wheel`

test: use `build_mode` in `test/test_dependency_versions.py` tests

* refactor: use build-backend instead

* Apply suggestions from code review

Co-authored-by: Joe Rickerby <joerick@mac.com>

* refactor: use literal types for build_frontend

* refactor(tests): build_mode -> build_frontend_env

* Update test/test_before_build.py

Co-authored-by: Joe Rickerby <joerick@mac.com>

cibuildwheel/__main__.py

@@ -20,6 +20,7 @@ from cibuildwheel.options import ConfigOptions
 from cibuildwheel.projectfiles import get_requires_python_str
 from cibuildwheel.typing import PLATFORMS, PlatformName, assert_never
 from cibuildwheel.util import (
+    BuildFrontend,
     BuildOptions,
     BuildSelector,
     DependencyConstraints,
@@ -184,6 +185,7 @@ def main() -> None:
 
     archs_config_str = args.archs or options("archs", sep=" ")
 
+    build_frontend_str = options("build-frontend", env_plat=False)
     environment_config = options("environment", table={"item": '{k}="{v}"', "sep": " "})
     before_all = options("before-all", sep=" && ")
     before_build = options("before-build", sep=" && ")
@@ -200,6 +202,16 @@ def main() -> None:
         os.environ.get("CIBW_PRERELEASE_PYTHONS", "0")
     )
 
+    build_frontend: BuildFrontend
+    if build_frontend_str == "build":
+        build_frontend = "build"
+    elif build_frontend_str == "pip":
+        build_frontend = "pip"
+    else:
+        msg = f"cibuildwheel: Unrecognised build frontend '{build_frontend}', only 'pip' and 'build' are supported"
+        print(msg, file=sys.stderr)
+        sys.exit(2)
+
     package_files = {"setup.py", "setup.cfg", "pyproject.toml"}
 
     if not any(package_dir.joinpath(name).exists() for name in package_files):
@@ -308,6 +320,7 @@ def main() -> None:
         environment=environment,
         dependency_constraints=dependency_constraints,
         manylinux_images=manylinux_images or None,
+        build_frontend=build_frontend,
     )
 
     # Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print'

cibuildwheel/linux.py

@@ -7,7 +7,7 @@ from typing import List, NamedTuple, Set
 from .architecture import Architecture
 from .docker_container import DockerContainer
 from .logger import log
-from .typing import PathOrStr
+from .typing import PathOrStr, assert_never
 from .util import (
     BuildOptions,
     BuildSelector,
@@ -145,7 +145,7 @@ def build(options: BuildOptions) -> None:
                         env, executor=docker.environment_executor
                     )
 
-                    # check config python and pip are still on PATH
+                    # check config python is still on PATH
                     which_python = docker.call(
                         ["which", "python"], env=env, capture_output=True
                     ).strip()
@@ -180,18 +180,38 @@ def build(options: BuildOptions) -> None:
                     docker.call(["rm", "-rf", built_wheel_dir])
                     docker.call(["mkdir", "-p", built_wheel_dir])
 
-                    docker.call(
-                        [
-                            "pip",
-                            "wheel",
-                            container_package_dir,
-                            "--wheel-dir",
-                            built_wheel_dir,
-                            "--no-deps",
-                            *get_build_verbosity_extra_flags(options.build_verbosity),
-                        ],
-                        env=env,
-                    )
+                    verbosity_flags = get_build_verbosity_extra_flags(options.build_verbosity)
+
+                    if options.build_frontend == "pip":
+                        docker.call(
+                            [
+                                "python",
+                                "-m",
+                                "pip",
+                                "wheel",
+                                container_package_dir,
+                                f"--wheel-dir={built_wheel_dir}",
+                                "--no-deps",
+                                *verbosity_flags,
+                            ],
+                            env=env,
+                        )
+                    elif options.build_frontend == "build":
+                        config_setting = " ".join(verbosity_flags)
+                        docker.call(
+                            [
+                                "python",
+                                "-m",
+                                "build",
+                                container_package_dir,
+                                "--wheel",
+                                f"--outdir={built_wheel_dir}",
+                                f"--config-setting={config_setting}",
+                            ],
+                            env=env,
+                        )
+                    else:
+                        assert_never(options.build_frontend)
 
                     built_wheel = docker.glob(built_wheel_dir, "*.whl")[0]
 
@@ -291,8 +311,11 @@ def build(options: BuildOptions) -> None:
 
 
 def troubleshoot(package_dir: Path, error: Exception) -> None:
-    if isinstance(error, subprocess.CalledProcessError) and error.cmd[0:2] == ["pip", "wheel"]:
-        # the 'pip wheel' step failed.
+    if isinstance(error, subprocess.CalledProcessError) and (
+        error.cmd[0:4] == ["python", "-m", "pip", "wheel"]
+        or error.cmd[0:3] == ["python", "-m", "build"]
+    ):
+        # the wheel build step failed
         print("Checking for common errors...")
         so_files = list(package_dir.glob("**/*.so"))
 

cibuildwheel/macos.py

@@ -12,13 +12,15 @@ from typing import Any, Dict, List, NamedTuple, Optional, Sequence, Set, Tuple,
 from .architecture import Architecture
 from .environment import ParsedEnvironment
 from .logger import log
-from .typing import Literal, PathOrStr
+from .typing import Literal, PathOrStr, assert_never
 from .util import (
+    BuildFrontend,
     BuildOptions,
     BuildSelector,
     NonPlatformWheelError,
     download,
     get_build_verbosity_extra_flags,
+    get_pip_version,
     install_certifi_script,
     prepare_command,
     read_python_configs,
@@ -178,7 +180,9 @@ def setup_python(
     python_configuration: PythonConfiguration,
     dependency_constraint_flags: Sequence[PathOrStr],
     environment: ParsedEnvironment,
+    build_frontend: BuildFrontend,
 ) -> Dict[str, str]:
+
     implementation_id = python_configuration.identifier.split("-")[0]
     log.step(f"Installing Python {implementation_id}...")
 
@@ -308,18 +312,33 @@ def setup_python(
             env.setdefault("SDKROOT", arm64_compatible_sdks[0])
 
     log.step("Installing build tools...")
-    call(
-        [
-            "pip",
-            "install",
-            "--upgrade",
-            "setuptools",
-            "wheel",
-            "delocate",
-            *dependency_constraint_flags,
-        ],
-        env=env,
-    )
+    if build_frontend == "pip":
+        call(
+            [
+                "pip",
+                "install",
+                "--upgrade",
+                "setuptools",
+                "wheel",
+                "delocate",
+                *dependency_constraint_flags,
+            ],
+            env=env,
+        )
+    elif build_frontend == "build":
+        call(
+            [
+                "pip",
+                "install",
+                "--upgrade",
+                "delocate",
+                "build[virtualenv]",
+                *dependency_constraint_flags,
+            ],
+            env=env,
+        )
+    else:
+        assert_never(build_frontend)
 
     return env
 
@@ -356,7 +375,12 @@ def build(options: BuildOptions) -> None:
                     options.dependency_constraints.get_for_python_version(config.version),
                 ]
 
-            env = setup_python(config, dependency_constraint_flags, options.environment)
+            env = setup_python(
+                config,
+                dependency_constraint_flags,
+                options.environment,
+                options.build_frontend,
+            )
 
             if options.before_build:
                 log.step("Running before_build...")
@@ -370,20 +394,46 @@ def build(options: BuildOptions) -> None:
                 shutil.rmtree(built_wheel_dir)
             built_wheel_dir.mkdir(parents=True)
 
-            # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
-            # see https://github.com/pypa/cibuildwheel/pull/369
-            call(
-                [
-                    "pip",
-                    "wheel",
-                    options.package_dir.resolve(),
-                    "--wheel-dir",
-                    built_wheel_dir,
-                    "--no-deps",
-                    *get_build_verbosity_extra_flags(options.build_verbosity),
-                ],
-                env=env,
-            )
+            verbosity_flags = get_build_verbosity_extra_flags(options.build_verbosity)
+
+            if options.build_frontend == "pip":
+                # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
+                # see https://github.com/pypa/cibuildwheel/pull/369
+                call(
+                    [
+                        "python",
+                        "-m",
+                        "pip",
+                        "wheel",
+                        options.package_dir.resolve(),
+                        f"--wheel-dir={built_wheel_dir}",
+                        "--no-deps",
+                        *verbosity_flags,
+                    ],
+                    env=env,
+                )
+            elif options.build_frontend == "build":
+                config_setting = " ".join(verbosity_flags)
+                build_env = env.copy()
+                if options.dependency_constraints:
+                    build_env["PIP_CONSTRAINT"] = str(
+                        options.dependency_constraints.get_for_python_version(config.version)
+                    )
+                build_env["VIRTUALENV_PIP"] = get_pip_version(env)
+                call(
+                    [
+                        "python",
+                        "-m",
+                        "build",
+                        options.package_dir,
+                        "--wheel",
+                        f"--outdir={built_wheel_dir}",
+                        f"--config-setting={config_setting}",
+                    ],
+                    env=build_env,
+                )
+            else:
+                assert_never(options.build_frontend)
 
             built_wheel = next(built_wheel_dir.glob("*.whl"))
 

cibuildwheel/resources/defaults.toml

@@ -4,6 +4,7 @@ skip = ""
 test-skip = ""
 
 archs = ["auto"]
+build-frontend = "pip"
 dependency-versions = "pinned"
 environment = {}
 build-verbosity = ""

cibuildwheel/util.py

@@ -4,6 +4,8 @@ import itertools
 import os
 import re
 import ssl
+import subprocess
+import sys
 import textwrap
 import time
 import urllib.request
@@ -20,12 +22,14 @@ from packaging.version import Version
 
 from .architecture import Architecture
 from .environment import ParsedEnvironment
-from .typing import PathOrStr, PlatformName
+from .typing import Literal, PathOrStr, PlatformName
 
 resources_dir = Path(__file__).parent / "resources"
 
 install_certifi_script = resources_dir / "install_certifi.py"
 
+BuildFrontend = Literal["pip", "build"]
+
 
 def prepare_command(command: str, **kwargs: PathOrStr) -> str:
     """
@@ -218,6 +222,7 @@ class BuildOptions(NamedTuple):
     test_requires: List[str]
     test_extras: str
     build_verbosity: int
+    build_frontend: BuildFrontend
 
 
 class NonPlatformWheelError(Exception):
@@ -300,3 +305,18 @@ def print_new_wheels(msg: str, output_dir: Path) -> Iterator[None]:
     s = time.time() - start_time
     m = s / 60
     print(msg.format(n=n, s=s, m=m), *sorted(f"  {f.name}" for f in new_contents), sep="\n")
+
+
+def get_pip_version(env: Dict[str, str]) -> str:
+    # we use shell=True here for windows, even though we don't need a shell due to a bug
+    # https://bugs.python.org/issue8557
+    shell = sys.platform.startswith("win")
+    versions_output_text = subprocess.check_output(
+        ["python", "-m", "pip", "freeze", "--all"], universal_newlines=True, shell=shell, env=env
+    )
+    (pip_version,) = (
+        version[5:]
+        for version in versions_output_text.strip().splitlines()
+        if version.startswith("pip==")
+    )
+    return pip_version

cibuildwheel/windows.py

@@ -10,13 +10,15 @@ from zipfile import ZipFile
 from .architecture import Architecture
 from .environment import ParsedEnvironment
 from .logger import log
-from .typing import PathOrStr
+from .typing import PathOrStr, assert_never
 from .util import (
+    BuildFrontend,
     BuildOptions,
     BuildSelector,
     NonPlatformWheelError,
     download,
     get_build_verbosity_extra_flags,
+    get_pip_version,
     prepare_command,
     read_python_configs,
 )
@@ -114,7 +116,9 @@ def setup_python(
     python_configuration: PythonConfiguration,
     dependency_constraint_flags: Sequence[PathOrStr],
     environment: ParsedEnvironment,
+    build_frontend: BuildFrontend,
 ) -> Dict[str, str]:
+
     nuget = Path("C:\\cibw\\nuget.exe")
     if not nuget.exists():
         log.step("Downloading nuget...")
@@ -214,10 +218,26 @@ def setup_python(
         sys.exit(1)
 
     call(["pip", "--version"], env=env)
-    call(
-        ["pip", "install", "--upgrade", "setuptools", "wheel", *dependency_constraint_flags],
-        env=env,
-    )
+
+    if build_frontend == "pip":
+        call(
+            [
+                "pip",
+                "install",
+                "--upgrade",
+                "setuptools",
+                "wheel",
+                *dependency_constraint_flags,
+            ],
+            env=env,
+        )
+    elif build_frontend == "build":
+        call(
+            ["pip", "install", "--upgrade", "build[virtualenv]", *dependency_constraint_flags],
+            env=env,
+        )
+    else:
+        assert_never(build_frontend)
 
     return env
 
@@ -251,7 +271,12 @@ def build(options: BuildOptions) -> None:
                 ]
 
             # install Python
-            env = setup_python(config, dependency_constraint_flags, options.environment)
+            env = setup_python(
+                config,
+                dependency_constraint_flags,
+                options.environment,
+                options.build_frontend,
+            )
 
             # run the before_build command
             if options.before_build:
@@ -265,20 +290,47 @@ def build(options: BuildOptions) -> None:
             if built_wheel_dir.exists():
                 shutil.rmtree(built_wheel_dir)
             built_wheel_dir.mkdir(parents=True)
-            # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
-            # see https://github.com/pypa/cibuildwheel/pull/369
-            call(
-                [
-                    "pip",
-                    "wheel",
-                    options.package_dir.resolve(),
-                    "-w",
-                    built_wheel_dir,
-                    "--no-deps",
-                    *get_build_verbosity_extra_flags(options.build_verbosity),
-                ],
-                env=env,
-            )
+
+            verbosity_flags = get_build_verbosity_extra_flags(options.build_verbosity)
+
+            if options.build_frontend == "pip":
+                # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
+                # see https://github.com/pypa/cibuildwheel/pull/369
+                call(
+                    [
+                        "python",
+                        "-m",
+                        "pip",
+                        "wheel",
+                        options.package_dir.resolve(),
+                        f"--wheel-dir={built_wheel_dir}",
+                        "--no-deps",
+                        *get_build_verbosity_extra_flags(options.build_verbosity),
+                    ],
+                    env=env,
+                )
+            elif options.build_frontend == "build":
+                config_setting = " ".join(verbosity_flags)
+                build_env = env.copy()
+                if options.dependency_constraints:
+                    build_env["PIP_CONSTRAINT"] = str(
+                        options.dependency_constraints.get_for_python_version(config.version)
+                    )
+                build_env["VIRTUALENV_PIP"] = get_pip_version(env)
+                call(
+                    [
+                        "python",
+                        "-m",
+                        "build",
+                        options.package_dir,
+                        "--wheel",
+                        f"--outdir={built_wheel_dir}",
+                        f"--config-setting={config_setting}",
+                    ],
+                    env=build_env,
+                )
+            else:
+                assert_never(options.build_frontend)
 
             built_wheel = next(built_wheel_dir.glob("*.whl"))