from __future__ import annotations import subprocess import sys import textwrap from collections.abc import Iterable, Iterator, Sequence, Set from dataclasses import dataclass from pathlib import Path, PurePath, PurePosixPath from typing import OrderedDict, Tuple from packaging.version import Version from . import errors from ._compat.typing import assert_never from .architecture import Architecture from .logger import log from .oci_container import OCIContainer, OCIContainerEngineConfig from .options import BuildOptions, Options from .typing import PathOrStr from .util import ( BuildFrontendConfig, BuildSelector, find_compatible_wheel, get_build_verbosity_extra_flags, prepare_command, read_python_configs, split_config_settings, test_fail_cwd_file, unwrap, ) @dataclass(frozen=True) class PythonConfiguration: version: str identifier: str path_str: str @property def path(self) -> PurePosixPath: return PurePosixPath(self.path_str) @dataclass(frozen=True) class BuildStep: platform_configs: list[PythonConfiguration] platform_tag: str container_engine: OCIContainerEngineConfig container_image: str def get_python_configurations( build_selector: BuildSelector, architectures: Set[Architecture], ) -> list[PythonConfiguration]: full_python_configs = read_python_configs("linux") python_configurations = [PythonConfiguration(**item) for item in full_python_configs] # return all configurations whose arch is in our `architectures` set, # and match the build/skip rules return [ c for c in python_configurations if any(c.identifier.endswith(arch.value) for arch in architectures) and build_selector(c.identifier) ] def container_image_for_python_configuration( config: PythonConfiguration, build_options: BuildOptions ) -> str: # e.g # identifier is 'cp310-manylinux_x86_64' # platform_tag is 'manylinux_x86_64' # platform_arch is 'x86_64' _, platform_tag = config.identifier.split("-", 1) _, platform_arch = platform_tag.split("_", 1) assert build_options.manylinux_images is not None assert build_options.musllinux_images is not None return ( build_options.manylinux_images[platform_arch] if platform_tag.startswith("manylinux") else build_options.musllinux_images[platform_arch] ) def get_build_steps( options: Options, python_configurations: list[PythonConfiguration] ) -> Iterator[BuildStep]: """ Groups PythonConfigurations into BuildSteps. Each BuildStep represents a separate container instance. """ steps = OrderedDict[Tuple[str, str, str, OCIContainerEngineConfig], BuildStep]() for config in python_configurations: _, platform_tag = config.identifier.split("-", 1) build_options = options.build_options(config.identifier) before_all = build_options.before_all container_image = container_image_for_python_configuration(config, build_options) container_engine = build_options.container_engine step_key = (platform_tag, container_image, before_all, container_engine) if step_key in steps: steps[step_key].platform_configs.append(config) else: steps[step_key] = BuildStep( platform_configs=[config], platform_tag=platform_tag, container_engine=container_engine, container_image=container_image, ) yield from steps.values() def check_all_python_exist( *, platform_configs: Iterable[PythonConfiguration], container: OCIContainer ) -> None: exist = True has_manylinux_interpreters = True messages = [] try: # use capture_output to keep quiet container.call(["manylinux-interpreters", "--help"], capture_output=True) except subprocess.CalledProcessError: has_manylinux_interpreters = False for config in platform_configs: python_path = config.path / "bin" / "python" try: if has_manylinux_interpreters: container.call(["manylinux-interpreters", "ensure", config.path.name]) container.call(["test", "-x", python_path]) except subprocess.CalledProcessError: messages.append( f" '{python_path}' executable doesn't exist in image '{container.image}' to build '{config.identifier}'." ) exist = False if not exist: message = "\n".join(messages) raise errors.FatalError(message) def build_in_container( *, options: Options, platform_configs: Sequence[PythonConfiguration], container: OCIContainer, container_project_path: PurePath, container_package_dir: PurePath, ) -> None: container_output_dir = PurePosixPath("/output") check_all_python_exist(platform_configs=platform_configs, container=container) log.step("Copying project into container...") container.copy_into(Path.cwd(), container_project_path) before_all_options_identifier = platform_configs[0].identifier before_all_options = options.build_options(before_all_options_identifier) if before_all_options.before_all: log.step("Running before_all...") env = container.get_environment() env["PATH"] = f'/opt/python/cp38-cp38/bin:{env["PATH"]}' env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" env["PIP_ROOT_USER_ACTION"] = "ignore" env = before_all_options.environment.as_dictionary( env, executor=container.environment_executor ) before_all_prepared = prepare_command( before_all_options.before_all, project=container_project_path, package=container_package_dir, ) container.call(["sh", "-c", before_all_prepared], env=env) built_wheels: list[PurePosixPath] = [] for config in platform_configs: log.build_start(config.identifier) build_options = options.build_options(config.identifier) build_frontend = build_options.build_frontend or BuildFrontendConfig("pip") use_uv = build_frontend.name == "build[uv]" and Version(config.version) >= Version("3.8") pip = ["uv", "pip"] if use_uv else ["pip"] dependency_constraint_flags: list[PathOrStr] = [] if build_options.dependency_constraints: constraints_file = build_options.dependency_constraints.get_for_python_version( config.version ) container_constraints_file = PurePosixPath("/constraints.txt") container.copy_into(constraints_file, container_constraints_file) dependency_constraint_flags = ["-c", container_constraints_file] log.step("Setting up build environment...") env = container.get_environment() env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" env["PIP_ROOT_USER_ACTION"] = "ignore" # put this config's python top of the list python_bin = config.path / "bin" env["PATH"] = f'{python_bin}:{env["PATH"]}' env = build_options.environment.as_dictionary(env, executor=container.environment_executor) # check config python is still on PATH which_python = container.call(["which", "python"], env=env, capture_output=True).strip() if PurePosixPath(which_python) != python_bin / "python": msg = "python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it." raise errors.FatalError(msg) if use_uv: which_uv = container.call(["which", "uv"], env=env, capture_output=True).strip() if not which_uv: msg = "uv not found on PATH. You must use a supported manylinux or musllinux environment with uv." raise errors.FatalError(msg) else: which_pip = container.call(["which", "pip"], env=env, capture_output=True).strip() if PurePosixPath(which_pip) != python_bin / "pip": msg = "pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it." raise errors.FatalError(msg) compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) if compatible_wheel: log.step_end() print( f"\nFound previously built wheel {compatible_wheel.name}, that's compatible with {config.identifier}. Skipping build step..." ) repaired_wheels = [compatible_wheel] else: if build_options.before_build: log.step("Running before_build...") before_build_prepared = prepare_command( build_options.before_build, project=container_project_path, package=container_package_dir, ) container.call(["sh", "-c", before_build_prepared], env=env) log.step("Building wheel...") temp_dir = PurePosixPath("/tmp/cibuildwheel") built_wheel_dir = temp_dir / "built_wheel" container.call(["rm", "-rf", built_wheel_dir]) container.call(["mkdir", "-p", built_wheel_dir]) extra_flags = split_config_settings(build_options.config_settings, build_frontend.name) extra_flags += build_frontend.args if build_frontend.name == "pip": extra_flags += get_build_verbosity_extra_flags(build_options.build_verbosity) container.call( [ "python", "-m", "pip", "wheel", container_package_dir, f"--wheel-dir={built_wheel_dir}", "--no-deps", *extra_flags, ], env=env, ) elif build_frontend.name == "build" or build_frontend.name == "build[uv]": if not 0 <= build_options.build_verbosity < 2: msg = f"build_verbosity {build_options.build_verbosity} is not supported for build frontend. Ignoring." log.warning(msg) if use_uv and "--no-isolation" not in extra_flags and "-n" not in extra_flags: extra_flags += ["--installer=uv"] container.call( [ "python", "-m", "build", container_package_dir, "--wheel", f"--outdir={built_wheel_dir}", *extra_flags, ], env=env, ) else: assert_never(build_frontend) built_wheel = container.glob(built_wheel_dir, "*.whl")[0] repaired_wheel_dir = temp_dir / "repaired_wheel" container.call(["rm", "-rf", repaired_wheel_dir]) container.call(["mkdir", "-p", repaired_wheel_dir]) if built_wheel.name.endswith("none-any.whl"): raise errors.NonPlatformWheelError() if build_options.repair_command: log.step("Repairing wheel...") repair_command_prepared = prepare_command( build_options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir ) container.call(["sh", "-c", repair_command_prepared], env=env) else: container.call(["mv", built_wheel, repaired_wheel_dir]) repaired_wheels = container.glob(repaired_wheel_dir, "*.whl") for repaired_wheel in repaired_wheels: if repaired_wheel.name in {wheel.name for wheel in built_wheels}: raise errors.AlreadyBuiltWheelError(repaired_wheel.name) if build_options.test_command and build_options.test_selector(config.identifier): log.step("Testing wheel...") # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. if not use_uv: container.call( ["pip", "install", "virtualenv", *dependency_constraint_flags], env=env ) testing_temp_dir = PurePosixPath( container.call(["mktemp", "-d"], capture_output=True).strip() ) venv_dir = testing_temp_dir / "venv" if use_uv: container.call(["uv", "venv", venv_dir], env=env) else: # Use embedded dependencies from virtualenv to ensure determinism venv_args = ["--no-periodic-update", "--pip=embed"] # In Python<3.12, setuptools & wheel are installed as well if Version(config.version) < Version("3.12"): venv_args.extend(("--setuptools=embed", "--wheel=embed")) container.call(["python", "-m", "virtualenv", *venv_args, venv_dir], env=env) virtualenv_env = env.copy() virtualenv_env["PATH"] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}" virtualenv_env["VIRTUAL_ENV"] = str(venv_dir) if build_options.before_test: before_test_prepared = prepare_command( build_options.before_test, project=container_project_path, package=container_package_dir, ) container.call(["sh", "-c", before_test_prepared], env=virtualenv_env) # Install the wheel we just built # Note: If auditwheel produced two wheels, it's because the earlier produced wheel # conforms to multiple manylinux standards. These multiple versions of the wheel are # functionally the same, differing only in name, wheel metadata, and possibly include # different external shared libraries. so it doesn't matter which one we run the tests on. # Let's just pick the first one. wheel_to_test = repaired_wheels[0] container.call( [*pip, "install", str(wheel_to_test) + build_options.test_extras], env=virtualenv_env, ) # Install any requirements to run the tests if build_options.test_requires: container.call([*pip, "install", *build_options.test_requires], env=virtualenv_env) # Run the tests from a different directory test_command_prepared = prepare_command( build_options.test_command, project=container_project_path, package=container_package_dir, wheel=wheel_to_test, ) test_cwd = testing_temp_dir / "test_cwd" container.call(["mkdir", "-p", test_cwd]) container.copy_into(test_fail_cwd_file, test_cwd / "test_fail.py") container.call(["sh", "-c", test_command_prepared], cwd=test_cwd, env=virtualenv_env) # clean up test environment container.call(["rm", "-rf", testing_temp_dir]) # move repaired wheels to output if compatible_wheel is None: container.call(["mkdir", "-p", container_output_dir]) container.call(["mv", *repaired_wheels, container_output_dir]) built_wheels.extend( container_output_dir / repaired_wheel.name for repaired_wheel in repaired_wheels ) log.build_end() log.step("Copying wheels back to host...") # copy the output back into the host container.copy_out(container_output_dir, options.globals.output_dir) log.step_end() def build(options: Options, tmp_path: Path) -> None: # noqa: ARG001 python_configurations = get_python_configurations( options.globals.build_selector, options.globals.architectures ) cwd = Path.cwd() abs_package_dir = options.globals.package_dir.resolve() if cwd != abs_package_dir and cwd not in abs_package_dir.parents: msg = "package_dir must be inside the working directory" raise Exception(msg) container_project_path = PurePosixPath("/project") container_package_dir = container_project_path / abs_package_dir.relative_to(cwd) for build_step in get_build_steps(options, python_configurations): try: # check the container engine is installed subprocess.run( [build_step.container_engine.name, "--version"], check=True, stdout=subprocess.DEVNULL, ) except subprocess.CalledProcessError as error: msg = unwrap( f""" cibuildwheel: {build_step.container_engine.name} not found. An OCI exe like Docker or Podman is required to run Linux builds. If you're building on Travis CI, add `services: [docker]` to your .travis.yml. If you're building on Circle CI in Linux, add a `setup_remote_docker` step to your .circleci/config.yml. If you're building on Cirrus CI, use `docker_builder` task. """ ) raise errors.ConfigurationError(msg) from error try: ids_to_build = [x.identifier for x in build_step.platform_configs] log.step(f"Starting container image {build_step.container_image}...") print(f"info: This container will host the build for {', '.join(ids_to_build)}...") with OCIContainer( image=build_step.container_image, enforce_32_bit=build_step.platform_tag.endswith("i686"), cwd=container_project_path, engine=build_step.container_engine, ) as container: build_in_container( options=options, platform_configs=build_step.platform_configs, container=container, container_project_path=container_project_path, container_package_dir=container_package_dir, ) except subprocess.CalledProcessError as error: troubleshoot(options, error) msg = f"Command {error.cmd} failed with code {error.returncode}. {error.stdout or ''}" raise errors.FatalError(msg) from error def _matches_prepared_command(error_cmd: Sequence[str], command_template: str) -> bool: if len(error_cmd) < 3 or error_cmd[0:2] != ["sh", "-c"]: return False command_prefix = command_template.split("{", maxsplit=1)[0].strip() return error_cmd[2].startswith(command_prefix) def troubleshoot(options: Options, error: Exception) -> None: if isinstance(error, subprocess.CalledProcessError) and ( error.cmd[0:4] == ["python", "-m", "pip", "wheel"] or error.cmd[0:3] == ["python", "-m", "build"] or _matches_prepared_command( error.cmd, options.build_options(None).repair_command ) # TODO allow matching of overrides too? ): # the wheel build step or the repair step failed so_files = list(options.globals.package_dir.glob("**/*.so")) if so_files: print( textwrap.dedent( """ NOTE: Shared object (.so) files found in this project. These files might be built against the wrong OS, causing problems with auditwheel. If possible, run cibuildwheel in a clean checkout. If you're using Cython and have previously done an in-place build, remove those build files (*.so and *.c) before starting cibuildwheel. setuptools uses the build/ folder to store its build cache. It may be necessary to remove those build files (*.so and *.o) before starting cibuildwheel. Files that belong to a virtual environment are probably not an issue unless you used a custom command telling cibuildwheel to activate it. """ ), file=sys.stderr, ) print(" Files detected:") print("\n".join(f" {f}" for f in so_files)) print()