from __future__ import annotations import subprocess import sys import textwrap from dataclasses import dataclass from pathlib import Path, PurePath, PurePosixPath from typing import Iterator, Tuple from .architecture import Architecture from .logger import log from .oci_container import OCIContainer from .options import Options from .typing import OrderedDict, PathOrStr, assert_never from .util import ( AlreadyBuiltWheelError, BuildSelector, NonPlatformWheelError, find_compatible_wheel, get_build_verbosity_extra_flags, prepare_command, read_python_configs, split_config_settings, unwrap, ) @dataclass(frozen=True) class PythonConfiguration: version: str identifier: str path_str: str @property def path(self) -> PurePosixPath: return PurePosixPath(self.path_str) @dataclass(frozen=True) class BuildStep: platform_configs: list[PythonConfiguration] platform_tag: str container_image: str def get_python_configurations( build_selector: BuildSelector, architectures: set[Architecture], ) -> list[PythonConfiguration]: full_python_configs = read_python_configs("linux") python_configurations = [PythonConfiguration(**item) for item in full_python_configs] # return all configurations whose arch is in our `architectures` set, # and match the build/skip rules return [ c for c in python_configurations if any(c.identifier.endswith(arch.value) for arch in architectures) and build_selector(c.identifier) ] def container_image_for_python_configuration(config: PythonConfiguration, options: Options) -> str: build_options = options.build_options(config.identifier) # e.g # identifier is 'cp310-manylinux_x86_64' # platform_tag is 'manylinux_x86_64' # platform_arch is 'x86_64' _, platform_tag = config.identifier.split("-", 1) _, platform_arch = platform_tag.split("_", 1) assert build_options.manylinux_images is not None assert build_options.musllinux_images is not None return ( build_options.manylinux_images[platform_arch] if platform_tag.startswith("manylinux") else build_options.musllinux_images[platform_arch] ) def get_build_steps( options: Options, python_configurations: list[PythonConfiguration] ) -> Iterator[BuildStep]: """ Groups PythonConfigurations into BuildSteps. Each BuildStep represents a separate container instance. """ steps = OrderedDict[Tuple[str, str, str], BuildStep]() for config in python_configurations: _, platform_tag = config.identifier.split("-", 1) before_all = options.build_options(config.identifier).before_all container_image = container_image_for_python_configuration(config, options) step_key = (platform_tag, container_image, before_all) if step_key in steps: steps[step_key].platform_configs.append(config) else: steps[step_key] = BuildStep( platform_configs=[config], platform_tag=platform_tag, container_image=container_image, ) yield from steps.values() def check_all_python_exist( *, platform_configs: list[PythonConfiguration], container: OCIContainer ) -> None: exist = True messages = [] for config in platform_configs: python_path = config.path / "bin" / "python" try: container.call(["test", "-x", python_path]) except subprocess.CalledProcessError: messages.append( f" '{python_path}' executable doesn't exist in image '{container.image}' to build '{config.identifier}'." ) exist = False if not exist: message = "\n".join(messages) print( f"cibuildwheel:\n{message}", file=sys.stderr, ) sys.exit(1) def build_in_container( *, options: Options, platform_configs: list[PythonConfiguration], container: OCIContainer, container_project_path: PurePath, container_package_dir: PurePath, ) -> None: container_output_dir = PurePosixPath("/output") check_all_python_exist(platform_configs=platform_configs, container=container) log.step("Copying project into container...") container.copy_into(Path.cwd(), container_project_path) before_all_options_identifier = platform_configs[0].identifier before_all_options = options.build_options(before_all_options_identifier) if before_all_options.before_all: log.step("Running before_all...") env = container.get_environment() env["PATH"] = f'/opt/python/cp38-cp38/bin:{env["PATH"]}' env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" env["PIP_ROOT_USER_ACTION"] = "ignore" env = before_all_options.environment.as_dictionary( env, executor=container.environment_executor ) before_all_prepared = prepare_command( before_all_options.before_all, project=container_project_path, package=container_package_dir, ) container.call(["sh", "-c", before_all_prepared], env=env) built_wheels: list[PurePosixPath] = [] for config in platform_configs: log.build_start(config.identifier) build_options = options.build_options(config.identifier) dependency_constraint_flags: list[PathOrStr] = [] if build_options.dependency_constraints: constraints_file = build_options.dependency_constraints.get_for_python_version( config.version ) container_constraints_file = PurePath("/constraints.txt") container.copy_into(constraints_file, container_constraints_file) dependency_constraint_flags = ["-c", container_constraints_file] log.step("Setting up build environment...") env = container.get_environment() env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" env["PIP_ROOT_USER_ACTION"] = "ignore" # put this config's python top of the list python_bin = config.path / "bin" env["PATH"] = f'{python_bin}:{env["PATH"]}' env = build_options.environment.as_dictionary(env, executor=container.environment_executor) # check config python is still on PATH which_python = container.call(["which", "python"], env=env, capture_output=True).strip() if PurePosixPath(which_python) != python_bin / "python": print( "cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr, ) sys.exit(1) which_pip = container.call(["which", "pip"], env=env, capture_output=True).strip() if PurePosixPath(which_pip) != python_bin / "pip": print( "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr, ) sys.exit(1) compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) if compatible_wheel: log.step_end() print( f"\nFound previously built wheel {compatible_wheel.name}, that's compatible with {config.identifier}. Skipping build step..." ) repaired_wheels = [compatible_wheel] else: if build_options.before_build: log.step("Running before_build...") before_build_prepared = prepare_command( build_options.before_build, project=container_project_path, package=container_package_dir, ) container.call(["sh", "-c", before_build_prepared], env=env) log.step("Building wheel...") temp_dir = PurePosixPath("/tmp/cibuildwheel") built_wheel_dir = temp_dir / "built_wheel" container.call(["rm", "-rf", built_wheel_dir]) container.call(["mkdir", "-p", built_wheel_dir]) verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity) extra_flags = split_config_settings(build_options.config_settings) if build_options.build_frontend == "pip": extra_flags += verbosity_flags container.call( [ "python", "-m", "pip", "wheel", container_package_dir, f"--wheel-dir={built_wheel_dir}", "--no-deps", *extra_flags, ], env=env, ) elif build_options.build_frontend == "build": verbosity_setting = " ".join(verbosity_flags) extra_flags += (f"--config-setting={verbosity_setting}",) container.call( [ "python", "-m", "build", container_package_dir, "--wheel", f"--outdir={built_wheel_dir}", *extra_flags, ], env=env, ) else: assert_never(build_options.build_frontend) built_wheel = container.glob(built_wheel_dir, "*.whl")[0] repaired_wheel_dir = temp_dir / "repaired_wheel" container.call(["rm", "-rf", repaired_wheel_dir]) container.call(["mkdir", "-p", repaired_wheel_dir]) if built_wheel.name.endswith("none-any.whl"): raise NonPlatformWheelError() if build_options.repair_command: log.step("Repairing wheel...") repair_command_prepared = prepare_command( build_options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir ) container.call(["sh", "-c", repair_command_prepared], env=env) else: container.call(["mv", built_wheel, repaired_wheel_dir]) repaired_wheels = container.glob(repaired_wheel_dir, "*.whl") for repaired_wheel in repaired_wheels: if repaired_wheel.name in {wheel.name for wheel in built_wheels}: raise AlreadyBuiltWheelError(repaired_wheel.name) if build_options.test_command and build_options.test_selector(config.identifier): log.step("Testing wheel...") # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. container.call(["pip", "install", "virtualenv", *dependency_constraint_flags], env=env) venv_dir = ( PurePath(container.call(["mktemp", "-d"], capture_output=True).strip()) / "venv" ) container.call(["python", "-m", "virtualenv", "--no-download", venv_dir], env=env) virtualenv_env = env.copy() virtualenv_env["PATH"] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}" if build_options.before_test: before_test_prepared = prepare_command( build_options.before_test, project=container_project_path, package=container_package_dir, ) container.call(["sh", "-c", before_test_prepared], env=virtualenv_env) # Install the wheel we just built # Note: If auditwheel produced two wheels, it's because the earlier produced wheel # conforms to multiple manylinux standards. These multiple versions of the wheel are # functionally the same, differing only in name, wheel metadata, and possibly include # different external shared libraries. so it doesn't matter which one we run the tests on. # Let's just pick the first one. wheel_to_test = repaired_wheels[0] container.call( ["pip", "install", str(wheel_to_test) + build_options.test_extras], env=virtualenv_env, ) # Install any requirements to run the tests if build_options.test_requires: container.call(["pip", "install", *build_options.test_requires], env=virtualenv_env) # Run the tests from a different directory test_command_prepared = prepare_command( build_options.test_command, project=container_project_path, package=container_package_dir, ) container.call(["sh", "-c", test_command_prepared], cwd="/root", env=virtualenv_env) # clean up test environment container.call(["rm", "-rf", venv_dir]) # move repaired wheels to output if compatible_wheel is None: container.call(["mkdir", "-p", container_output_dir]) container.call(["mv", *repaired_wheels, container_output_dir]) built_wheels.extend( container_output_dir / repaired_wheel.name for repaired_wheel in repaired_wheels ) log.build_end() log.step("Copying wheels back to host...") # copy the output back into the host container.copy_out(container_output_dir, options.globals.output_dir) log.step_end() def build(options: Options, tmp_path: Path) -> None: # pylint: disable=unused-argument try: # check the container engine is installed subprocess.run( [options.globals.container_engine, "--version"], check=True, stdout=subprocess.DEVNULL ) except subprocess.CalledProcessError: print( unwrap( f""" cibuildwheel: {options.globals.container_engine} not found. An OCI exe like Docker or Podman is required to run Linux builds. If you're building on Travis CI, add `services: [docker]` to your .travis.yml. If you're building on Circle CI in Linux, add a `setup_remote_docker` step to your .circleci/config.yml. If you're building on Cirrus CI, use `docker_builder` task. """ ), file=sys.stderr, ) sys.exit(2) python_configurations = get_python_configurations( options.globals.build_selector, options.globals.architectures ) cwd = Path.cwd() abs_package_dir = options.globals.package_dir.resolve() if cwd != abs_package_dir and cwd not in abs_package_dir.parents: msg = "package_dir must be inside the working directory" raise Exception(msg) container_project_path = PurePosixPath("/project") container_package_dir = container_project_path / abs_package_dir.relative_to(cwd) for build_step in get_build_steps(options, python_configurations): try: ids_to_build = [x.identifier for x in build_step.platform_configs] log.step(f"Starting container image {build_step.container_image}...") print(f"info: This container will host the build for {', '.join(ids_to_build)}...") with OCIContainer( image=build_step.container_image, simulate_32_bit=build_step.platform_tag.endswith("i686"), cwd=container_project_path, engine=options.globals.container_engine, ) as container: build_in_container( options=options, platform_configs=build_step.platform_configs, container=container, container_project_path=container_project_path, container_package_dir=container_package_dir, ) except subprocess.CalledProcessError as error: log.step_end_with_error( f"Command {error.cmd} failed with code {error.returncode}. {error.stdout}" ) troubleshoot(options, error) sys.exit(1) def _matches_prepared_command(error_cmd: list[str], command_template: str) -> bool: if len(error_cmd) < 3 or error_cmd[0:2] != ["sh", "-c"]: return False command_prefix = command_template.split("{", maxsplit=1)[0].strip() return error_cmd[2].startswith(command_prefix) def troubleshoot(options: Options, error: Exception) -> None: if isinstance(error, subprocess.CalledProcessError) and ( error.cmd[0:4] == ["python", "-m", "pip", "wheel"] or error.cmd[0:3] == ["python", "-m", "build"] or _matches_prepared_command( error.cmd, options.build_options(None).repair_command ) # TODO allow matching of overrides too? ): # the wheel build step or the repair step failed print("Checking for common errors...") so_files = list(options.globals.package_dir.glob("**/*.so")) if so_files: print( textwrap.dedent( """ NOTE: Shared object (.so) files found in this project. These files might be built against the wrong OS, causing problems with auditwheel. If possible, run cibuildwheel in a clean checkout. If you're using Cython and have previously done an in-place build, remove those build files (*.so and *.c) before starting cibuildwheel. setuptools uses the build/ folder to store its build cache. It may be necessary to remove those build files (*.so and *.o) before starting cibuildwheel. Files that belong to a virtual environment are probably not an issue unless you used a custom command telling cibuildwheel to activate it. """ ), file=sys.stderr, ) print(" Files detected:") print("\n".join(f" {f}" for f in so_files)) print()