diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index e601a8cf..8c8d85bd 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -1,3 +1,4 @@ +from cibuildwheel.logger import Logger import platform import subprocess import sys @@ -111,6 +112,8 @@ def build(options: BuildOptions) -> None: container_package_dir = container_project_path / abs_package_dir.relative_to(cwd) container_output_dir = PurePath('/output') + logger = Logger() + for implementation, platform_tag, docker_image in platforms: platform_configs = [c for c in python_configurations if c.identifier.startswith(implementation) and c.identifier.endswith(platform_tag)] if not platform_configs: @@ -121,114 +124,16 @@ def build(options: BuildOptions) -> None: docker.copy_into(Path.cwd(), container_project_path) if options.before_all: - env = docker.get_environment() - env['PATH'] = f'/opt/python/cp38-cp38:{env["PATH"]}' - env = options.environment.as_dictionary(env, executor=docker.environment_executor) + with logger.step('Running before_all...'): + env = docker.get_environment() + env['PATH'] = f'/opt/python/cp38-cp38:{env["PATH"]}' + env = options.environment.as_dictionary(env, executor=docker.environment_executor) - before_all_prepared = prepare_command(options.before_all, project=container_project_path, package=container_package_dir) - docker.call(['sh', '-c', before_all_prepared], env=env) + before_all_prepared = prepare_command(options.before_all, project=container_project_path, package=container_package_dir) + docker.call(['sh', '-c', before_all_prepared], env=env) for config in platform_configs: - dependency_constraint_flags: List[Union[str, PathLike]] = [] - - if options.dependency_constraints: - constraints_file = options.dependency_constraints.get_for_python_version(config.version) - container_constraints_file = PurePath('/constraints.txt') - - docker.copy_into(constraints_file, container_constraints_file) - dependency_constraint_flags = ['-c', container_constraints_file] - - env = docker.get_environment() - - # put this config's python top of the list - python_bin = config.path / 'bin' - env['PATH'] = f'{python_bin}:{env["PATH"]}' - - env = options.environment.as_dictionary(env, executor=docker.environment_executor) - - # check config python and pip are still on PATH - which_python = docker.call(['which', 'python'], env=env, capture_output=True).strip() - if PurePath(which_python) != python_bin / 'python': - print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr) - exit(1) - - which_pip = docker.call(['which', 'pip'], env=env, capture_output=True).strip() - if PurePath(which_pip) != python_bin / 'pip': - print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr) - exit(1) - - if options.before_build: - before_build_prepared = prepare_command(options.before_build, project=container_project_path, package=container_package_dir) - docker.call(['sh', '-c', before_build_prepared], env=env) - - temp_dir = PurePath('/tmp/cibuildwheel') - built_wheel_dir = temp_dir / 'built_wheel' - docker.call(['rm', '-rf', built_wheel_dir]) - docker.call(['mkdir', '-p', built_wheel_dir]) - - docker.call([ - 'pip', 'wheel', - container_package_dir, - '-w', built_wheel_dir, - '--no-deps', - *get_build_verbosity_extra_flags(options.build_verbosity) - ], env=env) - - built_wheel = docker.glob(built_wheel_dir, '*.whl')[0] - - repaired_wheel_dir = temp_dir / 'repaired_wheel' - docker.call(['rm', '-rf', repaired_wheel_dir]) - docker.call(['mkdir', '-p', repaired_wheel_dir]) - - if built_wheel.name.endswith('none-any.whl'): - raise NonPlatformWheelError() - - if options.repair_command: - repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) - docker.call(['sh', '-c', repair_command_prepared], env=env) - else: - docker.call(['mv', built_wheel, repaired_wheel_dir]) - - repaired_wheels = docker.glob(repaired_wheel_dir, '*.whl') - - if options.test_command: - # set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - docker.call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) - venv_dir = PurePath(docker.call(['mktemp', '-d'], capture_output=True).strip()) / 'venv' - - docker.call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) - - virtualenv_env = env.copy() - virtualenv_env['PATH'] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}" - - if options.before_test: - before_test_prepared = prepare_command(options.before_test, project=container_project_path, package=container_package_dir) - docker.call(['sh', '-c', before_test_prepared], env=virtualenv_env) - - # Install the wheel we just built - # Note: If auditwheel produced two wheels, it's because the earlier produced wheel - # conforms to multiple manylinux standards. These multiple versions of the wheel are - # functionally the same, differing only in name, wheel metadata, and possibly include - # different external shared libraries. so it doesn't matter which one we run the tests on. - # Let's just pick the first one. - wheel_to_test = repaired_wheels[0] - docker.call(['pip', 'install', str(wheel_to_test) + options.test_extras], env=virtualenv_env) - - # Install any requirements to run the tests - if options.test_requires: - docker.call(['pip', 'install', *options.test_requires], env=virtualenv_env) - - # Run the tests from a different directory - test_command_prepared = prepare_command(options.test_command, project=container_project_path, package=container_package_dir) - docker.call(['sh', '-c', test_command_prepared], cwd='/root', env=virtualenv_env) - - # clean up test environment - docker.call(['rm', '-rf', venv_dir]) - - # move repaired wheels to output - docker.call(['mkdir', '-p', container_output_dir]) - docker.call(['mv', *repaired_wheels, container_output_dir]) + build_one(options=options, config=config, docker=docker, container_project_path=container_project_path, container_package_dir=container_package_dir, container_output_dir=container_output_dir, logger=logger) # copy the output back into the host docker.copy_out(container_output_dir, options.output_dir) @@ -238,6 +143,116 @@ def build(options: BuildOptions) -> None: exit(1) +def build_one(options: BuildOptions, config: PythonConfiguration, docker: DockerContainer, + container_project_path: PurePath, container_package_dir: PurePath, + container_output_dir: PurePath, logger: Logger): + dependency_constraint_flags: List[Union[str, PathLike]] = [] + + if options.dependency_constraints: + constraints_file = options.dependency_constraints.get_for_python_version(config.version) + container_constraints_file = PurePath('/constraints.txt') + + docker.copy_into(constraints_file, container_constraints_file) + dependency_constraint_flags = ['-c', container_constraints_file] + + env = docker.get_environment() + + # put this config's python top of the list + python_bin = config.path / 'bin' + env['PATH'] = f'{python_bin}:{env["PATH"]}' + + with logger.step(' Setting up build environment...'): + env = options.environment.as_dictionary(env, executor=docker.environment_executor) + + # check config python and pip are still on PATH + which_python = docker.call(['which', 'python'], env=env, capture_output=True).strip() + if PurePath(which_python) != python_bin / 'python': + print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr) + exit(1) + + which_pip = docker.call(['which', 'pip'], env=env, capture_output=True).strip() + if PurePath(which_pip) != python_bin / 'pip': + print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr) + exit(1) + + if options.before_build: + with logger.step(' Running before_build...'): + before_build_prepared = prepare_command(options.before_build, project=container_project_path, package=container_package_dir) + docker.call(['sh', '-c', before_build_prepared], env=env) + + temp_dir = PurePath('/tmp/cibuildwheel') + built_wheel_dir = temp_dir / 'built_wheel' + docker.call(['rm', '-rf', built_wheel_dir]) + docker.call(['mkdir', '-p', built_wheel_dir]) + + with logger.step(' Building wheel...'): + docker.call([ + 'pip', 'wheel', + container_package_dir, + '-w', built_wheel_dir, + '--no-deps', + *get_build_verbosity_extra_flags(options.build_verbosity) + ], env=env) + + built_wheel = docker.glob(built_wheel_dir, '*.whl')[0] + + repaired_wheel_dir = temp_dir / 'repaired_wheel' + docker.call(['rm', '-rf', repaired_wheel_dir]) + docker.call(['mkdir', '-p', repaired_wheel_dir]) + + if built_wheel.name.endswith('none-any.whl'): + raise NonPlatformWheelError() + + if options.repair_command: + with logger.step(' Repairing wheel...'): + repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) + docker.call(['sh', '-c', repair_command_prepared], env=env) + else: + docker.call(['mv', built_wheel, repaired_wheel_dir]) + + repaired_wheels = docker.glob(repaired_wheel_dir, '*.whl') + + if options.test_command: + with logger.step(' Testing wheel...'): + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + docker.call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) + venv_dir = PurePath(docker.call(['mktemp', '-d'], capture_output=True).strip()) / 'venv' + + docker.call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) + + virtualenv_env = env.copy() + virtualenv_env['PATH'] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}" + + if options.before_test: + before_test_prepared = prepare_command(options.before_test, project=container_project_path, package=container_package_dir) + docker.call(['sh', '-c', before_test_prepared], env=virtualenv_env) + + # Install the wheel we just built + # Note: If auditwheel produced two wheels, it's because the earlier produced wheel + # conforms to multiple manylinux standards. These multiple versions of the wheel are + # functionally the same, differing only in name, wheel metadata, and possibly include + # different external shared libraries. so it doesn't matter which one we run the tests on. + # Let's just pick the first one. + wheel_to_test = repaired_wheels[0] + docker.call(['pip', 'install', str(wheel_to_test) + options.test_extras], env=virtualenv_env) + + # Install any requirements to run the tests + if options.test_requires: + docker.call(['pip', 'install', *options.test_requires], env=virtualenv_env) + + # Run the tests from a different directory + test_command_prepared = prepare_command(options.test_command, project=container_project_path, package=container_package_dir) + docker.call(['sh', '-c', test_command_prepared], cwd='/root', env=virtualenv_env) + + # clean up test environment + docker.call(['rm', '-rf', venv_dir]) + + # move repaired wheels to output + docker.call(['mkdir', '-p', container_output_dir]) + docker.call(['mv', *repaired_wheels, container_output_dir]) + + def troubleshoot(package_dir: Path, error: Exception) -> None: if (isinstance(error, subprocess.CalledProcessError) and error.cmd[0:2] == ['pip', 'wheel']): # the 'pip wheel' step failed. diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py new file mode 100644 index 00000000..772be2a2 --- /dev/null +++ b/cibuildwheel/logger.py @@ -0,0 +1,139 @@ +from contextlib import contextmanager +import os +import time +import typing + +FOLD_PATTERNS = { + 'azure': ['##[group]{name}', '##[endgroup]'], + 'travis': ['travis_fold:start:{name}', 'travis_fold:end:{name}'], + 'github': ['::group::{name}', '::endgroup::{name}'], +} + +PLATFORM_IDENTIFIER_DESCIPTIONS = { + 'manylinux_x86_64': 'Manylinux x86_64', + 'manylinux_i686': 'Manylinux i686', + 'manylinux_aarch64': 'Manylinux aarch64', + 'manylinux_ppc64le': 'Manylinux ppc64le', + 'manylinux_s390x': 'Manylinux s390x', + 'win32': 'Windows 32bit', + 'win_amd64': 'Windows 64bit', + 'macosx_x86_64': 'macOS x86_64', +} + + +class Logger: + def __init__(self): + if 'AZURE_HTTP_USER_AGENT' in os.environ: + self.fold_mode = 'azure' + self.colors_enabled = True + + elif 'GITHUB_ACTIONS' in os.environ: + self.fold_mode = 'github' + self.colors_enabled = True + + elif 'TRAVIS' in os.environ: + self.fold_mode = 'travis' + self.colors_enabled = True + + elif 'APPVEYOR' in os.environ: + self.fold_mode = 'disabled' + self.colors_enabled = True + + else: + self.fold_mode = 'disabled' + self.colors_enabled = False + + @contextmanager + def build(self, identifier: str): + c = self.colors + print(f'{c.bold}Building {build_description_from_identifier(identifier)} wheel{c.end}') + print(f'Identifier: {identifier}') + + start_time = time.time() + try: + yield + duration = time.time() - start_time + print(f'{c.green}Build {c.bg_grey}{identifier}{c.end}{c.green} completed in {duration:.2f}s{c.end}') + except Exception: + duration = time.time() - start_time + print(f'{c.red}Build {c.bg_grey}{identifier}{c.end}{c.red} failed in {duration:.2f}s{c.end}') + raise + + @contextmanager + def step(self, name: str): + c = self.colors + start_time = time.time() + + try: + with self.fold_group(name): + yield + duration = time.time() - start_time + print(f'{c.green}✓ {c.faint}[{duration:.2f}]{c.end}') + except Exception: + raise + + @contextmanager + def fold_group(self, name: str): + fold_start_pattern, fold_end_pattern = FOLD_PATTERNS.get(self.fold_mode, ('', '')) + print(fold_start_pattern.format(name=name)) + try: + yield + finally: + print(fold_end_pattern.format(name=name)) + + @property + def colors(self): + if self.colors_enabled: + return colors_enabled + else: + return colors_disabled + + +def build_description_from_identifier(identifier): + python_identifier, _, platform_identifier = identifier.partition('-') + + build_description = '' + + python_interpreter = python_identifier[0:2] + python_version = python_identifier[2:4] + + if python_interpreter == 'cp': + build_description += 'CPython' + elif python_interpreter == 'pp': + build_description += 'PyPy' + else: + raise Exception('unknown python') + + build_description += f' {python_version[0]}.{python_version[1]} ' + + try: + build_description += PLATFORM_IDENTIFIER_DESCIPTIONS[platform_identifier] + except KeyError as e: + raise Exception('unknown platform') from e + + return build_description + + +class Colors(): + red = '\033[31m' + green = '\033[32m' + yellow = '\033[33m' + blue = '\033[34m' + cyan = '\033[36m' + bright_green = '\033[92m' + white = '\033[37m\033[97m' + + bg_grey = '\033[48;5;244m' + + bold = '\033[1m' + faint = '\033[2m' + + end = '\033[0m' + + class Disabled: + def __getattr__(self, attr): + return '' + + +colors_enabled = Colors() +colors_disabled = Colors.Disabled()