From 38e7b5a91ded109ae44947b754cd47d28e3a26d5 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 1 Nov 2020 11:34:02 +0000 Subject: [PATCH] Change to an imperative-style API --- cibuildwheel/linux.py | 240 +++++++++++++++++++++-------------------- cibuildwheel/logger.py | 81 ++++++++------ 2 files changed, 175 insertions(+), 146 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 8c8d85bd..80ea93a0 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -112,7 +112,7 @@ def build(options: BuildOptions) -> None: container_package_dir = container_project_path / abs_package_dir.relative_to(cwd) container_output_dir = PurePath('/output') - logger = Logger() + log = Logger() for implementation, platform_tag, docker_image in platforms: platform_configs = [c for c in python_configurations if c.identifier.startswith(implementation) and c.identifier.endswith(platform_tag)] @@ -124,19 +124,135 @@ def build(options: BuildOptions) -> None: docker.copy_into(Path.cwd(), container_project_path) if options.before_all: - with logger.step('Running before_all...'): - env = docker.get_environment() - env['PATH'] = f'/opt/python/cp38-cp38:{env["PATH"]}' - env = options.environment.as_dictionary(env, executor=docker.environment_executor) + log.build_step('Running before_all...') - before_all_prepared = prepare_command(options.before_all, project=container_project_path, package=container_package_dir) - docker.call(['sh', '-c', before_all_prepared], env=env) + env = docker.get_environment() + env['PATH'] = f'/opt/python/cp38-cp38:{env["PATH"]}' + env = options.environment.as_dictionary(env, executor=docker.environment_executor) + + before_all_prepared = prepare_command(options.before_all, project=container_project_path, package=container_package_dir) + docker.call(['sh', '-c', before_all_prepared], env=env) + + log.build_step_end() for config in platform_configs: - build_one(options=options, config=config, docker=docker, container_project_path=container_project_path, container_package_dir=container_package_dir, container_output_dir=container_output_dir, logger=logger) + log.build_start(config.identifier) + dependency_constraint_flags: List[Union[str, PathLike]] = [] + + if options.dependency_constraints: + constraints_file = options.dependency_constraints.get_for_python_version(config.version) + container_constraints_file = PurePath('/constraints.txt') + + docker.copy_into(constraints_file, container_constraints_file) + dependency_constraint_flags = ['-c', container_constraints_file] + + env = docker.get_environment() + + # put this config's python top of the list + python_bin = config.path / 'bin' + env['PATH'] = f'{python_bin}:{env["PATH"]}' + + log.build_step('Setting up build environment...') + + env = options.environment.as_dictionary(env, executor=docker.environment_executor) + + # check config python and pip are still on PATH + which_python = docker.call(['which', 'python'], env=env, capture_output=True).strip() + if PurePath(which_python) != python_bin / 'python': + print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr) + exit(1) + + which_pip = docker.call(['which', 'pip'], env=env, capture_output=True).strip() + if PurePath(which_pip) != python_bin / 'pip': + print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr) + exit(1) + + if options.before_build: + log.build_step('Running before_build...') + before_build_prepared = prepare_command(options.before_build, project=container_project_path, package=container_package_dir) + docker.call(['sh', '-c', before_build_prepared], env=env) + + log.build_step('Building wheel...') + + temp_dir = PurePath('/tmp/cibuildwheel') + built_wheel_dir = temp_dir / 'built_wheel' + docker.call(['rm', '-rf', built_wheel_dir]) + docker.call(['mkdir', '-p', built_wheel_dir]) + + docker.call([ + 'pip', 'wheel', + container_package_dir, + '-w', built_wheel_dir, + '--no-deps', + *get_build_verbosity_extra_flags(options.build_verbosity) + ], env=env) + + built_wheel = docker.glob(built_wheel_dir, '*.whl')[0] + + repaired_wheel_dir = temp_dir / 'repaired_wheel' + docker.call(['rm', '-rf', repaired_wheel_dir]) + docker.call(['mkdir', '-p', repaired_wheel_dir]) + + if built_wheel.name.endswith('none-any.whl'): + raise NonPlatformWheelError() + + if options.repair_command: + log.build_step('Repairing wheel...') + repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) + docker.call(['sh', '-c', repair_command_prepared], env=env) + else: + docker.call(['mv', built_wheel, repaired_wheel_dir]) + + repaired_wheels = docker.glob(repaired_wheel_dir, '*.whl') + + if options.test_command: + log.build_step('Testing wheel...') + + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + docker.call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) + venv_dir = PurePath(docker.call(['mktemp', '-d'], capture_output=True).strip()) / 'venv' + + docker.call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) + + virtualenv_env = env.copy() + virtualenv_env['PATH'] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}" + + if options.before_test: + before_test_prepared = prepare_command(options.before_test, project=container_project_path, package=container_package_dir) + docker.call(['sh', '-c', before_test_prepared], env=virtualenv_env) + + # Install the wheel we just built + # Note: If auditwheel produced two wheels, it's because the earlier produced wheel + # conforms to multiple manylinux standards. These multiple versions of the wheel are + # functionally the same, differing only in name, wheel metadata, and possibly include + # different external shared libraries. so it doesn't matter which one we run the tests on. + # Let's just pick the first one. + wheel_to_test = repaired_wheels[0] + docker.call(['pip', 'install', str(wheel_to_test) + options.test_extras], env=virtualenv_env) + + # Install any requirements to run the tests + if options.test_requires: + docker.call(['pip', 'install', *options.test_requires], env=virtualenv_env) + + # Run the tests from a different directory + test_command_prepared = prepare_command(options.test_command, project=container_project_path, package=container_package_dir) + docker.call(['sh', '-c', test_command_prepared], cwd='/root', env=virtualenv_env) + + # clean up test environment + docker.call(['rm', '-rf', venv_dir]) + + # move repaired wheels to output + docker.call(['mkdir', '-p', container_output_dir]) + docker.call(['mv', *repaired_wheels, container_output_dir]) + + log.build_end() + + log.build_step('Copying wheels back to host...') # copy the output back into the host docker.copy_out(container_output_dir, options.output_dir) + log.build_step_end() except subprocess.CalledProcessError as error: print(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') troubleshoot(options.package_dir, error) @@ -145,112 +261,8 @@ def build(options: BuildOptions) -> None: def build_one(options: BuildOptions, config: PythonConfiguration, docker: DockerContainer, container_project_path: PurePath, container_package_dir: PurePath, - container_output_dir: PurePath, logger: Logger): - dependency_constraint_flags: List[Union[str, PathLike]] = [] - - if options.dependency_constraints: - constraints_file = options.dependency_constraints.get_for_python_version(config.version) - container_constraints_file = PurePath('/constraints.txt') - - docker.copy_into(constraints_file, container_constraints_file) - dependency_constraint_flags = ['-c', container_constraints_file] - - env = docker.get_environment() - - # put this config's python top of the list - python_bin = config.path / 'bin' - env['PATH'] = f'{python_bin}:{env["PATH"]}' - - with logger.step(' Setting up build environment...'): - env = options.environment.as_dictionary(env, executor=docker.environment_executor) - - # check config python and pip are still on PATH - which_python = docker.call(['which', 'python'], env=env, capture_output=True).strip() - if PurePath(which_python) != python_bin / 'python': - print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr) - exit(1) - - which_pip = docker.call(['which', 'pip'], env=env, capture_output=True).strip() - if PurePath(which_pip) != python_bin / 'pip': - print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr) - exit(1) - - if options.before_build: - with logger.step(' Running before_build...'): - before_build_prepared = prepare_command(options.before_build, project=container_project_path, package=container_package_dir) - docker.call(['sh', '-c', before_build_prepared], env=env) - - temp_dir = PurePath('/tmp/cibuildwheel') - built_wheel_dir = temp_dir / 'built_wheel' - docker.call(['rm', '-rf', built_wheel_dir]) - docker.call(['mkdir', '-p', built_wheel_dir]) - - with logger.step(' Building wheel...'): - docker.call([ - 'pip', 'wheel', - container_package_dir, - '-w', built_wheel_dir, - '--no-deps', - *get_build_verbosity_extra_flags(options.build_verbosity) - ], env=env) - - built_wheel = docker.glob(built_wheel_dir, '*.whl')[0] - - repaired_wheel_dir = temp_dir / 'repaired_wheel' - docker.call(['rm', '-rf', repaired_wheel_dir]) - docker.call(['mkdir', '-p', repaired_wheel_dir]) - - if built_wheel.name.endswith('none-any.whl'): - raise NonPlatformWheelError() - - if options.repair_command: - with logger.step(' Repairing wheel...'): - repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) - docker.call(['sh', '-c', repair_command_prepared], env=env) - else: - docker.call(['mv', built_wheel, repaired_wheel_dir]) - - repaired_wheels = docker.glob(repaired_wheel_dir, '*.whl') - - if options.test_command: - with logger.step(' Testing wheel...'): - # set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - docker.call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) - venv_dir = PurePath(docker.call(['mktemp', '-d'], capture_output=True).strip()) / 'venv' - - docker.call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) - - virtualenv_env = env.copy() - virtualenv_env['PATH'] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}" - - if options.before_test: - before_test_prepared = prepare_command(options.before_test, project=container_project_path, package=container_package_dir) - docker.call(['sh', '-c', before_test_prepared], env=virtualenv_env) - - # Install the wheel we just built - # Note: If auditwheel produced two wheels, it's because the earlier produced wheel - # conforms to multiple manylinux standards. These multiple versions of the wheel are - # functionally the same, differing only in name, wheel metadata, and possibly include - # different external shared libraries. so it doesn't matter which one we run the tests on. - # Let's just pick the first one. - wheel_to_test = repaired_wheels[0] - docker.call(['pip', 'install', str(wheel_to_test) + options.test_extras], env=virtualenv_env) - - # Install any requirements to run the tests - if options.test_requires: - docker.call(['pip', 'install', *options.test_requires], env=virtualenv_env) - - # Run the tests from a different directory - test_command_prepared = prepare_command(options.test_command, project=container_project_path, package=container_package_dir) - docker.call(['sh', '-c', test_command_prepared], cwd='/root', env=virtualenv_env) - - # clean up test environment - docker.call(['rm', '-rf', venv_dir]) - - # move repaired wheels to output - docker.call(['mkdir', '-p', container_output_dir]) - docker.call(['mv', *repaired_wheels, container_output_dir]) + container_output_dir: PurePath, log: Logger): + pass def troubleshoot(package_dir: Path, error: Exception) -> None: diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index 93a1fde3..4dde6e6a 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -1,6 +1,7 @@ -from contextlib import contextmanager import os import time +import re +from typing import Optional FOLD_PATTERNS = { 'azure': ['##[group]{name}', '##[endgroup]'], @@ -21,6 +22,13 @@ PLATFORM_IDENTIFIER_DESCIPTIONS = { class Logger: + fold_mode: str + colors_enabled: bool + active_build_identifier: Optional[str] = None + build_start_time: Optional[float] = 0 + step_start_time: Optional[float] = 0 + active_fold_group_id: Optional[str] = None + def __init__(self): if 'AZURE_HTTP_USER_AGENT' in os.environ: self.fold_mode = 'azure' @@ -42,43 +50,52 @@ class Logger: self.fold_mode = 'disabled' self.colors_enabled = False - @contextmanager - def build(self, identifier: str): + def build_start(self, identifier: str): c = self.colors + print() print(f'{c.bold}Building {build_description_from_identifier(identifier)} wheel{c.end}') print(f'Identifier: {identifier}') + print() - start_time = time.time() - try: - yield - duration = time.time() - start_time - print(f'{c.green}Build {c.bg_grey}{identifier}{c.end}{c.green} completed in {duration:.2f}s{c.end}') - except Exception: - duration = time.time() - start_time - print(f'{c.red}Build {c.bg_grey}{identifier}{c.end}{c.red} failed in {duration:.2f}s{c.end}') - raise + self.build_start_time = time.time() + self.active_build_identifier = identifier + + def build_end(self): + assert self.build_start_time is not None + self.build_step_end() - @contextmanager - def step(self, name: str): c = self.colors - start_time = time.time() + duration = time.time() - self.build_start_time + print(f'{c.green}Build {c.bg_grey}{self.active_build_identifier}{c.end}{c.green} completed in {duration:.2f}s{c.end}') + print() + print('---') + self.build_start_time = None - try: - with self.fold_group(name): - yield - duration = time.time() - start_time - print(f'{c.green}✓ {c.faint}[{duration:.2f}]{c.end}') - except Exception: - raise + def build_step(self, step_description: str): + self.build_step_end() + self.step_start_time = time.time() + self.start_fold_group(step_description) - @contextmanager - def fold_group(self, name: str): - fold_start_pattern, fold_end_pattern = FOLD_PATTERNS.get(self.fold_mode, ('', '')) - print(fold_start_pattern.format(name=name)) - try: - yield - finally: - print(fold_end_pattern.format(name=name)) + def build_step_end(self): + if self.step_start_time is not None: + self.end_fold_group() + c = self.colors + duration = time.time() - self.step_start_time + print(f'{c.green}✓ {c.faint}[{duration:.2f}s]{c.end}'.rjust(78)) + self.step_start_time = None + + def start_fold_group(self, name: str): + self.end_fold_group() + self.active_fold_group_id = re.sub(r'[^A-Za-z]', '', name) + fold_start_pattern = FOLD_PATTERNS.get(self.fold_mode, ('', ''))[0] + print(fold_start_pattern.format(name=self.active_fold_group_id)) + print() + + def end_fold_group(self): + if self.active_fold_group_id: + fold_start_pattern = FOLD_PATTERNS.get(self.fold_mode, ('', ''))[1] + print(fold_start_pattern.format(name=self.active_fold_group_id)) + self.active_fold_group_id = None @property def colors(self): @@ -88,7 +105,7 @@ class Logger: return colors_disabled -def build_description_from_identifier(identifier): +def build_description_from_identifier(identifier: str): python_identifier, _, platform_identifier = identifier.partition('-') build_description = '' @@ -130,7 +147,7 @@ class Colors(): end = '\033[0m' class Disabled: - def __getattr__(self, attr): + def __getattr__(self, attr: str): return ''