From 38297e6663bdd81ee8e67759aaaa0a4ccb2d0b05 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 13 Nov 2020 16:30:27 +0000 Subject: [PATCH] Add log steps to macOS/Windows --- cibuildwheel/linux.py | 4 +- cibuildwheel/logger.py | 11 ++- cibuildwheel/macos.py | 185 +++++++++++++++++++---------------- cibuildwheel/windows.py | 212 ++++++++++++++++++++++------------------ 4 files changed, 226 insertions(+), 186 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 6c6593a3..f404e9f9 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -1,4 +1,3 @@ -from cibuildwheel.logger import Logger import platform import subprocess import sys @@ -8,6 +7,7 @@ from pathlib import Path, PurePath from typing import List, NamedTuple, Union from .docker_container import DockerContainer +from .logger import log from .util import (BuildOptions, BuildSelector, NonPlatformWheelError, get_build_verbosity_extra_flags, prepare_command) @@ -112,8 +112,6 @@ def build(options: BuildOptions) -> None: container_package_dir = container_project_path / abs_package_dir.relative_to(cwd) container_output_dir = PurePath('/output') - log = Logger() - for implementation, platform_tag, docker_image in platforms: platform_configs = [c for c in python_configurations if c.identifier.startswith(implementation) and c.identifier.endswith(platform_tag)] if not platform_configs: diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index 7ccc71ff..df38cde8 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -1,7 +1,7 @@ import os -import time -import sys import re +import sys +import time from typing import Optional, Union DEFAULT_FOLD_PATTERN = ('{name}', '') @@ -142,6 +142,13 @@ class Logger: return colors_disabled +''' +Global instance of the Logger. +''' +# (there's only one stdout per-process, so a global instance is justified) +log = Logger() + + def build_description_from_identifier(identifier: str): python_identifier, _, platform_identifier = identifier.partition('-') diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 5dfd1cd6..eb5ce5aa 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -9,9 +9,10 @@ from pathlib import Path from typing import Dict, List, NamedTuple, Optional, Sequence, Union from .environment import ParsedEnvironment -from .util import (BuildOptions, BuildSelector, NonPlatformWheelError, download, - get_build_verbosity_extra_flags, get_pip_script, - prepare_command, install_certifi_script) +from .logger import log +from .util import (BuildOptions, BuildSelector, NonPlatformWheelError, + download, get_build_verbosity_extra_flags, get_pip_script, + install_certifi_script, prepare_command) def call(args: Union[str, Sequence[Union[str, PathLike]]], env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None, shell: bool = False) -> int: @@ -116,13 +117,18 @@ def install_pypy(version: str, url: str) -> Path: def setup_python(python_configuration: PythonConfiguration, dependency_constraint_flags: Sequence[Union[str, PathLike]], environment: ParsedEnvironment) -> Dict[str, str]: - if python_configuration.identifier.startswith('cp'): + implementation_id = python_configuration.identifier.split("-")[0] + log.step(f'Installing Python {implementation_id}...') + + if implementation_id.startswith('cp'): installation_bin_path = install_cpython(python_configuration.version, python_configuration.url) - elif python_configuration.identifier.startswith('pp'): + elif implementation_id.startswith('pp'): installation_bin_path = install_pypy(python_configuration.version, python_configuration.url) else: raise ValueError("Unknown Python implementation") + log.step('Setting up build environment...') + env = os.environ.copy() env['PATH'] = os.pathsep.join([ str(SYMLINKS_DIR), @@ -156,7 +162,6 @@ def setup_python(python_configuration: PythonConfiguration, if which_pip != '/tmp/cibw_bin/pip': print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr) exit(1) - call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate', *dependency_constraint_flags], env=env) # Set MACOSX_DEPLOYMENT_TARGET to 10.9, if the user didn't set it. # CPython 3.5 defaults to 10.6, and pypy defaults to 10.7, causing @@ -170,6 +175,9 @@ def setup_python(python_configuration: PythonConfiguration, # https://github.com/python/cpython/blob/a5ed2fe0eedefa1649aa93ee74a0bafc8e628a10/Lib/_osx_support.py#L260 env.setdefault('ARCHFLAGS', '-arch x86_64') + log.step('Installing build tools...') + call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate', *dependency_constraint_flags], env=env) + return env @@ -178,102 +186,111 @@ def build(options: BuildOptions) -> None: built_wheel_dir = temp_dir / 'built_wheel' repaired_wheel_dir = temp_dir / 'repaired_wheel' - if options.before_all: - env = options.environment.as_dictionary(prev_environment=os.environ) - before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) - call([before_all_prepared], shell=True, env=env) + try: + if options.before_all: + log.step('Running before_all...') + env = options.environment.as_dictionary(prev_environment=os.environ) + before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) + call([before_all_prepared], shell=True, env=env) - python_configurations = get_python_configurations(options.build_selector) + python_configurations = get_python_configurations(options.build_selector) - for config in python_configurations: - dependency_constraint_flags: Sequence[Union[str, PathLike]] = [] - if options.dependency_constraints: - dependency_constraint_flags = [ - '-c', options.dependency_constraints.get_for_python_version(config.version) - ] + for config in python_configurations: + log.build_start(config.identifier) - env = setup_python(config, dependency_constraint_flags, options.environment) + dependency_constraint_flags: Sequence[Union[str, PathLike]] = [] + if options.dependency_constraints: + dependency_constraint_flags = [ + '-c', options.dependency_constraints.get_for_python_version(config.version) + ] - # run the before_build command - if options.before_build: - before_build_prepared = prepare_command(options.before_build, project='.', package=options.package_dir) - call(before_build_prepared, env=env, shell=True) + env = setup_python(config, dependency_constraint_flags, options.environment) - # build the wheel - if built_wheel_dir.exists(): - shutil.rmtree(built_wheel_dir) - built_wheel_dir.mkdir(parents=True) + if options.before_build: + log.step('Running before_build...') + before_build_prepared = prepare_command(options.before_build, project='.', package=options.package_dir) + call(before_build_prepared, env=env, shell=True) - # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org - # see https://github.com/joerick/cibuildwheel/pull/369 - call([ - 'pip', 'wheel', - options.package_dir.resolve(), - '-w', built_wheel_dir, - '--no-deps', - *get_build_verbosity_extra_flags(options.build_verbosity) - ], env=env) + log.step('Building wheel...') + if built_wheel_dir.exists(): + shutil.rmtree(built_wheel_dir) + built_wheel_dir.mkdir(parents=True) - built_wheel = next(built_wheel_dir.glob('*.whl')) + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org + # see https://github.com/joerick/cibuildwheel/pull/369 + call([ + 'pip', 'wheel', + options.package_dir.resolve(), + '-w', built_wheel_dir, + '--no-deps', + *get_build_verbosity_extra_flags(options.build_verbosity) + ], env=env) - # repair the wheel - if repaired_wheel_dir.exists(): - shutil.rmtree(repaired_wheel_dir) - repaired_wheel_dir.mkdir(parents=True) + built_wheel = next(built_wheel_dir.glob('*.whl')) - if built_wheel.name.endswith('none-any.whl'): - raise NonPlatformWheelError() + if repaired_wheel_dir.exists(): + shutil.rmtree(repaired_wheel_dir) + repaired_wheel_dir.mkdir(parents=True) - if options.repair_command: - repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) - call(repair_command_prepared, env=env, shell=True) - else: - shutil.move(str(built_wheel), repaired_wheel_dir) + if built_wheel.name.endswith('none-any.whl'): + raise NonPlatformWheelError() - repaired_wheel = next(repaired_wheel_dir.glob('*.whl')) + if options.repair_command: + log.step('Repairing wheel...') + repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) + call(repair_command_prepared, env=env, shell=True) + else: + shutil.move(str(built_wheel), repaired_wheel_dir) - if options.test_command: - # set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) - venv_dir = Path(tempfile.mkdtemp()) + repaired_wheel = next(repaired_wheel_dir.glob('*.whl')) - # Use --no-download to ensure determinism by using seed libraries - # built into virtualenv - call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) + if options.test_command: + log.step('Testing wheel...') + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) + venv_dir = Path(tempfile.mkdtemp()) - virtualenv_env = env.copy() - virtualenv_env['PATH'] = os.pathsep.join([ - str(venv_dir / 'bin'), - virtualenv_env['PATH'], - ]) + # Use --no-download to ensure determinism by using seed libraries + # built into virtualenv + call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) - # check that we are using the Python from the virtual environment - call(['which', 'python'], env=virtualenv_env) + virtualenv_env = env.copy() + virtualenv_env['PATH'] = os.pathsep.join([ + str(venv_dir / 'bin'), + virtualenv_env['PATH'], + ]) - if options.before_test: - before_test_prepared = prepare_command(options.before_test, project='.', package=options.package_dir) - call(before_test_prepared, env=virtualenv_env, shell=True) + # check that we are using the Python from the virtual environment + call(['which', 'python'], env=virtualenv_env) - # install the wheel - call(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env) + if options.before_test: + before_test_prepared = prepare_command(options.before_test, project='.', package=options.package_dir) + call(before_test_prepared, env=virtualenv_env, shell=True) - # test the wheel - if options.test_requires: - call(['pip', 'install'] + options.test_requires, env=virtualenv_env) + # install the wheel + call(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env) - # run the tests from $HOME, with an absolute path in the command - # (this ensures that Python runs the tests against the installed wheel - # and not the repo code) - test_command_prepared = prepare_command( - options.test_command, - project=Path('.').resolve(), - package=options.package_dir.resolve() - ) - call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True) + # test the wheel + if options.test_requires: + call(['pip', 'install'] + options.test_requires, env=virtualenv_env) - # clean up - shutil.rmtree(venv_dir) + # run the tests from $HOME, with an absolute path in the command + # (this ensures that Python runs the tests against the installed wheel + # and not the repo code) + test_command_prepared = prepare_command( + options.test_command, + project=Path('.').resolve(), + package=options.package_dir.resolve() + ) + call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True) - # we're all done here; move it to output (overwrite existing) - shutil.move(str(repaired_wheel), options.output_dir) + # clean up + shutil.rmtree(venv_dir) + + # we're all done here; move it to output (overwrite existing) + shutil.move(str(repaired_wheel), options.output_dir) + log.build_end() + except subprocess.CalledProcessError as error: + log.error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') + exit(1) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 9e8c1df5..f9997927 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -7,11 +7,13 @@ from os import PathLike from pathlib import Path from typing import Dict, List, NamedTuple, Optional, Sequence, Union from zipfile import ZipFile + import toml from .environment import ParsedEnvironment -from .util import (BuildOptions, BuildSelector, NonPlatformWheelError, download, - get_build_verbosity_extra_flags, get_pip_script, +from .logger import log +from .util import (BuildOptions, BuildSelector, NonPlatformWheelError, + download, get_build_verbosity_extra_flags, get_pip_script, prepare_command) IS_RUNNING_ON_AZURE = Path('C:\\hostedtoolcache').exists() @@ -109,11 +111,15 @@ def install_pypy(version: str, arch: str, url: str) -> Path: def setup_python(python_configuration: PythonConfiguration, dependency_constraint_flags: Sequence[Union[str, PathLike]], environment: ParsedEnvironment) -> Dict[str, str]: nuget = Path('C:\\cibw\\nuget.exe') if not nuget.exists(): + log.step('Downloading nuget...') download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget) - if python_configuration.identifier.startswith('cp'): + implementation_id = python_configuration.identifier.split("-")[0] + log.step(f'Installing Python {implementation_id}...') + + if implementation_id.startswith('cp'): installation_path = install_cpython(python_configuration.version, python_configuration.arch, nuget) - elif python_configuration.identifier.startswith('pp'): + elif implementation_id.startswith('pp'): assert python_configuration.url is not None installation_path = install_pypy(python_configuration.version, python_configuration.arch, python_configuration.url) else: @@ -121,6 +127,8 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain assert (installation_path / 'python.exe').exists() + log.step('Setting up build environment...') + # set up PATH and environment variables for run_with_env env = os.environ.copy() env['PYTHON_VERSION'] = python_configuration.version @@ -151,7 +159,8 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr) exit(1) - # prepare the Python environment + log.step('Installing build tools...') + call(['python', '-m', 'pip', 'install', '--upgrade', 'pip', *dependency_constraint_flags], env=env) call(['pip', '--version'], env=env) call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', *dependency_constraint_flags], env=env) @@ -182,6 +191,7 @@ def pep_518_cp35_workaround(package_dir: Path, env: Dict[str, str]) -> None: ) if requirements: + log.step('Performing PEP518 workaround...') with tempfile.TemporaryDirectory() as d: reqfile = Path(d) / "requirements.txt" with reqfile.open("w") as f: @@ -195,114 +205,122 @@ def build(options: BuildOptions) -> None: built_wheel_dir = temp_dir / 'built_wheel' repaired_wheel_dir = temp_dir / 'repaired_wheel' - # install nuget as best way to provide python - nuget = Path('C:\\cibw\\nuget.exe') - download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget) + try: + if options.before_all: + log.step('Running before_all...') + env = options.environment.as_dictionary(prev_environment=os.environ) + before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) + shell(before_all_prepared, env=env) - if options.before_all: - env = options.environment.as_dictionary(prev_environment=os.environ) - before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) - shell(before_all_prepared, env=env) + python_configurations = get_python_configurations(options.build_selector) - python_configurations = get_python_configurations(options.build_selector) - for config in python_configurations: - dependency_constraint_flags: Sequence[Union[str, PathLike]] = [] - if options.dependency_constraints: - dependency_constraint_flags = [ - '-c', options.dependency_constraints.get_for_python_version(config.version) - ] + for config in python_configurations: + log.build_start(config.identifier) - # install Python - env = setup_python(config, dependency_constraint_flags, options.environment) + dependency_constraint_flags: Sequence[Union[str, PathLike]] = [] + if options.dependency_constraints: + dependency_constraint_flags = [ + '-c', options.dependency_constraints.get_for_python_version(config.version) + ] - # run the before_build command - if options.before_build: - before_build_prepared = prepare_command(options.before_build, project='.', package=options.package_dir) - shell(before_build_prepared, env=env) + # install Python + env = setup_python(config, dependency_constraint_flags, options.environment) - # activate the PEP 518 patch if on Windows Python 3.5 - # (will only have an effect if PEP 517 builds are used): - if config.version.startswith('3.5'): - pep_518_cp35_workaround(options.package_dir, env) + # run the before_build command + if options.before_build: + log.step('Running before_build...') + before_build_prepared = prepare_command(options.before_build, project='.', package=options.package_dir) + shell(before_build_prepared, env=env) - # build the wheel - if built_wheel_dir.exists(): - shutil.rmtree(built_wheel_dir) - built_wheel_dir.mkdir(parents=True) - # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org - # see https://github.com/joerick/cibuildwheel/pull/369 - call([ - 'pip', 'wheel', - options.package_dir.resolve(), - '-w', built_wheel_dir, - '--no-deps', - *get_build_verbosity_extra_flags(options.build_verbosity) - ], env=env) + # activate the PEP 518 patch if on Windows Python 3.5 + # (will only have an effect if PEP 517 builds are used): + if config.version.startswith('3.5'): + pep_518_cp35_workaround(options.package_dir, env) - built_wheel = next(built_wheel_dir.glob('*.whl')) + log.step('Building wheel...') + if built_wheel_dir.exists(): + shutil.rmtree(built_wheel_dir) + built_wheel_dir.mkdir(parents=True) + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org + # see https://github.com/joerick/cibuildwheel/pull/369 + call([ + 'pip', 'wheel', + options.package_dir.resolve(), + '-w', built_wheel_dir, + '--no-deps', + *get_build_verbosity_extra_flags(options.build_verbosity) + ], env=env) - # repair the wheel - if repaired_wheel_dir.exists(): - shutil.rmtree(repaired_wheel_dir) - repaired_wheel_dir.mkdir(parents=True) + built_wheel = next(built_wheel_dir.glob('*.whl')) - if built_wheel.name.endswith('none-any.whl'): - raise NonPlatformWheelError() + # repair the wheel + if repaired_wheel_dir.exists(): + shutil.rmtree(repaired_wheel_dir) + repaired_wheel_dir.mkdir(parents=True) - if options.repair_command: - repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) - shell(repair_command_prepared, env=env) - else: - shutil.move(str(built_wheel), repaired_wheel_dir) + if built_wheel.name.endswith('none-any.whl'): + raise NonPlatformWheelError() - repaired_wheel = next(repaired_wheel_dir.glob('*.whl')) + if options.repair_command: + log.step('Repairing wheel...') + repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) + shell(repair_command_prepared, env=env) + else: + shutil.move(str(built_wheel), repaired_wheel_dir) - if options.test_command: - # set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) - venv_dir = Path(tempfile.mkdtemp()) + repaired_wheel = next(repaired_wheel_dir.glob('*.whl')) - # Use --no-download to ensure determinism by using seed libraries - # built into virtualenv - call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) + if options.test_command: + log.step('Testing wheel...') + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) + venv_dir = Path(tempfile.mkdtemp()) - virtualenv_env = env.copy() - virtualenv_env['PATH'] = os.pathsep.join([ - str(venv_dir / 'Scripts'), - virtualenv_env['PATH'], - ]) + # Use --no-download to ensure determinism by using seed libraries + # built into virtualenv + call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) - # check that we are using the Python from the virtual environment - call(['which', 'python'], env=virtualenv_env) + virtualenv_env = env.copy() + virtualenv_env['PATH'] = os.pathsep.join([ + str(venv_dir / 'Scripts'), + virtualenv_env['PATH'], + ]) - if options.before_test: - before_test_prepared = prepare_command( - options.before_test, - project='.', - package=options.package_dir + # check that we are using the Python from the virtual environment + call(['which', 'python'], env=virtualenv_env) + + if options.before_test: + before_test_prepared = prepare_command( + options.before_test, + project='.', + package=options.package_dir + ) + shell(before_test_prepared, env=virtualenv_env) + + # install the wheel + call(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env) + + # test the wheel + if options.test_requires: + call(['pip', 'install'] + options.test_requires, env=virtualenv_env) + + # run the tests from c:\, with an absolute path in the command + # (this ensures that Python runs the tests against the installed wheel + # and not the repo code) + test_command_prepared = prepare_command( + options.test_command, + project=Path('.').resolve(), + package=options.package_dir.resolve() ) - shell(before_test_prepared, env=virtualenv_env) + shell(test_command_prepared, cwd='c:\\', env=virtualenv_env) - # install the wheel - call(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env) + # clean up + shutil.rmtree(venv_dir) - # test the wheel - if options.test_requires: - call(['pip', 'install'] + options.test_requires, env=virtualenv_env) - - # run the tests from c:\, with an absolute path in the command - # (this ensures that Python runs the tests against the installed wheel - # and not the repo code) - test_command_prepared = prepare_command( - options.test_command, - project=Path('.').resolve(), - package=options.package_dir.resolve() - ) - shell(test_command_prepared, cwd='c:\\', env=virtualenv_env) - - # clean up - shutil.rmtree(venv_dir) - - # we're all done here; move it to output (remove if already exists) - shutil.move(str(repaired_wheel), options.output_dir) + # we're all done here; move it to output (remove if already exists) + shutil.move(str(repaired_wheel), options.output_dir) + log.build_end() + except subprocess.CalledProcessError as error: + log.error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') + exit(1)