From 461aa35f99f82d8e762c9d77379c4fdc04d69a69 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 3 Aug 2020 17:38:07 +0100 Subject: [PATCH 01/63] Initial commit of logger functionality --- cibuildwheel/linux.py | 225 ++++++++++++++++++++++------------------- cibuildwheel/logger.py | 139 +++++++++++++++++++++++++ 2 files changed, 259 insertions(+), 105 deletions(-) create mode 100644 cibuildwheel/logger.py diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index e601a8cf..8c8d85bd 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -1,3 +1,4 @@ +from cibuildwheel.logger import Logger import platform import subprocess import sys @@ -111,6 +112,8 @@ def build(options: BuildOptions) -> None: container_package_dir = container_project_path / abs_package_dir.relative_to(cwd) container_output_dir = PurePath('/output') + logger = Logger() + for implementation, platform_tag, docker_image in platforms: platform_configs = [c for c in python_configurations if c.identifier.startswith(implementation) and c.identifier.endswith(platform_tag)] if not platform_configs: @@ -121,114 +124,16 @@ def build(options: BuildOptions) -> None: docker.copy_into(Path.cwd(), container_project_path) if options.before_all: - env = docker.get_environment() - env['PATH'] = f'/opt/python/cp38-cp38:{env["PATH"]}' - env = options.environment.as_dictionary(env, executor=docker.environment_executor) + with logger.step('Running before_all...'): + env = docker.get_environment() + env['PATH'] = f'/opt/python/cp38-cp38:{env["PATH"]}' + env = options.environment.as_dictionary(env, executor=docker.environment_executor) - before_all_prepared = prepare_command(options.before_all, project=container_project_path, package=container_package_dir) - docker.call(['sh', '-c', before_all_prepared], env=env) + before_all_prepared = prepare_command(options.before_all, project=container_project_path, package=container_package_dir) + docker.call(['sh', '-c', before_all_prepared], env=env) for config in platform_configs: - dependency_constraint_flags: List[Union[str, PathLike]] = [] - - if options.dependency_constraints: - constraints_file = options.dependency_constraints.get_for_python_version(config.version) - container_constraints_file = PurePath('/constraints.txt') - - docker.copy_into(constraints_file, container_constraints_file) - dependency_constraint_flags = ['-c', container_constraints_file] - - env = docker.get_environment() - - # put this config's python top of the list - python_bin = config.path / 'bin' - env['PATH'] = f'{python_bin}:{env["PATH"]}' - - env = options.environment.as_dictionary(env, executor=docker.environment_executor) - - # check config python and pip are still on PATH - which_python = docker.call(['which', 'python'], env=env, capture_output=True).strip() - if PurePath(which_python) != python_bin / 'python': - print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr) - exit(1) - - which_pip = docker.call(['which', 'pip'], env=env, capture_output=True).strip() - if PurePath(which_pip) != python_bin / 'pip': - print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr) - exit(1) - - if options.before_build: - before_build_prepared = prepare_command(options.before_build, project=container_project_path, package=container_package_dir) - docker.call(['sh', '-c', before_build_prepared], env=env) - - temp_dir = PurePath('/tmp/cibuildwheel') - built_wheel_dir = temp_dir / 'built_wheel' - docker.call(['rm', '-rf', built_wheel_dir]) - docker.call(['mkdir', '-p', built_wheel_dir]) - - docker.call([ - 'pip', 'wheel', - container_package_dir, - '-w', built_wheel_dir, - '--no-deps', - *get_build_verbosity_extra_flags(options.build_verbosity) - ], env=env) - - built_wheel = docker.glob(built_wheel_dir, '*.whl')[0] - - repaired_wheel_dir = temp_dir / 'repaired_wheel' - docker.call(['rm', '-rf', repaired_wheel_dir]) - docker.call(['mkdir', '-p', repaired_wheel_dir]) - - if built_wheel.name.endswith('none-any.whl'): - raise NonPlatformWheelError() - - if options.repair_command: - repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) - docker.call(['sh', '-c', repair_command_prepared], env=env) - else: - docker.call(['mv', built_wheel, repaired_wheel_dir]) - - repaired_wheels = docker.glob(repaired_wheel_dir, '*.whl') - - if options.test_command: - # set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - docker.call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) - venv_dir = PurePath(docker.call(['mktemp', '-d'], capture_output=True).strip()) / 'venv' - - docker.call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) - - virtualenv_env = env.copy() - virtualenv_env['PATH'] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}" - - if options.before_test: - before_test_prepared = prepare_command(options.before_test, project=container_project_path, package=container_package_dir) - docker.call(['sh', '-c', before_test_prepared], env=virtualenv_env) - - # Install the wheel we just built - # Note: If auditwheel produced two wheels, it's because the earlier produced wheel - # conforms to multiple manylinux standards. These multiple versions of the wheel are - # functionally the same, differing only in name, wheel metadata, and possibly include - # different external shared libraries. so it doesn't matter which one we run the tests on. - # Let's just pick the first one. - wheel_to_test = repaired_wheels[0] - docker.call(['pip', 'install', str(wheel_to_test) + options.test_extras], env=virtualenv_env) - - # Install any requirements to run the tests - if options.test_requires: - docker.call(['pip', 'install', *options.test_requires], env=virtualenv_env) - - # Run the tests from a different directory - test_command_prepared = prepare_command(options.test_command, project=container_project_path, package=container_package_dir) - docker.call(['sh', '-c', test_command_prepared], cwd='/root', env=virtualenv_env) - - # clean up test environment - docker.call(['rm', '-rf', venv_dir]) - - # move repaired wheels to output - docker.call(['mkdir', '-p', container_output_dir]) - docker.call(['mv', *repaired_wheels, container_output_dir]) + build_one(options=options, config=config, docker=docker, container_project_path=container_project_path, container_package_dir=container_package_dir, container_output_dir=container_output_dir, logger=logger) # copy the output back into the host docker.copy_out(container_output_dir, options.output_dir) @@ -238,6 +143,116 @@ def build(options: BuildOptions) -> None: exit(1) +def build_one(options: BuildOptions, config: PythonConfiguration, docker: DockerContainer, + container_project_path: PurePath, container_package_dir: PurePath, + container_output_dir: PurePath, logger: Logger): + dependency_constraint_flags: List[Union[str, PathLike]] = [] + + if options.dependency_constraints: + constraints_file = options.dependency_constraints.get_for_python_version(config.version) + container_constraints_file = PurePath('/constraints.txt') + + docker.copy_into(constraints_file, container_constraints_file) + dependency_constraint_flags = ['-c', container_constraints_file] + + env = docker.get_environment() + + # put this config's python top of the list + python_bin = config.path / 'bin' + env['PATH'] = f'{python_bin}:{env["PATH"]}' + + with logger.step(' Setting up build environment...'): + env = options.environment.as_dictionary(env, executor=docker.environment_executor) + + # check config python and pip are still on PATH + which_python = docker.call(['which', 'python'], env=env, capture_output=True).strip() + if PurePath(which_python) != python_bin / 'python': + print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr) + exit(1) + + which_pip = docker.call(['which', 'pip'], env=env, capture_output=True).strip() + if PurePath(which_pip) != python_bin / 'pip': + print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr) + exit(1) + + if options.before_build: + with logger.step(' Running before_build...'): + before_build_prepared = prepare_command(options.before_build, project=container_project_path, package=container_package_dir) + docker.call(['sh', '-c', before_build_prepared], env=env) + + temp_dir = PurePath('/tmp/cibuildwheel') + built_wheel_dir = temp_dir / 'built_wheel' + docker.call(['rm', '-rf', built_wheel_dir]) + docker.call(['mkdir', '-p', built_wheel_dir]) + + with logger.step(' Building wheel...'): + docker.call([ + 'pip', 'wheel', + container_package_dir, + '-w', built_wheel_dir, + '--no-deps', + *get_build_verbosity_extra_flags(options.build_verbosity) + ], env=env) + + built_wheel = docker.glob(built_wheel_dir, '*.whl')[0] + + repaired_wheel_dir = temp_dir / 'repaired_wheel' + docker.call(['rm', '-rf', repaired_wheel_dir]) + docker.call(['mkdir', '-p', repaired_wheel_dir]) + + if built_wheel.name.endswith('none-any.whl'): + raise NonPlatformWheelError() + + if options.repair_command: + with logger.step(' Repairing wheel...'): + repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) + docker.call(['sh', '-c', repair_command_prepared], env=env) + else: + docker.call(['mv', built_wheel, repaired_wheel_dir]) + + repaired_wheels = docker.glob(repaired_wheel_dir, '*.whl') + + if options.test_command: + with logger.step(' Testing wheel...'): + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + docker.call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) + venv_dir = PurePath(docker.call(['mktemp', '-d'], capture_output=True).strip()) / 'venv' + + docker.call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) + + virtualenv_env = env.copy() + virtualenv_env['PATH'] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}" + + if options.before_test: + before_test_prepared = prepare_command(options.before_test, project=container_project_path, package=container_package_dir) + docker.call(['sh', '-c', before_test_prepared], env=virtualenv_env) + + # Install the wheel we just built + # Note: If auditwheel produced two wheels, it's because the earlier produced wheel + # conforms to multiple manylinux standards. These multiple versions of the wheel are + # functionally the same, differing only in name, wheel metadata, and possibly include + # different external shared libraries. so it doesn't matter which one we run the tests on. + # Let's just pick the first one. + wheel_to_test = repaired_wheels[0] + docker.call(['pip', 'install', str(wheel_to_test) + options.test_extras], env=virtualenv_env) + + # Install any requirements to run the tests + if options.test_requires: + docker.call(['pip', 'install', *options.test_requires], env=virtualenv_env) + + # Run the tests from a different directory + test_command_prepared = prepare_command(options.test_command, project=container_project_path, package=container_package_dir) + docker.call(['sh', '-c', test_command_prepared], cwd='/root', env=virtualenv_env) + + # clean up test environment + docker.call(['rm', '-rf', venv_dir]) + + # move repaired wheels to output + docker.call(['mkdir', '-p', container_output_dir]) + docker.call(['mv', *repaired_wheels, container_output_dir]) + + def troubleshoot(package_dir: Path, error: Exception) -> None: if (isinstance(error, subprocess.CalledProcessError) and error.cmd[0:2] == ['pip', 'wheel']): # the 'pip wheel' step failed. diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py new file mode 100644 index 00000000..772be2a2 --- /dev/null +++ b/cibuildwheel/logger.py @@ -0,0 +1,139 @@ +from contextlib import contextmanager +import os +import time +import typing + +FOLD_PATTERNS = { + 'azure': ['##[group]{name}', '##[endgroup]'], + 'travis': ['travis_fold:start:{name}', 'travis_fold:end:{name}'], + 'github': ['::group::{name}', '::endgroup::{name}'], +} + +PLATFORM_IDENTIFIER_DESCIPTIONS = { + 'manylinux_x86_64': 'Manylinux x86_64', + 'manylinux_i686': 'Manylinux i686', + 'manylinux_aarch64': 'Manylinux aarch64', + 'manylinux_ppc64le': 'Manylinux ppc64le', + 'manylinux_s390x': 'Manylinux s390x', + 'win32': 'Windows 32bit', + 'win_amd64': 'Windows 64bit', + 'macosx_x86_64': 'macOS x86_64', +} + + +class Logger: + def __init__(self): + if 'AZURE_HTTP_USER_AGENT' in os.environ: + self.fold_mode = 'azure' + self.colors_enabled = True + + elif 'GITHUB_ACTIONS' in os.environ: + self.fold_mode = 'github' + self.colors_enabled = True + + elif 'TRAVIS' in os.environ: + self.fold_mode = 'travis' + self.colors_enabled = True + + elif 'APPVEYOR' in os.environ: + self.fold_mode = 'disabled' + self.colors_enabled = True + + else: + self.fold_mode = 'disabled' + self.colors_enabled = False + + @contextmanager + def build(self, identifier: str): + c = self.colors + print(f'{c.bold}Building {build_description_from_identifier(identifier)} wheel{c.end}') + print(f'Identifier: {identifier}') + + start_time = time.time() + try: + yield + duration = time.time() - start_time + print(f'{c.green}Build {c.bg_grey}{identifier}{c.end}{c.green} completed in {duration:.2f}s{c.end}') + except Exception: + duration = time.time() - start_time + print(f'{c.red}Build {c.bg_grey}{identifier}{c.end}{c.red} failed in {duration:.2f}s{c.end}') + raise + + @contextmanager + def step(self, name: str): + c = self.colors + start_time = time.time() + + try: + with self.fold_group(name): + yield + duration = time.time() - start_time + print(f'{c.green}✓ {c.faint}[{duration:.2f}]{c.end}') + except Exception: + raise + + @contextmanager + def fold_group(self, name: str): + fold_start_pattern, fold_end_pattern = FOLD_PATTERNS.get(self.fold_mode, ('', '')) + print(fold_start_pattern.format(name=name)) + try: + yield + finally: + print(fold_end_pattern.format(name=name)) + + @property + def colors(self): + if self.colors_enabled: + return colors_enabled + else: + return colors_disabled + + +def build_description_from_identifier(identifier): + python_identifier, _, platform_identifier = identifier.partition('-') + + build_description = '' + + python_interpreter = python_identifier[0:2] + python_version = python_identifier[2:4] + + if python_interpreter == 'cp': + build_description += 'CPython' + elif python_interpreter == 'pp': + build_description += 'PyPy' + else: + raise Exception('unknown python') + + build_description += f' {python_version[0]}.{python_version[1]} ' + + try: + build_description += PLATFORM_IDENTIFIER_DESCIPTIONS[platform_identifier] + except KeyError as e: + raise Exception('unknown platform') from e + + return build_description + + +class Colors(): + red = '\033[31m' + green = '\033[32m' + yellow = '\033[33m' + blue = '\033[34m' + cyan = '\033[36m' + bright_green = '\033[92m' + white = '\033[37m\033[97m' + + bg_grey = '\033[48;5;244m' + + bold = '\033[1m' + faint = '\033[2m' + + end = '\033[0m' + + class Disabled: + def __getattr__(self, attr): + return '' + + +colors_enabled = Colors() +colors_disabled = Colors.Disabled() From 136adebf09adf0b4b3c2aa9734f9ff0f8855bdbc Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 31 Oct 2020 18:13:31 +0000 Subject: [PATCH 02/63] Add a github action to run a sample build to check output --- .github/workflows/sample_build.yml | 28 ++++++++++++++++++++++++++++ test/test_projects/__main__.py | 21 ++++++++++++++------- 2 files changed, 42 insertions(+), 7 deletions(-) create mode 100644 .github/workflows/sample_build.yml diff --git a/.github/workflows/sample_build.yml b/.github/workflows/sample_build.yml new file mode 100644 index 00000000..1f7f5cf7 --- /dev/null +++ b/.github/workflows/sample_build.yml @@ -0,0 +1,28 @@ +name: Sample build + +on: [push, pull_request] + +jobs: + build_wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-18.04] + + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-python@v2 + name: Install Python + with: + python-version: '3.7' + + - name: Generate sample project + run: | + python -m test.test_projects test_0_basic.basic_project sample_project + + - name: Build wheels + run: | + cd sample_project + python -m cibuildwheel --output-dir wheelhouse diff --git a/test/test_projects/__main__.py b/test/test_projects/__main__.py index 06fe696f..070cc2a2 100644 --- a/test/test_projects/__main__.py +++ b/test/test_projects/__main__.py @@ -11,27 +11,34 @@ def main(): prog="python -m test.test_projects", description='Generate a test project to check it out' ) + parser.add_argument('--open', action='store_true', help=''' + Open the generated project in a file explorer + ''') parser.add_argument('PROJECT', help=''' Python path to a project object. E.g. test.test_0_basic.basic_project ''') + parser.add_argument('OUTPUT', nargs='?', help=''' + Path to output dir. If no dir is passed, a tempdir will be generated. + ''') options = parser.parse_args() module, _, name = options.PROJECT.rpartition('.') project = getattr(importlib.import_module(module), name) - project_dir = Path(tempfile.mkdtemp()) + project_dir = Path(options.OUTPUT or tempfile.mkdtemp()) project.generate(project_dir) print('Project generated at', project_dir) print() - if sys.platform == 'darwin': - subprocess.check_call(['open', '--', project_dir]) - elif sys.platform == 'linux2': - subprocess.check_call(['xdg-open', '--', project_dir]) - elif sys.platform == 'win32': - subprocess.check_call(['explorer', project_dir]) + if options.open: + if sys.platform == 'darwin': + subprocess.check_call(['open', '--', project_dir]) + elif sys.platform == 'linux2': + subprocess.check_call(['xdg-open', '--', project_dir]) + elif sys.platform == 'win32': + subprocess.check_call(['explorer', project_dir]) if __name__ == '__main__': From 4ec28cb37e70d9ea8ab0cd806c753ce2a4289566 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 31 Oct 2020 18:16:23 +0000 Subject: [PATCH 03/63] Fix flake8 --- cibuildwheel/logger.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index 772be2a2..93a1fde3 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -1,7 +1,6 @@ from contextlib import contextmanager import os import time -import typing FOLD_PATTERNS = { 'azure': ['##[group]{name}', '##[endgroup]'], From 0191496fc9d31e9ae8a4499d5a3c575a78ad2264 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 31 Oct 2020 18:17:36 +0000 Subject: [PATCH 04/63] Install dependencies in sample build --- .github/workflows/sample_build.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/sample_build.yml b/.github/workflows/sample_build.yml index 1f7f5cf7..9eee786b 100644 --- a/.github/workflows/sample_build.yml +++ b/.github/workflows/sample_build.yml @@ -18,6 +18,10 @@ jobs: with: python-version: '3.7' + - name: Install dependencies + run: | + python -m pip install -r requirements-dev.txt + - name: Generate sample project run: | python -m test.test_projects test_0_basic.basic_project sample_project From 946306b21dee8411cc27807a9f47a7e91ecf1b5d Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 31 Oct 2020 18:19:02 +0000 Subject: [PATCH 05/63] Fix sample project gen command --- .github/workflows/sample_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sample_build.yml b/.github/workflows/sample_build.yml index 9eee786b..d08714fd 100644 --- a/.github/workflows/sample_build.yml +++ b/.github/workflows/sample_build.yml @@ -24,7 +24,7 @@ jobs: - name: Generate sample project run: | - python -m test.test_projects test_0_basic.basic_project sample_project + python -m test.test_projects test.test_0_basic.basic_project sample_project - name: Build wheels run: | From 38e7b5a91ded109ae44947b754cd47d28e3a26d5 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 1 Nov 2020 11:34:02 +0000 Subject: [PATCH 06/63] Change to an imperative-style API --- cibuildwheel/linux.py | 240 +++++++++++++++++++++-------------------- cibuildwheel/logger.py | 81 ++++++++------ 2 files changed, 175 insertions(+), 146 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 8c8d85bd..80ea93a0 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -112,7 +112,7 @@ def build(options: BuildOptions) -> None: container_package_dir = container_project_path / abs_package_dir.relative_to(cwd) container_output_dir = PurePath('/output') - logger = Logger() + log = Logger() for implementation, platform_tag, docker_image in platforms: platform_configs = [c for c in python_configurations if c.identifier.startswith(implementation) and c.identifier.endswith(platform_tag)] @@ -124,19 +124,135 @@ def build(options: BuildOptions) -> None: docker.copy_into(Path.cwd(), container_project_path) if options.before_all: - with logger.step('Running before_all...'): - env = docker.get_environment() - env['PATH'] = f'/opt/python/cp38-cp38:{env["PATH"]}' - env = options.environment.as_dictionary(env, executor=docker.environment_executor) + log.build_step('Running before_all...') - before_all_prepared = prepare_command(options.before_all, project=container_project_path, package=container_package_dir) - docker.call(['sh', '-c', before_all_prepared], env=env) + env = docker.get_environment() + env['PATH'] = f'/opt/python/cp38-cp38:{env["PATH"]}' + env = options.environment.as_dictionary(env, executor=docker.environment_executor) + + before_all_prepared = prepare_command(options.before_all, project=container_project_path, package=container_package_dir) + docker.call(['sh', '-c', before_all_prepared], env=env) + + log.build_step_end() for config in platform_configs: - build_one(options=options, config=config, docker=docker, container_project_path=container_project_path, container_package_dir=container_package_dir, container_output_dir=container_output_dir, logger=logger) + log.build_start(config.identifier) + dependency_constraint_flags: List[Union[str, PathLike]] = [] + + if options.dependency_constraints: + constraints_file = options.dependency_constraints.get_for_python_version(config.version) + container_constraints_file = PurePath('/constraints.txt') + + docker.copy_into(constraints_file, container_constraints_file) + dependency_constraint_flags = ['-c', container_constraints_file] + + env = docker.get_environment() + + # put this config's python top of the list + python_bin = config.path / 'bin' + env['PATH'] = f'{python_bin}:{env["PATH"]}' + + log.build_step('Setting up build environment...') + + env = options.environment.as_dictionary(env, executor=docker.environment_executor) + + # check config python and pip are still on PATH + which_python = docker.call(['which', 'python'], env=env, capture_output=True).strip() + if PurePath(which_python) != python_bin / 'python': + print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr) + exit(1) + + which_pip = docker.call(['which', 'pip'], env=env, capture_output=True).strip() + if PurePath(which_pip) != python_bin / 'pip': + print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr) + exit(1) + + if options.before_build: + log.build_step('Running before_build...') + before_build_prepared = prepare_command(options.before_build, project=container_project_path, package=container_package_dir) + docker.call(['sh', '-c', before_build_prepared], env=env) + + log.build_step('Building wheel...') + + temp_dir = PurePath('/tmp/cibuildwheel') + built_wheel_dir = temp_dir / 'built_wheel' + docker.call(['rm', '-rf', built_wheel_dir]) + docker.call(['mkdir', '-p', built_wheel_dir]) + + docker.call([ + 'pip', 'wheel', + container_package_dir, + '-w', built_wheel_dir, + '--no-deps', + *get_build_verbosity_extra_flags(options.build_verbosity) + ], env=env) + + built_wheel = docker.glob(built_wheel_dir, '*.whl')[0] + + repaired_wheel_dir = temp_dir / 'repaired_wheel' + docker.call(['rm', '-rf', repaired_wheel_dir]) + docker.call(['mkdir', '-p', repaired_wheel_dir]) + + if built_wheel.name.endswith('none-any.whl'): + raise NonPlatformWheelError() + + if options.repair_command: + log.build_step('Repairing wheel...') + repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) + docker.call(['sh', '-c', repair_command_prepared], env=env) + else: + docker.call(['mv', built_wheel, repaired_wheel_dir]) + + repaired_wheels = docker.glob(repaired_wheel_dir, '*.whl') + + if options.test_command: + log.build_step('Testing wheel...') + + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + docker.call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) + venv_dir = PurePath(docker.call(['mktemp', '-d'], capture_output=True).strip()) / 'venv' + + docker.call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) + + virtualenv_env = env.copy() + virtualenv_env['PATH'] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}" + + if options.before_test: + before_test_prepared = prepare_command(options.before_test, project=container_project_path, package=container_package_dir) + docker.call(['sh', '-c', before_test_prepared], env=virtualenv_env) + + # Install the wheel we just built + # Note: If auditwheel produced two wheels, it's because the earlier produced wheel + # conforms to multiple manylinux standards. These multiple versions of the wheel are + # functionally the same, differing only in name, wheel metadata, and possibly include + # different external shared libraries. so it doesn't matter which one we run the tests on. + # Let's just pick the first one. + wheel_to_test = repaired_wheels[0] + docker.call(['pip', 'install', str(wheel_to_test) + options.test_extras], env=virtualenv_env) + + # Install any requirements to run the tests + if options.test_requires: + docker.call(['pip', 'install', *options.test_requires], env=virtualenv_env) + + # Run the tests from a different directory + test_command_prepared = prepare_command(options.test_command, project=container_project_path, package=container_package_dir) + docker.call(['sh', '-c', test_command_prepared], cwd='/root', env=virtualenv_env) + + # clean up test environment + docker.call(['rm', '-rf', venv_dir]) + + # move repaired wheels to output + docker.call(['mkdir', '-p', container_output_dir]) + docker.call(['mv', *repaired_wheels, container_output_dir]) + + log.build_end() + + log.build_step('Copying wheels back to host...') # copy the output back into the host docker.copy_out(container_output_dir, options.output_dir) + log.build_step_end() except subprocess.CalledProcessError as error: print(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') troubleshoot(options.package_dir, error) @@ -145,112 +261,8 @@ def build(options: BuildOptions) -> None: def build_one(options: BuildOptions, config: PythonConfiguration, docker: DockerContainer, container_project_path: PurePath, container_package_dir: PurePath, - container_output_dir: PurePath, logger: Logger): - dependency_constraint_flags: List[Union[str, PathLike]] = [] - - if options.dependency_constraints: - constraints_file = options.dependency_constraints.get_for_python_version(config.version) - container_constraints_file = PurePath('/constraints.txt') - - docker.copy_into(constraints_file, container_constraints_file) - dependency_constraint_flags = ['-c', container_constraints_file] - - env = docker.get_environment() - - # put this config's python top of the list - python_bin = config.path / 'bin' - env['PATH'] = f'{python_bin}:{env["PATH"]}' - - with logger.step(' Setting up build environment...'): - env = options.environment.as_dictionary(env, executor=docker.environment_executor) - - # check config python and pip are still on PATH - which_python = docker.call(['which', 'python'], env=env, capture_output=True).strip() - if PurePath(which_python) != python_bin / 'python': - print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr) - exit(1) - - which_pip = docker.call(['which', 'pip'], env=env, capture_output=True).strip() - if PurePath(which_pip) != python_bin / 'pip': - print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr) - exit(1) - - if options.before_build: - with logger.step(' Running before_build...'): - before_build_prepared = prepare_command(options.before_build, project=container_project_path, package=container_package_dir) - docker.call(['sh', '-c', before_build_prepared], env=env) - - temp_dir = PurePath('/tmp/cibuildwheel') - built_wheel_dir = temp_dir / 'built_wheel' - docker.call(['rm', '-rf', built_wheel_dir]) - docker.call(['mkdir', '-p', built_wheel_dir]) - - with logger.step(' Building wheel...'): - docker.call([ - 'pip', 'wheel', - container_package_dir, - '-w', built_wheel_dir, - '--no-deps', - *get_build_verbosity_extra_flags(options.build_verbosity) - ], env=env) - - built_wheel = docker.glob(built_wheel_dir, '*.whl')[0] - - repaired_wheel_dir = temp_dir / 'repaired_wheel' - docker.call(['rm', '-rf', repaired_wheel_dir]) - docker.call(['mkdir', '-p', repaired_wheel_dir]) - - if built_wheel.name.endswith('none-any.whl'): - raise NonPlatformWheelError() - - if options.repair_command: - with logger.step(' Repairing wheel...'): - repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) - docker.call(['sh', '-c', repair_command_prepared], env=env) - else: - docker.call(['mv', built_wheel, repaired_wheel_dir]) - - repaired_wheels = docker.glob(repaired_wheel_dir, '*.whl') - - if options.test_command: - with logger.step(' Testing wheel...'): - # set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - docker.call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) - venv_dir = PurePath(docker.call(['mktemp', '-d'], capture_output=True).strip()) / 'venv' - - docker.call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) - - virtualenv_env = env.copy() - virtualenv_env['PATH'] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}" - - if options.before_test: - before_test_prepared = prepare_command(options.before_test, project=container_project_path, package=container_package_dir) - docker.call(['sh', '-c', before_test_prepared], env=virtualenv_env) - - # Install the wheel we just built - # Note: If auditwheel produced two wheels, it's because the earlier produced wheel - # conforms to multiple manylinux standards. These multiple versions of the wheel are - # functionally the same, differing only in name, wheel metadata, and possibly include - # different external shared libraries. so it doesn't matter which one we run the tests on. - # Let's just pick the first one. - wheel_to_test = repaired_wheels[0] - docker.call(['pip', 'install', str(wheel_to_test) + options.test_extras], env=virtualenv_env) - - # Install any requirements to run the tests - if options.test_requires: - docker.call(['pip', 'install', *options.test_requires], env=virtualenv_env) - - # Run the tests from a different directory - test_command_prepared = prepare_command(options.test_command, project=container_project_path, package=container_package_dir) - docker.call(['sh', '-c', test_command_prepared], cwd='/root', env=virtualenv_env) - - # clean up test environment - docker.call(['rm', '-rf', venv_dir]) - - # move repaired wheels to output - docker.call(['mkdir', '-p', container_output_dir]) - docker.call(['mv', *repaired_wheels, container_output_dir]) + container_output_dir: PurePath, log: Logger): + pass def troubleshoot(package_dir: Path, error: Exception) -> None: diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index 93a1fde3..4dde6e6a 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -1,6 +1,7 @@ -from contextlib import contextmanager import os import time +import re +from typing import Optional FOLD_PATTERNS = { 'azure': ['##[group]{name}', '##[endgroup]'], @@ -21,6 +22,13 @@ PLATFORM_IDENTIFIER_DESCIPTIONS = { class Logger: + fold_mode: str + colors_enabled: bool + active_build_identifier: Optional[str] = None + build_start_time: Optional[float] = 0 + step_start_time: Optional[float] = 0 + active_fold_group_id: Optional[str] = None + def __init__(self): if 'AZURE_HTTP_USER_AGENT' in os.environ: self.fold_mode = 'azure' @@ -42,43 +50,52 @@ class Logger: self.fold_mode = 'disabled' self.colors_enabled = False - @contextmanager - def build(self, identifier: str): + def build_start(self, identifier: str): c = self.colors + print() print(f'{c.bold}Building {build_description_from_identifier(identifier)} wheel{c.end}') print(f'Identifier: {identifier}') + print() - start_time = time.time() - try: - yield - duration = time.time() - start_time - print(f'{c.green}Build {c.bg_grey}{identifier}{c.end}{c.green} completed in {duration:.2f}s{c.end}') - except Exception: - duration = time.time() - start_time - print(f'{c.red}Build {c.bg_grey}{identifier}{c.end}{c.red} failed in {duration:.2f}s{c.end}') - raise + self.build_start_time = time.time() + self.active_build_identifier = identifier + + def build_end(self): + assert self.build_start_time is not None + self.build_step_end() - @contextmanager - def step(self, name: str): c = self.colors - start_time = time.time() + duration = time.time() - self.build_start_time + print(f'{c.green}Build {c.bg_grey}{self.active_build_identifier}{c.end}{c.green} completed in {duration:.2f}s{c.end}') + print() + print('---') + self.build_start_time = None - try: - with self.fold_group(name): - yield - duration = time.time() - start_time - print(f'{c.green}✓ {c.faint}[{duration:.2f}]{c.end}') - except Exception: - raise + def build_step(self, step_description: str): + self.build_step_end() + self.step_start_time = time.time() + self.start_fold_group(step_description) - @contextmanager - def fold_group(self, name: str): - fold_start_pattern, fold_end_pattern = FOLD_PATTERNS.get(self.fold_mode, ('', '')) - print(fold_start_pattern.format(name=name)) - try: - yield - finally: - print(fold_end_pattern.format(name=name)) + def build_step_end(self): + if self.step_start_time is not None: + self.end_fold_group() + c = self.colors + duration = time.time() - self.step_start_time + print(f'{c.green}✓ {c.faint}[{duration:.2f}s]{c.end}'.rjust(78)) + self.step_start_time = None + + def start_fold_group(self, name: str): + self.end_fold_group() + self.active_fold_group_id = re.sub(r'[^A-Za-z]', '', name) + fold_start_pattern = FOLD_PATTERNS.get(self.fold_mode, ('', ''))[0] + print(fold_start_pattern.format(name=self.active_fold_group_id)) + print() + + def end_fold_group(self): + if self.active_fold_group_id: + fold_start_pattern = FOLD_PATTERNS.get(self.fold_mode, ('', ''))[1] + print(fold_start_pattern.format(name=self.active_fold_group_id)) + self.active_fold_group_id = None @property def colors(self): @@ -88,7 +105,7 @@ class Logger: return colors_disabled -def build_description_from_identifier(identifier): +def build_description_from_identifier(identifier: str): python_identifier, _, platform_identifier = identifier.partition('-') build_description = '' @@ -130,7 +147,7 @@ class Colors(): end = '\033[0m' class Disabled: - def __getattr__(self, attr): + def __getattr__(self, attr: str): return '' From f48a84d92f63b32f17cc3e9697c2a05811bd36d1 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 1 Nov 2020 11:43:28 +0000 Subject: [PATCH 07/63] Improve some formatting --- cibuildwheel/linux.py | 23 ++++++++++++++--------- cibuildwheel/logger.py | 30 ++++++++++++++++-------------- 2 files changed, 30 insertions(+), 23 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 80ea93a0..500307b7 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -120,11 +120,16 @@ def build(options: BuildOptions) -> None: continue try: + log.step(f'Starting Docker image {docker_image}...') with DockerContainer(docker_image, simulate_32_bit=platform_tag.endswith('i686'), cwd=container_project_path) as docker: + log.step_end() + + log.step('Copying project into Docker...') docker.copy_into(Path.cwd(), container_project_path) + log.step_end() if options.before_all: - log.build_step('Running before_all...') + log.step('Running before_all...') env = docker.get_environment() env['PATH'] = f'/opt/python/cp38-cp38:{env["PATH"]}' @@ -133,7 +138,7 @@ def build(options: BuildOptions) -> None: before_all_prepared = prepare_command(options.before_all, project=container_project_path, package=container_package_dir) docker.call(['sh', '-c', before_all_prepared], env=env) - log.build_step_end() + log.step_end() for config in platform_configs: log.build_start(config.identifier) @@ -153,7 +158,7 @@ def build(options: BuildOptions) -> None: python_bin = config.path / 'bin' env['PATH'] = f'{python_bin}:{env["PATH"]}' - log.build_step('Setting up build environment...') + log.step('Setting up build environment...') env = options.environment.as_dictionary(env, executor=docker.environment_executor) @@ -169,11 +174,11 @@ def build(options: BuildOptions) -> None: exit(1) if options.before_build: - log.build_step('Running before_build...') + log.step('Running before_build...') before_build_prepared = prepare_command(options.before_build, project=container_project_path, package=container_package_dir) docker.call(['sh', '-c', before_build_prepared], env=env) - log.build_step('Building wheel...') + log.step('Building wheel...') temp_dir = PurePath('/tmp/cibuildwheel') built_wheel_dir = temp_dir / 'built_wheel' @@ -198,7 +203,7 @@ def build(options: BuildOptions) -> None: raise NonPlatformWheelError() if options.repair_command: - log.build_step('Repairing wheel...') + log.step('Repairing wheel...') repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) docker.call(['sh', '-c', repair_command_prepared], env=env) else: @@ -207,7 +212,7 @@ def build(options: BuildOptions) -> None: repaired_wheels = docker.glob(repaired_wheel_dir, '*.whl') if options.test_command: - log.build_step('Testing wheel...') + log.step('Testing wheel...') # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. @@ -249,10 +254,10 @@ def build(options: BuildOptions) -> None: log.build_end() - log.build_step('Copying wheels back to host...') + log.step('Copying wheels back to host...') # copy the output back into the host docker.copy_out(container_output_dir, options.output_dir) - log.build_step_end() + log.step_end() except subprocess.CalledProcessError as error: print(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') troubleshoot(options.package_dir, error) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index 4dde6e6a..cdbaa762 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -25,9 +25,9 @@ class Logger: fold_mode: str colors_enabled: bool active_build_identifier: Optional[str] = None - build_start_time: Optional[float] = 0 - step_start_time: Optional[float] = 0 - active_fold_group_id: Optional[str] = None + build_start_time: Optional[float] = None + step_start_time: Optional[float] = None + active_fold_group_name: Optional[str] = None def __init__(self): if 'AZURE_HTTP_USER_AGENT' in os.environ: @@ -54,7 +54,7 @@ class Logger: c = self.colors print() print(f'{c.bold}Building {build_description_from_identifier(identifier)} wheel{c.end}') - print(f'Identifier: {identifier}') + print(f'Identifier: {c.bg_grey}{identifier}{c.end}') print() self.build_start_time = time.time() @@ -62,21 +62,23 @@ class Logger: def build_end(self): assert self.build_start_time is not None - self.build_step_end() + self.step_end() c = self.colors duration = time.time() - self.build_start_time + print() print(f'{c.green}Build {c.bg_grey}{self.active_build_identifier}{c.end}{c.green} completed in {duration:.2f}s{c.end}') print() print('---') + print() self.build_start_time = None - def build_step(self, step_description: str): - self.build_step_end() + def step(self, step_description: str): + self.step_end() self.step_start_time = time.time() self.start_fold_group(step_description) - def build_step_end(self): + def step_end(self): if self.step_start_time is not None: self.end_fold_group() c = self.colors @@ -86,16 +88,16 @@ class Logger: def start_fold_group(self, name: str): self.end_fold_group() - self.active_fold_group_id = re.sub(r'[^A-Za-z]', '', name) + self.active_fold_group_name = name fold_start_pattern = FOLD_PATTERNS.get(self.fold_mode, ('', ''))[0] - print(fold_start_pattern.format(name=self.active_fold_group_id)) + print(fold_start_pattern.format(name=self.active_fold_group_name)) print() def end_fold_group(self): - if self.active_fold_group_id: + if self.active_fold_group_name: fold_start_pattern = FOLD_PATTERNS.get(self.fold_mode, ('', ''))[1] - print(fold_start_pattern.format(name=self.active_fold_group_id)) - self.active_fold_group_id = None + print(fold_start_pattern.format(name=self.active_fold_group_name)) + self.active_fold_group_name = None @property def colors(self): @@ -139,7 +141,7 @@ class Colors(): bright_green = '\033[92m' white = '\033[37m\033[97m' - bg_grey = '\033[48;5;244m' + bg_grey = '\033[48;5;235m' bold = '\033[1m' faint = '\033[2m' From dac400608326ff5efac2a7513a3d96237f150af7 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 1 Nov 2020 12:00:59 +0000 Subject: [PATCH 08/63] More formatting tweaks --- cibuildwheel/logger.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index cdbaa762..fce9ce22 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -1,6 +1,6 @@ import os import time -import re +import sys from typing import Optional FOLD_PATTERNS = { @@ -64,10 +64,6 @@ class Logger: assert self.build_start_time is not None self.step_end() - c = self.colors - duration = time.time() - self.build_start_time - print() - print(f'{c.green}Build {c.bg_grey}{self.active_build_identifier}{c.end}{c.green} completed in {duration:.2f}s{c.end}') print() print('---') print() @@ -83,20 +79,22 @@ class Logger: self.end_fold_group() c = self.colors duration = time.time() - self.step_start_time - print(f'{c.green}✓ {c.faint}[{duration:.2f}s]{c.end}'.rjust(78)) + print(f'{c.green}✓ {c.end}{duration:.2f}s'.rjust(78)) self.step_start_time = None def start_fold_group(self, name: str): self.end_fold_group() self.active_fold_group_name = name fold_start_pattern = FOLD_PATTERNS.get(self.fold_mode, ('', ''))[0] - print(fold_start_pattern.format(name=self.active_fold_group_name)) + + # output to stderr because stdout is a little slower + print(fold_start_pattern.format(name=self.active_fold_group_name), file=sys.stderr) print() def end_fold_group(self): if self.active_fold_group_name: fold_start_pattern = FOLD_PATTERNS.get(self.fold_mode, ('', ''))[1] - print(fold_start_pattern.format(name=self.active_fold_group_name)) + print(fold_start_pattern.format(name=self.active_fold_group_name), file=sys.stderr) self.active_fold_group_name = None @property From 45c723cdeadb209a6069343a8f2b951e015a1e0b Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 1 Nov 2020 12:03:21 +0000 Subject: [PATCH 09/63] Fix folding --- cibuildwheel/logger.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index fce9ce22..f188a71b 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -87,14 +87,16 @@ class Logger: self.active_fold_group_name = name fold_start_pattern = FOLD_PATTERNS.get(self.fold_mode, ('', ''))[0] - # output to stderr because stdout is a little slower - print(fold_start_pattern.format(name=self.active_fold_group_name), file=sys.stderr) + print(fold_start_pattern.format(name=self.active_fold_group_name)) print() + # flush to ensure it's output before continuing + sys.stdout.flush() def end_fold_group(self): if self.active_fold_group_name: fold_start_pattern = FOLD_PATTERNS.get(self.fold_mode, ('', ''))[1] - print(fold_start_pattern.format(name=self.active_fold_group_name), file=sys.stderr) + print(fold_start_pattern.format(name=self.active_fold_group_name)) + sys.stdout.flush() self.active_fold_group_name = None @property From f51c8dc1e3c58c2dafa8a5577ee88e230ca16498 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 1 Nov 2020 12:33:14 +0000 Subject: [PATCH 10/63] More tweaks --- cibuildwheel/linux.py | 4 ++-- cibuildwheel/logger.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 500307b7..dba5c38d 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -152,14 +152,14 @@ def build(options: BuildOptions) -> None: docker.copy_into(constraints_file, container_constraints_file) dependency_constraint_flags = ['-c', container_constraints_file] + log.step('Setting up build environment...') + env = docker.get_environment() # put this config's python top of the list python_bin = config.path / 'bin' env['PATH'] = f'{python_bin}:{env["PATH"]}' - log.step('Setting up build environment...') - env = options.environment.as_dictionary(env, executor=docker.environment_executor) # check config python and pip are still on PATH diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index f188a71b..0ecfdf3c 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -52,9 +52,10 @@ class Logger: def build_start(self, identifier: str): c = self.colors + description = build_description_from_identifier(identifier) print() - print(f'{c.bold}Building {build_description_from_identifier(identifier)} wheel{c.end}') - print(f'Identifier: {c.bg_grey}{identifier}{c.end}') + print(f'{c.bold}{c.blue}Building {identifier} wheel{c.end}') + print(f'{description}') print() self.build_start_time = time.time() @@ -66,7 +67,6 @@ class Logger: print() print('---') - print() self.build_start_time = None def step(self, step_description: str): From 152483bcb470979aaedc71ae7b1d182f6c690efb Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 1 Nov 2020 15:16:28 +0000 Subject: [PATCH 11/63] Try putting all build output on stdout --- cibuildwheel/docker_container.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cibuildwheel/docker_container.py b/cibuildwheel/docker_container.py index a58eb750..7511f9e0 100644 --- a/cibuildwheel/docker_container.py +++ b/cibuildwheel/docker_container.py @@ -57,6 +57,9 @@ class DockerContainer: ], stdin=subprocess.PIPE, stdout=subprocess.PIPE, + # output the subcommand stderr to our stdout. Avoids ordering + # problems between log messages on stderr and stdout + stderr=sys.stdout, ) assert self.process.stdin and self.process.stdout From 39b94b7deccadfae22be9d4dc8349a4e4692cce4 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 1 Nov 2020 15:19:25 +0000 Subject: [PATCH 12/63] Tidy up log output --- cibuildwheel/linux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index dba5c38d..c28bd6b2 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -84,7 +84,7 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi def build(options: BuildOptions) -> None: try: - subprocess.check_call(['docker', '--version']) + subprocess.check_output(['docker', '--version']) except Exception: print('cibuildwheel: Docker not found. Docker is required to run Linux builds. ' 'If you\'re building on Travis CI, add `services: [docker]` to your .travis.yml.' From b22001b26dc2b17bb3ad04a50324bda37e7c44d8 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 1 Nov 2020 15:19:42 +0000 Subject: [PATCH 13/63] Add build end message --- cibuildwheel/logger.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index 0ecfdf3c..ad1d3c34 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -63,11 +63,16 @@ class Logger: def build_end(self): assert self.build_start_time is not None + assert self.active_build_identifier is not None self.step_end() + c = self.colors + duration = time.time() - self.build_start_time + print() - print('---') + print(f'{c.green}✓ {c.end}{self.active_build_identifier} finished in {duration:.2f}s') self.build_start_time = None + self.active_build_identifier = None def step(self, step_description: str): self.step_end() From 0a32e3bc0e6625050a3900ca80af2697d2971376 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 1 Nov 2020 16:57:07 +0000 Subject: [PATCH 14/63] Improve appearance outside CI --- cibuildwheel/logger.py | 31 +++++++++++++++++++++++-------- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index ad1d3c34..67d4f745 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -3,10 +3,11 @@ import time import sys from typing import Optional +DEFAULT_FOLD_PATTERN = ('{name}', '') FOLD_PATTERNS = { - 'azure': ['##[group]{name}', '##[endgroup]'], - 'travis': ['travis_fold:start:{name}', 'travis_fold:end:{name}'], - 'github': ['::group::{name}', '::endgroup::{name}'], + 'azure': ('##[group]{name}', '##[endgroup]'), + 'travis': ('travis_fold:start:{name}', 'travis_fold:end:{name}'), + 'github': ('::group::{name}', '::endgroup::{name}'), } PLATFORM_IDENTIFIER_DESCIPTIONS = { @@ -48,7 +49,7 @@ class Logger: else: self.fold_mode = 'disabled' - self.colors_enabled = False + self.colors_enabled = file_supports_color(sys.stdout) def build_start(self, identifier: str): c = self.colors @@ -90,16 +91,14 @@ class Logger: def start_fold_group(self, name: str): self.end_fold_group() self.active_fold_group_name = name - fold_start_pattern = FOLD_PATTERNS.get(self.fold_mode, ('', ''))[0] + fold_start_pattern = FOLD_PATTERNS.get(self.fold_mode, DEFAULT_FOLD_PATTERN)[0] print(fold_start_pattern.format(name=self.active_fold_group_name)) print() - # flush to ensure it's output before continuing - sys.stdout.flush() def end_fold_group(self): if self.active_fold_group_name: - fold_start_pattern = FOLD_PATTERNS.get(self.fold_mode, ('', ''))[1] + fold_start_pattern = FOLD_PATTERNS.get(self.fold_mode, DEFAULT_FOLD_PATTERN)[1] print(fold_start_pattern.format(name=self.active_fold_group_name)) sys.stdout.flush() self.active_fold_group_name = None @@ -160,3 +159,19 @@ class Colors(): colors_enabled = Colors() colors_disabled = Colors.Disabled() + + +def file_supports_color(file_obj): + """ + Returns True if the running system's terminal supports color. + """ + plat = sys.platform + supported_platform = (plat != 'win32' or 'ANSICON' in os.environ) + + is_a_tty = file_is_a_tty(file_obj) + + return (supported_platform and is_a_tty) + + +def file_is_a_tty(file_obj): + return hasattr(file_obj, 'isatty') and file_obj.isatty() From 4eff3bc7a85b07bf66a3b6793a45b133b85bc42e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 1 Nov 2020 17:06:55 +0000 Subject: [PATCH 15/63] Add a failure example build --- .github/workflows/sample_build.yml | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/.github/workflows/sample_build.yml b/.github/workflows/sample_build.yml index d08714fd..4ff80b52 100644 --- a/.github/workflows/sample_build.yml +++ b/.github/workflows/sample_build.yml @@ -4,11 +4,11 @@ on: [push, pull_request] jobs: build_wheels: - name: Build wheels on ${{ matrix.os }} - runs-on: ${{ matrix.os }} + name: Build wheels [${{ matrix.result }}] + runs-on: ubuntu-18.04 strategy: matrix: - os: [ubuntu-18.04] + result: [success, failure] steps: - uses: actions/checkout@v2 @@ -27,6 +27,15 @@ jobs: python -m test.test_projects test.test_0_basic.basic_project sample_project - name: Build wheels + if: matrix.result == 'success' run: | cd sample_project python -m cibuildwheel --output-dir wheelhouse + + - name: Build wheels [expected failure] + if: matrix.result == 'failure' + run: | + cd sample_project + # sabotage the project to cause an error + echo '>' >> setup.py + python -m cibuildwheel --output-dir wheelhouse || true From 4b402e05934a67fb6001e47294672d0fa3b5f1cb Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 1 Nov 2020 20:44:38 +0000 Subject: [PATCH 16/63] Experiment --- cibuildwheel/linux.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index c28bd6b2..fcec1553 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -259,6 +259,7 @@ def build(options: BuildOptions) -> None: docker.copy_out(container_output_dir, options.output_dir) log.step_end() except subprocess.CalledProcessError as error: + print(f'::error::{error}') print(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') troubleshoot(options.package_dir, error) exit(1) From 4a736cdcb1776c98fa79192ec327d09d5baa818c Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 1 Nov 2020 21:17:25 +0000 Subject: [PATCH 17/63] Improve options output --- cibuildwheel/util.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index c24f9bf5..61c28301 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -42,7 +42,10 @@ class BuildSelector: return match_any(self.build_patterns) and not match_any(self.skip_patterns) def __repr__(self) -> str: - return f'BuildSelector({" ".join(self.build_patterns)!r} - {" ".join(self.skip_patterns)!r})' + if not self.skip_patterns: + return f'BuildSelector({" ".join(self.build_patterns)!r})' + else: + return f'BuildSelector({" ".join(self.build_patterns)!r} - {" ".join(self.skip_patterns)!r})' # Taken from https://stackoverflow.com/a/107717 @@ -113,8 +116,8 @@ class DependencyConstraints: else: return self.base_file_path - def __str__(self): - return f"File '{self.base_file_path}'" + def __repr__(self): + return f'DependencyConstraints({self.base_file_path!r})' class BuildOptions(NamedTuple): From fa4a0cbd33b6604414cb212f22b89bf839978c51 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 1 Nov 2020 21:36:44 +0000 Subject: [PATCH 18/63] Improve error output --- cibuildwheel/linux.py | 3 +-- cibuildwheel/logger.py | 31 +++++++++++++++++++++++-------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index fcec1553..dd7bce32 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -259,8 +259,7 @@ def build(options: BuildOptions) -> None: docker.copy_out(container_output_dir, options.output_dir) log.step_end() except subprocess.CalledProcessError as error: - print(f'::error::{error}') - print(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') + log.error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') troubleshoot(options.package_dir, error) exit(1) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index 67d4f745..5f73d894 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -1,7 +1,7 @@ import os import time import sys -from typing import Optional +from typing import Optional, Union DEFAULT_FOLD_PATTERN = ('{name}', '') FOLD_PATTERNS = { @@ -78,25 +78,39 @@ class Logger: def step(self, step_description: str): self.step_end() self.step_start_time = time.time() - self.start_fold_group(step_description) + self._start_fold_group(step_description) - def step_end(self): + def step_end(self, success=True): if self.step_start_time is not None: - self.end_fold_group() + self._end_fold_group() c = self.colors duration = time.time() - self.step_start_time - print(f'{c.green}✓ {c.end}{duration:.2f}s'.rjust(78)) + if success: + print(f'{c.green}✓ {c.end}{duration:.2f}s'.rjust(78)) + else: + print(f'{c.red}✕ {c.end}{duration:.2f}s'.rjust(78)) + self.step_start_time = None - def start_fold_group(self, name: str): - self.end_fold_group() + def error(self, error: Union[Exception, str]): + self.step_end(success=False) + print() + + if self.fold_mode == 'github': + print(f'::error::{error}') + else: + c = self.colors + print(f'{c.bright_red}Error{c.end} {error}') + + def _start_fold_group(self, name: str): + self._end_fold_group() self.active_fold_group_name = name fold_start_pattern = FOLD_PATTERNS.get(self.fold_mode, DEFAULT_FOLD_PATTERN)[0] print(fold_start_pattern.format(name=self.active_fold_group_name)) print() - def end_fold_group(self): + def _end_fold_group(self): if self.active_fold_group_name: fold_start_pattern = FOLD_PATTERNS.get(self.fold_mode, DEFAULT_FOLD_PATTERN)[1] print(fold_start_pattern.format(name=self.active_fold_group_name)) @@ -142,6 +156,7 @@ class Colors(): yellow = '\033[33m' blue = '\033[34m' cyan = '\033[36m' + bright_red = '\033[91m' bright_green = '\033[92m' white = '\033[37m\033[97m' From 8808add708ce8ce162e095b2f20b9c85cedce610 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 3 Nov 2020 08:51:45 +0000 Subject: [PATCH 19/63] Try allow failure Github job --- .github/workflows/sample_build.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sample_build.yml b/.github/workflows/sample_build.yml index 4ff80b52..906d140a 100644 --- a/.github/workflows/sample_build.yml +++ b/.github/workflows/sample_build.yml @@ -6,9 +6,14 @@ jobs: build_wheels: name: Build wheels [${{ matrix.result }}] runs-on: ubuntu-18.04 + continue-on-error: ${{ matrix.allow_fail }} strategy: + fail_fast: false matrix: - result: [success, failure] + include: + - result: success + - result: failure + allow_fail: true steps: - uses: actions/checkout@v2 From 225342c777dcd53936c90d63c87626f6da885c9b Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 3 Nov 2020 09:12:19 +0000 Subject: [PATCH 20/63] fix syntax --- .github/workflows/sample_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sample_build.yml b/.github/workflows/sample_build.yml index 906d140a..e8754870 100644 --- a/.github/workflows/sample_build.yml +++ b/.github/workflows/sample_build.yml @@ -8,7 +8,7 @@ jobs: runs-on: ubuntu-18.04 continue-on-error: ${{ matrix.allow_fail }} strategy: - fail_fast: false + fail-fast: false matrix: include: - result: success From 53ee55bad7bbbb424d26666920969c7a72463850 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 3 Nov 2020 09:13:42 +0000 Subject: [PATCH 21/63] and again --- .github/workflows/sample_build.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/sample_build.yml b/.github/workflows/sample_build.yml index e8754870..5e15218e 100644 --- a/.github/workflows/sample_build.yml +++ b/.github/workflows/sample_build.yml @@ -12,6 +12,7 @@ jobs: matrix: include: - result: success + allow_fail: false - result: failure allow_fail: true From c5bbfb3cc6f35762e151e65a9fff88c9b6e5e21f Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 3 Nov 2020 09:15:22 +0000 Subject: [PATCH 22/63] ...and make that step fail --- .github/workflows/sample_build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/sample_build.yml b/.github/workflows/sample_build.yml index 5e15218e..99791631 100644 --- a/.github/workflows/sample_build.yml +++ b/.github/workflows/sample_build.yml @@ -44,4 +44,4 @@ jobs: cd sample_project # sabotage the project to cause an error echo '>' >> setup.py - python -m cibuildwheel --output-dir wheelhouse || true + python -m cibuildwheel --output-dir wheelhouse From 75c6e6a0a497f79ba87126b948d1378bc07930b6 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 4 Nov 2020 11:11:42 +0000 Subject: [PATCH 23/63] Improvements from code review --- cibuildwheel/linux.py | 6 ------ cibuildwheel/logger.py | 11 +++++------ 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index dd7bce32..971a69f9 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -264,12 +264,6 @@ def build(options: BuildOptions) -> None: exit(1) -def build_one(options: BuildOptions, config: PythonConfiguration, docker: DockerContainer, - container_project_path: PurePath, container_package_dir: PurePath, - container_output_dir: PurePath, log: Logger): - pass - - def troubleshoot(package_dir: Path, error: Exception) -> None: if (isinstance(error, subprocess.CalledProcessError) and error.cmd[0:2] == ['pip', 'wheel']): # the 'pip wheel' step failed. diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index 5f73d894..5be30243 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -11,11 +11,11 @@ FOLD_PATTERNS = { } PLATFORM_IDENTIFIER_DESCIPTIONS = { - 'manylinux_x86_64': 'Manylinux x86_64', - 'manylinux_i686': 'Manylinux i686', - 'manylinux_aarch64': 'Manylinux aarch64', - 'manylinux_ppc64le': 'Manylinux ppc64le', - 'manylinux_s390x': 'Manylinux s390x', + 'manylinux_x86_64': 'manylinux x86_64', + 'manylinux_i686': 'manylinux i686', + 'manylinux_aarch64': 'manylinux aarch64', + 'manylinux_ppc64le': 'manylinux ppc64le', + 'manylinux_s390x': 'manylinux s390x', 'win32': 'Windows 32bit', 'win_amd64': 'Windows 64bit', 'macosx_x86_64': 'macOS x86_64', @@ -108,7 +108,6 @@ class Logger: fold_start_pattern = FOLD_PATTERNS.get(self.fold_mode, DEFAULT_FOLD_PATTERN)[0] print(fold_start_pattern.format(name=self.active_fold_group_name)) - print() def _end_fold_group(self): if self.active_fold_group_name: From 2f9ee793f0c70ff058e29859c5f62ba80770b688 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 4 Nov 2020 11:38:03 +0000 Subject: [PATCH 24/63] Adjust sample build to show passing/failing test Drop the 'allow fail' approach, it shows up as red in PR/commit status --- .github/workflows/sample_build.yml | 22 ++++++++++------------ 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/workflows/sample_build.yml b/.github/workflows/sample_build.yml index 99791631..6dc8de88 100644 --- a/.github/workflows/sample_build.yml +++ b/.github/workflows/sample_build.yml @@ -4,17 +4,11 @@ on: [push, pull_request] jobs: build_wheels: - name: Build wheels [${{ matrix.result }}] + name: Sample build [${{ matrix.result }}] runs-on: ubuntu-18.04 - continue-on-error: ${{ matrix.allow_fail }} strategy: - fail-fast: false matrix: - include: - - result: success - allow_fail: false - - result: failure - allow_fail: true + result: [success, failure] steps: - uses: actions/checkout@v2 @@ -30,7 +24,7 @@ jobs: - name: Generate sample project run: | - python -m test.test_projects test.test_0_basic.basic_project sample_project + python -m test.test_projects test.test_testing.project_with_a_test sample_project - name: Build wheels if: matrix.result == 'success' @@ -42,6 +36,10 @@ jobs: if: matrix.result == 'failure' run: | cd sample_project - # sabotage the project to cause an error - echo '>' >> setup.py - python -m cibuildwheel --output-dir wheelhouse + # adjust the project to cause an error + echo ' + class AnotherTest: + def test_something(self): + self.fail("sabotage!") + ' >> test/spam_test.py + python -m cibuildwheel --output-dir wheelhouse || true From 06b682363711fdcbf3febfaee26ce3f487e654da Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 4 Nov 2020 11:48:12 +0000 Subject: [PATCH 25/63] Remove some unnecessary step_end calls --- cibuildwheel/linux.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 971a69f9..6c6593a3 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -122,11 +122,9 @@ def build(options: BuildOptions) -> None: try: log.step(f'Starting Docker image {docker_image}...') with DockerContainer(docker_image, simulate_32_bit=platform_tag.endswith('i686'), cwd=container_project_path) as docker: - log.step_end() log.step('Copying project into Docker...') docker.copy_into(Path.cwd(), container_project_path) - log.step_end() if options.before_all: log.step('Running before_all...') @@ -138,8 +136,6 @@ def build(options: BuildOptions) -> None: before_all_prepared = prepare_command(options.before_all, project=container_project_path, package=container_package_dir) docker.call(['sh', '-c', before_all_prepared], env=env) - log.step_end() - for config in platform_configs: log.build_start(config.identifier) From 888da8274c96c4b27053f7797a45075cccbf1f88 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 4 Nov 2020 15:43:27 +0000 Subject: [PATCH 26/63] GHA build tweak --- .github/workflows/sample_build.yml | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/.github/workflows/sample_build.yml b/.github/workflows/sample_build.yml index 6dc8de88..47a73f9f 100644 --- a/.github/workflows/sample_build.yml +++ b/.github/workflows/sample_build.yml @@ -26,20 +26,26 @@ jobs: run: | python -m test.test_projects test.test_testing.project_with_a_test sample_project - - name: Build wheels - if: matrix.result == 'success' - run: | - cd sample_project - python -m cibuildwheel --output-dir wheelhouse - - - name: Build wheels [expected failure] + - name: Add a failing test if: matrix.result == 'failure' run: | cd sample_project - # adjust the project to cause an error echo ' class AnotherTest: def test_something(self): self.fail("sabotage!") ' >> test/spam_test.py - python -m cibuildwheel --output-dir wheelhouse || true + + - name: Build & test wheels + run: | + cd sample_project + if $should_fail; then + # prefix with ! to expect failure + ! python -m cibuildwheel --output-dir wheelhouse + else + python -m cibuildwheel --output-dir wheelhouse + fi + env: + CIBW_TEST_COMMAND: nosetests {project}/test + CIBW_TEST_REQUIRES: nose + should_fail: ${{ matrix.result == 'failure' }} From fbbc29590107ae0a3afb09295c3dd7457d393646 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 4 Nov 2020 16:08:24 +0000 Subject: [PATCH 27/63] Starting a build implicitly closes a log step --- cibuildwheel/logger.py | 1 + 1 file changed, 1 insertion(+) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index 5be30243..fcd77751 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -52,6 +52,7 @@ class Logger: self.colors_enabled = file_supports_color(sys.stdout) def build_start(self, identifier: str): + self.step_end() c = self.colors description = build_description_from_identifier(identifier) print() From c9ea0251fef356ef3e1d6cd8eb3338edc45d2a11 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 4 Nov 2020 16:08:40 +0000 Subject: [PATCH 28/63] Sample build uses a testcase failure project --- .github/workflows/sample_build.yml | 18 +++++++----------- test/test_testing.py | 15 +++++++++++++-- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/.github/workflows/sample_build.yml b/.github/workflows/sample_build.yml index 47a73f9f..e25e8fa6 100644 --- a/.github/workflows/sample_build.yml +++ b/.github/workflows/sample_build.yml @@ -24,17 +24,13 @@ jobs: - name: Generate sample project run: | - python -m test.test_projects test.test_testing.project_with_a_test sample_project - - - name: Add a failing test - if: matrix.result == 'failure' - run: | - cd sample_project - echo ' - class AnotherTest: - def test_something(self): - self.fail("sabotage!") - ' >> test/spam_test.py + if $should_fail; then + python -m test.test_projects test.test_testing.project_with_a_failing_test sample_project + else + python -m test.test_projects test.test_testing.project_with_a_test sample_project + fi + env: + should_fail: ${{ matrix.result == 'failure' }} - name: Build & test wheels run: | diff --git a/test/test_testing.py b/test/test_testing.py index 58a38224..6136c4aa 100644 --- a/test/test_testing.py +++ b/test/test_testing.py @@ -103,15 +103,26 @@ def test_extras_require(tmp_path): assert set(actual_wheels) == set(expected_wheels) +project_with_a_failing_test = test_projects.new_c_project() +project_with_a_failing_test.files['test/spam_test.py'] = r''' +from unittest import TestCase + +class TestSpam(TestCase): + def test_something(self): + self.fail('this test is supposed to fail') +''' + + def test_failing_test(tmp_path): """Ensure a failing test causes cibuildwheel to error out and exit""" project_dir = tmp_path / 'project' output_dir = tmp_path / 'output' - project_with_a_test.generate(project_dir) + project_with_a_failing_test.generate(project_dir) with pytest.raises(subprocess.CalledProcessError): utils.cibuildwheel_run(project_dir, output_dir=output_dir, add_env={ - 'CIBW_TEST_COMMAND': 'false', + 'CIBW_TEST_REQUIRES': 'nose', + 'CIBW_TEST_COMMAND': 'nosetests {project}/test', # manylinux1 has a version of bash that's been shown to have # problems with this, so let's check that. 'CIBW_MANYLINUX_I686_IMAGE': 'manylinux1', From 8108deacfe0605c4c56da857305862754c5d4de1 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 5 Nov 2020 23:09:28 +0000 Subject: [PATCH 29/63] Don't capture the output of the first test --- test/test_0_basic.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/test_0_basic.py b/test/test_0_basic.py index 60b309ab..cd17cef8 100644 --- a/test/test_0_basic.py +++ b/test/test_0_basic.py @@ -13,12 +13,14 @@ basic_project = test_projects.new_c_project( ) -def test(tmp_path): +def test(tmp_path, capfd): project_dir = tmp_path / 'project' basic_project.generate(project_dir) - # build the wheels - actual_wheels = utils.cibuildwheel_run(project_dir) + # build the wheels, and let the output passthrough to the caller, so + # we can see how it looks + with capfd.disabled(): + actual_wheels = utils.cibuildwheel_run(project_dir) # check that the expected wheels are produced expected_wheels = utils.expected_wheels('spam', '0.1.0') From 5e99d962f50a84ae561efdc53c1267d9816c04fd Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 13 Nov 2020 15:22:43 +0000 Subject: [PATCH 30/63] Wrap the sample build output in a fold group --- test/test_0_basic.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/test_0_basic.py b/test/test_0_basic.py index cd17cef8..e4bcbc19 100644 --- a/test/test_0_basic.py +++ b/test/test_0_basic.py @@ -1,3 +1,4 @@ +from cibuildwheel.logger import Logger import textwrap import platform from . import test_projects @@ -20,8 +21,13 @@ def test(tmp_path, capfd): # build the wheels, and let the output passthrough to the caller, so # we can see how it looks with capfd.disabled(): + logger = Logger() + logger._start_fold_group('Sample build') + actual_wheels = utils.cibuildwheel_run(project_dir) + logger._end_fold_group() + # check that the expected wheels are produced expected_wheels = utils.expected_wheels('spam', '0.1.0') assert set(actual_wheels) == set(expected_wheels) From 7914432a68b3b2d2dd334782caac265042d8d11c Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 13 Nov 2020 15:25:52 +0000 Subject: [PATCH 31/63] fix appearance on Travis with headers and identifiers --- cibuildwheel/logger.py | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index fcd77751..7ccc71ff 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -1,12 +1,13 @@ import os import time import sys +import re from typing import Optional, Union DEFAULT_FOLD_PATTERN = ('{name}', '') FOLD_PATTERNS = { 'azure': ('##[group]{name}', '##[endgroup]'), - 'travis': ('travis_fold:start:{name}', 'travis_fold:end:{name}'), + 'travis': ('travis_fold:start:{identifier}\n{name}', 'travis_fold:end:{identifier}'), 'github': ('::group::{name}', '::endgroup::{name}'), } @@ -107,16 +108,32 @@ class Logger: self._end_fold_group() self.active_fold_group_name = name fold_start_pattern = FOLD_PATTERNS.get(self.fold_mode, DEFAULT_FOLD_PATTERN)[0] + identifier = self._fold_group_identifier(name) - print(fold_start_pattern.format(name=self.active_fold_group_name)) + print(fold_start_pattern.format(name=self.active_fold_group_name, identifier=identifier)) def _end_fold_group(self): if self.active_fold_group_name: fold_start_pattern = FOLD_PATTERNS.get(self.fold_mode, DEFAULT_FOLD_PATTERN)[1] - print(fold_start_pattern.format(name=self.active_fold_group_name)) + identifier = self._fold_group_identifier(self.active_fold_group_name) + print(fold_start_pattern.format(name=self.active_fold_group_name, identifier=identifier)) sys.stdout.flush() self.active_fold_group_name = None + def _fold_group_identifier(self, name: str): + ''' + Travis doesn't like fold groups identifiers that have spaces in. This + method converts them to ascii identifiers + ''' + # whitespace to dashes + identifier = re.sub(r'\s+', '-', name) + # remove non-alphanum + identifier = re.sub(r'[^A-Za-z\d]+', r'', identifier) + # trim dashes + identifier = identifier.strip('-') + # lowercase + return identifier.lower() + @property def colors(self): if self.colors_enabled: From 246aa0848ee9f6593aa3811a4d85057b0974176a Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 13 Nov 2020 15:38:34 +0000 Subject: [PATCH 32/63] Update cibuildwheel/util.py Co-authored-by: Henry Schreiner --- cibuildwheel/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 61c28301..4f6d005d 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -117,7 +117,7 @@ class DependencyConstraints: return self.base_file_path def __repr__(self): - return f'DependencyConstraints({self.base_file_path!r})' + return f'{self.__class__.__name__}{self.base_file_path!r})' class BuildOptions(NamedTuple): From 38297e6663bdd81ee8e67759aaaa0a4ccb2d0b05 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 13 Nov 2020 16:30:27 +0000 Subject: [PATCH 33/63] Add log steps to macOS/Windows --- cibuildwheel/linux.py | 4 +- cibuildwheel/logger.py | 11 ++- cibuildwheel/macos.py | 185 +++++++++++++++++++---------------- cibuildwheel/windows.py | 212 ++++++++++++++++++++++------------------ 4 files changed, 226 insertions(+), 186 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 6c6593a3..f404e9f9 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -1,4 +1,3 @@ -from cibuildwheel.logger import Logger import platform import subprocess import sys @@ -8,6 +7,7 @@ from pathlib import Path, PurePath from typing import List, NamedTuple, Union from .docker_container import DockerContainer +from .logger import log from .util import (BuildOptions, BuildSelector, NonPlatformWheelError, get_build_verbosity_extra_flags, prepare_command) @@ -112,8 +112,6 @@ def build(options: BuildOptions) -> None: container_package_dir = container_project_path / abs_package_dir.relative_to(cwd) container_output_dir = PurePath('/output') - log = Logger() - for implementation, platform_tag, docker_image in platforms: platform_configs = [c for c in python_configurations if c.identifier.startswith(implementation) and c.identifier.endswith(platform_tag)] if not platform_configs: diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index 7ccc71ff..df38cde8 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -1,7 +1,7 @@ import os -import time -import sys import re +import sys +import time from typing import Optional, Union DEFAULT_FOLD_PATTERN = ('{name}', '') @@ -142,6 +142,13 @@ class Logger: return colors_disabled +''' +Global instance of the Logger. +''' +# (there's only one stdout per-process, so a global instance is justified) +log = Logger() + + def build_description_from_identifier(identifier: str): python_identifier, _, platform_identifier = identifier.partition('-') diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 5dfd1cd6..eb5ce5aa 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -9,9 +9,10 @@ from pathlib import Path from typing import Dict, List, NamedTuple, Optional, Sequence, Union from .environment import ParsedEnvironment -from .util import (BuildOptions, BuildSelector, NonPlatformWheelError, download, - get_build_verbosity_extra_flags, get_pip_script, - prepare_command, install_certifi_script) +from .logger import log +from .util import (BuildOptions, BuildSelector, NonPlatformWheelError, + download, get_build_verbosity_extra_flags, get_pip_script, + install_certifi_script, prepare_command) def call(args: Union[str, Sequence[Union[str, PathLike]]], env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None, shell: bool = False) -> int: @@ -116,13 +117,18 @@ def install_pypy(version: str, url: str) -> Path: def setup_python(python_configuration: PythonConfiguration, dependency_constraint_flags: Sequence[Union[str, PathLike]], environment: ParsedEnvironment) -> Dict[str, str]: - if python_configuration.identifier.startswith('cp'): + implementation_id = python_configuration.identifier.split("-")[0] + log.step(f'Installing Python {implementation_id}...') + + if implementation_id.startswith('cp'): installation_bin_path = install_cpython(python_configuration.version, python_configuration.url) - elif python_configuration.identifier.startswith('pp'): + elif implementation_id.startswith('pp'): installation_bin_path = install_pypy(python_configuration.version, python_configuration.url) else: raise ValueError("Unknown Python implementation") + log.step('Setting up build environment...') + env = os.environ.copy() env['PATH'] = os.pathsep.join([ str(SYMLINKS_DIR), @@ -156,7 +162,6 @@ def setup_python(python_configuration: PythonConfiguration, if which_pip != '/tmp/cibw_bin/pip': print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr) exit(1) - call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate', *dependency_constraint_flags], env=env) # Set MACOSX_DEPLOYMENT_TARGET to 10.9, if the user didn't set it. # CPython 3.5 defaults to 10.6, and pypy defaults to 10.7, causing @@ -170,6 +175,9 @@ def setup_python(python_configuration: PythonConfiguration, # https://github.com/python/cpython/blob/a5ed2fe0eedefa1649aa93ee74a0bafc8e628a10/Lib/_osx_support.py#L260 env.setdefault('ARCHFLAGS', '-arch x86_64') + log.step('Installing build tools...') + call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate', *dependency_constraint_flags], env=env) + return env @@ -178,102 +186,111 @@ def build(options: BuildOptions) -> None: built_wheel_dir = temp_dir / 'built_wheel' repaired_wheel_dir = temp_dir / 'repaired_wheel' - if options.before_all: - env = options.environment.as_dictionary(prev_environment=os.environ) - before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) - call([before_all_prepared], shell=True, env=env) + try: + if options.before_all: + log.step('Running before_all...') + env = options.environment.as_dictionary(prev_environment=os.environ) + before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) + call([before_all_prepared], shell=True, env=env) - python_configurations = get_python_configurations(options.build_selector) + python_configurations = get_python_configurations(options.build_selector) - for config in python_configurations: - dependency_constraint_flags: Sequence[Union[str, PathLike]] = [] - if options.dependency_constraints: - dependency_constraint_flags = [ - '-c', options.dependency_constraints.get_for_python_version(config.version) - ] + for config in python_configurations: + log.build_start(config.identifier) - env = setup_python(config, dependency_constraint_flags, options.environment) + dependency_constraint_flags: Sequence[Union[str, PathLike]] = [] + if options.dependency_constraints: + dependency_constraint_flags = [ + '-c', options.dependency_constraints.get_for_python_version(config.version) + ] - # run the before_build command - if options.before_build: - before_build_prepared = prepare_command(options.before_build, project='.', package=options.package_dir) - call(before_build_prepared, env=env, shell=True) + env = setup_python(config, dependency_constraint_flags, options.environment) - # build the wheel - if built_wheel_dir.exists(): - shutil.rmtree(built_wheel_dir) - built_wheel_dir.mkdir(parents=True) + if options.before_build: + log.step('Running before_build...') + before_build_prepared = prepare_command(options.before_build, project='.', package=options.package_dir) + call(before_build_prepared, env=env, shell=True) - # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org - # see https://github.com/joerick/cibuildwheel/pull/369 - call([ - 'pip', 'wheel', - options.package_dir.resolve(), - '-w', built_wheel_dir, - '--no-deps', - *get_build_verbosity_extra_flags(options.build_verbosity) - ], env=env) + log.step('Building wheel...') + if built_wheel_dir.exists(): + shutil.rmtree(built_wheel_dir) + built_wheel_dir.mkdir(parents=True) - built_wheel = next(built_wheel_dir.glob('*.whl')) + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org + # see https://github.com/joerick/cibuildwheel/pull/369 + call([ + 'pip', 'wheel', + options.package_dir.resolve(), + '-w', built_wheel_dir, + '--no-deps', + *get_build_verbosity_extra_flags(options.build_verbosity) + ], env=env) - # repair the wheel - if repaired_wheel_dir.exists(): - shutil.rmtree(repaired_wheel_dir) - repaired_wheel_dir.mkdir(parents=True) + built_wheel = next(built_wheel_dir.glob('*.whl')) - if built_wheel.name.endswith('none-any.whl'): - raise NonPlatformWheelError() + if repaired_wheel_dir.exists(): + shutil.rmtree(repaired_wheel_dir) + repaired_wheel_dir.mkdir(parents=True) - if options.repair_command: - repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) - call(repair_command_prepared, env=env, shell=True) - else: - shutil.move(str(built_wheel), repaired_wheel_dir) + if built_wheel.name.endswith('none-any.whl'): + raise NonPlatformWheelError() - repaired_wheel = next(repaired_wheel_dir.glob('*.whl')) + if options.repair_command: + log.step('Repairing wheel...') + repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) + call(repair_command_prepared, env=env, shell=True) + else: + shutil.move(str(built_wheel), repaired_wheel_dir) - if options.test_command: - # set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) - venv_dir = Path(tempfile.mkdtemp()) + repaired_wheel = next(repaired_wheel_dir.glob('*.whl')) - # Use --no-download to ensure determinism by using seed libraries - # built into virtualenv - call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) + if options.test_command: + log.step('Testing wheel...') + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) + venv_dir = Path(tempfile.mkdtemp()) - virtualenv_env = env.copy() - virtualenv_env['PATH'] = os.pathsep.join([ - str(venv_dir / 'bin'), - virtualenv_env['PATH'], - ]) + # Use --no-download to ensure determinism by using seed libraries + # built into virtualenv + call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) - # check that we are using the Python from the virtual environment - call(['which', 'python'], env=virtualenv_env) + virtualenv_env = env.copy() + virtualenv_env['PATH'] = os.pathsep.join([ + str(venv_dir / 'bin'), + virtualenv_env['PATH'], + ]) - if options.before_test: - before_test_prepared = prepare_command(options.before_test, project='.', package=options.package_dir) - call(before_test_prepared, env=virtualenv_env, shell=True) + # check that we are using the Python from the virtual environment + call(['which', 'python'], env=virtualenv_env) - # install the wheel - call(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env) + if options.before_test: + before_test_prepared = prepare_command(options.before_test, project='.', package=options.package_dir) + call(before_test_prepared, env=virtualenv_env, shell=True) - # test the wheel - if options.test_requires: - call(['pip', 'install'] + options.test_requires, env=virtualenv_env) + # install the wheel + call(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env) - # run the tests from $HOME, with an absolute path in the command - # (this ensures that Python runs the tests against the installed wheel - # and not the repo code) - test_command_prepared = prepare_command( - options.test_command, - project=Path('.').resolve(), - package=options.package_dir.resolve() - ) - call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True) + # test the wheel + if options.test_requires: + call(['pip', 'install'] + options.test_requires, env=virtualenv_env) - # clean up - shutil.rmtree(venv_dir) + # run the tests from $HOME, with an absolute path in the command + # (this ensures that Python runs the tests against the installed wheel + # and not the repo code) + test_command_prepared = prepare_command( + options.test_command, + project=Path('.').resolve(), + package=options.package_dir.resolve() + ) + call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True) - # we're all done here; move it to output (overwrite existing) - shutil.move(str(repaired_wheel), options.output_dir) + # clean up + shutil.rmtree(venv_dir) + + # we're all done here; move it to output (overwrite existing) + shutil.move(str(repaired_wheel), options.output_dir) + log.build_end() + except subprocess.CalledProcessError as error: + log.error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') + exit(1) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 9e8c1df5..f9997927 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -7,11 +7,13 @@ from os import PathLike from pathlib import Path from typing import Dict, List, NamedTuple, Optional, Sequence, Union from zipfile import ZipFile + import toml from .environment import ParsedEnvironment -from .util import (BuildOptions, BuildSelector, NonPlatformWheelError, download, - get_build_verbosity_extra_flags, get_pip_script, +from .logger import log +from .util import (BuildOptions, BuildSelector, NonPlatformWheelError, + download, get_build_verbosity_extra_flags, get_pip_script, prepare_command) IS_RUNNING_ON_AZURE = Path('C:\\hostedtoolcache').exists() @@ -109,11 +111,15 @@ def install_pypy(version: str, arch: str, url: str) -> Path: def setup_python(python_configuration: PythonConfiguration, dependency_constraint_flags: Sequence[Union[str, PathLike]], environment: ParsedEnvironment) -> Dict[str, str]: nuget = Path('C:\\cibw\\nuget.exe') if not nuget.exists(): + log.step('Downloading nuget...') download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget) - if python_configuration.identifier.startswith('cp'): + implementation_id = python_configuration.identifier.split("-")[0] + log.step(f'Installing Python {implementation_id}...') + + if implementation_id.startswith('cp'): installation_path = install_cpython(python_configuration.version, python_configuration.arch, nuget) - elif python_configuration.identifier.startswith('pp'): + elif implementation_id.startswith('pp'): assert python_configuration.url is not None installation_path = install_pypy(python_configuration.version, python_configuration.arch, python_configuration.url) else: @@ -121,6 +127,8 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain assert (installation_path / 'python.exe').exists() + log.step('Setting up build environment...') + # set up PATH and environment variables for run_with_env env = os.environ.copy() env['PYTHON_VERSION'] = python_configuration.version @@ -151,7 +159,8 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr) exit(1) - # prepare the Python environment + log.step('Installing build tools...') + call(['python', '-m', 'pip', 'install', '--upgrade', 'pip', *dependency_constraint_flags], env=env) call(['pip', '--version'], env=env) call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', *dependency_constraint_flags], env=env) @@ -182,6 +191,7 @@ def pep_518_cp35_workaround(package_dir: Path, env: Dict[str, str]) -> None: ) if requirements: + log.step('Performing PEP518 workaround...') with tempfile.TemporaryDirectory() as d: reqfile = Path(d) / "requirements.txt" with reqfile.open("w") as f: @@ -195,114 +205,122 @@ def build(options: BuildOptions) -> None: built_wheel_dir = temp_dir / 'built_wheel' repaired_wheel_dir = temp_dir / 'repaired_wheel' - # install nuget as best way to provide python - nuget = Path('C:\\cibw\\nuget.exe') - download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget) + try: + if options.before_all: + log.step('Running before_all...') + env = options.environment.as_dictionary(prev_environment=os.environ) + before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) + shell(before_all_prepared, env=env) - if options.before_all: - env = options.environment.as_dictionary(prev_environment=os.environ) - before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) - shell(before_all_prepared, env=env) + python_configurations = get_python_configurations(options.build_selector) - python_configurations = get_python_configurations(options.build_selector) - for config in python_configurations: - dependency_constraint_flags: Sequence[Union[str, PathLike]] = [] - if options.dependency_constraints: - dependency_constraint_flags = [ - '-c', options.dependency_constraints.get_for_python_version(config.version) - ] + for config in python_configurations: + log.build_start(config.identifier) - # install Python - env = setup_python(config, dependency_constraint_flags, options.environment) + dependency_constraint_flags: Sequence[Union[str, PathLike]] = [] + if options.dependency_constraints: + dependency_constraint_flags = [ + '-c', options.dependency_constraints.get_for_python_version(config.version) + ] - # run the before_build command - if options.before_build: - before_build_prepared = prepare_command(options.before_build, project='.', package=options.package_dir) - shell(before_build_prepared, env=env) + # install Python + env = setup_python(config, dependency_constraint_flags, options.environment) - # activate the PEP 518 patch if on Windows Python 3.5 - # (will only have an effect if PEP 517 builds are used): - if config.version.startswith('3.5'): - pep_518_cp35_workaround(options.package_dir, env) + # run the before_build command + if options.before_build: + log.step('Running before_build...') + before_build_prepared = prepare_command(options.before_build, project='.', package=options.package_dir) + shell(before_build_prepared, env=env) - # build the wheel - if built_wheel_dir.exists(): - shutil.rmtree(built_wheel_dir) - built_wheel_dir.mkdir(parents=True) - # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org - # see https://github.com/joerick/cibuildwheel/pull/369 - call([ - 'pip', 'wheel', - options.package_dir.resolve(), - '-w', built_wheel_dir, - '--no-deps', - *get_build_verbosity_extra_flags(options.build_verbosity) - ], env=env) + # activate the PEP 518 patch if on Windows Python 3.5 + # (will only have an effect if PEP 517 builds are used): + if config.version.startswith('3.5'): + pep_518_cp35_workaround(options.package_dir, env) - built_wheel = next(built_wheel_dir.glob('*.whl')) + log.step('Building wheel...') + if built_wheel_dir.exists(): + shutil.rmtree(built_wheel_dir) + built_wheel_dir.mkdir(parents=True) + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org + # see https://github.com/joerick/cibuildwheel/pull/369 + call([ + 'pip', 'wheel', + options.package_dir.resolve(), + '-w', built_wheel_dir, + '--no-deps', + *get_build_verbosity_extra_flags(options.build_verbosity) + ], env=env) - # repair the wheel - if repaired_wheel_dir.exists(): - shutil.rmtree(repaired_wheel_dir) - repaired_wheel_dir.mkdir(parents=True) + built_wheel = next(built_wheel_dir.glob('*.whl')) - if built_wheel.name.endswith('none-any.whl'): - raise NonPlatformWheelError() + # repair the wheel + if repaired_wheel_dir.exists(): + shutil.rmtree(repaired_wheel_dir) + repaired_wheel_dir.mkdir(parents=True) - if options.repair_command: - repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) - shell(repair_command_prepared, env=env) - else: - shutil.move(str(built_wheel), repaired_wheel_dir) + if built_wheel.name.endswith('none-any.whl'): + raise NonPlatformWheelError() - repaired_wheel = next(repaired_wheel_dir.glob('*.whl')) + if options.repair_command: + log.step('Repairing wheel...') + repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) + shell(repair_command_prepared, env=env) + else: + shutil.move(str(built_wheel), repaired_wheel_dir) - if options.test_command: - # set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) - venv_dir = Path(tempfile.mkdtemp()) + repaired_wheel = next(repaired_wheel_dir.glob('*.whl')) - # Use --no-download to ensure determinism by using seed libraries - # built into virtualenv - call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) + if options.test_command: + log.step('Testing wheel...') + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) + venv_dir = Path(tempfile.mkdtemp()) - virtualenv_env = env.copy() - virtualenv_env['PATH'] = os.pathsep.join([ - str(venv_dir / 'Scripts'), - virtualenv_env['PATH'], - ]) + # Use --no-download to ensure determinism by using seed libraries + # built into virtualenv + call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) - # check that we are using the Python from the virtual environment - call(['which', 'python'], env=virtualenv_env) + virtualenv_env = env.copy() + virtualenv_env['PATH'] = os.pathsep.join([ + str(venv_dir / 'Scripts'), + virtualenv_env['PATH'], + ]) - if options.before_test: - before_test_prepared = prepare_command( - options.before_test, - project='.', - package=options.package_dir + # check that we are using the Python from the virtual environment + call(['which', 'python'], env=virtualenv_env) + + if options.before_test: + before_test_prepared = prepare_command( + options.before_test, + project='.', + package=options.package_dir + ) + shell(before_test_prepared, env=virtualenv_env) + + # install the wheel + call(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env) + + # test the wheel + if options.test_requires: + call(['pip', 'install'] + options.test_requires, env=virtualenv_env) + + # run the tests from c:\, with an absolute path in the command + # (this ensures that Python runs the tests against the installed wheel + # and not the repo code) + test_command_prepared = prepare_command( + options.test_command, + project=Path('.').resolve(), + package=options.package_dir.resolve() ) - shell(before_test_prepared, env=virtualenv_env) + shell(test_command_prepared, cwd='c:\\', env=virtualenv_env) - # install the wheel - call(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env) + # clean up + shutil.rmtree(venv_dir) - # test the wheel - if options.test_requires: - call(['pip', 'install'] + options.test_requires, env=virtualenv_env) - - # run the tests from c:\, with an absolute path in the command - # (this ensures that Python runs the tests against the installed wheel - # and not the repo code) - test_command_prepared = prepare_command( - options.test_command, - project=Path('.').resolve(), - package=options.package_dir.resolve() - ) - shell(test_command_prepared, cwd='c:\\', env=virtualenv_env) - - # clean up - shutil.rmtree(venv_dir) - - # we're all done here; move it to output (remove if already exists) - shutil.move(str(repaired_wheel), options.output_dir) + # we're all done here; move it to output (remove if already exists) + shutil.move(str(repaired_wheel), options.output_dir) + log.build_end() + except subprocess.CalledProcessError as error: + log.error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') + exit(1) From 8b8d6d749c825764cb1457753490fb2df8523eab Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 13 Nov 2020 16:33:59 +0000 Subject: [PATCH 34/63] Prevent double-builds of the sample --- .github/workflows/sample_build.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/sample_build.yml b/.github/workflows/sample_build.yml index e25e8fa6..da08449d 100644 --- a/.github/workflows/sample_build.yml +++ b/.github/workflows/sample_build.yml @@ -1,6 +1,12 @@ name: Sample build -on: [push, pull_request] +on: + push: {} + pull_request: + branches: + # PRs from branches on the origin repo get a build from the 'push' + # trigger. So the pull_request trigger is limited to fork branches + - '**:**' jobs: build_wheels: From 931f8e106e92633fc8850916c171a77777c2f484 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 13 Nov 2020 16:46:58 +0000 Subject: [PATCH 35/63] Shorten the travis IDs --- cibuildwheel/logger.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index df38cde8..0fd0d61e 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -125,14 +125,14 @@ class Logger: Travis doesn't like fold groups identifiers that have spaces in. This method converts them to ascii identifiers ''' - # whitespace to dashes - identifier = re.sub(r'\s+', '-', name) + # whitespace to underscores + identifier = re.sub(r'\s+', '_', name) # remove non-alphanum - identifier = re.sub(r'[^A-Za-z\d]+', r'', identifier) - # trim dashes - identifier = identifier.strip('-') - # lowercase - return identifier.lower() + identifier = re.sub(r'[^A-Za-z\d_]+', '', identifier) + # trim underscores + identifier = identifier.strip('_') + # lowercase, shorten + return identifier.lower()[:20] @property def colors(self): From 1ab1130fe1dbec23966fe92c3a7d5cae32417caf Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 13 Nov 2020 16:51:26 +0000 Subject: [PATCH 36/63] Move initialization of the global logger to the end of the file --- cibuildwheel/logger.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index 0fd0d61e..451f632f 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -142,13 +142,6 @@ class Logger: return colors_disabled -''' -Global instance of the Logger. -''' -# (there's only one stdout per-process, so a global instance is justified) -log = Logger() - - def build_description_from_identifier(identifier: str): python_identifier, _, platform_identifier = identifier.partition('-') @@ -214,3 +207,10 @@ def file_supports_color(file_obj): def file_is_a_tty(file_obj): return hasattr(file_obj, 'isatty') and file_obj.isatty() + + +''' +Global instance of the Logger. +''' +# (there's only one stdout per-process, so a global instance is justified) +log = Logger() From 57d38efd4ddee12a8019b061b21305910480a85d Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 13 Nov 2020 17:20:15 +0000 Subject: [PATCH 37/63] Add mac/windows sample builds --- .github/workflows/sample_build.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/sample_build.yml b/.github/workflows/sample_build.yml index da08449d..9cd98be4 100644 --- a/.github/workflows/sample_build.yml +++ b/.github/workflows/sample_build.yml @@ -10,10 +10,11 @@ on: jobs: build_wheels: - name: Sample build [${{ matrix.result }}] - runs-on: ubuntu-18.04 + name: Sample build [${{ matrix.os }},${{ matrix.result }}] + runs-on: ${{ matrix.os }} strategy: matrix: + os: [ubuntu-18.04, windows-latest, macos-latest] result: [success, failure] steps: @@ -28,6 +29,11 @@ jobs: run: | python -m pip install -r requirements-dev.txt + - name: Install Visual C++ for Python 2.7 + if: runner.os == 'Windows' + run: | + choco install vcpython27 -f -y + - name: Generate sample project run: | if $should_fail; then From 3d59c5658489efcb0229e570c8af21ff0c741bd2 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 13 Nov 2020 17:32:49 +0000 Subject: [PATCH 38/63] Ensure sample build shell is bash on Windows --- .github/workflows/sample_build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/sample_build.yml b/.github/workflows/sample_build.yml index 9cd98be4..926b1215 100644 --- a/.github/workflows/sample_build.yml +++ b/.github/workflows/sample_build.yml @@ -35,6 +35,7 @@ jobs: choco install vcpython27 -f -y - name: Generate sample project + shell: bash run: | if $should_fail; then python -m test.test_projects test.test_testing.project_with_a_failing_test sample_project @@ -45,6 +46,7 @@ jobs: should_fail: ${{ matrix.result == 'failure' }} - name: Build & test wheels + shell: bash run: | cd sample_project if $should_fail; then From 8e2381bfb13231ae50ad4fc2776c613452b5fd71 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 13 Nov 2020 17:40:07 +0000 Subject: [PATCH 39/63] add some encoding debug code --- cibuildwheel/__main__.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 4e25befa..ea12f8b5 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -91,6 +91,17 @@ def main() -> None: args = parser.parse_args() + # TODO remove this encoding stuff + import locale + import os + print(sys.stdout.encoding) + print(sys.stdout.isatty()) + print(locale.getpreferredencoding()) + print(sys.getfilesystemencoding()) + print(os.environ["PYTHONIOENCODING"]) + print(chr(246), chr(9786), chr(9787)) + # TODO end + detect_obsolete_options() if args.platform != 'auto': From df71e028a948989749db6f47c40418c3dab72946 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 13 Nov 2020 17:41:43 +0000 Subject: [PATCH 40/63] grumble --- cibuildwheel/__main__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index ea12f8b5..c30fba9a 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -93,7 +93,6 @@ def main() -> None: # TODO remove this encoding stuff import locale - import os print(sys.stdout.encoding) print(sys.stdout.isatty()) print(locale.getpreferredencoding()) From 0227b9d941320c04e62e60312fd518d75faf8ffb Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 13 Nov 2020 17:44:13 +0000 Subject: [PATCH 41/63] Improve debug code --- cibuildwheel/__main__.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index c30fba9a..d4f51964 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -93,12 +93,12 @@ def main() -> None: # TODO remove this encoding stuff import locale - print(sys.stdout.encoding) - print(sys.stdout.isatty()) - print(locale.getpreferredencoding()) - print(sys.getfilesystemencoding()) - print(os.environ["PYTHONIOENCODING"]) - print(chr(246), chr(9786), chr(9787)) + print('sys.stdout.encoding', sys.stdout.encoding) + print('sys.stdout.isatty()', sys.stdout.isatty()) + print('locale.getpreferredencoding()', locale.getpreferredencoding()) + print('sys.getfilesystemencoding()', sys.getfilesystemencoding()) + print('os.environ["PYTHONIOENCODING"]', os.environ.get("PYTHONIOENCODING")) + print('chr(246), chr(9786), chr(9787)', chr(246), chr(9786), chr(9787)) # TODO end detect_obsolete_options() From 54858bbd64d2eb2f0883452a9e05268eb7db3802 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 13 Nov 2020 18:37:30 +0000 Subject: [PATCH 42/63] Support non-unicode stdout --- cibuildwheel/logger.py | 78 ++++++++++++++++++++++++++++++------------ 1 file changed, 57 insertions(+), 21 deletions(-) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index 451f632f..5dbf3631 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -1,3 +1,4 @@ +import codecs import os import re import sys @@ -26,12 +27,15 @@ PLATFORM_IDENTIFIER_DESCIPTIONS = { class Logger: fold_mode: str colors_enabled: bool + unicode_enabled: bool active_build_identifier: Optional[str] = None build_start_time: Optional[float] = None step_start_time: Optional[float] = None active_fold_group_name: Optional[str] = None def __init__(self): + self.unicode_enabled = file_supports_unicode(sys.stdout) + if 'AZURE_HTTP_USER_AGENT' in os.environ: self.fold_mode = 'azure' self.colors_enabled = True @@ -70,10 +74,11 @@ class Logger: self.step_end() c = self.colors + s = self.symbols duration = time.time() - self.build_start_time print() - print(f'{c.green}✓ {c.end}{self.active_build_identifier} finished in {duration:.2f}s') + print(f'{c.green}{s.done} {c.end}{self.active_build_identifier} finished in {duration:.2f}s') self.build_start_time = None self.active_build_identifier = None @@ -86,11 +91,12 @@ class Logger: if self.step_start_time is not None: self._end_fold_group() c = self.colors + s = self.symbols duration = time.time() - self.step_start_time if success: - print(f'{c.green}✓ {c.end}{duration:.2f}s'.rjust(78)) + print(f'{c.green}{s.done} {c.end}{duration:.2f}s'.rjust(78)) else: - print(f'{c.red}✕ {c.end}{duration:.2f}s'.rjust(78)) + print(f'{c.red}{s.error} {c.end}{duration:.2f}s'.rjust(78)) self.step_start_time = None @@ -137,9 +143,16 @@ class Logger: @property def colors(self): if self.colors_enabled: - return colors_enabled + return Colors.enabled else: - return colors_disabled + return Colors.disabled + + @property + def symbols(self): + if self.unicode_enabled: + return Symbols.unicode + else: + return Symbols.ascii def build_description_from_identifier(identifier: str): @@ -167,30 +180,43 @@ def build_description_from_identifier(identifier: str): return build_description -class Colors(): - red = '\033[31m' - green = '\033[32m' - yellow = '\033[33m' - blue = '\033[34m' - cyan = '\033[36m' - bright_red = '\033[91m' - bright_green = '\033[92m' - white = '\033[37m\033[97m' +class Colors: + class Enabled: + red = '\033[31m' + green = '\033[32m' + yellow = '\033[33m' + blue = '\033[34m' + cyan = '\033[36m' + bright_red = '\033[91m' + bright_green = '\033[92m' + white = '\033[37m\033[97m' - bg_grey = '\033[48;5;235m' + bg_grey = '\033[48;5;235m' - bold = '\033[1m' - faint = '\033[2m' + bold = '\033[1m' + faint = '\033[2m' - end = '\033[0m' + end = '\033[0m' class Disabled: - def __getattr__(self, attr: str): + def __getattr__(self, attr: str) -> str: return '' + enabled = Enabled() + disabled = Disabled() -colors_enabled = Colors() -colors_disabled = Colors.Disabled() + +class Symbols: + class Unicode: + done = '✓' + error = '✕' + + class Ascii: + done = 'done' + error = 'failed' + + unicode = Unicode() + ascii = Ascii() def file_supports_color(file_obj): @@ -209,6 +235,16 @@ def file_is_a_tty(file_obj): return hasattr(file_obj, 'isatty') and file_obj.isatty() +def file_supports_unicode(file_obj): + encoding = getattr(file_obj, 'encoding', None) + if not encoding: + return False + + codec_info = codecs.lookup(encoding) + + return ('utf' in codec_info.name) + + ''' Global instance of the Logger. ''' From 113350b7fb9fdec75bc17a271c402e8174b7ff06 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 13 Nov 2020 18:37:48 +0000 Subject: [PATCH 43/63] Remove unicode test --- cibuildwheel/__main__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index d4f51964..ad000dd2 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -98,7 +98,6 @@ def main() -> None: print('locale.getpreferredencoding()', locale.getpreferredencoding()) print('sys.getfilesystemencoding()', sys.getfilesystemencoding()) print('os.environ["PYTHONIOENCODING"]', os.environ.get("PYTHONIOENCODING")) - print('chr(246), chr(9786), chr(9787)', chr(246), chr(9786), chr(9787)) # TODO end detect_obsolete_options() From fba324816300f7d22bd437584fa371087c5ce87a Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 13 Nov 2020 18:46:41 +0000 Subject: [PATCH 44/63] Remove encoding debug code --- cibuildwheel/__main__.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index ad000dd2..4e25befa 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -91,15 +91,6 @@ def main() -> None: args = parser.parse_args() - # TODO remove this encoding stuff - import locale - print('sys.stdout.encoding', sys.stdout.encoding) - print('sys.stdout.isatty()', sys.stdout.isatty()) - print('locale.getpreferredencoding()', locale.getpreferredencoding()) - print('sys.getfilesystemencoding()', sys.getfilesystemencoding()) - print('os.environ["PYTHONIOENCODING"]', os.environ.get("PYTHONIOENCODING")) - # TODO end - detect_obsolete_options() if args.platform != 'auto': From e8aef6962adae50b05710d5b687cc31be288cd4d Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 13 Nov 2020 18:57:07 +0000 Subject: [PATCH 45/63] Set output to UTF8 on windows (on Python >= 3.7) --- cibuildwheel/__main__.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 4e25befa..86ea12f0 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -54,6 +54,11 @@ def strtobool(val: str) -> bool: def main() -> None: + if sys.platform == 'win32' and hasattr(sys.stdout, 'reconfigure'): + # the encoding on Windows can be a 1-byte charmap, but all CIs support + # utf8, so we hardcode that + sys.stdout.reconfigure(encoding='utf8') + parser = argparse.ArgumentParser( description='Build wheels for all the platforms.', epilog=''' From a7325250841e4ea8cefa4052b0497d935a384721 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 13 Nov 2020 19:04:02 +0000 Subject: [PATCH 46/63] Move encoding switch to before the unicode decision is made --- cibuildwheel/__main__.py | 5 ----- cibuildwheel/logger.py | 5 +++++ 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 86ea12f0..4e25befa 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -54,11 +54,6 @@ def strtobool(val: str) -> bool: def main() -> None: - if sys.platform == 'win32' and hasattr(sys.stdout, 'reconfigure'): - # the encoding on Windows can be a 1-byte charmap, but all CIs support - # utf8, so we hardcode that - sys.stdout.reconfigure(encoding='utf8') - parser = argparse.ArgumentParser( description='Build wheels for all the platforms.', epilog=''' diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index 5dbf3631..66e84f20 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -34,6 +34,11 @@ class Logger: active_fold_group_name: Optional[str] = None def __init__(self): + if sys.platform == 'win32' and hasattr(sys.stdout, 'reconfigure'): + # the encoding on Windows can be a 1-byte charmap, but all CIs + # support utf8, so we hardcode that + sys.stdout.reconfigure(encoding='utf8') + self.unicode_enabled = file_supports_unicode(sys.stdout) if 'AZURE_HTTP_USER_AGENT' in os.environ: From 4c94e7ef20d96a58c02b97313af63a3581485cb3 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 14 Nov 2020 14:28:17 +0000 Subject: [PATCH 47/63] Add back newline after stage header. Also flush stdout to help with ordering --- cibuildwheel/logger.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index 66e84f20..d3f86077 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -122,6 +122,8 @@ class Logger: identifier = self._fold_group_identifier(name) print(fold_start_pattern.format(name=self.active_fold_group_name, identifier=identifier)) + print() + sys.stdout.flush() def _end_fold_group(self): if self.active_fold_group_name: From 8606c4771ec98ed6315c5f1840d0ebe8db24b75d Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 14 Nov 2020 14:28:47 +0000 Subject: [PATCH 48/63] Add explicit sample build test to keep the first test simple --- test/test_0_basic.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/test/test_0_basic.py b/test/test_0_basic.py index e4bcbc19..6b589b45 100644 --- a/test/test_0_basic.py +++ b/test/test_0_basic.py @@ -14,7 +14,19 @@ basic_project = test_projects.new_c_project( ) -def test(tmp_path, capfd): +def test(tmp_path): + project_dir = tmp_path / 'project' + basic_project.generate(project_dir) + + # build the wheels + actual_wheels = utils.cibuildwheel_run(project_dir) + + # check that the expected wheels are produced + expected_wheels = utils.expected_wheels('spam', '0.1.0') + assert set(actual_wheels) == set(expected_wheels) + + +def test_sample_build(tmp_path, capfd): project_dir = tmp_path / 'project' basic_project.generate(project_dir) @@ -22,15 +34,11 @@ def test(tmp_path, capfd): # we can see how it looks with capfd.disabled(): logger = Logger() - logger._start_fold_group('Sample build') - - actual_wheels = utils.cibuildwheel_run(project_dir) - - logger._end_fold_group() - - # check that the expected wheels are produced - expected_wheels = utils.expected_wheels('spam', '0.1.0') - assert set(actual_wheels) == set(expected_wheels) + logger.step('test_sample_build') + try: + utils.cibuildwheel_run(project_dir) + finally: + logger.step_end() def test_build_identifiers(tmp_path): From 91906b2aea0d7fd032f86ef4a5d16be2b63ebd82 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 14 Nov 2020 14:50:29 +0000 Subject: [PATCH 49/63] Skip the example output test --- test/test_0_basic.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/test/test_0_basic.py b/test/test_0_basic.py index 6b589b45..fff1b1dc 100644 --- a/test/test_0_basic.py +++ b/test/test_0_basic.py @@ -1,8 +1,9 @@ -from cibuildwheel.logger import Logger -import textwrap import platform -from . import test_projects -from . import utils +import textwrap +import pytest + +from cibuildwheel.logger import Logger +from . import test_projects, utils basic_project = test_projects.new_c_project( setup_py_add=textwrap.dedent(''' @@ -26,6 +27,7 @@ def test(tmp_path): assert set(actual_wheels) == set(expected_wheels) +@pytest.mark.skip(reason='to keep test output clean') def test_sample_build(tmp_path, capfd): project_dir = tmp_path / 'project' basic_project.generate(project_dir) From 9d731cd440d68e6b19941e2cd1ddb34071b8c5b7 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 14 Nov 2020 14:52:20 +0000 Subject: [PATCH 50/63] Make the sample build only happen from a manual trigger --- .github/workflows/sample_build.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/sample_build.yml b/.github/workflows/sample_build.yml index 926b1215..f12b30e5 100644 --- a/.github/workflows/sample_build.yml +++ b/.github/workflows/sample_build.yml @@ -1,12 +1,7 @@ name: Sample build on: - push: {} - pull_request: - branches: - # PRs from branches on the origin repo get a build from the 'push' - # trigger. So the pull_request trigger is limited to fork branches - - '**:**' + workflow_dispatch: {} jobs: build_wheels: From a93cee1a305899ad1dca58a5ca5c882c190fc1cb Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 22 Nov 2020 01:15:44 +0000 Subject: [PATCH 51/63] Run a sample build before tests on PRs with this label --- .github/workflows/test.yml | 11 +++++++++++ bin/sample_build.py | 23 +++++++++++++++++++++++ 2 files changed, 34 insertions(+) create mode 100755 bin/sample_build.py diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fdd20d0f..d65a52b7 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,10 +37,21 @@ jobs: - name: Install dependencies run: | python -m pip install -r requirements-dev.txt + - name: Install Visual C++ for Python 2.7 if: runner.os == 'Windows' run: | choco install vcpython27 -f -y + + - name: Get PR labels + id: pr-labels + uses: joerick/pr-labels-action@v1.0.5 + + - name: Sample build + if: contains(steps.pr-labels.outputs.labels, ' ci-sample-build ') + run: | + python bin/sample_build.py + - name: Test cibuildwheel run: | python ./bin/run_tests.py diff --git a/bin/sample_build.py b/bin/sample_build.py new file mode 100755 index 00000000..c8a13e67 --- /dev/null +++ b/bin/sample_build.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 + +import os +import subprocess +import sys +import argparse +import tempfile +from pathlib import Path + +if __name__ == '__main__': + # move cwd to the project root + os.chdir(Path(__file__).resolve().parents[1]) + + parser = argparse.ArgumentParser(description='Runs a sample build') + parser.add_argument('PROJECT_PYTHON_PATH', nargs='?', default='test.test_0_basic.basic_project') + + options = parser.parse_args() + print(options) + + project_dir = tempfile.mkdtemp() + subprocess.run([sys.executable, '-m', 'test.test_projects', options.PROJECT_PYTHON_PATH, project_dir], check=True) + + subprocess.run(['cibuildwheel'], check=True, cwd=project_dir) From e84c6b2de0f0a033baad5d856340b1fb24e79ac0 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 22 Nov 2020 01:20:14 +0000 Subject: [PATCH 52/63] Remove old sample build workflow --- .github/workflows/sample_build.yml | 56 ------------------------------ bin/sample_build.py | 2 +- 2 files changed, 1 insertion(+), 57 deletions(-) delete mode 100644 .github/workflows/sample_build.yml diff --git a/.github/workflows/sample_build.yml b/.github/workflows/sample_build.yml deleted file mode 100644 index f12b30e5..00000000 --- a/.github/workflows/sample_build.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: Sample build - -on: - workflow_dispatch: {} - -jobs: - build_wheels: - name: Sample build [${{ matrix.os }},${{ matrix.result }}] - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: [ubuntu-18.04, windows-latest, macos-latest] - result: [success, failure] - - steps: - - uses: actions/checkout@v2 - - - uses: actions/setup-python@v2 - name: Install Python - with: - python-version: '3.7' - - - name: Install dependencies - run: | - python -m pip install -r requirements-dev.txt - - - name: Install Visual C++ for Python 2.7 - if: runner.os == 'Windows' - run: | - choco install vcpython27 -f -y - - - name: Generate sample project - shell: bash - run: | - if $should_fail; then - python -m test.test_projects test.test_testing.project_with_a_failing_test sample_project - else - python -m test.test_projects test.test_testing.project_with_a_test sample_project - fi - env: - should_fail: ${{ matrix.result == 'failure' }} - - - name: Build & test wheels - shell: bash - run: | - cd sample_project - if $should_fail; then - # prefix with ! to expect failure - ! python -m cibuildwheel --output-dir wheelhouse - else - python -m cibuildwheel --output-dir wheelhouse - fi - env: - CIBW_TEST_COMMAND: nosetests {project}/test - CIBW_TEST_REQUIRES: nose - should_fail: ${{ matrix.result == 'failure' }} diff --git a/bin/sample_build.py b/bin/sample_build.py index c8a13e67..204bbf0c 100755 --- a/bin/sample_build.py +++ b/bin/sample_build.py @@ -20,4 +20,4 @@ if __name__ == '__main__': project_dir = tempfile.mkdtemp() subprocess.run([sys.executable, '-m', 'test.test_projects', options.PROJECT_PYTHON_PATH, project_dir], check=True) - subprocess.run(['cibuildwheel'], check=True, cwd=project_dir) + exit(subprocess.run(['cibuildwheel'], cwd=project_dir).returncode) From 05d7c142e9b808abdf1704efbeaa516500085262 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 22 Nov 2020 10:35:40 +0000 Subject: [PATCH 53/63] Subprocess stderr goes to stdout on Mac and Windows too --- cibuildwheel/macos.py | 3 ++- cibuildwheel/windows.py | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index eb5ce5aa..e2cddb6e 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -22,7 +22,8 @@ def call(args: Union[str, Sequence[Union[str, PathLike]]], env: Optional[Dict[st else: print('+ ' + ' '.join(shlex.quote(str(a)) for a in args)) - return subprocess.check_call(args, env=env, cwd=cwd, shell=shell) + # run the process. Subprocess stderr is routed to stdout to keep output interleaved + return subprocess.check_call(args, env=env, cwd=cwd, shell=shell, stderr=subprocess.STDOUT) class PythonConfiguration(NamedTuple): diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index f9997927..4532e1bb 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -25,12 +25,13 @@ def call(args: Sequence[Union[str, PathLike]], env: Optional[Dict[str, str]] = N print('+ ' + ' '.join(str(a) for a in args)) # we use shell=True here, even though we don't need a shell due to a bug # https://bugs.python.org/issue8557 - subprocess.check_call([str(a) for a in args], env=env, cwd=cwd, shell=True) + subprocess.check_call([str(a) for a in args], env=env, cwd=cwd, shell=True, stderr=subprocess.STDOUT) def shell(command: str, env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None) -> None: print(f'+ {command}') - subprocess.check_call(command, env=env, cwd=cwd, shell=True) + # route stderr to stdout so output remains interleaved + subprocess.check_call(command, env=env, cwd=cwd, shell=True, stderr=subprocess.STDOUT) def get_nuget_args(version: str, arch: str) -> List[str]: From ece358f29df85f38c006b3348d3480ac1a184628 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 23 Nov 2020 20:24:08 +0000 Subject: [PATCH 54/63] lowercase for command line argument --- bin/sample_build.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/bin/sample_build.py b/bin/sample_build.py index 204bbf0c..10fc6c1f 100755 --- a/bin/sample_build.py +++ b/bin/sample_build.py @@ -12,12 +12,15 @@ if __name__ == '__main__': os.chdir(Path(__file__).resolve().parents[1]) parser = argparse.ArgumentParser(description='Runs a sample build') - parser.add_argument('PROJECT_PYTHON_PATH', nargs='?', default='test.test_0_basic.basic_project') + parser.add_argument('project_python_path', nargs='?', default='test.test_0_basic.basic_project') options = parser.parse_args() print(options) project_dir = tempfile.mkdtemp() - subprocess.run([sys.executable, '-m', 'test.test_projects', options.PROJECT_PYTHON_PATH, project_dir], check=True) + subprocess.run([ + sys.executable, '-m', 'test.test_projects', + options.project_python_path, project_dir + ], check=True) exit(subprocess.run(['cibuildwheel'], cwd=project_dir).returncode) From 95fecc2d58b3ebcadb0cd0cb003467ecd64f469a Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 23 Nov 2020 20:56:24 +0000 Subject: [PATCH 55/63] Remove debug code --- bin/sample_build.py | 1 - 1 file changed, 1 deletion(-) diff --git a/bin/sample_build.py b/bin/sample_build.py index 10fc6c1f..d2adb022 100755 --- a/bin/sample_build.py +++ b/bin/sample_build.py @@ -15,7 +15,6 @@ if __name__ == '__main__': parser.add_argument('project_python_path', nargs='?', default='test.test_0_basic.basic_project') options = parser.parse_args() - print(options) project_dir = tempfile.mkdtemp() subprocess.run([ From f8a4fbe42ddc6a672d51701c7b037dec4c63b0ac Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 23 Nov 2020 21:00:22 +0000 Subject: [PATCH 56/63] Share CI platform detection --- cibuildwheel/__main__.py | 22 ++++++++----------- cibuildwheel/logger.py | 12 +++++++---- cibuildwheel/util.py | 46 ++++++++++++++++++++++++++++++++++++---- 3 files changed, 59 insertions(+), 21 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 4e25befa..e171ab20 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -21,6 +21,7 @@ from cibuildwheel.util import ( BuildSelector, DependencyConstraints, Unbuffered, + detect_ci_provider, resources_dir, ) @@ -47,12 +48,6 @@ def get_option_from_environment(option_name: str, platform: Optional[str] = None return os.environ.get(option_name, default) -def strtobool(val: str) -> bool: - if val.lower() in ('y', 'yes', 't', 'true', 'on', '1'): - return True - return False - - def main() -> None: parser = argparse.ArgumentParser( description='Build wheels for all the platforms.', @@ -96,13 +91,14 @@ def main() -> None: if args.platform != 'auto': platform = args.platform else: - ci = strtobool(os.environ.get('CI', 'false')) or 'BITRISE_BUILD_NUMBER' in os.environ or 'AZURE_HTTP_USER_AGENT' in os.environ or 'GITHUB_WORKFLOW' in os.environ - if not ci: - print('cibuildwheel: Unable to detect platform. cibuildwheel should run on your CI server, ' - 'Travis CI, AppVeyor, Azure Pipelines, GitHub Actions and CircleCI are supported. You ' - 'can run on your development machine or other CI providers using the --platform argument. ' - 'Check --help output for more information.', - file=sys.stderr) + ci_provider = detect_ci_provider() + if ci_provider is None: + print(textwrap.dedent(''' + cibuildwheel: Unable to detect platform. cibuildwheel should run on your CI server; + Travis CI, AppVeyor, Azure Pipelines, GitHub Actions, CircleCI, and Gitlab are + supported. You can run on your development machine or other CI providers using the + --platform argument. Check --help output for more information. + '''), file=sys.stderr) exit(2) if sys.platform.startswith('linux'): platform = 'linux' diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index d3f86077..a8289e76 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -5,6 +5,8 @@ import sys import time from typing import Optional, Union +from cibuildwheel.util import CIProvider, detect_ci_provider + DEFAULT_FOLD_PATTERN = ('{name}', '') FOLD_PATTERNS = { 'azure': ('##[group]{name}', '##[endgroup]'), @@ -41,19 +43,21 @@ class Logger: self.unicode_enabled = file_supports_unicode(sys.stdout) - if 'AZURE_HTTP_USER_AGENT' in os.environ: + ci_provider = detect_ci_provider() + + if ci_provider == CIProvider.azure_pipelines: self.fold_mode = 'azure' self.colors_enabled = True - elif 'GITHUB_ACTIONS' in os.environ: + elif ci_provider == CIProvider.github_actions: self.fold_mode = 'github' self.colors_enabled = True - elif 'TRAVIS' in os.environ: + elif ci_provider == CIProvider.travis_ci: self.fold_mode = 'travis' self.colors_enabled = True - elif 'APPVEYOR' in os.environ: + elif ci_provider == CIProvider.appveyor: self.fold_mode = 'disabled' self.colors_enabled = True diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 4f6d005d..9cd2187e 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -1,14 +1,15 @@ import os -import textwrap -import certifi -import urllib.request import ssl +import textwrap +import urllib.request +from enum import Enum from fnmatch import fnmatch from pathlib import Path from time import sleep - from typing import Dict, List, NamedTuple, Optional, Union +import certifi + from .environment import ParsedEnvironment @@ -155,3 +156,40 @@ class NonPlatformWheelError(Exception): ''') super().__init__(message) + + +def strtobool(val: str) -> bool: + if val.lower() in ('y', 'yes', 't', 'true', 'on', '1'): + return True + return False + + +class CIProvider(str, Enum): + travis_ci = 'travis' + appveyor = 'appveyor' + circle_ci = 'circle_ci' + azure_pipelines = 'azure_pipelines' + github_actions = 'github_actions' + gitlab = 'gitlab' + other = 'other' + + +def detect_ci_provider() -> Optional[CIProvider]: + if 'TRAVIS' in os.environ: + return CIProvider.travis_ci + elif 'APPVEYOR' in os.environ: + return CIProvider.appveyor + elif 'CIRCLECI' in os.environ: + return CIProvider.circle_ci + elif 'AZURE_HTTP_USER_AGENT' in os.environ: + return CIProvider.azure_pipelines + elif 'GITHUB_ACTIONS' in os.environ: + return CIProvider.github_actions + elif 'GITLAB_CI' in os.environ: + return CIProvider.gitlab + elif strtobool(os.environ.get('CI', 'false')): + return CIProvider.other + else: + return None + + From c2a7cdf98a2b231fa95fb201af9694bc281de910 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 23 Nov 2020 21:06:36 +0000 Subject: [PATCH 57/63] Fix flake8 errors --- cibuildwheel/util.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 9cd2187e..87ce5d17 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -191,5 +191,3 @@ def detect_ci_provider() -> Optional[CIProvider]: return CIProvider.other else: return None - - From bfd9bb482b620bfbcc5cebf473cdb117484772ae Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 23 Nov 2020 21:17:28 +0000 Subject: [PATCH 58/63] Fix unit test --- unit_test/main_tests/main_platform_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/unit_test/main_tests/main_platform_test.py b/unit_test/main_tests/main_platform_test.py index 009fb1b4..004855e8 100644 --- a/unit_test/main_tests/main_platform_test.py +++ b/unit_test/main_tests/main_platform_test.py @@ -11,7 +11,11 @@ def test_unknown_platform_non_ci(monkeypatch, capsys): monkeypatch.delenv('CI', raising=False) monkeypatch.delenv('BITRISE_BUILD_NUMBER', raising=False) monkeypatch.delenv('AZURE_HTTP_USER_AGENT', raising=False) - monkeypatch.delenv('GITHUB_WORKFLOW', raising=False) + monkeypatch.delenv('TRAVIS', raising=False) + monkeypatch.delenv('APPVEYOR', raising=False) + monkeypatch.delenv('GITHUB_ACTIONS', raising=False) + monkeypatch.delenv('GITLAB_CI', raising=False) + monkeypatch.delenv('CIRCLECI', raising=False) monkeypatch.delenv('CIBW_PLATFORM', raising=False) with pytest.raises(SystemExit) as exit: From ba01ec546ead2bcf4a6875d779883a67922b70d4 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 24 Nov 2020 09:05:14 +0000 Subject: [PATCH 59/63] Update pr-labels-action --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d65a52b7..3ab6656c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -45,7 +45,7 @@ jobs: - name: Get PR labels id: pr-labels - uses: joerick/pr-labels-action@v1.0.5 + uses: joerick/pr-labels-action@v1.0.6 - name: Sample build if: contains(steps.pr-labels.outputs.labels, ' ci-sample-build ') From 4c035a441b5cb1db62c796650fb4ceea46174c2d Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 24 Nov 2020 18:48:27 +0000 Subject: [PATCH 60/63] Try using an Unbuffered sys.stderr --- cibuildwheel/__main__.py | 1 + cibuildwheel/docker_container.py | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index e171ab20..3ec57195 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -220,6 +220,7 @@ def main() -> None: # Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print' sys.stdout = Unbuffered(sys.stdout) # type: ignore + sys.stderr = Unbuffered(sys.stderr) # type: ignore print_preamble(platform, build_options) diff --git a/cibuildwheel/docker_container.py b/cibuildwheel/docker_container.py index 7511f9e0..a58eb750 100644 --- a/cibuildwheel/docker_container.py +++ b/cibuildwheel/docker_container.py @@ -57,9 +57,6 @@ class DockerContainer: ], stdin=subprocess.PIPE, stdout=subprocess.PIPE, - # output the subcommand stderr to our stdout. Avoids ordering - # problems between log messages on stderr and stdout - stderr=sys.stdout, ) assert self.process.stdin and self.process.stdout From 5665b61cbf069a7084dddcb91b64540ff1aa8c78 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 24 Nov 2020 18:53:43 +0000 Subject: [PATCH 61/63] Double-check this --- cibuildwheel/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 3ec57195..f7753b87 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -220,7 +220,7 @@ def main() -> None: # Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print' sys.stdout = Unbuffered(sys.stdout) # type: ignore - sys.stderr = Unbuffered(sys.stderr) # type: ignore + # sys.stderr = Unbuffered(sys.stderr) # type: ignore print_preamble(platform, build_options) From 27bda4fa64166fc0e8d994e53424d4a6b86cf0d1 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 24 Nov 2020 19:02:50 +0000 Subject: [PATCH 62/63] Hide some errant output --- cibuildwheel/docker_container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/docker_container.py b/cibuildwheel/docker_container.py index a58eb750..d127d01d 100644 --- a/cibuildwheel/docker_container.py +++ b/cibuildwheel/docker_container.py @@ -73,7 +73,7 @@ class DockerContainer: self.process.terminate() self.process.wait() - subprocess.run(['docker', 'rm', '--force', '-v', self.name]) + subprocess.run(['docker', 'rm', '--force', '-v', self.name], stdout=subprocess.DEVNULL) self.name = None def copy_into(self, from_path: Path, to_path: PurePath) -> None: From c065e3ce2de8b80563ffe7e57728dfbd16d92bfc Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 24 Nov 2020 19:03:14 +0000 Subject: [PATCH 63/63] Scrap the stderr->stdout redirect (we don't seem to need it) Let's keep an eye on this and if we see output outside of the fold groups we can add it back --- cibuildwheel/__main__.py | 1 - cibuildwheel/macos.py | 3 +-- cibuildwheel/windows.py | 5 ++--- 3 files changed, 3 insertions(+), 6 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index f7753b87..e171ab20 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -220,7 +220,6 @@ def main() -> None: # Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print' sys.stdout = Unbuffered(sys.stdout) # type: ignore - # sys.stderr = Unbuffered(sys.stderr) # type: ignore print_preamble(platform, build_options) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index e2cddb6e..eb5ce5aa 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -22,8 +22,7 @@ def call(args: Union[str, Sequence[Union[str, PathLike]]], env: Optional[Dict[st else: print('+ ' + ' '.join(shlex.quote(str(a)) for a in args)) - # run the process. Subprocess stderr is routed to stdout to keep output interleaved - return subprocess.check_call(args, env=env, cwd=cwd, shell=shell, stderr=subprocess.STDOUT) + return subprocess.check_call(args, env=env, cwd=cwd, shell=shell) class PythonConfiguration(NamedTuple): diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 4532e1bb..f9997927 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -25,13 +25,12 @@ def call(args: Sequence[Union[str, PathLike]], env: Optional[Dict[str, str]] = N print('+ ' + ' '.join(str(a) for a in args)) # we use shell=True here, even though we don't need a shell due to a bug # https://bugs.python.org/issue8557 - subprocess.check_call([str(a) for a in args], env=env, cwd=cwd, shell=True, stderr=subprocess.STDOUT) + subprocess.check_call([str(a) for a in args], env=env, cwd=cwd, shell=True) def shell(command: str, env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None) -> None: print(f'+ {command}') - # route stderr to stdout so output remains interleaved - subprocess.check_call(command, env=env, cwd=cwd, shell=True, stderr=subprocess.STDOUT) + subprocess.check_call(command, env=env, cwd=cwd, shell=True) def get_nuget_args(version: str, arch: str) -> List[str]: