diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index fdd20d0f..3ab6656c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -37,10 +37,21 @@ jobs: - name: Install dependencies run: | python -m pip install -r requirements-dev.txt + - name: Install Visual C++ for Python 2.7 if: runner.os == 'Windows' run: | choco install vcpython27 -f -y + + - name: Get PR labels + id: pr-labels + uses: joerick/pr-labels-action@v1.0.6 + + - name: Sample build + if: contains(steps.pr-labels.outputs.labels, ' ci-sample-build ') + run: | + python bin/sample_build.py + - name: Test cibuildwheel run: | python ./bin/run_tests.py diff --git a/bin/sample_build.py b/bin/sample_build.py new file mode 100755 index 00000000..d2adb022 --- /dev/null +++ b/bin/sample_build.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 + +import os +import subprocess +import sys +import argparse +import tempfile +from pathlib import Path + +if __name__ == '__main__': + # move cwd to the project root + os.chdir(Path(__file__).resolve().parents[1]) + + parser = argparse.ArgumentParser(description='Runs a sample build') + parser.add_argument('project_python_path', nargs='?', default='test.test_0_basic.basic_project') + + options = parser.parse_args() + + project_dir = tempfile.mkdtemp() + subprocess.run([ + sys.executable, '-m', 'test.test_projects', + options.project_python_path, project_dir + ], check=True) + + exit(subprocess.run(['cibuildwheel'], cwd=project_dir).returncode) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 4e25befa..e171ab20 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -21,6 +21,7 @@ from cibuildwheel.util import ( BuildSelector, DependencyConstraints, Unbuffered, + detect_ci_provider, resources_dir, ) @@ -47,12 +48,6 @@ def get_option_from_environment(option_name: str, platform: Optional[str] = None return os.environ.get(option_name, default) -def strtobool(val: str) -> bool: - if val.lower() in ('y', 'yes', 't', 'true', 'on', '1'): - return True - return False - - def main() -> None: parser = argparse.ArgumentParser( description='Build wheels for all the platforms.', @@ -96,13 +91,14 @@ def main() -> None: if args.platform != 'auto': platform = args.platform else: - ci = strtobool(os.environ.get('CI', 'false')) or 'BITRISE_BUILD_NUMBER' in os.environ or 'AZURE_HTTP_USER_AGENT' in os.environ or 'GITHUB_WORKFLOW' in os.environ - if not ci: - print('cibuildwheel: Unable to detect platform. cibuildwheel should run on your CI server, ' - 'Travis CI, AppVeyor, Azure Pipelines, GitHub Actions and CircleCI are supported. You ' - 'can run on your development machine or other CI providers using the --platform argument. ' - 'Check --help output for more information.', - file=sys.stderr) + ci_provider = detect_ci_provider() + if ci_provider is None: + print(textwrap.dedent(''' + cibuildwheel: Unable to detect platform. cibuildwheel should run on your CI server; + Travis CI, AppVeyor, Azure Pipelines, GitHub Actions, CircleCI, and Gitlab are + supported. You can run on your development machine or other CI providers using the + --platform argument. Check --help output for more information. + '''), file=sys.stderr) exit(2) if sys.platform.startswith('linux'): platform = 'linux' diff --git a/cibuildwheel/docker_container.py b/cibuildwheel/docker_container.py index a58eb750..d127d01d 100644 --- a/cibuildwheel/docker_container.py +++ b/cibuildwheel/docker_container.py @@ -73,7 +73,7 @@ class DockerContainer: self.process.terminate() self.process.wait() - subprocess.run(['docker', 'rm', '--force', '-v', self.name]) + subprocess.run(['docker', 'rm', '--force', '-v', self.name], stdout=subprocess.DEVNULL) self.name = None def copy_into(self, from_path: Path, to_path: PurePath) -> None: diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index d1ca37fa..cfd4952c 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -7,6 +7,7 @@ from pathlib import Path, PurePath from typing import List, NamedTuple, Union from .docker_container import DockerContainer +from .logger import log from .util import (BuildOptions, BuildSelector, NonPlatformWheelError, get_build_verbosity_extra_flags, prepare_command) @@ -83,7 +84,7 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi def build(options: BuildOptions) -> None: try: - subprocess.check_call(['docker', '--version']) + subprocess.check_output(['docker', '--version']) except Exception: print('cibuildwheel: Docker not found. Docker is required to run Linux builds. ' 'If you\'re building on Travis CI, add `services: [docker]` to your .travis.yml.' @@ -117,10 +118,15 @@ def build(options: BuildOptions) -> None: continue try: + log.step(f'Starting Docker image {docker_image}...') with DockerContainer(docker_image, simulate_32_bit=platform_tag.endswith('i686'), cwd=container_project_path) as docker: + + log.step('Copying project into Docker...') docker.copy_into(Path.cwd(), container_project_path) if options.before_all: + log.step('Running before_all...') + env = docker.get_environment() env['PATH'] = f'/opt/python/cp38-cp38/bin:{env["PATH"]}' env = options.environment.as_dictionary(env, executor=docker.environment_executor) @@ -129,6 +135,8 @@ def build(options: BuildOptions) -> None: docker.call(['sh', '-c', before_all_prepared], env=env) for config in platform_configs: + log.build_start(config.identifier) + dependency_constraint_flags: List[Union[str, PathLike]] = [] if options.dependency_constraints: @@ -138,6 +146,8 @@ def build(options: BuildOptions) -> None: docker.copy_into(constraints_file, container_constraints_file) dependency_constraint_flags = ['-c', container_constraints_file] + log.step('Setting up build environment...') + env = docker.get_environment() # put this config's python top of the list @@ -158,9 +168,12 @@ def build(options: BuildOptions) -> None: exit(1) if options.before_build: + log.step('Running before_build...') before_build_prepared = prepare_command(options.before_build, project=container_project_path, package=container_package_dir) docker.call(['sh', '-c', before_build_prepared], env=env) + log.step('Building wheel...') + temp_dir = PurePath('/tmp/cibuildwheel') built_wheel_dir = temp_dir / 'built_wheel' docker.call(['rm', '-rf', built_wheel_dir]) @@ -184,6 +197,7 @@ def build(options: BuildOptions) -> None: raise NonPlatformWheelError() if options.repair_command: + log.step('Repairing wheel...') repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) docker.call(['sh', '-c', repair_command_prepared], env=env) else: @@ -192,6 +206,8 @@ def build(options: BuildOptions) -> None: repaired_wheels = docker.glob(repaired_wheel_dir, '*.whl') if options.test_command: + log.step('Testing wheel...') + # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. docker.call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) @@ -230,10 +246,14 @@ def build(options: BuildOptions) -> None: docker.call(['mkdir', '-p', container_output_dir]) docker.call(['mv', *repaired_wheels, container_output_dir]) + log.build_end() + + log.step('Copying wheels back to host...') # copy the output back into the host docker.copy_out(container_output_dir, options.output_dir) + log.step_end() except subprocess.CalledProcessError as error: - print(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') + log.error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') troubleshoot(options.package_dir, error) exit(1) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py new file mode 100644 index 00000000..a8289e76 --- /dev/null +++ b/cibuildwheel/logger.py @@ -0,0 +1,263 @@ +import codecs +import os +import re +import sys +import time +from typing import Optional, Union + +from cibuildwheel.util import CIProvider, detect_ci_provider + +DEFAULT_FOLD_PATTERN = ('{name}', '') +FOLD_PATTERNS = { + 'azure': ('##[group]{name}', '##[endgroup]'), + 'travis': ('travis_fold:start:{identifier}\n{name}', 'travis_fold:end:{identifier}'), + 'github': ('::group::{name}', '::endgroup::{name}'), +} + +PLATFORM_IDENTIFIER_DESCIPTIONS = { + 'manylinux_x86_64': 'manylinux x86_64', + 'manylinux_i686': 'manylinux i686', + 'manylinux_aarch64': 'manylinux aarch64', + 'manylinux_ppc64le': 'manylinux ppc64le', + 'manylinux_s390x': 'manylinux s390x', + 'win32': 'Windows 32bit', + 'win_amd64': 'Windows 64bit', + 'macosx_x86_64': 'macOS x86_64', +} + + +class Logger: + fold_mode: str + colors_enabled: bool + unicode_enabled: bool + active_build_identifier: Optional[str] = None + build_start_time: Optional[float] = None + step_start_time: Optional[float] = None + active_fold_group_name: Optional[str] = None + + def __init__(self): + if sys.platform == 'win32' and hasattr(sys.stdout, 'reconfigure'): + # the encoding on Windows can be a 1-byte charmap, but all CIs + # support utf8, so we hardcode that + sys.stdout.reconfigure(encoding='utf8') + + self.unicode_enabled = file_supports_unicode(sys.stdout) + + ci_provider = detect_ci_provider() + + if ci_provider == CIProvider.azure_pipelines: + self.fold_mode = 'azure' + self.colors_enabled = True + + elif ci_provider == CIProvider.github_actions: + self.fold_mode = 'github' + self.colors_enabled = True + + elif ci_provider == CIProvider.travis_ci: + self.fold_mode = 'travis' + self.colors_enabled = True + + elif ci_provider == CIProvider.appveyor: + self.fold_mode = 'disabled' + self.colors_enabled = True + + else: + self.fold_mode = 'disabled' + self.colors_enabled = file_supports_color(sys.stdout) + + def build_start(self, identifier: str): + self.step_end() + c = self.colors + description = build_description_from_identifier(identifier) + print() + print(f'{c.bold}{c.blue}Building {identifier} wheel{c.end}') + print(f'{description}') + print() + + self.build_start_time = time.time() + self.active_build_identifier = identifier + + def build_end(self): + assert self.build_start_time is not None + assert self.active_build_identifier is not None + self.step_end() + + c = self.colors + s = self.symbols + duration = time.time() - self.build_start_time + + print() + print(f'{c.green}{s.done} {c.end}{self.active_build_identifier} finished in {duration:.2f}s') + self.build_start_time = None + self.active_build_identifier = None + + def step(self, step_description: str): + self.step_end() + self.step_start_time = time.time() + self._start_fold_group(step_description) + + def step_end(self, success=True): + if self.step_start_time is not None: + self._end_fold_group() + c = self.colors + s = self.symbols + duration = time.time() - self.step_start_time + if success: + print(f'{c.green}{s.done} {c.end}{duration:.2f}s'.rjust(78)) + else: + print(f'{c.red}{s.error} {c.end}{duration:.2f}s'.rjust(78)) + + self.step_start_time = None + + def error(self, error: Union[Exception, str]): + self.step_end(success=False) + print() + + if self.fold_mode == 'github': + print(f'::error::{error}') + else: + c = self.colors + print(f'{c.bright_red}Error{c.end} {error}') + + def _start_fold_group(self, name: str): + self._end_fold_group() + self.active_fold_group_name = name + fold_start_pattern = FOLD_PATTERNS.get(self.fold_mode, DEFAULT_FOLD_PATTERN)[0] + identifier = self._fold_group_identifier(name) + + print(fold_start_pattern.format(name=self.active_fold_group_name, identifier=identifier)) + print() + sys.stdout.flush() + + def _end_fold_group(self): + if self.active_fold_group_name: + fold_start_pattern = FOLD_PATTERNS.get(self.fold_mode, DEFAULT_FOLD_PATTERN)[1] + identifier = self._fold_group_identifier(self.active_fold_group_name) + print(fold_start_pattern.format(name=self.active_fold_group_name, identifier=identifier)) + sys.stdout.flush() + self.active_fold_group_name = None + + def _fold_group_identifier(self, name: str): + ''' + Travis doesn't like fold groups identifiers that have spaces in. This + method converts them to ascii identifiers + ''' + # whitespace to underscores + identifier = re.sub(r'\s+', '_', name) + # remove non-alphanum + identifier = re.sub(r'[^A-Za-z\d_]+', '', identifier) + # trim underscores + identifier = identifier.strip('_') + # lowercase, shorten + return identifier.lower()[:20] + + @property + def colors(self): + if self.colors_enabled: + return Colors.enabled + else: + return Colors.disabled + + @property + def symbols(self): + if self.unicode_enabled: + return Symbols.unicode + else: + return Symbols.ascii + + +def build_description_from_identifier(identifier: str): + python_identifier, _, platform_identifier = identifier.partition('-') + + build_description = '' + + python_interpreter = python_identifier[0:2] + python_version = python_identifier[2:4] + + if python_interpreter == 'cp': + build_description += 'CPython' + elif python_interpreter == 'pp': + build_description += 'PyPy' + else: + raise Exception('unknown python') + + build_description += f' {python_version[0]}.{python_version[1]} ' + + try: + build_description += PLATFORM_IDENTIFIER_DESCIPTIONS[platform_identifier] + except KeyError as e: + raise Exception('unknown platform') from e + + return build_description + + +class Colors: + class Enabled: + red = '\033[31m' + green = '\033[32m' + yellow = '\033[33m' + blue = '\033[34m' + cyan = '\033[36m' + bright_red = '\033[91m' + bright_green = '\033[92m' + white = '\033[37m\033[97m' + + bg_grey = '\033[48;5;235m' + + bold = '\033[1m' + faint = '\033[2m' + + end = '\033[0m' + + class Disabled: + def __getattr__(self, attr: str) -> str: + return '' + + enabled = Enabled() + disabled = Disabled() + + +class Symbols: + class Unicode: + done = '✓' + error = '✕' + + class Ascii: + done = 'done' + error = 'failed' + + unicode = Unicode() + ascii = Ascii() + + +def file_supports_color(file_obj): + """ + Returns True if the running system's terminal supports color. + """ + plat = sys.platform + supported_platform = (plat != 'win32' or 'ANSICON' in os.environ) + + is_a_tty = file_is_a_tty(file_obj) + + return (supported_platform and is_a_tty) + + +def file_is_a_tty(file_obj): + return hasattr(file_obj, 'isatty') and file_obj.isatty() + + +def file_supports_unicode(file_obj): + encoding = getattr(file_obj, 'encoding', None) + if not encoding: + return False + + codec_info = codecs.lookup(encoding) + + return ('utf' in codec_info.name) + + +''' +Global instance of the Logger. +''' +# (there's only one stdout per-process, so a global instance is justified) +log = Logger() diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 8541ec72..9ffca89d 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -9,9 +9,10 @@ from pathlib import Path from typing import Dict, List, NamedTuple, Optional, Sequence, Union from .environment import ParsedEnvironment -from .util import (BuildOptions, BuildSelector, NonPlatformWheelError, download, - get_build_verbosity_extra_flags, get_pip_script, - prepare_command, install_certifi_script) +from .logger import log +from .util import (BuildOptions, BuildSelector, NonPlatformWheelError, + download, get_build_verbosity_extra_flags, get_pip_script, + install_certifi_script, prepare_command) def call(args: Union[str, Sequence[Union[str, PathLike]]], env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None, shell: bool = False) -> int: @@ -116,13 +117,18 @@ def install_pypy(version: str, url: str) -> Path: def setup_python(python_configuration: PythonConfiguration, dependency_constraint_flags: Sequence[Union[str, PathLike]], environment: ParsedEnvironment) -> Dict[str, str]: - if python_configuration.identifier.startswith('cp'): + implementation_id = python_configuration.identifier.split("-")[0] + log.step(f'Installing Python {implementation_id}...') + + if implementation_id.startswith('cp'): installation_bin_path = install_cpython(python_configuration.version, python_configuration.url) - elif python_configuration.identifier.startswith('pp'): + elif implementation_id.startswith('pp'): installation_bin_path = install_pypy(python_configuration.version, python_configuration.url) else: raise ValueError("Unknown Python implementation") + log.step('Setting up build environment...') + env = os.environ.copy() env['PATH'] = os.pathsep.join([ str(SYMLINKS_DIR), @@ -156,7 +162,6 @@ def setup_python(python_configuration: PythonConfiguration, if which_pip != '/tmp/cibw_bin/pip': print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr) exit(1) - call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate', *dependency_constraint_flags], env=env) # Set MACOSX_DEPLOYMENT_TARGET to 10.9, if the user didn't set it. # CPython 3.5 defaults to 10.6, and pypy defaults to 10.7, causing @@ -170,6 +175,9 @@ def setup_python(python_configuration: PythonConfiguration, # https://github.com/python/cpython/blob/a5ed2fe0eedefa1649aa93ee74a0bafc8e628a10/Lib/_osx_support.py#L260 env.setdefault('ARCHFLAGS', '-arch x86_64') + log.step('Installing build tools...') + call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate', *dependency_constraint_flags], env=env) + return env @@ -178,102 +186,111 @@ def build(options: BuildOptions) -> None: built_wheel_dir = temp_dir / 'built_wheel' repaired_wheel_dir = temp_dir / 'repaired_wheel' - if options.before_all: - env = options.environment.as_dictionary(prev_environment=os.environ) - before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) - call([before_all_prepared], shell=True, env=env) + try: + if options.before_all: + log.step('Running before_all...') + env = options.environment.as_dictionary(prev_environment=os.environ) + before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) + call([before_all_prepared], shell=True, env=env) - python_configurations = get_python_configurations(options.build_selector) + python_configurations = get_python_configurations(options.build_selector) - for config in python_configurations: - dependency_constraint_flags: Sequence[Union[str, PathLike]] = [] - if options.dependency_constraints: - dependency_constraint_flags = [ - '-c', options.dependency_constraints.get_for_python_version(config.version) - ] + for config in python_configurations: + log.build_start(config.identifier) - env = setup_python(config, dependency_constraint_flags, options.environment) + dependency_constraint_flags: Sequence[Union[str, PathLike]] = [] + if options.dependency_constraints: + dependency_constraint_flags = [ + '-c', options.dependency_constraints.get_for_python_version(config.version) + ] - # run the before_build command - if options.before_build: - before_build_prepared = prepare_command(options.before_build, project='.', package=options.package_dir) - call(before_build_prepared, env=env, shell=True) + env = setup_python(config, dependency_constraint_flags, options.environment) - # build the wheel - if built_wheel_dir.exists(): - shutil.rmtree(built_wheel_dir) - built_wheel_dir.mkdir(parents=True) + if options.before_build: + log.step('Running before_build...') + before_build_prepared = prepare_command(options.before_build, project='.', package=options.package_dir) + call(before_build_prepared, env=env, shell=True) - # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org - # see https://github.com/joerick/cibuildwheel/pull/369 - call([ - 'pip', 'wheel', - options.package_dir.resolve(), - '-w', built_wheel_dir, - '--no-deps', - *get_build_verbosity_extra_flags(options.build_verbosity) - ], env=env) + log.step('Building wheel...') + if built_wheel_dir.exists(): + shutil.rmtree(built_wheel_dir) + built_wheel_dir.mkdir(parents=True) - built_wheel = next(built_wheel_dir.glob('*.whl')) + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org + # see https://github.com/joerick/cibuildwheel/pull/369 + call([ + 'pip', 'wheel', + options.package_dir.resolve(), + '-w', built_wheel_dir, + '--no-deps', + *get_build_verbosity_extra_flags(options.build_verbosity) + ], env=env) - # repair the wheel - if repaired_wheel_dir.exists(): - shutil.rmtree(repaired_wheel_dir) - repaired_wheel_dir.mkdir(parents=True) + built_wheel = next(built_wheel_dir.glob('*.whl')) - if built_wheel.name.endswith('none-any.whl'): - raise NonPlatformWheelError() + if repaired_wheel_dir.exists(): + shutil.rmtree(repaired_wheel_dir) + repaired_wheel_dir.mkdir(parents=True) - if options.repair_command: - repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) - call(repair_command_prepared, env=env, shell=True) - else: - shutil.move(str(built_wheel), repaired_wheel_dir) + if built_wheel.name.endswith('none-any.whl'): + raise NonPlatformWheelError() - repaired_wheel = next(repaired_wheel_dir.glob('*.whl')) + if options.repair_command: + log.step('Repairing wheel...') + repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) + call(repair_command_prepared, env=env, shell=True) + else: + shutil.move(str(built_wheel), repaired_wheel_dir) - if options.test_command: - # set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) - venv_dir = Path(tempfile.mkdtemp()) + repaired_wheel = next(repaired_wheel_dir.glob('*.whl')) - # Use --no-download to ensure determinism by using seed libraries - # built into virtualenv - call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) + if options.test_command: + log.step('Testing wheel...') + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) + venv_dir = Path(tempfile.mkdtemp()) - virtualenv_env = env.copy() - virtualenv_env['PATH'] = os.pathsep.join([ - str(venv_dir / 'bin'), - virtualenv_env['PATH'], - ]) + # Use --no-download to ensure determinism by using seed libraries + # built into virtualenv + call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) - # check that we are using the Python from the virtual environment - call(['which', 'python'], env=virtualenv_env) + virtualenv_env = env.copy() + virtualenv_env['PATH'] = os.pathsep.join([ + str(venv_dir / 'bin'), + virtualenv_env['PATH'], + ]) - if options.before_test: - before_test_prepared = prepare_command(options.before_test, project='.', package=options.package_dir) - call(before_test_prepared, env=virtualenv_env, shell=True) + # check that we are using the Python from the virtual environment + call(['which', 'python'], env=virtualenv_env) - # install the wheel - call(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env) + if options.before_test: + before_test_prepared = prepare_command(options.before_test, project='.', package=options.package_dir) + call(before_test_prepared, env=virtualenv_env, shell=True) - # test the wheel - if options.test_requires: - call(['pip', 'install'] + options.test_requires, env=virtualenv_env) + # install the wheel + call(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env) - # run the tests from $HOME, with an absolute path in the command - # (this ensures that Python runs the tests against the installed wheel - # and not the repo code) - test_command_prepared = prepare_command( - options.test_command, - project=Path('.').resolve(), - package=options.package_dir.resolve() - ) - call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True) + # test the wheel + if options.test_requires: + call(['pip', 'install'] + options.test_requires, env=virtualenv_env) - # clean up - shutil.rmtree(venv_dir) + # run the tests from $HOME, with an absolute path in the command + # (this ensures that Python runs the tests against the installed wheel + # and not the repo code) + test_command_prepared = prepare_command( + options.test_command, + project=Path('.').resolve(), + package=options.package_dir.resolve() + ) + call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True) - # we're all done here; move it to output (overwrite existing) - shutil.move(str(repaired_wheel), options.output_dir) + # clean up + shutil.rmtree(venv_dir) + + # we're all done here; move it to output (overwrite existing) + shutil.move(str(repaired_wheel), options.output_dir) + log.build_end() + except subprocess.CalledProcessError as error: + log.error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') + exit(1) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index c24f9bf5..87ce5d17 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -1,14 +1,15 @@ import os -import textwrap -import certifi -import urllib.request import ssl +import textwrap +import urllib.request +from enum import Enum from fnmatch import fnmatch from pathlib import Path from time import sleep - from typing import Dict, List, NamedTuple, Optional, Union +import certifi + from .environment import ParsedEnvironment @@ -42,7 +43,10 @@ class BuildSelector: return match_any(self.build_patterns) and not match_any(self.skip_patterns) def __repr__(self) -> str: - return f'BuildSelector({" ".join(self.build_patterns)!r} - {" ".join(self.skip_patterns)!r})' + if not self.skip_patterns: + return f'BuildSelector({" ".join(self.build_patterns)!r})' + else: + return f'BuildSelector({" ".join(self.build_patterns)!r} - {" ".join(self.skip_patterns)!r})' # Taken from https://stackoverflow.com/a/107717 @@ -113,8 +117,8 @@ class DependencyConstraints: else: return self.base_file_path - def __str__(self): - return f"File '{self.base_file_path}'" + def __repr__(self): + return f'{self.__class__.__name__}{self.base_file_path!r})' class BuildOptions(NamedTuple): @@ -152,3 +156,38 @@ class NonPlatformWheelError(Exception): ''') super().__init__(message) + + +def strtobool(val: str) -> bool: + if val.lower() in ('y', 'yes', 't', 'true', 'on', '1'): + return True + return False + + +class CIProvider(str, Enum): + travis_ci = 'travis' + appveyor = 'appveyor' + circle_ci = 'circle_ci' + azure_pipelines = 'azure_pipelines' + github_actions = 'github_actions' + gitlab = 'gitlab' + other = 'other' + + +def detect_ci_provider() -> Optional[CIProvider]: + if 'TRAVIS' in os.environ: + return CIProvider.travis_ci + elif 'APPVEYOR' in os.environ: + return CIProvider.appveyor + elif 'CIRCLECI' in os.environ: + return CIProvider.circle_ci + elif 'AZURE_HTTP_USER_AGENT' in os.environ: + return CIProvider.azure_pipelines + elif 'GITHUB_ACTIONS' in os.environ: + return CIProvider.github_actions + elif 'GITLAB_CI' in os.environ: + return CIProvider.gitlab + elif strtobool(os.environ.get('CI', 'false')): + return CIProvider.other + else: + return None diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 2679324f..4331915a 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -7,11 +7,13 @@ from os import PathLike from pathlib import Path from typing import Dict, List, NamedTuple, Optional, Sequence, Union from zipfile import ZipFile + import toml from .environment import ParsedEnvironment -from .util import (BuildOptions, BuildSelector, NonPlatformWheelError, download, - get_build_verbosity_extra_flags, get_pip_script, +from .logger import log +from .util import (BuildOptions, BuildSelector, NonPlatformWheelError, + download, get_build_verbosity_extra_flags, get_pip_script, prepare_command) IS_RUNNING_ON_AZURE = Path('C:\\hostedtoolcache').exists() @@ -109,11 +111,15 @@ def install_pypy(version: str, arch: str, url: str) -> Path: def setup_python(python_configuration: PythonConfiguration, dependency_constraint_flags: Sequence[Union[str, PathLike]], environment: ParsedEnvironment) -> Dict[str, str]: nuget = Path('C:\\cibw\\nuget.exe') if not nuget.exists(): + log.step('Downloading nuget...') download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget) - if python_configuration.identifier.startswith('cp'): + implementation_id = python_configuration.identifier.split("-")[0] + log.step(f'Installing Python {implementation_id}...') + + if implementation_id.startswith('cp'): installation_path = install_cpython(python_configuration.version, python_configuration.arch, nuget) - elif python_configuration.identifier.startswith('pp'): + elif implementation_id.startswith('pp'): assert python_configuration.url is not None installation_path = install_pypy(python_configuration.version, python_configuration.arch, python_configuration.url) else: @@ -121,6 +127,8 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain assert (installation_path / 'python.exe').exists() + log.step('Setting up build environment...') + # set up PATH and environment variables for run_with_env env = os.environ.copy() env['PYTHON_VERSION'] = python_configuration.version @@ -151,7 +159,8 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr) exit(1) - # prepare the Python environment + log.step('Installing build tools...') + call(['python', '-m', 'pip', 'install', '--upgrade', 'pip', *dependency_constraint_flags], env=env) call(['pip', '--version'], env=env) call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', *dependency_constraint_flags], env=env) @@ -182,6 +191,7 @@ def pep_518_cp35_workaround(package_dir: Path, env: Dict[str, str]) -> None: ) if requirements: + log.step('Performing PEP518 workaround...') with tempfile.TemporaryDirectory() as d: reqfile = Path(d) / "requirements.txt" with reqfile.open("w") as f: @@ -195,114 +205,122 @@ def build(options: BuildOptions) -> None: built_wheel_dir = temp_dir / 'built_wheel' repaired_wheel_dir = temp_dir / 'repaired_wheel' - # install nuget as best way to provide python - nuget = Path('C:\\cibw\\nuget.exe') - download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget) + try: + if options.before_all: + log.step('Running before_all...') + env = options.environment.as_dictionary(prev_environment=os.environ) + before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) + shell(before_all_prepared, env=env) - if options.before_all: - env = options.environment.as_dictionary(prev_environment=os.environ) - before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) - shell(before_all_prepared, env=env) + python_configurations = get_python_configurations(options.build_selector) - python_configurations = get_python_configurations(options.build_selector) - for config in python_configurations: - dependency_constraint_flags: Sequence[Union[str, PathLike]] = [] - if options.dependency_constraints: - dependency_constraint_flags = [ - '-c', options.dependency_constraints.get_for_python_version(config.version) - ] + for config in python_configurations: + log.build_start(config.identifier) - # install Python - env = setup_python(config, dependency_constraint_flags, options.environment) + dependency_constraint_flags: Sequence[Union[str, PathLike]] = [] + if options.dependency_constraints: + dependency_constraint_flags = [ + '-c', options.dependency_constraints.get_for_python_version(config.version) + ] - # run the before_build command - if options.before_build: - before_build_prepared = prepare_command(options.before_build, project='.', package=options.package_dir) - shell(before_build_prepared, env=env) + # install Python + env = setup_python(config, dependency_constraint_flags, options.environment) - # activate the PEP 518 patch if on Windows Python 3.5 - # (will only have an effect if PEP 517 builds are used): - if config.version.startswith('3.5'): - pep_518_cp35_workaround(options.package_dir, env) + # run the before_build command + if options.before_build: + log.step('Running before_build...') + before_build_prepared = prepare_command(options.before_build, project='.', package=options.package_dir) + shell(before_build_prepared, env=env) - # build the wheel - if built_wheel_dir.exists(): - shutil.rmtree(built_wheel_dir) - built_wheel_dir.mkdir(parents=True) - # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org - # see https://github.com/joerick/cibuildwheel/pull/369 - call([ - 'pip', 'wheel', - options.package_dir.resolve(), - '-w', built_wheel_dir, - '--no-deps', - *get_build_verbosity_extra_flags(options.build_verbosity) - ], env=env) + # activate the PEP 518 patch if on Windows Python 3.5 + # (will only have an effect if PEP 517 builds are used): + if config.version.startswith('3.5'): + pep_518_cp35_workaround(options.package_dir, env) - built_wheel = next(built_wheel_dir.glob('*.whl')) + log.step('Building wheel...') + if built_wheel_dir.exists(): + shutil.rmtree(built_wheel_dir) + built_wheel_dir.mkdir(parents=True) + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org + # see https://github.com/joerick/cibuildwheel/pull/369 + call([ + 'pip', 'wheel', + options.package_dir.resolve(), + '-w', built_wheel_dir, + '--no-deps', + *get_build_verbosity_extra_flags(options.build_verbosity) + ], env=env) - # repair the wheel - if repaired_wheel_dir.exists(): - shutil.rmtree(repaired_wheel_dir) - repaired_wheel_dir.mkdir(parents=True) + built_wheel = next(built_wheel_dir.glob('*.whl')) - if built_wheel.name.endswith('none-any.whl'): - raise NonPlatformWheelError() + # repair the wheel + if repaired_wheel_dir.exists(): + shutil.rmtree(repaired_wheel_dir) + repaired_wheel_dir.mkdir(parents=True) - if options.repair_command: - repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) - shell(repair_command_prepared, env=env) - else: - shutil.move(str(built_wheel), repaired_wheel_dir) + if built_wheel.name.endswith('none-any.whl'): + raise NonPlatformWheelError() - repaired_wheel = next(repaired_wheel_dir.glob('*.whl')) + if options.repair_command: + log.step('Repairing wheel...') + repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) + shell(repair_command_prepared, env=env) + else: + shutil.move(str(built_wheel), repaired_wheel_dir) - if options.test_command: - # set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) - venv_dir = Path(tempfile.mkdtemp()) + repaired_wheel = next(repaired_wheel_dir.glob('*.whl')) - # Use --no-download to ensure determinism by using seed libraries - # built into virtualenv - call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) + if options.test_command: + log.step('Testing wheel...') + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) + venv_dir = Path(tempfile.mkdtemp()) - virtualenv_env = env.copy() - virtualenv_env['PATH'] = os.pathsep.join([ - str(venv_dir / 'Scripts'), - virtualenv_env['PATH'], - ]) + # Use --no-download to ensure determinism by using seed libraries + # built into virtualenv + call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) - # check that we are using the Python from the virtual environment - call(['which', 'python'], env=virtualenv_env) + virtualenv_env = env.copy() + virtualenv_env['PATH'] = os.pathsep.join([ + str(venv_dir / 'Scripts'), + virtualenv_env['PATH'], + ]) - if options.before_test: - before_test_prepared = prepare_command( - options.before_test, - project='.', - package=options.package_dir + # check that we are using the Python from the virtual environment + call(['which', 'python'], env=virtualenv_env) + + if options.before_test: + before_test_prepared = prepare_command( + options.before_test, + project='.', + package=options.package_dir + ) + shell(before_test_prepared, env=virtualenv_env) + + # install the wheel + call(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env) + + # test the wheel + if options.test_requires: + call(['pip', 'install'] + options.test_requires, env=virtualenv_env) + + # run the tests from c:\, with an absolute path in the command + # (this ensures that Python runs the tests against the installed wheel + # and not the repo code) + test_command_prepared = prepare_command( + options.test_command, + project=Path('.').resolve(), + package=options.package_dir.resolve() ) - shell(before_test_prepared, env=virtualenv_env) + shell(test_command_prepared, cwd='c:\\', env=virtualenv_env) - # install the wheel - call(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env) + # clean up + shutil.rmtree(venv_dir) - # test the wheel - if options.test_requires: - call(['pip', 'install'] + options.test_requires, env=virtualenv_env) - - # run the tests from c:\, with an absolute path in the command - # (this ensures that Python runs the tests against the installed wheel - # and not the repo code) - test_command_prepared = prepare_command( - options.test_command, - project=Path('.').resolve(), - package=options.package_dir.resolve() - ) - shell(test_command_prepared, cwd='c:\\', env=virtualenv_env) - - # clean up - shutil.rmtree(venv_dir) - - # we're all done here; move it to output (remove if already exists) - shutil.move(str(repaired_wheel), options.output_dir) + # we're all done here; move it to output (remove if already exists) + shutil.move(str(repaired_wheel), options.output_dir) + log.build_end() + except subprocess.CalledProcessError as error: + log.error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') + exit(1) diff --git a/test/test_0_basic.py b/test/test_0_basic.py index 60b309ab..fff1b1dc 100644 --- a/test/test_0_basic.py +++ b/test/test_0_basic.py @@ -1,7 +1,9 @@ -import textwrap import platform -from . import test_projects -from . import utils +import textwrap +import pytest + +from cibuildwheel.logger import Logger +from . import test_projects, utils basic_project = test_projects.new_c_project( setup_py_add=textwrap.dedent(''' @@ -25,6 +27,22 @@ def test(tmp_path): assert set(actual_wheels) == set(expected_wheels) +@pytest.mark.skip(reason='to keep test output clean') +def test_sample_build(tmp_path, capfd): + project_dir = tmp_path / 'project' + basic_project.generate(project_dir) + + # build the wheels, and let the output passthrough to the caller, so + # we can see how it looks + with capfd.disabled(): + logger = Logger() + logger.step('test_sample_build') + try: + utils.cibuildwheel_run(project_dir) + finally: + logger.step_end() + + def test_build_identifiers(tmp_path): project_dir = tmp_path / 'project' basic_project.generate(project_dir) diff --git a/test/test_projects/__main__.py b/test/test_projects/__main__.py index 06fe696f..070cc2a2 100644 --- a/test/test_projects/__main__.py +++ b/test/test_projects/__main__.py @@ -11,27 +11,34 @@ def main(): prog="python -m test.test_projects", description='Generate a test project to check it out' ) + parser.add_argument('--open', action='store_true', help=''' + Open the generated project in a file explorer + ''') parser.add_argument('PROJECT', help=''' Python path to a project object. E.g. test.test_0_basic.basic_project ''') + parser.add_argument('OUTPUT', nargs='?', help=''' + Path to output dir. If no dir is passed, a tempdir will be generated. + ''') options = parser.parse_args() module, _, name = options.PROJECT.rpartition('.') project = getattr(importlib.import_module(module), name) - project_dir = Path(tempfile.mkdtemp()) + project_dir = Path(options.OUTPUT or tempfile.mkdtemp()) project.generate(project_dir) print('Project generated at', project_dir) print() - if sys.platform == 'darwin': - subprocess.check_call(['open', '--', project_dir]) - elif sys.platform == 'linux2': - subprocess.check_call(['xdg-open', '--', project_dir]) - elif sys.platform == 'win32': - subprocess.check_call(['explorer', project_dir]) + if options.open: + if sys.platform == 'darwin': + subprocess.check_call(['open', '--', project_dir]) + elif sys.platform == 'linux2': + subprocess.check_call(['xdg-open', '--', project_dir]) + elif sys.platform == 'win32': + subprocess.check_call(['explorer', project_dir]) if __name__ == '__main__': diff --git a/test/test_testing.py b/test/test_testing.py index 58a38224..6136c4aa 100644 --- a/test/test_testing.py +++ b/test/test_testing.py @@ -103,15 +103,26 @@ def test_extras_require(tmp_path): assert set(actual_wheels) == set(expected_wheels) +project_with_a_failing_test = test_projects.new_c_project() +project_with_a_failing_test.files['test/spam_test.py'] = r''' +from unittest import TestCase + +class TestSpam(TestCase): + def test_something(self): + self.fail('this test is supposed to fail') +''' + + def test_failing_test(tmp_path): """Ensure a failing test causes cibuildwheel to error out and exit""" project_dir = tmp_path / 'project' output_dir = tmp_path / 'output' - project_with_a_test.generate(project_dir) + project_with_a_failing_test.generate(project_dir) with pytest.raises(subprocess.CalledProcessError): utils.cibuildwheel_run(project_dir, output_dir=output_dir, add_env={ - 'CIBW_TEST_COMMAND': 'false', + 'CIBW_TEST_REQUIRES': 'nose', + 'CIBW_TEST_COMMAND': 'nosetests {project}/test', # manylinux1 has a version of bash that's been shown to have # problems with this, so let's check that. 'CIBW_MANYLINUX_I686_IMAGE': 'manylinux1', diff --git a/unit_test/main_tests/main_platform_test.py b/unit_test/main_tests/main_platform_test.py index 009fb1b4..004855e8 100644 --- a/unit_test/main_tests/main_platform_test.py +++ b/unit_test/main_tests/main_platform_test.py @@ -11,7 +11,11 @@ def test_unknown_platform_non_ci(monkeypatch, capsys): monkeypatch.delenv('CI', raising=False) monkeypatch.delenv('BITRISE_BUILD_NUMBER', raising=False) monkeypatch.delenv('AZURE_HTTP_USER_AGENT', raising=False) - monkeypatch.delenv('GITHUB_WORKFLOW', raising=False) + monkeypatch.delenv('TRAVIS', raising=False) + monkeypatch.delenv('APPVEYOR', raising=False) + monkeypatch.delenv('GITHUB_ACTIONS', raising=False) + monkeypatch.delenv('GITLAB_CI', raising=False) + monkeypatch.delenv('CIRCLECI', raising=False) monkeypatch.delenv('CIBW_PLATFORM', raising=False) with pytest.raises(SystemExit) as exit: