Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Change to an imperative-style API

Browse Source
This commit is contained in:
Joe Rickerby
2020-11-01 11:34:02 +00:00
parent 946306b21d
commit 38e7b5a91d
2 changed files with 175 additions and 146 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cibuildwheel/linux.py
+126 -114
View File
@@ -112,7 +112,7 @@ def build(options: BuildOptions) -> None:
container_package_dir = container_project_path / abs_package_dir.relative_to(cwd)
container_output_dir = PurePath('/output')
logger = Logger()
log = Logger()
for implementation, platform_tag, docker_image in platforms:
platform_configs = [c for c in python_configurations if c.identifier.startswith(implementation) and c.identifier.endswith(platform_tag)]
@@ -124,19 +124,135 @@ def build(options: BuildOptions) -> None:
docker.copy_into(Path.cwd(), container_project_path)
if options.before_all:
with logger.step('Running before_all...'):
env = docker.get_environment()
env['PATH'] = f'/opt/python/cp38-cp38:{env["PATH"]}'
env = options.environment.as_dictionary(env, executor=docker.environment_executor)
log.build_step('Running before_all...')
before_all_prepared = prepare_command(options.before_all, project=container_project_path, package=container_package_dir)
docker.call(['sh', '-c', before_all_prepared], env=env)
env = docker.get_environment()
env['PATH'] = f'/opt/python/cp38-cp38:{env["PATH"]}'
env = options.environment.as_dictionary(env, executor=docker.environment_executor)
before_all_prepared = prepare_command(options.before_all, project=container_project_path, package=container_package_dir)
docker.call(['sh', '-c', before_all_prepared], env=env)
log.build_step_end()
for config in platform_configs:
build_one(options=options, config=config, docker=docker, container_project_path=container_project_path, container_package_dir=container_package_dir, container_output_dir=container_output_dir, logger=logger)
log.build_start(config.identifier)
dependency_constraint_flags: List[Union[str, PathLike]] = []
if options.dependency_constraints:
constraints_file = options.dependency_constraints.get_for_python_version(config.version)
container_constraints_file = PurePath('/constraints.txt')
docker.copy_into(constraints_file, container_constraints_file)
dependency_constraint_flags = ['-c', container_constraints_file]
env = docker.get_environment()
# put this config's python top of the list
python_bin = config.path / 'bin'
env['PATH'] = f'{python_bin}:{env["PATH"]}'
log.build_step('Setting up build environment...')
env = options.environment.as_dictionary(env, executor=docker.environment_executor)
# check config python and pip are still on PATH
which_python = docker.call(['which', 'python'], env=env, capture_output=True).strip()
if PurePath(which_python) != python_bin / 'python':
print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr)
exit(1)
which_pip = docker.call(['which', 'pip'], env=env, capture_output=True).strip()
if PurePath(which_pip) != python_bin / 'pip':
print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr)
exit(1)
if options.before_build:
log.build_step('Running before_build...')
before_build_prepared = prepare_command(options.before_build, project=container_project_path, package=container_package_dir)
docker.call(['sh', '-c', before_build_prepared], env=env)
log.build_step('Building wheel...')
temp_dir = PurePath('/tmp/cibuildwheel')
built_wheel_dir = temp_dir / 'built_wheel'
docker.call(['rm', '-rf', built_wheel_dir])
docker.call(['mkdir', '-p', built_wheel_dir])
docker.call([
'pip', 'wheel',
container_package_dir,
'-w', built_wheel_dir,
'--no-deps',
*get_build_verbosity_extra_flags(options.build_verbosity)
], env=env)
built_wheel = docker.glob(built_wheel_dir, '*.whl')[0]
repaired_wheel_dir = temp_dir / 'repaired_wheel'
docker.call(['rm', '-rf', repaired_wheel_dir])
docker.call(['mkdir', '-p', repaired_wheel_dir])
if built_wheel.name.endswith('none-any.whl'):
raise NonPlatformWheelError()
if options.repair_command:
log.build_step('Repairing wheel...')
repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir)
docker.call(['sh', '-c', repair_command_prepared], env=env)
else:
docker.call(['mv', built_wheel, repaired_wheel_dir])
repaired_wheels = docker.glob(repaired_wheel_dir, '*.whl')
if options.test_command:
log.build_step('Testing wheel...')
# set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
docker.call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env)
venv_dir = PurePath(docker.call(['mktemp', '-d'], capture_output=True).strip()) / 'venv'
docker.call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env)
virtualenv_env = env.copy()
virtualenv_env['PATH'] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}"
if options.before_test:
before_test_prepared = prepare_command(options.before_test, project=container_project_path, package=container_package_dir)
docker.call(['sh', '-c', before_test_prepared], env=virtualenv_env)
# Install the wheel we just built
# Note: If auditwheel produced two wheels, it's because the earlier produced wheel
# conforms to multiple manylinux standards. These multiple versions of the wheel are
# functionally the same, differing only in name, wheel metadata, and possibly include
# different external shared libraries. so it doesn't matter which one we run the tests on.
# Let's just pick the first one.
wheel_to_test = repaired_wheels[0]
docker.call(['pip', 'install', str(wheel_to_test) + options.test_extras], env=virtualenv_env)
# Install any requirements to run the tests
if options.test_requires:
docker.call(['pip', 'install', *options.test_requires], env=virtualenv_env)
# Run the tests from a different directory
test_command_prepared = prepare_command(options.test_command, project=container_project_path, package=container_package_dir)
docker.call(['sh', '-c', test_command_prepared], cwd='/root', env=virtualenv_env)
# clean up test environment
docker.call(['rm', '-rf', venv_dir])
# move repaired wheels to output
docker.call(['mkdir', '-p', container_output_dir])
docker.call(['mv', *repaired_wheels, container_output_dir])
log.build_end()
log.build_step('Copying wheels back to host...')
# copy the output back into the host
docker.copy_out(container_output_dir, options.output_dir)
log.build_step_end()
except subprocess.CalledProcessError as error:
print(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}')
troubleshoot(options.package_dir, error)
@@ -145,112 +261,8 @@ def build(options: BuildOptions) -> None:
def build_one(options: BuildOptions, config: PythonConfiguration, docker: DockerContainer,
container_project_path: PurePath, container_package_dir: PurePath,
container_output_dir: PurePath, logger: Logger):
dependency_constraint_flags: List[Union[str, PathLike]] = []
if options.dependency_constraints:
constraints_file = options.dependency_constraints.get_for_python_version(config.version)
container_constraints_file = PurePath('/constraints.txt')
docker.copy_into(constraints_file, container_constraints_file)
dependency_constraint_flags = ['-c', container_constraints_file]
env = docker.get_environment()
# put this config's python top of the list
python_bin = config.path / 'bin'
env['PATH'] = f'{python_bin}:{env["PATH"]}'
with logger.step(' Setting up build environment...'):
env = options.environment.as_dictionary(env, executor=docker.environment_executor)
# check config python and pip are still on PATH
which_python = docker.call(['which', 'python'], env=env, capture_output=True).strip()
if PurePath(which_python) != python_bin / 'python':
print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr)
exit(1)
which_pip = docker.call(['which', 'pip'], env=env, capture_output=True).strip()
if PurePath(which_pip) != python_bin / 'pip':
print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr)
exit(1)
if options.before_build:
with logger.step(' Running before_build...'):
before_build_prepared = prepare_command(options.before_build, project=container_project_path, package=container_package_dir)
docker.call(['sh', '-c', before_build_prepared], env=env)
temp_dir = PurePath('/tmp/cibuildwheel')
built_wheel_dir = temp_dir / 'built_wheel'
docker.call(['rm', '-rf', built_wheel_dir])
docker.call(['mkdir', '-p', built_wheel_dir])
with logger.step(' Building wheel...'):
docker.call([
'pip', 'wheel',
container_package_dir,
'-w', built_wheel_dir,
'--no-deps',
*get_build_verbosity_extra_flags(options.build_verbosity)
], env=env)
built_wheel = docker.glob(built_wheel_dir, '*.whl')[0]
repaired_wheel_dir = temp_dir / 'repaired_wheel'
docker.call(['rm', '-rf', repaired_wheel_dir])
docker.call(['mkdir', '-p', repaired_wheel_dir])
if built_wheel.name.endswith('none-any.whl'):
raise NonPlatformWheelError()
if options.repair_command:
with logger.step(' Repairing wheel...'):
repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir)
docker.call(['sh', '-c', repair_command_prepared], env=env)
else:
docker.call(['mv', built_wheel, repaired_wheel_dir])
repaired_wheels = docker.glob(repaired_wheel_dir, '*.whl')
if options.test_command:
with logger.step(' Testing wheel...'):
# set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
docker.call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env)
venv_dir = PurePath(docker.call(['mktemp', '-d'], capture_output=True).strip()) / 'venv'
docker.call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env)
virtualenv_env = env.copy()
virtualenv_env['PATH'] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}"
if options.before_test:
before_test_prepared = prepare_command(options.before_test, project=container_project_path, package=container_package_dir)
docker.call(['sh', '-c', before_test_prepared], env=virtualenv_env)
# Install the wheel we just built
# Note: If auditwheel produced two wheels, it's because the earlier produced wheel
# conforms to multiple manylinux standards. These multiple versions of the wheel are
# functionally the same, differing only in name, wheel metadata, and possibly include
# different external shared libraries. so it doesn't matter which one we run the tests on.
# Let's just pick the first one.
wheel_to_test = repaired_wheels[0]
docker.call(['pip', 'install', str(wheel_to_test) + options.test_extras], env=virtualenv_env)
# Install any requirements to run the tests
if options.test_requires:
docker.call(['pip', 'install', *options.test_requires], env=virtualenv_env)
# Run the tests from a different directory
test_command_prepared = prepare_command(options.test_command, project=container_project_path, package=container_package_dir)
docker.call(['sh', '-c', test_command_prepared], cwd='/root', env=virtualenv_env)
# clean up test environment
docker.call(['rm', '-rf', venv_dir])
# move repaired wheels to output
docker.call(['mkdir', '-p', container_output_dir])
docker.call(['mv', *repaired_wheels, container_output_dir])
container_output_dir: PurePath, log: Logger):
pass
def troubleshoot(package_dir: Path, error: Exception) -> None:
cibuildwheel/logger.py
+49 -32
View File
@@ -1,6 +1,7 @@
from contextlib import contextmanager
import os
import time
import re
from typing import Optional
FOLD_PATTERNS = {
'azure': ['##[group]{name}', '##[endgroup]'],
@@ -21,6 +22,13 @@ PLATFORM_IDENTIFIER_DESCIPTIONS = {
class Logger:
fold_mode: str
colors_enabled: bool
active_build_identifier: Optional[str] = None
build_start_time: Optional[float] = 0
step_start_time: Optional[float] = 0
active_fold_group_id: Optional[str] = None
def __init__(self):
if 'AZURE_HTTP_USER_AGENT' in os.environ:
self.fold_mode = 'azure'
@@ -42,43 +50,52 @@ class Logger:
self.fold_mode = 'disabled'
self.colors_enabled = False
@contextmanager
def build(self, identifier: str):
def build_start(self, identifier: str):
c = self.colors
print()
print(f'{c.bold}Building {build_description_from_identifier(identifier)} wheel{c.end}')
print(f'Identifier: {identifier}')
print()
start_time = time.time()
try:
yield
duration = time.time() - start_time
print(f'{c.green}Build {c.bg_grey}{identifier}{c.end}{c.green} completed in {duration:.2f}s{c.end}')
except Exception:
duration = time.time() - start_time
print(f'{c.red}Build {c.bg_grey}{identifier}{c.end}{c.red} failed in {duration:.2f}s{c.end}')
raise
self.build_start_time = time.time()
self.active_build_identifier = identifier
def build_end(self):
assert self.build_start_time is not None
self.build_step_end()
@contextmanager
def step(self, name: str):
c = self.colors
start_time = time.time()
duration = time.time() - self.build_start_time
print(f'{c.green}Build {c.bg_grey}{self.active_build_identifier}{c.end}{c.green} completed in {duration:.2f}s{c.end}')
print()
print('---')
self.build_start_time = None
try:
with self.fold_group(name):
yield
duration = time.time() - start_time
print(f'{c.green}✓ {c.faint}[{duration:.2f}]{c.end}')
except Exception:
raise
def build_step(self, step_description: str):
self.build_step_end()
self.step_start_time = time.time()
self.start_fold_group(step_description)
@contextmanager
def fold_group(self, name: str):
fold_start_pattern, fold_end_pattern = FOLD_PATTERNS.get(self.fold_mode, ('', ''))
print(fold_start_pattern.format(name=name))
try:
yield
finally:
print(fold_end_pattern.format(name=name))
def build_step_end(self):
if self.step_start_time is not None:
self.end_fold_group()
c = self.colors
duration = time.time() - self.step_start_time
print(f'{c.green}✓ {c.faint}[{duration:.2f}s]{c.end}'.rjust(78))
self.step_start_time = None
def start_fold_group(self, name: str):
self.end_fold_group()
self.active_fold_group_id = re.sub(r'[^A-Za-z]', '', name)
fold_start_pattern = FOLD_PATTERNS.get(self.fold_mode, ('', ''))[0]
print(fold_start_pattern.format(name=self.active_fold_group_id))
print()
def end_fold_group(self):
if self.active_fold_group_id:
fold_start_pattern = FOLD_PATTERNS.get(self.fold_mode, ('', ''))[1]
print(fold_start_pattern.format(name=self.active_fold_group_id))
self.active_fold_group_id = None
@property
def colors(self):
@@ -88,7 +105,7 @@ class Logger:
return colors_disabled
def build_description_from_identifier(identifier):
def build_description_from_identifier(identifier: str):
python_identifier, _, platform_identifier = identifier.partition('-')
build_description = ''
@@ -130,7 +147,7 @@ class Colors():
end = '\033[0m'
class Disabled:
def __getattr__(self, attr):
def __getattr__(self, attr: str):
return ''