Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add log steps to macOS/Windows

Browse Source
This commit is contained in:
Joe Rickerby
2020-11-13 16:30:35 +00:00
parent 246aa0848e
commit 38297e6663
4 changed files with 226 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cibuildwheel/linux.py
+1 -3
View File
@@ -1,4 +1,3 @@
from cibuildwheel.logger import Logger
import platform
import subprocess
import sys
@@ -8,6 +7,7 @@ from pathlib import Path, PurePath
from typing import List, NamedTuple, Union
from .docker_container import DockerContainer
from .logger import log
from .util import (BuildOptions, BuildSelector, NonPlatformWheelError,
get_build_verbosity_extra_flags, prepare_command)
@@ -112,8 +112,6 @@ def build(options: BuildOptions) -> None:
container_package_dir = container_project_path / abs_package_dir.relative_to(cwd)
container_output_dir = PurePath('/output')
log = Logger()
for implementation, platform_tag, docker_image in platforms:
platform_configs = [c for c in python_configurations if c.identifier.startswith(implementation) and c.identifier.endswith(platform_tag)]
if not platform_configs:
cibuildwheel/logger.py
+9 -2
View File
@@ -1,7 +1,7 @@
import os
import time
import sys
import re
import sys
import time
from typing import Optional, Union
DEFAULT_FOLD_PATTERN = ('{name}', '')
@@ -142,6 +142,13 @@ class Logger:
return colors_disabled
'''
Global instance of the Logger.
'''
# (there's only one stdout per-process, so a global instance is justified)
log = Logger()
def build_description_from_identifier(identifier: str):
python_identifier, _, platform_identifier = identifier.partition('-')
cibuildwheel/macos.py
+101 -84
View File
@@ -9,9 +9,10 @@ from pathlib import Path
from typing import Dict, List, NamedTuple, Optional, Sequence, Union
from .environment import ParsedEnvironment
from .util import (BuildOptions, BuildSelector, NonPlatformWheelError, download,
get_build_verbosity_extra_flags, get_pip_script,
prepare_command, install_certifi_script)
from .logger import log
from .util import (BuildOptions, BuildSelector, NonPlatformWheelError,
download, get_build_verbosity_extra_flags, get_pip_script,
install_certifi_script, prepare_command)
def call(args: Union[str, Sequence[Union[str, PathLike]]], env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None, shell: bool = False) -> int:
@@ -116,13 +117,18 @@ def install_pypy(version: str, url: str) -> Path:
def setup_python(python_configuration: PythonConfiguration,
dependency_constraint_flags: Sequence[Union[str, PathLike]],
environment: ParsedEnvironment) -> Dict[str, str]:
if python_configuration.identifier.startswith('cp'):
implementation_id = python_configuration.identifier.split("-")[0]
log.step(f'Installing Python {implementation_id}...')
if implementation_id.startswith('cp'):
installation_bin_path = install_cpython(python_configuration.version, python_configuration.url)
elif python_configuration.identifier.startswith('pp'):
elif implementation_id.startswith('pp'):
installation_bin_path = install_pypy(python_configuration.version, python_configuration.url)
else:
raise ValueError("Unknown Python implementation")
log.step('Setting up build environment...')
env = os.environ.copy()
env['PATH'] = os.pathsep.join([
str(SYMLINKS_DIR),
@@ -156,7 +162,6 @@ def setup_python(python_configuration: PythonConfiguration,
if which_pip != '/tmp/cibw_bin/pip':
print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr)
exit(1)
call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate', *dependency_constraint_flags], env=env)
# Set MACOSX_DEPLOYMENT_TARGET to 10.9, if the user didn't set it.
# CPython 3.5 defaults to 10.6, and pypy defaults to 10.7, causing
@@ -170,6 +175,9 @@ def setup_python(python_configuration: PythonConfiguration,
# https://github.com/python/cpython/blob/a5ed2fe0eedefa1649aa93ee74a0bafc8e628a10/Lib/_osx_support.py#L260
env.setdefault('ARCHFLAGS', '-arch x86_64')
log.step('Installing build tools...')
call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate', *dependency_constraint_flags], env=env)
return env
@@ -178,102 +186,111 @@ def build(options: BuildOptions) -> None:
built_wheel_dir = temp_dir / 'built_wheel'
repaired_wheel_dir = temp_dir / 'repaired_wheel'
if options.before_all:
env = options.environment.as_dictionary(prev_environment=os.environ)
before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir)
call([before_all_prepared], shell=True, env=env)
try:
if options.before_all:
log.step('Running before_all...')
env = options.environment.as_dictionary(prev_environment=os.environ)
before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir)
call([before_all_prepared], shell=True, env=env)
python_configurations = get_python_configurations(options.build_selector)
python_configurations = get_python_configurations(options.build_selector)
for config in python_configurations:
dependency_constraint_flags: Sequence[Union[str, PathLike]] = []
if options.dependency_constraints:
dependency_constraint_flags = [
'-c', options.dependency_constraints.get_for_python_version(config.version)
]
for config in python_configurations:
log.build_start(config.identifier)
env = setup_python(config, dependency_constraint_flags, options.environment)
dependency_constraint_flags: Sequence[Union[str, PathLike]] = []
if options.dependency_constraints:
dependency_constraint_flags = [
'-c', options.dependency_constraints.get_for_python_version(config.version)
]
# run the before_build command
if options.before_build:
before_build_prepared = prepare_command(options.before_build, project='.', package=options.package_dir)
call(before_build_prepared, env=env, shell=True)
env = setup_python(config, dependency_constraint_flags, options.environment)
# build the wheel
if built_wheel_dir.exists():
shutil.rmtree(built_wheel_dir)
built_wheel_dir.mkdir(parents=True)
if options.before_build:
log.step('Running before_build...')
before_build_prepared = prepare_command(options.before_build, project='.', package=options.package_dir)
call(before_build_prepared, env=env, shell=True)
# Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
# see https://github.com/joerick/cibuildwheel/pull/369
call([
'pip', 'wheel',
options.package_dir.resolve(),
'-w', built_wheel_dir,
'--no-deps',
*get_build_verbosity_extra_flags(options.build_verbosity)
], env=env)
log.step('Building wheel...')
if built_wheel_dir.exists():
shutil.rmtree(built_wheel_dir)
built_wheel_dir.mkdir(parents=True)
built_wheel = next(built_wheel_dir.glob('*.whl'))
# Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
# see https://github.com/joerick/cibuildwheel/pull/369
call([
'pip', 'wheel',
options.package_dir.resolve(),
'-w', built_wheel_dir,
'--no-deps',
*get_build_verbosity_extra_flags(options.build_verbosity)
], env=env)
# repair the wheel
if repaired_wheel_dir.exists():
shutil.rmtree(repaired_wheel_dir)
repaired_wheel_dir.mkdir(parents=True)
built_wheel = next(built_wheel_dir.glob('*.whl'))
if built_wheel.name.endswith('none-any.whl'):
raise NonPlatformWheelError()
if repaired_wheel_dir.exists():
shutil.rmtree(repaired_wheel_dir)
repaired_wheel_dir.mkdir(parents=True)
if options.repair_command:
repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir)
call(repair_command_prepared, env=env, shell=True)
else:
shutil.move(str(built_wheel), repaired_wheel_dir)
if built_wheel.name.endswith('none-any.whl'):
raise NonPlatformWheelError()
repaired_wheel = next(repaired_wheel_dir.glob('*.whl'))
if options.repair_command:
log.step('Repairing wheel...')
repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir)
call(repair_command_prepared, env=env, shell=True)
else:
shutil.move(str(built_wheel), repaired_wheel_dir)
if options.test_command:
# set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env)
venv_dir = Path(tempfile.mkdtemp())
repaired_wheel = next(repaired_wheel_dir.glob('*.whl'))
# Use --no-download to ensure determinism by using seed libraries
# built into virtualenv
call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env)
if options.test_command:
log.step('Testing wheel...')
# set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env)
venv_dir = Path(tempfile.mkdtemp())
virtualenv_env = env.copy()
virtualenv_env['PATH'] = os.pathsep.join([
str(venv_dir / 'bin'),
virtualenv_env['PATH'],
])
# Use --no-download to ensure determinism by using seed libraries
# built into virtualenv
call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env)
# check that we are using the Python from the virtual environment
call(['which', 'python'], env=virtualenv_env)
virtualenv_env = env.copy()
virtualenv_env['PATH'] = os.pathsep.join([
str(venv_dir / 'bin'),
virtualenv_env['PATH'],
])
if options.before_test:
before_test_prepared = prepare_command(options.before_test, project='.', package=options.package_dir)
call(before_test_prepared, env=virtualenv_env, shell=True)
# check that we are using the Python from the virtual environment
call(['which', 'python'], env=virtualenv_env)
# install the wheel
call(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env)
if options.before_test:
before_test_prepared = prepare_command(options.before_test, project='.', package=options.package_dir)
call(before_test_prepared, env=virtualenv_env, shell=True)
# test the wheel
if options.test_requires:
call(['pip', 'install'] + options.test_requires, env=virtualenv_env)
# install the wheel
call(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env)
# run the tests from $HOME, with an absolute path in the command
# (this ensures that Python runs the tests against the installed wheel
# and not the repo code)
test_command_prepared = prepare_command(
options.test_command,
project=Path('.').resolve(),
package=options.package_dir.resolve()
)
call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True)
# test the wheel
if options.test_requires:
call(['pip', 'install'] + options.test_requires, env=virtualenv_env)
# clean up
shutil.rmtree(venv_dir)
# run the tests from $HOME, with an absolute path in the command
# (this ensures that Python runs the tests against the installed wheel
# and not the repo code)
test_command_prepared = prepare_command(
options.test_command,
project=Path('.').resolve(),
package=options.package_dir.resolve()
)
call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True)
# we're all done here; move it to output (overwrite existing)
shutil.move(str(repaired_wheel), options.output_dir)
# clean up
shutil.rmtree(venv_dir)
# we're all done here; move it to output (overwrite existing)
shutil.move(str(repaired_wheel), options.output_dir)
log.build_end()
except subprocess.CalledProcessError as error:
log.error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}')
exit(1)
cibuildwheel/windows.py
+115 -97
View File
@@ -7,11 +7,13 @@ from os import PathLike
from pathlib import Path
from typing import Dict, List, NamedTuple, Optional, Sequence, Union
from zipfile import ZipFile
import toml
from .environment import ParsedEnvironment
from .util import (BuildOptions, BuildSelector, NonPlatformWheelError, download,
get_build_verbosity_extra_flags, get_pip_script,
from .logger import log
from .util import (BuildOptions, BuildSelector, NonPlatformWheelError,
download, get_build_verbosity_extra_flags, get_pip_script,
prepare_command)
IS_RUNNING_ON_AZURE = Path('C:\\hostedtoolcache').exists()
@@ -109,11 +111,15 @@ def install_pypy(version: str, arch: str, url: str) -> Path:
def setup_python(python_configuration: PythonConfiguration, dependency_constraint_flags: Sequence[Union[str, PathLike]], environment: ParsedEnvironment) -> Dict[str, str]:
nuget = Path('C:\\cibw\\nuget.exe')
if not nuget.exists():
log.step('Downloading nuget...')
download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget)
if python_configuration.identifier.startswith('cp'):
implementation_id = python_configuration.identifier.split("-")[0]
log.step(f'Installing Python {implementation_id}...')
if implementation_id.startswith('cp'):
installation_path = install_cpython(python_configuration.version, python_configuration.arch, nuget)
elif python_configuration.identifier.startswith('pp'):
elif implementation_id.startswith('pp'):
assert python_configuration.url is not None
installation_path = install_pypy(python_configuration.version, python_configuration.arch, python_configuration.url)
else:
@@ -121,6 +127,8 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain
assert (installation_path / 'python.exe').exists()
log.step('Setting up build environment...')
# set up PATH and environment variables for run_with_env
env = os.environ.copy()
env['PYTHON_VERSION'] = python_configuration.version
@@ -151,7 +159,8 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain
print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr)
exit(1)
# prepare the Python environment
log.step('Installing build tools...')
call(['python', '-m', 'pip', 'install', '--upgrade', 'pip', *dependency_constraint_flags], env=env)
call(['pip', '--version'], env=env)
call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', *dependency_constraint_flags], env=env)
@@ -182,6 +191,7 @@ def pep_518_cp35_workaround(package_dir: Path, env: Dict[str, str]) -> None:
)
if requirements:
log.step('Performing PEP518 workaround...')
with tempfile.TemporaryDirectory() as d:
reqfile = Path(d) / "requirements.txt"
with reqfile.open("w") as f:
@@ -195,114 +205,122 @@ def build(options: BuildOptions) -> None:
built_wheel_dir = temp_dir / 'built_wheel'
repaired_wheel_dir = temp_dir / 'repaired_wheel'
# install nuget as best way to provide python
nuget = Path('C:\\cibw\\nuget.exe')
download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget)
try:
if options.before_all:
log.step('Running before_all...')
env = options.environment.as_dictionary(prev_environment=os.environ)
before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir)
shell(before_all_prepared, env=env)
if options.before_all:
env = options.environment.as_dictionary(prev_environment=os.environ)
before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir)
shell(before_all_prepared, env=env)
python_configurations = get_python_configurations(options.build_selector)
python_configurations = get_python_configurations(options.build_selector)
for config in python_configurations:
dependency_constraint_flags: Sequence[Union[str, PathLike]] = []
if options.dependency_constraints:
dependency_constraint_flags = [
'-c', options.dependency_constraints.get_for_python_version(config.version)
]
for config in python_configurations:
log.build_start(config.identifier)
# install Python
env = setup_python(config, dependency_constraint_flags, options.environment)
dependency_constraint_flags: Sequence[Union[str, PathLike]] = []
if options.dependency_constraints:
dependency_constraint_flags = [
'-c', options.dependency_constraints.get_for_python_version(config.version)
]
# run the before_build command
if options.before_build:
before_build_prepared = prepare_command(options.before_build, project='.', package=options.package_dir)
shell(before_build_prepared, env=env)
# install Python
env = setup_python(config, dependency_constraint_flags, options.environment)
# activate the PEP 518 patch if on Windows Python 3.5
# (will only have an effect if PEP 517 builds are used):
if config.version.startswith('3.5'):
pep_518_cp35_workaround(options.package_dir, env)
# run the before_build command
if options.before_build:
log.step('Running before_build...')
before_build_prepared = prepare_command(options.before_build, project='.', package=options.package_dir)
shell(before_build_prepared, env=env)
# build the wheel
if built_wheel_dir.exists():
shutil.rmtree(built_wheel_dir)
built_wheel_dir.mkdir(parents=True)
# Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
# see https://github.com/joerick/cibuildwheel/pull/369
call([
'pip', 'wheel',
options.package_dir.resolve(),
'-w', built_wheel_dir,
'--no-deps',
*get_build_verbosity_extra_flags(options.build_verbosity)
], env=env)
# activate the PEP 518 patch if on Windows Python 3.5
# (will only have an effect if PEP 517 builds are used):
if config.version.startswith('3.5'):
pep_518_cp35_workaround(options.package_dir, env)
built_wheel = next(built_wheel_dir.glob('*.whl'))
log.step('Building wheel...')
if built_wheel_dir.exists():
shutil.rmtree(built_wheel_dir)
built_wheel_dir.mkdir(parents=True)
# Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
# see https://github.com/joerick/cibuildwheel/pull/369
call([
'pip', 'wheel',
options.package_dir.resolve(),
'-w', built_wheel_dir,
'--no-deps',
*get_build_verbosity_extra_flags(options.build_verbosity)
], env=env)
# repair the wheel
if repaired_wheel_dir.exists():
shutil.rmtree(repaired_wheel_dir)
repaired_wheel_dir.mkdir(parents=True)
built_wheel = next(built_wheel_dir.glob('*.whl'))
if built_wheel.name.endswith('none-any.whl'):
raise NonPlatformWheelError()
# repair the wheel
if repaired_wheel_dir.exists():
shutil.rmtree(repaired_wheel_dir)
repaired_wheel_dir.mkdir(parents=True)
if options.repair_command:
repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir)
shell(repair_command_prepared, env=env)
else:
shutil.move(str(built_wheel), repaired_wheel_dir)
if built_wheel.name.endswith('none-any.whl'):
raise NonPlatformWheelError()
repaired_wheel = next(repaired_wheel_dir.glob('*.whl'))
if options.repair_command:
log.step('Repairing wheel...')
repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir)
shell(repair_command_prepared, env=env)
else:
shutil.move(str(built_wheel), repaired_wheel_dir)
if options.test_command:
# set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env)
venv_dir = Path(tempfile.mkdtemp())
repaired_wheel = next(repaired_wheel_dir.glob('*.whl'))
# Use --no-download to ensure determinism by using seed libraries
# built into virtualenv
call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env)
if options.test_command:
log.step('Testing wheel...')
# set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env)
venv_dir = Path(tempfile.mkdtemp())
virtualenv_env = env.copy()
virtualenv_env['PATH'] = os.pathsep.join([
str(venv_dir / 'Scripts'),
virtualenv_env['PATH'],
])
# Use --no-download to ensure determinism by using seed libraries
# built into virtualenv
call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env)
# check that we are using the Python from the virtual environment
call(['which', 'python'], env=virtualenv_env)
virtualenv_env = env.copy()
virtualenv_env['PATH'] = os.pathsep.join([
str(venv_dir / 'Scripts'),
virtualenv_env['PATH'],
])
if options.before_test:
before_test_prepared = prepare_command(
options.before_test,
project='.',
package=options.package_dir
# check that we are using the Python from the virtual environment
call(['which', 'python'], env=virtualenv_env)
if options.before_test:
before_test_prepared = prepare_command(
options.before_test,
project='.',
package=options.package_dir
)
shell(before_test_prepared, env=virtualenv_env)
# install the wheel
call(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env)
# test the wheel
if options.test_requires:
call(['pip', 'install'] + options.test_requires, env=virtualenv_env)
# run the tests from c:\, with an absolute path in the command
# (this ensures that Python runs the tests against the installed wheel
# and not the repo code)
test_command_prepared = prepare_command(
options.test_command,
project=Path('.').resolve(),
package=options.package_dir.resolve()
)
shell(before_test_prepared, env=virtualenv_env)
shell(test_command_prepared, cwd='c:\\', env=virtualenv_env)
# install the wheel
call(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env)
# clean up
shutil.rmtree(venv_dir)
# test the wheel
if options.test_requires:
call(['pip', 'install'] + options.test_requires, env=virtualenv_env)
# run the tests from c:\, with an absolute path in the command
# (this ensures that Python runs the tests against the installed wheel
# and not the repo code)
test_command_prepared = prepare_command(
options.test_command,
project=Path('.').resolve(),
package=options.package_dir.resolve()
)
shell(test_command_prepared, cwd='c:\\', env=virtualenv_env)
# clean up
shutil.rmtree(venv_dir)
# we're all done here; move it to output (remove if already exists)
shutil.move(str(repaired_wheel), options.output_dir)
# we're all done here; move it to output (remove if already exists)
shutil.move(str(repaired_wheel), options.output_dir)
log.build_end()
except subprocess.CalledProcessError as error:
log.error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}')
exit(1)