Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #835 from henryiii/henryiii/refactor/prepover

Browse Source 
refactor: better code factorization
This commit is contained in:
Matthieu Darbois
2021-09-25 11:24:28 +02:00
committed by GitHub
parent 7833e45f5a 62b72198b4
commit 0e6334e135
5 changed files with 514 additions and 441 deletions
Download Patch File Download Diff File Expand all files Collapse all files
bin/run_tests.py
+2 -3
View File
@@ -16,10 +16,9 @@ if __name__ == "__main__":
unit_test_args += ["--run-docker"]
subprocess.run(unit_test_args, check=True)
xdist_test_args = ["-n", "2"] if sys.platform.startswith("linux") else []
# run the integration tests
xdist_test_args = []
if sys.platform.startswith("linux"):
xdist_test_args = ["-n", "2"]
subprocess.run(
[
sys.executable,
cibuildwheel/__main__.py
+14 -205
View File
@@ -2,12 +2,8 @@ import argparse
import os
import sys
import textwrap
import traceback
from configparser import ConfigParser
from pathlib import Path
from typing import Dict, List, Optional, Set, Union
from packaging.specifiers import SpecifierSet
from typing import List, Set, Union
import cibuildwheel
import cibuildwheel.linux
@@ -15,38 +11,13 @@ import cibuildwheel.macos
import cibuildwheel.util
import cibuildwheel.windows
from cibuildwheel.architecture import Architecture, allowed_architectures_check
from cibuildwheel.environment import EnvironmentParseError, parse_environment
from cibuildwheel.options import ConfigOptions
from cibuildwheel.projectfiles import get_requires_python_str
from cibuildwheel.options import compute_options
from cibuildwheel.typing import PLATFORMS, PlatformName, assert_never
from cibuildwheel.util import (
BuildFrontend,
BuildOptions,
BuildSelector,
DependencyConstraints,
TestSelector,
Unbuffered,
detect_ci_provider,
resources_dir,
)
MANYLINUX_ARCHS = (
"x86_64",
"i686",
"pypy_x86_64",
"aarch64",
"ppc64le",
"s390x",
"pypy_aarch64",
"pypy_i686",
)
MUSLLINUX_ARCHS = (
"x86_64",
"i686",
"aarch64",
"ppc64le",
"s390x",
)
@@ -171,184 +142,28 @@ def main() -> None:
sys.exit(2)
package_dir = Path(args.package_dir)
manylinux_identifiers = {
f"manylinux-{build_platform}-image" for build_platform in MANYLINUX_ARCHS
}
musllinux_identifiers = {
f"musllinux-{build_platform}-image" for build_platform in MUSLLINUX_ARCHS
}
disallow = {
"linux": {"dependency-versions"},
"macos": manylinux_identifiers | musllinux_identifiers,
"windows": manylinux_identifiers | musllinux_identifiers,
}
options = ConfigOptions(package_dir, args.config_file, platform=platform, disallow=disallow)
output_dir = Path(
args.output_dir
if args.output_dir is not None
else os.environ.get("CIBW_OUTPUT_DIR", "wheelhouse")
)
build_config = options("build", env_plat=False, sep=" ") or "*"
skip_config = options("skip", env_plat=False, sep=" ")
test_skip = options("test-skip", env_plat=False, sep=" ")
archs_config_str = args.archs or options("archs", sep=" ")
build_frontend_str = options("build-frontend", env_plat=False)
environment_config = options("environment", table={"item": '{k}="{v}"', "sep": " "})
before_all = options("before-all", sep=" && ")
before_build = options("before-build", sep=" && ")
repair_command = options("repair-wheel-command", sep=" && ")
dependency_versions = options("dependency-versions")
test_command = options("test-command", sep=" && ")
before_test = options("before-test", sep=" && ")
test_requires = options("test-requires", sep=" ").split()
test_extras = options("test-extras", sep=",")
build_verbosity_str = options("build-verbosity")
prerelease_pythons = args.prerelease_pythons or cibuildwheel.util.strtobool(
os.environ.get("CIBW_PRERELEASE_PYTHONS", "0")
build_options = compute_options(
platform, package_dir, output_dir, args.config_file, args.archs, args.prerelease_pythons
)
build_frontend: BuildFrontend
if build_frontend_str == "build":
build_frontend = "build"
elif build_frontend_str == "pip":
build_frontend = "pip"
else:
msg = f"cibuildwheel: Unrecognised build frontend '{build_frontend}', only 'pip' and 'build' are supported"
print(msg, file=sys.stderr)
sys.exit(2)
package_files = {"setup.py", "setup.cfg", "pyproject.toml"}
if not any(package_dir.joinpath(name).exists() for name in package_files):
names = ", ".join(sorted(package_files, reverse=True))
msg = f"cibuildwheel: Could not find any of {{{names}}} at root of package"
print(msg, file=sys.stderr)
sys.exit(2)
# This is not supported in tool.cibuildwheel, as it comes from a standard location.
# Passing this in as an environment variable will override pyproject.toml, setup.cfg, or setup.py
requires_python_str: Optional[str] = os.environ.get(
"CIBW_PROJECT_REQUIRES_PYTHON"
) or get_requires_python_str(package_dir)
requires_python = None if requires_python_str is None else SpecifierSet(requires_python_str)
deprecated_selectors("CIBW_BUILD", build_config, error=True)
deprecated_selectors("CIBW_SKIP", skip_config)
deprecated_selectors("CIBW_TEST_SKIP", test_skip)
build_selector = BuildSelector(
build_config=build_config,
skip_config=skip_config,
requires_python=requires_python,
prerelease_pythons=prerelease_pythons,
identifiers = get_build_identifiers(
platform, build_options.build_selector, build_options.architectures
)
test_selector = TestSelector(skip_config=test_skip)
try:
environment = parse_environment(environment_config)
except (EnvironmentParseError, ValueError):
print(f'cibuildwheel: Malformed environment option "{environment_config}"', file=sys.stderr)
traceback.print_exc(None, sys.stderr)
sys.exit(2)
if dependency_versions == "pinned":
dependency_constraints: Optional[
DependencyConstraints
] = DependencyConstraints.with_defaults()
elif dependency_versions == "latest":
dependency_constraints = None
else:
dependency_versions_path = Path(dependency_versions)
dependency_constraints = DependencyConstraints(dependency_versions_path)
if test_extras:
test_extras = f"[{test_extras}]"
try:
build_verbosity = min(3, max(-3, int(build_verbosity_str)))
except ValueError:
build_verbosity = 0
# Add CIBUILDWHEEL environment variable
# This needs to be passed on to the docker container in linux.py
os.environ["CIBUILDWHEEL"] = "1"
archs = Architecture.parse_config(archs_config_str, platform=platform)
identifiers = get_build_identifiers(platform, build_selector, archs)
if args.print_build_identifiers:
for identifier in identifiers:
print(identifier)
sys.exit(0)
manylinux_images: Dict[str, str] = {}
musllinux_images: Dict[str, str] = {}
if platform == "linux":
pinned_docker_images_file = resources_dir / "pinned_docker_images.cfg"
all_pinned_docker_images = ConfigParser()
all_pinned_docker_images.read(pinned_docker_images_file)
# all_pinned_docker_images looks like a dict of dicts, e.g.
# { 'x86_64': {'manylinux1': '...', 'manylinux2010': '...', 'manylinux2014': '...'},
# 'i686': {'manylinux1': '...', 'manylinux2010': '...', 'manylinux2014': '...'},
# 'pypy_x86_64': {'manylinux2010': '...' }
# ... }
for build_platform in MANYLINUX_ARCHS:
pinned_images = all_pinned_docker_images[build_platform]
config_value = options(f"manylinux-{build_platform}-image", ignore_empty=True)
if not config_value:
# default to manylinux2010 if it's available, otherwise manylinux2014
image = pinned_images.get("manylinux2010") or pinned_images.get("manylinux2014")
elif config_value in pinned_images:
image = pinned_images[config_value]
else:
image = config_value
manylinux_images[build_platform] = image
for build_platform in MUSLLINUX_ARCHS:
pinned_images = all_pinned_docker_images[build_platform]
config_value = options(f"musllinux-{build_platform}-image")
if config_value is None:
image = pinned_images.get("musllinux_1_1")
elif config_value in pinned_images:
image = pinned_images[config_value]
else:
image = config_value
musllinux_images[build_platform] = image
build_options = BuildOptions(
architectures=archs,
package_dir=package_dir,
output_dir=output_dir,
test_command=test_command,
test_requires=test_requires,
test_extras=test_extras,
before_test=before_test,
before_build=before_build,
before_all=before_all,
build_verbosity=build_verbosity,
build_selector=build_selector,
test_selector=test_selector,
repair_command=repair_command,
environment=environment,
dependency_constraints=dependency_constraints,
manylinux_images=manylinux_images or None,
musllinux_images=musllinux_images or None,
build_frontend=build_frontend,
)
# Add CIBUILDWHEEL environment variable
# This needs to be passed on to the docker container in linux.py
os.environ["CIBUILDWHEEL"] = "1"
# Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print'
sys.stdout = Unbuffered(sys.stdout) # type: ignore
@@ -362,7 +177,10 @@ def main() -> None:
sys.exit(4)
if not identifiers:
print(f"cibuildwheel: No build identifiers selected: {build_selector}", file=sys.stderr)
print(
f"cibuildwheel: No build identifiers selected: {build_options.build_selector}",
file=sys.stderr,
)
if not args.allow_empty:
sys.exit(3)
@@ -382,14 +200,6 @@ def main() -> None:
assert_never(platform)
def deprecated_selectors(name: str, selector: str, *, error: bool = False) -> None:
if "p2" in selector or "p35" in selector:
msg = f"cibuildwheel 2.x no longer supports Python < 3.6. Please use the 1.x series or update {name}"
print(msg, file=sys.stderr)
if error:
sys.exit(4)
def print_preamble(platform: str, build_options: BuildOptions) -> None:
print(
textwrap.dedent(
@@ -406,8 +216,7 @@ def print_preamble(platform: str, build_options: BuildOptions) -> None:
print("Build options:")
print(f" platform: {platform!r}")
for option, value in sorted(build_options._asdict().items()):
print(f" {option}: {value!r}")
print(textwrap.indent(str(build_options), " "))
warnings = detect_warnings(platform, build_options)
if warnings:
cibuildwheel/linux.py
+254 -231
View File
@@ -2,7 +2,7 @@ import subprocess
import sys
import textwrap
from pathlib import Path, PurePath
from typing import List, NamedTuple, Set
from typing import Iterator, List, NamedTuple, Set
from .architecture import Architecture
from .docker_container import DockerContainer
@@ -28,6 +28,12 @@ class PythonConfiguration(NamedTuple):
return PurePath(self.path_str)
class BuildStep(NamedTuple):
platform_configs: List[PythonConfiguration]
platform_tag: str
docker_image: str
def get_python_configurations(
build_selector: BuildSelector,
architectures: Set[Architecture],
@@ -47,6 +53,241 @@ def get_python_configurations(
]
def get_build_steps(
options: BuildOptions, python_configurations: List[PythonConfiguration]
) -> Iterator[BuildStep]:
platforms = [
("cp", "manylinux_x86_64", "x86_64"),
("cp", "manylinux_i686", "i686"),
("cp", "manylinux_aarch64", "aarch64"),
("cp", "manylinux_ppc64le", "ppc64le"),
("cp", "manylinux_s390x", "s390x"),
("pp", "manylinux_x86_64", "pypy_x86_64"),
("pp", "manylinux_aarch64", "pypy_aarch64"),
("pp", "manylinux_i686", "pypy_i686"),
("cp", "musllinux_x86_64", "x86_64"),
("cp", "musllinux_i686", "i686"),
("cp", "musllinux_aarch64", "aarch64"),
("cp", "musllinux_ppc64le", "ppc64le"),
("cp", "musllinux_s390x", "s390x"),
]
for implementation, platform_tag, platform_arch in platforms:
assert options.manylinux_images is not None
assert options.musllinux_images is not None
docker_image = (
options.manylinux_images[platform_arch]
if platform_tag.startswith("manylinux")
else options.musllinux_images[platform_arch]
)
platform_configs = [
c
for c in python_configurations
if c.identifier.startswith(implementation) and c.identifier.endswith(platform_tag)
]
if not platform_configs:
continue
yield BuildStep(platform_configs, platform_tag, docker_image)
def build_on_docker(
options: BuildOptions,
platform_configs: List[PythonConfiguration],
docker: DockerContainer,
container_project_path: PurePath,
container_package_dir: PurePath,
) -> None:
container_output_dir = PurePath("/output")
log.step("Copying project into Docker...")
docker.copy_into(Path.cwd(), container_project_path)
if options.before_all:
log.step("Running before_all...")
env = docker.get_environment()
env["PATH"] = f'/opt/python/cp38-cp38/bin:{env["PATH"]}'
env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
env = options.environment.as_dictionary(env, executor=docker.environment_executor)
before_all_prepared = prepare_command(
options.before_all,
project=container_project_path,
package=container_package_dir,
)
docker.call(["sh", "-c", before_all_prepared], env=env)
for config in platform_configs:
log.build_start(config.identifier)
dependency_constraint_flags: List[PathOrStr] = []
if options.dependency_constraints:
constraints_file = options.dependency_constraints.get_for_python_version(config.version)
container_constraints_file = PurePath("/constraints.txt")
docker.copy_into(constraints_file, container_constraints_file)
dependency_constraint_flags = ["-c", container_constraints_file]
log.step("Setting up build environment...")
env = docker.get_environment()
# put this config's python top of the list
python_bin = config.path / "bin"
env["PATH"] = f'{python_bin}:{env["PATH"]}'
env = options.environment.as_dictionary(env, executor=docker.environment_executor)
# check config python is still on PATH
which_python = docker.call(["which", "python"], env=env, capture_output=True).strip()
if PurePath(which_python) != python_bin / "python":
print(
"cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.",
file=sys.stderr,
)
sys.exit(1)
which_pip = docker.call(["which", "pip"], env=env, capture_output=True).strip()
if PurePath(which_pip) != python_bin / "pip":
print(
"cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.",
file=sys.stderr,
)
sys.exit(1)
if options.before_build:
log.step("Running before_build...")
before_build_prepared = prepare_command(
options.before_build,
project=container_project_path,
package=container_package_dir,
)
docker.call(["sh", "-c", before_build_prepared], env=env)
log.step("Building wheel...")
temp_dir = PurePath("/tmp/cibuildwheel")
built_wheel_dir = temp_dir / "built_wheel"
docker.call(["rm", "-rf", built_wheel_dir])
docker.call(["mkdir", "-p", built_wheel_dir])
verbosity_flags = get_build_verbosity_extra_flags(options.build_verbosity)
if options.build_frontend == "pip":
docker.call(
[
"python",
"-m",
"pip",
"wheel",
container_package_dir,
f"--wheel-dir={built_wheel_dir}",
"--no-deps",
*verbosity_flags,
],
env=env,
)
elif options.build_frontend == "build":
config_setting = " ".join(verbosity_flags)
docker.call(
[
"python",
"-m",
"build",
container_package_dir,
"--wheel",
f"--outdir={built_wheel_dir}",
f"--config-setting={config_setting}",
],
env=env,
)
else:
assert_never(options.build_frontend)
built_wheel = docker.glob(built_wheel_dir, "*.whl")[0]
repaired_wheel_dir = temp_dir / "repaired_wheel"
docker.call(["rm", "-rf", repaired_wheel_dir])
docker.call(["mkdir", "-p", repaired_wheel_dir])
if built_wheel.name.endswith("none-any.whl"):
raise NonPlatformWheelError()
if options.repair_command:
log.step("Repairing wheel...")
repair_command_prepared = prepare_command(
options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir
)
docker.call(["sh", "-c", repair_command_prepared], env=env)
else:
docker.call(["mv", built_wheel, repaired_wheel_dir])
repaired_wheels = docker.glob(repaired_wheel_dir, "*.whl")
if options.test_command and options.test_selector(config.identifier):
log.step("Testing wheel...")
# set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
docker.call(["pip", "install", "virtualenv", *dependency_constraint_flags], env=env)
venv_dir = PurePath(docker.call(["mktemp", "-d"], capture_output=True).strip()) / "venv"
docker.call(["python", "-m", "virtualenv", "--no-download", venv_dir], env=env)
virtualenv_env = env.copy()
virtualenv_env["PATH"] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}"
if options.before_test:
before_test_prepared = prepare_command(
options.before_test,
project=container_project_path,
package=container_package_dir,
)
docker.call(["sh", "-c", before_test_prepared], env=virtualenv_env)
# Install the wheel we just built
# Note: If auditwheel produced two wheels, it's because the earlier produced wheel
# conforms to multiple manylinux standards. These multiple versions of the wheel are
# functionally the same, differing only in name, wheel metadata, and possibly include
# different external shared libraries. so it doesn't matter which one we run the tests on.
# Let's just pick the first one.
wheel_to_test = repaired_wheels[0]
docker.call(
["pip", "install", str(wheel_to_test) + options.test_extras],
env=virtualenv_env,
)
# Install any requirements to run the tests
if options.test_requires:
docker.call(["pip", "install", *options.test_requires], env=virtualenv_env)
# Run the tests from a different directory
test_command_prepared = prepare_command(
options.test_command,
project=container_project_path,
package=container_package_dir,
)
docker.call(["sh", "-c", test_command_prepared], cwd="/root", env=virtualenv_env)
# clean up test environment
docker.call(["rm", "-rf", venv_dir])
# move repaired wheels to output
docker.call(["mkdir", "-p", container_output_dir])
docker.call(["mv", *repaired_wheels, container_output_dir])
log.build_end()
log.step("Copying wheels back to host...")
# copy the output back into the host
docker.copy_out(container_output_dir, options.output_dir)
log.step_end()
def build(options: BuildOptions) -> None:
try:
# check docker is installed
@@ -63,21 +304,6 @@ def build(options: BuildOptions) -> None:
assert options.manylinux_images is not None
assert options.musllinux_images is not None
python_configurations = get_python_configurations(options.build_selector, options.architectures)
platforms = [
("cp", "manylinux_x86_64", options.manylinux_images["x86_64"]),
("cp", "manylinux_i686", options.manylinux_images["i686"]),
("cp", "manylinux_aarch64", options.manylinux_images["aarch64"]),
("cp", "manylinux_ppc64le", options.manylinux_images["ppc64le"]),
("cp", "manylinux_s390x", options.manylinux_images["s390x"]),
("pp", "manylinux_x86_64", options.manylinux_images["pypy_x86_64"]),
("pp", "manylinux_aarch64", options.manylinux_images["pypy_aarch64"]),
("pp", "manylinux_i686", options.manylinux_images["pypy_i686"]),
("cp", "musllinux_x86_64", options.musllinux_images["x86_64"]),
("cp", "musllinux_i686", options.musllinux_images["i686"]),
("cp", "musllinux_aarch64", options.musllinux_images["aarch64"]),
("cp", "musllinux_ppc64le", options.musllinux_images["ppc64le"]),
("cp", "musllinux_s390x", options.musllinux_images["s390x"]),
]
cwd = Path.cwd()
abs_package_dir = options.package_dir.resolve()
@@ -86,228 +312,25 @@ def build(options: BuildOptions) -> None:
container_project_path = PurePath("/project")
container_package_dir = container_project_path / abs_package_dir.relative_to(cwd)
container_output_dir = PurePath("/output")
for implementation, platform_tag, docker_image in platforms:
platform_configs = [
c
for c in python_configurations
if c.identifier.startswith(implementation) and c.identifier.endswith(platform_tag)
]
if not platform_configs:
continue
for build_step in get_build_steps(options, python_configurations):
try:
log.step(f"Starting Docker image {docker_image}...")
log.step(f"Starting Docker image {build_step.docker_image}...")
with DockerContainer(
docker_image,
simulate_32_bit=platform_tag.endswith("i686"),
build_step.docker_image,
simulate_32_bit=build_step.platform_tag.endswith("i686"),
cwd=container_project_path,
) as docker:
log.step("Copying project into Docker...")
docker.copy_into(Path.cwd(), container_project_path)
build_on_docker(
options,
build_step.platform_configs,
docker,
container_project_path,
container_package_dir,
)
if options.before_all:
log.step("Running before_all...")
env = docker.get_environment()
env["PATH"] = f'/opt/python/cp38-cp38/bin:{env["PATH"]}'
env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
env = options.environment.as_dictionary(
env, executor=docker.environment_executor
)
before_all_prepared = prepare_command(
options.before_all,
project=container_project_path,
package=container_package_dir,
)
docker.call(["sh", "-c", before_all_prepared], env=env)
for config in platform_configs:
log.build_start(config.identifier)
dependency_constraint_flags: List[PathOrStr] = []
if options.dependency_constraints:
constraints_file = options.dependency_constraints.get_for_python_version(
config.version
)
container_constraints_file = PurePath("/constraints.txt")
docker.copy_into(constraints_file, container_constraints_file)
dependency_constraint_flags = ["-c", container_constraints_file]
log.step("Setting up build environment...")
env = docker.get_environment()
# put this config's python top of the list
python_bin = config.path / "bin"
env["PATH"] = f'{python_bin}:{env["PATH"]}'
env = options.environment.as_dictionary(
env, executor=docker.environment_executor
)
# check config python is still on PATH
which_python = docker.call(
["which", "python"], env=env, capture_output=True
).strip()
if PurePath(which_python) != python_bin / "python":
print(
"cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.",
file=sys.stderr,
)
sys.exit(1)
which_pip = docker.call(["which", "pip"], env=env, capture_output=True).strip()
if PurePath(which_pip) != python_bin / "pip":
print(
"cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.",
file=sys.stderr,
)
sys.exit(1)
if options.before_build:
log.step("Running before_build...")
before_build_prepared = prepare_command(
options.before_build,
project=container_project_path,
package=container_package_dir,
)
docker.call(["sh", "-c", before_build_prepared], env=env)
log.step("Building wheel...")
temp_dir = PurePath("/tmp/cibuildwheel")
built_wheel_dir = temp_dir / "built_wheel"
docker.call(["rm", "-rf", built_wheel_dir])
docker.call(["mkdir", "-p", built_wheel_dir])
verbosity_flags = get_build_verbosity_extra_flags(options.build_verbosity)
if options.build_frontend == "pip":
docker.call(
[
"python",
"-m",
"pip",
"wheel",
container_package_dir,
f"--wheel-dir={built_wheel_dir}",
"--no-deps",
*verbosity_flags,
],
env=env,
)
elif options.build_frontend == "build":
config_setting = " ".join(verbosity_flags)
docker.call(
[
"python",
"-m",
"build",
container_package_dir,
"--wheel",
f"--outdir={built_wheel_dir}",
f"--config-setting={config_setting}",
],
env=env,
)
else:
assert_never(options.build_frontend)
built_wheel = docker.glob(built_wheel_dir, "*.whl")[0]
repaired_wheel_dir = temp_dir / "repaired_wheel"
docker.call(["rm", "-rf", repaired_wheel_dir])
docker.call(["mkdir", "-p", repaired_wheel_dir])
if built_wheel.name.endswith("none-any.whl"):
raise NonPlatformWheelError()
if options.repair_command:
log.step("Repairing wheel...")
repair_command_prepared = prepare_command(
options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir
)
docker.call(["sh", "-c", repair_command_prepared], env=env)
else:
docker.call(["mv", built_wheel, repaired_wheel_dir])
repaired_wheels = docker.glob(repaired_wheel_dir, "*.whl")
if options.test_command and options.test_selector(config.identifier):
log.step("Testing wheel...")
# set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
docker.call(
["pip", "install", "virtualenv", *dependency_constraint_flags], env=env
)
venv_dir = (
PurePath(docker.call(["mktemp", "-d"], capture_output=True).strip())
/ "venv"
)
docker.call(
["python", "-m", "virtualenv", "--no-download", venv_dir], env=env
)
virtualenv_env = env.copy()
virtualenv_env["PATH"] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}"
if options.before_test:
before_test_prepared = prepare_command(
options.before_test,
project=container_project_path,
package=container_package_dir,
)
docker.call(["sh", "-c", before_test_prepared], env=virtualenv_env)
# Install the wheel we just built
# Note: If auditwheel produced two wheels, it's because the earlier produced wheel
# conforms to multiple manylinux standards. These multiple versions of the wheel are
# functionally the same, differing only in name, wheel metadata, and possibly include
# different external shared libraries. so it doesn't matter which one we run the tests on.
# Let's just pick the first one.
wheel_to_test = repaired_wheels[0]
docker.call(
["pip", "install", str(wheel_to_test) + options.test_extras],
env=virtualenv_env,
)
# Install any requirements to run the tests
if options.test_requires:
docker.call(
["pip", "install", *options.test_requires], env=virtualenv_env
)
# Run the tests from a different directory
test_command_prepared = prepare_command(
options.test_command,
project=container_project_path,
package=container_package_dir,
)
docker.call(
["sh", "-c", test_command_prepared], cwd="/root", env=virtualenv_env
)
# clean up test environment
docker.call(["rm", "-rf", venv_dir])
# move repaired wheels to output
docker.call(["mkdir", "-p", container_output_dir])
docker.call(["mv", *repaired_wheels, container_output_dir])
log.build_end()
log.step("Copying wheels back to host...")
# copy the output back into the host
docker.copy_out(container_output_dir, options.output_dir)
log.step_end()
except subprocess.CalledProcessError as error:
log.step_end_with_error(
f"Command {error.cmd} failed with code {error.returncode}. {error.stdout}"
cibuildwheel/options.py
+221 -2
View File
@@ -1,11 +1,28 @@
import os
import sys
import traceback
from configparser import ConfigParser
from pathlib import Path
from typing import Any, Dict, List, Mapping, Optional, Set, Tuple, Union
import toml
from packaging.specifiers import SpecifierSet
from .typing import PLATFORMS, TypedDict
from .util import resources_dir
from .architecture import Architecture
from .environment import EnvironmentParseError, parse_environment
from .projectfiles import get_requires_python_str
from .typing import PLATFORMS, PlatformName, TypedDict
from .util import (
MANYLINUX_ARCHS,
MUSLLINUX_ARCHS,
BuildFrontend,
BuildOptions,
BuildSelector,
DependencyConstraints,
TestSelector,
resources_dir,
strtobool,
)
Setting = Union[Dict[str, str], List[str], str]
@@ -183,3 +200,205 @@ class ConfigOptions:
return str(result)
else:
return result
def compute_options(
platform: PlatformName,
package_dir: Path,
output_dir: Path,
config_file: Optional[str],
args_archs: Optional[str],
prerelease_pythons: bool,
) -> BuildOptions:
"""
Compute the options from the environment and configuration file.
"""
manylinux_identifiers = {
f"manylinux-{build_platform}-image" for build_platform in MANYLINUX_ARCHS
}
musllinux_identifiers = {
f"musllinux-{build_platform}-image" for build_platform in MUSLLINUX_ARCHS
}
disallow = {
"linux": {"dependency-versions"},
"macos": manylinux_identifiers | musllinux_identifiers,
"windows": manylinux_identifiers | musllinux_identifiers,
}
options = ConfigOptions(package_dir, config_file, platform=platform, disallow=disallow)
build_config = options("build", env_plat=False, sep=" ") or "*"
skip_config = options("skip", env_plat=False, sep=" ")
test_skip = options("test-skip", env_plat=False, sep=" ")
prerelease_pythons = prerelease_pythons or strtobool(
os.environ.get("CIBW_PRERELEASE_PYTHONS", "0")
)
deprecated_selectors("CIBW_BUILD", build_config, error=True)
deprecated_selectors("CIBW_SKIP", skip_config)
deprecated_selectors("CIBW_TEST_SKIP", test_skip)
package_files = {"setup.py", "setup.cfg", "pyproject.toml"}
if not any(package_dir.joinpath(name).exists() for name in package_files):
names = ", ".join(sorted(package_files, reverse=True))
msg = f"cibuildwheel: Could not find any of {{{names}}} at root of package"
print(msg, file=sys.stderr)
sys.exit(2)
# This is not supported in tool.cibuildwheel, as it comes from a standard location.
# Passing this in as an environment variable will override pyproject.toml, setup.cfg, or setup.py
requires_python_str: Optional[str] = os.environ.get(
"CIBW_PROJECT_REQUIRES_PYTHON"
) or get_requires_python_str(package_dir)
requires_python = None if requires_python_str is None else SpecifierSet(requires_python_str)
build_selector = BuildSelector(
build_config=build_config,
skip_config=skip_config,
requires_python=requires_python,
prerelease_pythons=prerelease_pythons,
)
test_selector = TestSelector(skip_config=test_skip)
return _compute_single_options(
options, args_archs, build_selector, test_selector, platform, package_dir, output_dir
)
def _compute_single_options(
options: ConfigOptions,
args_archs: Optional[str],
build_selector: BuildSelector,
test_selector: TestSelector,
platform: PlatformName,
package_dir: Path,
output_dir: Path,
) -> BuildOptions:
"""
Compute BuildOptions for a single run configuration.
"""
# Can't be configured per selector
before_all = options("before-all", sep=" && ")
archs_config_str = args_archs or options("archs", sep=" ")
build_frontend_str = options("build-frontend", env_plat=False)
environment_config = options("environment", table={"item": '{k}="{v}"', "sep": " "})
before_build = options("before-build", sep=" && ")
repair_command = options("repair-wheel-command", sep=" && ")
dependency_versions = options("dependency-versions")
test_command = options("test-command", sep=" && ")
before_test = options("before-test", sep=" && ")
test_requires = options("test-requires", sep=" ").split()
test_extras = options("test-extras", sep=",")
build_verbosity_str = options("build-verbosity")
build_frontend: BuildFrontend
if build_frontend_str == "build":
build_frontend = "build"
elif build_frontend_str == "pip":
build_frontend = "pip"
else:
msg = f"cibuildwheel: Unrecognised build frontend '{build_frontend}', only 'pip' and 'build' are supported"
print(msg, file=sys.stderr)
sys.exit(2)
try:
environment = parse_environment(environment_config)
except (EnvironmentParseError, ValueError):
print(f'cibuildwheel: Malformed environment option "{environment_config}"', file=sys.stderr)
traceback.print_exc(None, sys.stderr)
sys.exit(2)
if dependency_versions == "pinned":
dependency_constraints: Optional[
DependencyConstraints
] = DependencyConstraints.with_defaults()
elif dependency_versions == "latest":
dependency_constraints = None
else:
dependency_versions_path = Path(dependency_versions)
dependency_constraints = DependencyConstraints(dependency_versions_path)
if test_extras:
test_extras = f"[{test_extras}]"
try:
build_verbosity = min(3, max(-3, int(build_verbosity_str)))
except ValueError:
build_verbosity = 0
archs = Architecture.parse_config(archs_config_str, platform=platform)
manylinux_images: Dict[str, str] = {}
musllinux_images: Dict[str, str] = {}
if platform == "linux":
pinned_docker_images_file = resources_dir / "pinned_docker_images.cfg"
all_pinned_docker_images = ConfigParser()
all_pinned_docker_images.read(pinned_docker_images_file)
# all_pinned_docker_images looks like a dict of dicts, e.g.
# { 'x86_64': {'manylinux1': '...', 'manylinux2010': '...', 'manylinux2014': '...'},
# 'i686': {'manylinux1': '...', 'manylinux2010': '...', 'manylinux2014': '...'},
# 'pypy_x86_64': {'manylinux2010': '...' }
# ... }
for build_platform in MANYLINUX_ARCHS:
pinned_images = all_pinned_docker_images[build_platform]
config_value = options(f"manylinux-{build_platform}-image", ignore_empty=True)
if not config_value:
# default to manylinux2010 if it's available, otherwise manylinux2014
image = pinned_images.get("manylinux2010") or pinned_images.get("manylinux2014")
elif config_value in pinned_images:
image = pinned_images[config_value]
else:
image = config_value
manylinux_images[build_platform] = image
for build_platform in MUSLLINUX_ARCHS:
pinned_images = all_pinned_docker_images[build_platform]
config_value = options(f"musllinux-{build_platform}-image")
if config_value is None:
image = pinned_images.get("musllinux_1_1")
elif config_value in pinned_images:
image = pinned_images[config_value]
else:
image = config_value
musllinux_images[build_platform] = image
return BuildOptions(
architectures=archs,
package_dir=package_dir,
output_dir=output_dir,
test_command=test_command,
test_requires=test_requires,
test_extras=test_extras,
before_test=before_test,
before_build=before_build,
before_all=before_all,
build_verbosity=build_verbosity,
build_selector=build_selector,
test_selector=test_selector,
repair_command=repair_command,
environment=environment,
dependency_constraints=dependency_constraints,
manylinux_images=manylinux_images or None,
musllinux_images=musllinux_images or None,
build_frontend=build_frontend,
)
def deprecated_selectors(name: str, selector: str, *, error: bool = False) -> None:
if "p2" in selector or "p35" in selector:
msg = f"cibuildwheel 2.x no longer supports Python < 3.6. Please use the 1.x series or update {name}"
print(msg, file=sys.stderr)
if error:
sys.exit(4)
cibuildwheel/util.py
+23
View File
@@ -30,6 +30,25 @@ install_certifi_script = resources_dir / "install_certifi.py"
BuildFrontend = Literal["pip", "build"]
MANYLINUX_ARCHS = (
"x86_64",
"i686",
"pypy_x86_64",
"aarch64",
"ppc64le",
"s390x",
"pypy_aarch64",
"pypy_i686",
)
MUSLLINUX_ARCHS = (
"x86_64",
"i686",
"aarch64",
"ppc64le",
"s390x",
)
def prepare_command(command: str, **kwargs: PathOrStr) -> str:
"""
@@ -225,6 +244,10 @@ class BuildOptions(NamedTuple):
build_verbosity: int
build_frontend: BuildFrontend
def __str__(self) -> str:
res = (f"{option}: {value!r}" for option, value in sorted(self._asdict().items()))
return "\n".join(res)
class NonPlatformWheelError(Exception):
def __init__(self) -> None: