Henry SchreinerandJoe Rickerby
6ab9906ff0
* chore: rename joerick/cibuildwheel -> pypa/cibuildwheel * Update Appveyor status badge markup Note that it still has joerick/cibuildwheel - confusingly it's still within my appveyor account, but this is the new project that points to the pypa github repo * Update Appveyor badge markup again I went through a few passes to enable that project, here's the latest. * Update Azure badge, also * Update Travis badge to reference the .com Co-authored-by: Joe Rickerby <joerick@mac.com>
461 lines
16 KiB
Python
461 lines
16 KiB
Python
import argparse
|
|
import os
|
|
import sys
|
|
import textwrap
|
|
import traceback
|
|
from configparser import ConfigParser
|
|
from pathlib import Path
|
|
from typing import Dict, List, Optional, Set, Union, overload
|
|
|
|
from packaging.specifiers import SpecifierSet
|
|
|
|
import cibuildwheel
|
|
import cibuildwheel.linux
|
|
import cibuildwheel.macos
|
|
import cibuildwheel.util
|
|
import cibuildwheel.windows
|
|
from cibuildwheel.architecture import Architecture, allowed_architectures_check
|
|
from cibuildwheel.environment import EnvironmentParseError, parse_environment
|
|
from cibuildwheel.projectfiles import get_requires_python_str
|
|
from cibuildwheel.typing import PLATFORMS, PlatformName, assert_never
|
|
from cibuildwheel.util import (
|
|
BuildOptions,
|
|
BuildSelector,
|
|
DependencyConstraints,
|
|
TestSelector,
|
|
Unbuffered,
|
|
detect_ci_provider,
|
|
resources_dir,
|
|
)
|
|
|
|
|
|
@overload
|
|
def get_option_from_environment(
|
|
option_name: str, *, platform: Optional[str] = None, default: str
|
|
) -> str:
|
|
... # noqa: E704
|
|
|
|
|
|
@overload
|
|
def get_option_from_environment(
|
|
option_name: str, *, platform: Optional[str] = None, default: None = None
|
|
) -> Optional[str]:
|
|
... # noqa: E704 E302
|
|
|
|
|
|
def get_option_from_environment(
|
|
option_name: str, *, platform: Optional[str] = None, default: Optional[str] = None
|
|
) -> Optional[str]: # noqa: E302
|
|
"""
|
|
Returns an option from the environment, optionally scoped by the platform.
|
|
|
|
Example:
|
|
get_option_from_environment('CIBW_COLOR', platform='macos')
|
|
|
|
This will return the value of CIBW_COLOR_MACOS if it exists, otherwise the value of
|
|
CIBW_COLOR.
|
|
"""
|
|
if platform:
|
|
option = os.environ.get(f"{option_name}_{platform.upper()}")
|
|
if option is not None:
|
|
return option
|
|
|
|
return os.environ.get(option_name, default)
|
|
|
|
|
|
def main() -> None:
|
|
platform: PlatformName
|
|
|
|
parser = argparse.ArgumentParser(
|
|
description="Build wheels for all the platforms.",
|
|
epilog="""
|
|
Most options are supplied via environment variables.
|
|
See https://github.com/pypa/cibuildwheel#options for info.
|
|
""",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--platform",
|
|
choices=["auto", "linux", "macos", "windows"],
|
|
default=os.environ.get("CIBW_PLATFORM", "auto"),
|
|
help="""
|
|
Platform to build for. For "linux" you need docker running, on Mac
|
|
or Linux. For "macos", you need a Mac machine, and note that this
|
|
script is going to automatically install MacPython on your system,
|
|
so don't run on your development machine. For "windows", you need to
|
|
run in Windows, and it will build and test for all versions of
|
|
Python. Default: auto.
|
|
""",
|
|
)
|
|
|
|
arch_list_str = ", ".join(a.name for a in Architecture)
|
|
parser.add_argument(
|
|
"--archs",
|
|
default=None,
|
|
help=f"""
|
|
Comma-separated list of CPU architectures to build for.
|
|
When set to 'auto', builds the architectures natively supported
|
|
on this machine. Set this option to build an architecture
|
|
via emulation, for example, using binfmt_misc and QEMU.
|
|
Default: auto.
|
|
Choices: auto, auto64, auto32, native, all, {arch_list_str}
|
|
""",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--output-dir",
|
|
default=os.environ.get("CIBW_OUTPUT_DIR", "wheelhouse"),
|
|
help="Destination folder for the wheels.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"package_dir",
|
|
default=".",
|
|
nargs="?",
|
|
help="""
|
|
Path to the package that you want wheels for. Must be a subdirectory of
|
|
the working directory. When set, the working directory is still
|
|
considered the 'project' and is copied into the Docker container on
|
|
Linux. Default: the working directory.
|
|
""",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--print-build-identifiers",
|
|
action="store_true",
|
|
help="Print the build identifiers matched by the current invocation and exit.",
|
|
)
|
|
|
|
parser.add_argument(
|
|
"--allow-empty",
|
|
action="store_true",
|
|
help="Do not report an error code if the build does not match any wheels.",
|
|
)
|
|
|
|
args = parser.parse_args()
|
|
|
|
detect_obsolete_options()
|
|
|
|
if args.platform != "auto":
|
|
platform = args.platform
|
|
else:
|
|
ci_provider = detect_ci_provider()
|
|
if ci_provider is None:
|
|
print(
|
|
textwrap.dedent(
|
|
"""
|
|
cibuildwheel: Unable to detect platform. cibuildwheel should run on your CI server;
|
|
Travis CI, AppVeyor, Azure Pipelines, GitHub Actions, CircleCI, and Gitlab are
|
|
supported. You can run on your development machine or other CI providers using the
|
|
--platform argument. Check --help output for more information.
|
|
"""
|
|
),
|
|
file=sys.stderr,
|
|
)
|
|
sys.exit(2)
|
|
if sys.platform.startswith("linux"):
|
|
platform = "linux"
|
|
elif sys.platform == "darwin":
|
|
platform = "macos"
|
|
elif sys.platform == "win32":
|
|
platform = "windows"
|
|
else:
|
|
print(
|
|
'cibuildwheel: Unable to detect platform from "sys.platform" in a CI environment. You can run '
|
|
"cibuildwheel using the --platform argument. Check --help output for more information.",
|
|
file=sys.stderr,
|
|
)
|
|
sys.exit(2)
|
|
|
|
if platform not in PLATFORMS:
|
|
print(f"cibuildwheel: Unsupported platform: {platform}", file=sys.stderr)
|
|
sys.exit(2)
|
|
|
|
package_dir = Path(args.package_dir)
|
|
output_dir = Path(args.output_dir)
|
|
|
|
if platform == "linux":
|
|
repair_command_default = "auditwheel repair -w {dest_dir} {wheel}"
|
|
elif platform == "macos":
|
|
repair_command_default = "delocate-listdeps {wheel} && delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}"
|
|
elif platform == "windows":
|
|
repair_command_default = ""
|
|
else:
|
|
assert_never(platform)
|
|
|
|
build_config = os.environ.get("CIBW_BUILD") or "*"
|
|
skip_config = os.environ.get("CIBW_SKIP", "")
|
|
test_skip = os.environ.get("CIBW_TEST_SKIP", "")
|
|
environment_config = get_option_from_environment(
|
|
"CIBW_ENVIRONMENT", platform=platform, default=""
|
|
)
|
|
before_all = get_option_from_environment("CIBW_BEFORE_ALL", platform=platform, default="")
|
|
before_build = get_option_from_environment("CIBW_BEFORE_BUILD", platform=platform)
|
|
repair_command = get_option_from_environment(
|
|
"CIBW_REPAIR_WHEEL_COMMAND", platform=platform, default=repair_command_default
|
|
)
|
|
dependency_versions = get_option_from_environment(
|
|
"CIBW_DEPENDENCY_VERSIONS", platform=platform, default="pinned"
|
|
)
|
|
test_command = get_option_from_environment("CIBW_TEST_COMMAND", platform=platform)
|
|
before_test = get_option_from_environment("CIBW_BEFORE_TEST", platform=platform)
|
|
test_requires = get_option_from_environment(
|
|
"CIBW_TEST_REQUIRES", platform=platform, default=""
|
|
).split()
|
|
test_extras = get_option_from_environment("CIBW_TEST_EXTRAS", platform=platform, default="")
|
|
build_verbosity_str = get_option_from_environment(
|
|
"CIBW_BUILD_VERBOSITY", platform=platform, default=""
|
|
)
|
|
|
|
package_files = {"setup.py", "setup.cfg", "pyproject.toml"}
|
|
|
|
if not any(package_dir.joinpath(name).exists() for name in package_files):
|
|
names = ", ".join(sorted(package_files, reverse=True))
|
|
print(
|
|
f"cibuildwheel: Could not find any of {{{names}}} at root of package", file=sys.stderr
|
|
)
|
|
sys.exit(2)
|
|
|
|
# Passing this in as an environment variable will override pyproject.toml, setup.cfg, or setup.py
|
|
requires_python_str: Optional[str] = os.environ.get(
|
|
"CIBW_PROJECT_REQUIRES_PYTHON"
|
|
) or get_requires_python_str(package_dir)
|
|
requires_python = None if requires_python_str is None else SpecifierSet(requires_python_str)
|
|
|
|
build_selector = BuildSelector(
|
|
build_config=build_config, skip_config=skip_config, requires_python=requires_python
|
|
)
|
|
test_selector = TestSelector(skip_config=test_skip)
|
|
|
|
try:
|
|
environment = parse_environment(environment_config)
|
|
except (EnvironmentParseError, ValueError):
|
|
print(f'cibuildwheel: Malformed environment option "{environment_config}"', file=sys.stderr)
|
|
traceback.print_exc(None, sys.stderr)
|
|
sys.exit(2)
|
|
|
|
if dependency_versions == "pinned":
|
|
dependency_constraints: Optional[
|
|
DependencyConstraints
|
|
] = DependencyConstraints.with_defaults()
|
|
elif dependency_versions == "latest":
|
|
dependency_constraints = None
|
|
else:
|
|
dependency_versions_path = Path(dependency_versions)
|
|
dependency_constraints = DependencyConstraints(dependency_versions_path)
|
|
|
|
if test_extras:
|
|
test_extras = f"[{test_extras}]"
|
|
|
|
try:
|
|
build_verbosity = min(3, max(-3, int(build_verbosity_str)))
|
|
except ValueError:
|
|
build_verbosity = 0
|
|
|
|
# Add CIBUILDWHEEL environment variable
|
|
# This needs to be passed on to the docker container in linux.py
|
|
os.environ["CIBUILDWHEEL"] = "1"
|
|
|
|
if args.archs is not None:
|
|
archs_config_str = args.archs
|
|
else:
|
|
archs_config_str = get_option_from_environment(
|
|
"CIBW_ARCHS", platform=platform, default="auto"
|
|
)
|
|
|
|
archs = Architecture.parse_config(archs_config_str, platform=platform)
|
|
|
|
identifiers = get_build_identifiers(platform, build_selector, archs)
|
|
|
|
if args.print_build_identifiers:
|
|
for identifier in identifiers:
|
|
print(identifier)
|
|
sys.exit(0)
|
|
|
|
manylinux_images: Optional[Dict[str, str]] = None
|
|
if platform == "linux":
|
|
pinned_docker_images_file = resources_dir / "pinned_docker_images.cfg"
|
|
all_pinned_docker_images = ConfigParser()
|
|
all_pinned_docker_images.read(pinned_docker_images_file)
|
|
# all_pinned_docker_images looks like a dict of dicts, e.g.
|
|
# { 'x86_64': {'manylinux1': '...', 'manylinux2010': '...', 'manylinux2014': '...'},
|
|
# 'i686': {'manylinux1': '...', 'manylinux2010': '...', 'manylinux2014': '...'},
|
|
# 'pypy_x86_64': {'manylinux2010': '...' }
|
|
# ... }
|
|
|
|
manylinux_images = {}
|
|
|
|
for build_platform in ["x86_64", "i686", "pypy_x86_64", "aarch64", "ppc64le", "s390x"]:
|
|
pinned_images = all_pinned_docker_images[build_platform]
|
|
|
|
config_name = f"CIBW_MANYLINUX_{build_platform.upper()}_IMAGE"
|
|
config_value = os.environ.get(config_name)
|
|
|
|
if config_value is None:
|
|
# default to manylinux2010 if it's available, otherwise manylinux2014
|
|
image = pinned_images.get("manylinux2010") or pinned_images.get("manylinux2014")
|
|
elif config_value in pinned_images:
|
|
image = pinned_images[config_value]
|
|
else:
|
|
image = config_value
|
|
|
|
manylinux_images[build_platform] = image
|
|
|
|
build_options = BuildOptions(
|
|
architectures=archs,
|
|
package_dir=package_dir,
|
|
output_dir=output_dir,
|
|
test_command=test_command,
|
|
test_requires=test_requires,
|
|
test_extras=test_extras,
|
|
before_test=before_test,
|
|
before_build=before_build,
|
|
before_all=before_all,
|
|
build_verbosity=build_verbosity,
|
|
build_selector=build_selector,
|
|
test_selector=test_selector,
|
|
repair_command=repair_command,
|
|
environment=environment,
|
|
dependency_constraints=dependency_constraints,
|
|
manylinux_images=manylinux_images,
|
|
)
|
|
|
|
# Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print'
|
|
sys.stdout = Unbuffered(sys.stdout) # type: ignore
|
|
|
|
print_preamble(platform, build_options)
|
|
|
|
try:
|
|
allowed_architectures_check(platform, build_options.architectures)
|
|
except ValueError as err:
|
|
print("cibuildwheel:", *err.args, file=sys.stderr)
|
|
sys.exit(4)
|
|
|
|
if not identifiers:
|
|
print(f"cibuildwheel: No build identifiers selected: {build_selector}", file=sys.stderr)
|
|
if not args.allow_empty:
|
|
sys.exit(3)
|
|
|
|
if not output_dir.exists():
|
|
output_dir.mkdir(parents=True)
|
|
|
|
with cibuildwheel.util.print_new_wheels(
|
|
"\n{n} wheels produced in {m:.0f} minutes:", output_dir
|
|
):
|
|
if platform == "linux":
|
|
cibuildwheel.linux.build(build_options)
|
|
elif platform == "windows":
|
|
cibuildwheel.windows.build(build_options)
|
|
elif platform == "macos":
|
|
cibuildwheel.macos.build(build_options)
|
|
else:
|
|
assert_never(platform)
|
|
|
|
|
|
def detect_obsolete_options() -> None:
|
|
# Check the old 'MANYLINUX1_*_IMAGE' options
|
|
for (deprecated, alternative) in [
|
|
("CIBW_MANYLINUX1_X86_64_IMAGE", "CIBW_MANYLINUX_X86_64_IMAGE"),
|
|
("CIBW_MANYLINUX1_I686_IMAGE", "CIBW_MANYLINUX_I686_IMAGE"),
|
|
]:
|
|
if deprecated in os.environ:
|
|
print(
|
|
f"'{deprecated}' has been deprecated, and will be removed in a future release. Use the option '{alternative}' instead."
|
|
)
|
|
if alternative not in os.environ:
|
|
print(f"Using value of option '{deprecated}' as replacement for '{alternative}'")
|
|
os.environ[alternative] = os.environ[deprecated]
|
|
else:
|
|
print(f"Option '{alternative}' is not empty. Please unset '{deprecated}'")
|
|
sys.exit(2)
|
|
|
|
# Check for deprecated identifiers in 'CIBW_BUILD' and 'CIBW_SKIP' options
|
|
for option in ["CIBW_BUILD", "CIBW_SKIP"]:
|
|
for deprecated, alternative in [
|
|
("manylinux1", "manylinux"),
|
|
("macosx_10_6_intel", "macosx_x86_64"),
|
|
("macosx_10_9_x86_64", "macosx_x86_64"),
|
|
]:
|
|
if option in os.environ and deprecated in os.environ[option]:
|
|
print(
|
|
f"Build identifiers with '{deprecated}' have been deprecated. Replacing all occurences of '{deprecated}' with '{alternative}' in the option '{option}'"
|
|
)
|
|
os.environ[option] = os.environ[option].replace(deprecated, alternative)
|
|
|
|
|
|
def print_preamble(platform: str, build_options: BuildOptions) -> None:
|
|
print(
|
|
textwrap.dedent(
|
|
"""
|
|
_ _ _ _ _ _ _
|
|
___|_| |_ _ _|_| |_| |_ _ _| |_ ___ ___| |
|
|
| _| | . | | | | | . | | | | | -_| -_| |
|
|
|___|_|___|___|_|_|___|_____|_|_|___|___|_|
|
|
"""
|
|
)
|
|
)
|
|
|
|
print(f"cibuildwheel version {cibuildwheel.__version__}\n")
|
|
|
|
print("Build options:")
|
|
print(f" platform: {platform!r}")
|
|
for option, value in sorted(build_options._asdict().items()):
|
|
print(f" {option}: {value!r}")
|
|
|
|
warnings = detect_warnings(platform, build_options)
|
|
if warnings:
|
|
print("\nWarnings:")
|
|
for warning in warnings:
|
|
print(" " + warning)
|
|
|
|
print("\nHere we go!\n")
|
|
|
|
|
|
def get_build_identifiers(
|
|
platform: PlatformName, build_selector: BuildSelector, architectures: Set[Architecture]
|
|
) -> List[str]:
|
|
python_configurations: Union[
|
|
List[cibuildwheel.linux.PythonConfiguration],
|
|
List[cibuildwheel.windows.PythonConfiguration],
|
|
List[cibuildwheel.macos.PythonConfiguration],
|
|
]
|
|
|
|
if platform == "linux":
|
|
python_configurations = cibuildwheel.linux.get_python_configurations(
|
|
build_selector, architectures
|
|
)
|
|
elif platform == "windows":
|
|
python_configurations = cibuildwheel.windows.get_python_configurations(
|
|
build_selector, architectures
|
|
)
|
|
elif platform == "macos":
|
|
python_configurations = cibuildwheel.macos.get_python_configurations(
|
|
build_selector, architectures
|
|
)
|
|
else:
|
|
assert_never(platform)
|
|
|
|
return [config.identifier for config in python_configurations]
|
|
|
|
|
|
def detect_warnings(platform: str, build_options: BuildOptions) -> List[str]:
|
|
warnings = []
|
|
|
|
# warn about deprecated {python} and {pip}
|
|
for option_name in ["test_command", "before_build"]:
|
|
option_value = getattr(build_options, option_name)
|
|
|
|
if option_value and ("{python}" in option_value or "{pip}" in option_value):
|
|
# Reminder: in an f-string, double braces means literal single brace
|
|
msg = (
|
|
f"{option_name}: '{{python}}' and '{{pip}}' are no longer needed, "
|
|
"and will be removed in a future release. Simply use 'python' or 'pip' instead."
|
|
)
|
|
warnings.append(msg)
|
|
|
|
return warnings
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|