Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: pull out compute_options

Browse Source
This commit is contained in:
Henry Schreiner
2021-09-20 12:45:48 -04:00
parent a07edd9ddc
commit a7047a6955
3 changed files with 183 additions and 154 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cibuildwheel/__main__.py
+19 -152
View File
@@ -2,10 +2,8 @@ import argparse
import os
import sys
import textwrap
import traceback
from configparser import ConfigParser
from pathlib import Path
from typing import Dict, List, Optional, Set, Union
from typing import List, Optional, Set, Union
from packaging.specifiers import SpecifierSet
@@ -15,38 +13,17 @@ import cibuildwheel.macos
import cibuildwheel.util
import cibuildwheel.windows
from cibuildwheel.architecture import Architecture, allowed_architectures_check
from cibuildwheel.environment import EnvironmentParseError, parse_environment
from cibuildwheel.options import ConfigOptions
from cibuildwheel.options import ConfigOptions, compute_options
from cibuildwheel.projectfiles import get_requires_python_str
from cibuildwheel.typing import PLATFORMS, PlatformName, assert_never
from cibuildwheel.util import (
BuildFrontend,
MANYLINUX_ARCHS,
MUSLLINUX_ARCHS,
BuildOptions,
BuildSelector,
DependencyConstraints,
TestSelector,
Unbuffered,
detect_ci_provider,
resources_dir,
)
MANYLINUX_ARCHS = (
"x86_64",
"i686",
"pypy_x86_64",
"aarch64",
"ppc64le",
"s390x",
"pypy_aarch64",
"pypy_i686",
)
MUSLLINUX_ARCHS = (
"x86_64",
"i686",
"aarch64",
"ppc64le",
"s390x",
)
@@ -171,6 +148,11 @@ def main() -> None:
sys.exit(2)
package_dir = Path(args.package_dir)
output_dir = Path(
args.output_dir
if args.output_dir is not None
else os.environ.get("CIBW_OUTPUT_DIR", "wheelhouse")
)
manylinux_identifiers = {
f"manylinux-{build_platform}-image" for build_platform in MANYLINUX_ARCHS
@@ -184,44 +166,18 @@ def main() -> None:
"windows": manylinux_identifiers | musllinux_identifiers,
}
options = ConfigOptions(package_dir, args.config_file, platform=platform, disallow=disallow)
output_dir = Path(
args.output_dir
if args.output_dir is not None
else os.environ.get("CIBW_OUTPUT_DIR", "wheelhouse")
)
build_config = options("build", env_plat=False, sep=" ") or "*"
skip_config = options("skip", env_plat=False, sep=" ")
test_skip = options("test-skip", env_plat=False, sep=" ")
archs_config_str = args.archs or options("archs", sep=" ")
build_frontend_str = options("build-frontend", env_plat=False)
environment_config = options("environment", table={"item": '{k}="{v}"', "sep": " "})
before_all = options("before-all", sep=" && ")
before_build = options("before-build", sep=" && ")
repair_command = options("repair-wheel-command", sep=" && ")
dependency_versions = options("dependency-versions")
test_command = options("test-command", sep=" && ")
before_test = options("before-test", sep=" && ")
test_requires = options("test-requires", sep=" ").split()
test_extras = options("test-extras", sep=",")
build_verbosity_str = options("build-verbosity")
prerelease_pythons = args.prerelease_pythons or cibuildwheel.util.strtobool(
os.environ.get("CIBW_PRERELEASE_PYTHONS", "0")
)
build_frontend: BuildFrontend
if build_frontend_str == "build":
build_frontend = "build"
elif build_frontend_str == "pip":
build_frontend = "pip"
else:
msg = f"cibuildwheel: Unrecognised build frontend '{build_frontend}', only 'pip' and 'build' are supported"
print(msg, file=sys.stderr)
sys.exit(2)
deprecated_selectors("CIBW_BUILD", build_config, error=True)
deprecated_selectors("CIBW_SKIP", skip_config)
deprecated_selectors("CIBW_TEST_SKIP", test_skip)
package_files = {"setup.py", "setup.cfg", "pyproject.toml"}
@@ -238,10 +194,6 @@ def main() -> None:
) or get_requires_python_str(package_dir)
requires_python = None if requires_python_str is None else SpecifierSet(requires_python_str)
deprecated_selectors("CIBW_BUILD", build_config, error=True)
deprecated_selectors("CIBW_SKIP", skip_config)
deprecated_selectors("CIBW_TEST_SKIP", test_skip)
build_selector = BuildSelector(
build_config=build_config,
skip_config=skip_config,
@@ -250,105 +202,20 @@ def main() -> None:
)
test_selector = TestSelector(skip_config=test_skip)
try:
environment = parse_environment(environment_config)
except (EnvironmentParseError, ValueError):
print(f'cibuildwheel: Malformed environment option "{environment_config}"', file=sys.stderr)
traceback.print_exc(None, sys.stderr)
sys.exit(2)
build_options = compute_options(
options, args.archs, build_selector, test_selector, platform, package_dir, output_dir
)
if dependency_versions == "pinned":
dependency_constraints: Optional[
DependencyConstraints
] = DependencyConstraints.with_defaults()
elif dependency_versions == "latest":
dependency_constraints = None
else:
dependency_versions_path = Path(dependency_versions)
dependency_constraints = DependencyConstraints(dependency_versions_path)
if test_extras:
test_extras = f"[{test_extras}]"
try:
build_verbosity = min(3, max(-3, int(build_verbosity_str)))
except ValueError:
build_verbosity = 0
# Add CIBUILDWHEEL environment variable
# This needs to be passed on to the docker container in linux.py
os.environ["CIBUILDWHEEL"] = "1"
archs = Architecture.parse_config(archs_config_str, platform=platform)
identifiers = get_build_identifiers(platform, build_selector, archs)
identifiers = get_build_identifiers(platform, build_selector, build_options.architectures)
if args.print_build_identifiers:
for identifier in identifiers:
print(identifier)
sys.exit(0)
manylinux_images: Dict[str, str] = {}
musllinux_images: Dict[str, str] = {}
if platform == "linux":
pinned_docker_images_file = resources_dir / "pinned_docker_images.cfg"
all_pinned_docker_images = ConfigParser()
all_pinned_docker_images.read(pinned_docker_images_file)
# all_pinned_docker_images looks like a dict of dicts, e.g.
# { 'x86_64': {'manylinux1': '...', 'manylinux2010': '...', 'manylinux2014': '...'},
# 'i686': {'manylinux1': '...', 'manylinux2010': '...', 'manylinux2014': '...'},
# 'pypy_x86_64': {'manylinux2010': '...' }
# ... }
for build_platform in MANYLINUX_ARCHS:
pinned_images = all_pinned_docker_images[build_platform]
config_value = options(f"manylinux-{build_platform}-image", ignore_empty=True)
if not config_value:
# default to manylinux2010 if it's available, otherwise manylinux2014
image = pinned_images.get("manylinux2010") or pinned_images.get("manylinux2014")
elif config_value in pinned_images:
image = pinned_images[config_value]
else:
image = config_value
manylinux_images[build_platform] = image
for build_platform in MUSLLINUX_ARCHS:
pinned_images = all_pinned_docker_images[build_platform]
config_value = options(f"musllinux-{build_platform}-image")
if config_value is None:
image = pinned_images.get("musllinux_1_1")
elif config_value in pinned_images:
image = pinned_images[config_value]
else:
image = config_value
musllinux_images[build_platform] = image
build_options = BuildOptions(
architectures=archs,
package_dir=package_dir,
output_dir=output_dir,
test_command=test_command,
test_requires=test_requires,
test_extras=test_extras,
before_test=before_test,
before_build=before_build,
before_all=before_all,
build_verbosity=build_verbosity,
build_selector=build_selector,
test_selector=test_selector,
repair_command=repair_command,
environment=environment,
dependency_constraints=dependency_constraints,
manylinux_images=manylinux_images or None,
musllinux_images=musllinux_images or None,
build_frontend=build_frontend,
)
# Add CIBUILDWHEEL environment variable
# This needs to be passed on to the docker container in linux.py
os.environ["CIBUILDWHEEL"] = "1"
# Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print'
sys.stdout = Unbuffered(sys.stdout) # type: ignore
cibuildwheel/options.py
+145 -2
View File
@@ -1,11 +1,25 @@
import os
import sys
import traceback
from configparser import ConfigParser
from pathlib import Path
from typing import Any, Dict, List, Mapping, Optional, Set, Tuple, Union
import toml
from .typing import PLATFORMS, TypedDict
from .util import resources_dir
from .architecture import Architecture
from .environment import EnvironmentParseError, parse_environment
from .typing import PLATFORMS, PlatformName, TypedDict
from .util import (
MANYLINUX_ARCHS,
MUSLLINUX_ARCHS,
BuildFrontend,
BuildOptions,
BuildSelector,
DependencyConstraints,
TestSelector,
resources_dir,
)
Setting = Union[Dict[str, str], List[str], str]
@@ -183,3 +197,132 @@ class ConfigOptions:
return str(result)
else:
return result
def compute_options(
options: ConfigOptions,
args_archs: Optional[str],
build_selector: BuildSelector,
test_selector: TestSelector,
platform: PlatformName,
package_dir: Path,
output_dir: Path,
) -> BuildOptions:
"""
Gather options from the command line, environment, and configuration file.
"""
# Can't be configured per selector
before_all = options("before-all", sep=" && ")
archs_config_str = args_archs or options("archs", sep=" ")
build_frontend_str = options("build-frontend", env_plat=False)
environment_config = options("environment", table={"item": '{k}="{v}"', "sep": " "})
before_build = options("before-build", sep=" && ")
repair_command = options("repair-wheel-command", sep=" && ")
dependency_versions = options("dependency-versions")
test_command = options("test-command", sep=" && ")
before_test = options("before-test", sep=" && ")
test_requires = options("test-requires", sep=" ").split()
test_extras = options("test-extras", sep=",")
build_verbosity_str = options("build-verbosity")
build_frontend: BuildFrontend
if build_frontend_str == "build":
build_frontend = "build"
elif build_frontend_str == "pip":
build_frontend = "pip"
else:
msg = f"cibuildwheel: Unrecognised build frontend '{build_frontend}', only 'pip' and 'build' are supported"
print(msg, file=sys.stderr)
sys.exit(2)
try:
environment = parse_environment(environment_config)
except (EnvironmentParseError, ValueError):
print(f'cibuildwheel: Malformed environment option "{environment_config}"', file=sys.stderr)
traceback.print_exc(None, sys.stderr)
sys.exit(2)
if dependency_versions == "pinned":
dependency_constraints: Optional[
DependencyConstraints
] = DependencyConstraints.with_defaults()
elif dependency_versions == "latest":
dependency_constraints = None
else:
dependency_versions_path = Path(dependency_versions)
dependency_constraints = DependencyConstraints(dependency_versions_path)
if test_extras:
test_extras = f"[{test_extras}]"
try:
build_verbosity = min(3, max(-3, int(build_verbosity_str)))
except ValueError:
build_verbosity = 0
archs = Architecture.parse_config(archs_config_str, platform=platform)
manylinux_images: Dict[str, str] = {}
musllinux_images: Dict[str, str] = {}
if platform == "linux":
pinned_docker_images_file = resources_dir / "pinned_docker_images.cfg"
all_pinned_docker_images = ConfigParser()
all_pinned_docker_images.read(pinned_docker_images_file)
# all_pinned_docker_images looks like a dict of dicts, e.g.
# { 'x86_64': {'manylinux1': '...', 'manylinux2010': '...', 'manylinux2014': '...'},
# 'i686': {'manylinux1': '...', 'manylinux2010': '...', 'manylinux2014': '...'},
# 'pypy_x86_64': {'manylinux2010': '...' }
# ... }
for build_platform in MANYLINUX_ARCHS:
pinned_images = all_pinned_docker_images[build_platform]
config_value = options(f"manylinux-{build_platform}-image", ignore_empty=True)
if not config_value:
# default to manylinux2010 if it's available, otherwise manylinux2014
image = pinned_images.get("manylinux2010") or pinned_images.get("manylinux2014")
elif config_value in pinned_images:
image = pinned_images[config_value]
else:
image = config_value
manylinux_images[build_platform] = image
for build_platform in MUSLLINUX_ARCHS:
pinned_images = all_pinned_docker_images[build_platform]
config_value = options(f"musllinux-{build_platform}-image")
if config_value is None:
image = pinned_images.get("musllinux_1_1")
elif config_value in pinned_images:
image = pinned_images[config_value]
else:
image = config_value
musllinux_images[build_platform] = image
return BuildOptions(
architectures=archs,
package_dir=package_dir,
output_dir=output_dir,
test_command=test_command,
test_requires=test_requires,
test_extras=test_extras,
before_test=before_test,
before_build=before_build,
before_all=before_all,
build_verbosity=build_verbosity,
build_selector=build_selector,
test_selector=test_selector,
repair_command=repair_command,
environment=environment,
dependency_constraints=dependency_constraints,
manylinux_images=manylinux_images or None,
musllinux_images=musllinux_images or None,
build_frontend=build_frontend,
)
cibuildwheel/util.py
+19
View File
@@ -30,6 +30,25 @@ install_certifi_script = resources_dir / "install_certifi.py"
BuildFrontend = Literal["pip", "build"]
MANYLINUX_ARCHS = (
"x86_64",
"i686",
"pypy_x86_64",
"aarch64",
"ppc64le",
"s390x",
"pypy_aarch64",
"pypy_i686",
)
MUSLLINUX_ARCHS = (
"x86_64",
"i686",
"aarch64",
"ppc64le",
"s390x",
)
def prepare_command(command: str, **kwargs: PathOrStr) -> str:
"""