diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 750c5fa4..75f15052 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -36,7 +36,7 @@ repos: exclude: ^cibuildwheel/resources/.*py$ - repo: https://github.com/psf/black - rev: 22.6.0 + rev: 22.8.0 hooks: - id: black diff --git a/bin/update_docker.py b/bin/update_docker.py index 4d1af209..fa91cd00 100755 --- a/bin/update_docker.py +++ b/bin/update_docker.py @@ -50,6 +50,7 @@ images = [ Image("manylinux_2_28", "x86_64", "quay.io/pypa/manylinux_2_28_x86_64", None), Image("manylinux_2_28", "aarch64", "quay.io/pypa/manylinux_2_28_aarch64", None), Image("manylinux_2_28", "ppc64le", "quay.io/pypa/manylinux_2_28_ppc64le", None), + Image("manylinux_2_28", "s390x", "quay.io/pypa/manylinux_2_28_s390x", None), Image("manylinux_2_28", "pypy_x86_64", "quay.io/pypa/manylinux_2_28_x86_64", None), Image("manylinux_2_28", "pypy_aarch64", "quay.io/pypa/manylinux_2_28_aarch64", None), # musllinux_1_1 images diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 3853ba28..f3b44261 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -5,7 +5,6 @@ import os import shutil import sys import tarfile -import tempfile import textwrap from pathlib import Path from tempfile import mkdtemp @@ -130,8 +129,8 @@ def main() -> None: return # Tarfile builds require extraction and changing the directory - with tempfile.TemporaryDirectory(prefix="cibw-sdist-") as temp_dir_str: - temp_dir = Path(temp_dir_str) + temp_dir = Path(mkdtemp(prefix="cibw-sdist-")).resolve(strict=True) + try: with tarfile.open(args.package_dir) as tar: tar.extractall(path=temp_dir) @@ -146,6 +145,12 @@ def main() -> None: with chdir(temp_dir): build_in_directory(args) + finally: + # avoid https://github.com/python/cpython/issues/86962 by performing + # cleanup manually + shutil.rmtree(temp_dir, ignore_errors=sys.platform.startswith("win")) + if temp_dir.exists(): + log.warning(f"Can't delete temporary folder '{str(temp_dir)}'") def build_in_directory(args: CommandLineArguments) -> None: @@ -253,6 +258,8 @@ def build_in_directory(args: CommandLineArguments) -> None: else: assert_never(platform) finally: + # avoid https://github.com/python/cpython/issues/86962 by performing + # cleanup manually shutil.rmtree(tmp_path, ignore_errors=sys.platform.startswith("win")) if tmp_path.exists(): log.warning(f"Can't delete temporary folder '{str(tmp_path)}'") diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index d04ebf58..5720074d 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -20,6 +20,7 @@ from .util import ( get_build_verbosity_extra_flags, prepare_command, read_python_configs, + split_config_settings, unwrap, ) @@ -212,8 +213,10 @@ def build_in_container( container.call(["mkdir", "-p", built_wheel_dir]) verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity) + extra_flags = split_config_settings(build_options.config_settings) if build_options.build_frontend == "pip": + extra_flags += verbosity_flags container.call( [ "python", @@ -223,12 +226,13 @@ def build_in_container( container_package_dir, f"--wheel-dir={built_wheel_dir}", "--no-deps", - *verbosity_flags, + *extra_flags, ], env=env, ) elif build_options.build_frontend == "build": - config_setting = " ".join(verbosity_flags) + verbosity_setting = " ".join(verbosity_flags) + extra_flags += (f"--config-setting={verbosity_setting}",) container.call( [ "python", @@ -237,7 +241,7 @@ def build_in_container( container_package_dir, "--wheel", f"--outdir={built_wheel_dir}", - f"--config-setting={config_setting}", + *extra_flags, ], env=env, ) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 00820674..f15a04f8 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -34,6 +34,7 @@ from .util import ( prepare_command, read_python_configs, shell, + split_config_settings, unwrap, virtualenv, ) @@ -345,8 +346,10 @@ def build(options: Options, tmp_path: Path) -> None: built_wheel_dir.mkdir() verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity) + extra_flags = split_config_settings(build_options.config_settings) if build_options.build_frontend == "pip": + extra_flags += verbosity_flags # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org # see https://github.com/pypa/cibuildwheel/pull/369 call( @@ -357,11 +360,12 @@ def build(options: Options, tmp_path: Path) -> None: build_options.package_dir.resolve(), f"--wheel-dir={built_wheel_dir}", "--no-deps", - *verbosity_flags, + *extra_flags, env=env, ) elif build_options.build_frontend == "build": - config_setting = " ".join(verbosity_flags) + verbosity_setting = " ".join(verbosity_flags) + extra_flags += (f"--config-setting={verbosity_setting}",) build_env = env.copy() if build_options.dependency_constraints: constraint_path = ( @@ -378,7 +382,7 @@ def build(options: Options, tmp_path: Path) -> None: build_options.package_dir, "--wheel", f"--outdir={built_wheel_dir}", - f"--config-setting={config_setting}", + *extra_flags, env=build_env, ) else: diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index c8d050fa..241f0e0e 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -3,13 +3,14 @@ from __future__ import annotations import difflib import functools import os +import shlex import sys import traceback from configparser import ConfigParser from contextlib import contextmanager from dataclasses import asdict, dataclass from pathlib import Path -from typing import Any, Dict, Generator, List, Mapping, Union, cast +from typing import Any, Dict, Generator, Iterator, List, Mapping, Union, cast if sys.version_info >= (3, 11): import tomllib @@ -77,6 +78,7 @@ class BuildOptions: test_extras: str build_verbosity: int build_frontend: BuildFrontend + config_settings: str @property def package_dir(self) -> Path: @@ -293,8 +295,9 @@ class OptionsReader: accept platform versions of the environment variable. If this is an array it will be merged with "sep" before returning. If it is a table, it will be formatted with "table['item']" using {k} and {v} and merged - with "table['sep']". Empty variables will not override if ignore_empty - is True. + with "table['sep']". If sep is also given, it will be used for arrays + inside the table (must match table['sep']). Empty variables will not + override if ignore_empty is True. """ if name not in self.default_options and name not in self.default_platform_options: @@ -324,7 +327,9 @@ class OptionsReader: if isinstance(result, dict): if table is None: raise ConfigOptionError(f"{name!r} does not accept a table") - return table["sep"].join(table["item"].format(k=k, v=v) for k, v in result.items()) + return table["sep"].join( + item for k, v in result.items() for item in _inner_fmt(k, v, table["item"]) + ) if isinstance(result, list): if sep is None: @@ -337,6 +342,16 @@ class OptionsReader: return result +def _inner_fmt(k: str, v: Any, table_item: str) -> Iterator[str]: + if isinstance(v, list): + for inner_v in v: + qv = shlex.quote(inner_v) + yield table_item.format(k=k, v=qv) + else: + qv = shlex.quote(v) + yield table_item.format(k=k, v=qv) + + class Options: def __init__(self, platform: PlatformName, command_line_arguments: CommandLineArguments): self.platform = platform @@ -427,11 +442,14 @@ class Options: build_frontend_str = self.reader.get("build-frontend", env_plat=False) environment_config = self.reader.get( - "environment", table={"item": '{k}="{v}"', "sep": " "} + "environment", table={"item": "{k}={v}", "sep": " "} ) environment_pass = self.reader.get("environment-pass", sep=" ").split() before_build = self.reader.get("before-build", sep=" && ") repair_command = self.reader.get("repair-wheel-command", sep=" && ") + config_settings = self.reader.get( + "config-settings", table={"item": "{k}={v}", "sep": " "} + ) dependency_versions = self.reader.get("dependency-versions") test_command = self.reader.get("test-command", sep=" && ") @@ -537,6 +555,7 @@ class Options: manylinux_images=manylinux_images or None, musllinux_images=musllinux_images or None, build_frontend=build_frontend, + config_settings=config_settings, ) def check_for_invalid_configuration(self, identifiers: list[str]) -> None: diff --git a/cibuildwheel/resources/constraints-python310.txt b/cibuildwheel/resources/constraints-python310.txt index 79006c38..be1480a4 100644 --- a/cibuildwheel/resources/constraints-python310.txt +++ b/cibuildwheel/resources/constraints-python310.txt @@ -6,7 +6,7 @@ # delocate==0.10.2 # via -r cibuildwheel/resources/constraints.in -distlib==0.3.5 +distlib==0.3.6 # via virtualenv filelock==3.8.0 # via virtualenv @@ -14,7 +14,7 @@ platformdirs==2.5.2 # via virtualenv typing-extensions==4.3.0 # via delocate -virtualenv==20.16.3 +virtualenv==20.16.4 # via -r cibuildwheel/resources/constraints.in wheel==0.37.1 # via @@ -24,5 +24,5 @@ wheel==0.37.1 # The following packages are considered to be unsafe in a requirements file: pip==22.2.2 # via -r cibuildwheel/resources/constraints.in -setuptools==65.2.0 +setuptools==65.3.0 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python311.txt b/cibuildwheel/resources/constraints-python311.txt index 05abac2e..51136b16 100644 --- a/cibuildwheel/resources/constraints-python311.txt +++ b/cibuildwheel/resources/constraints-python311.txt @@ -6,7 +6,7 @@ # delocate==0.10.2 # via -r cibuildwheel/resources/constraints.in -distlib==0.3.5 +distlib==0.3.6 # via virtualenv filelock==3.8.0 # via virtualenv @@ -14,7 +14,7 @@ platformdirs==2.5.2 # via virtualenv typing-extensions==4.3.0 # via delocate -virtualenv==20.16.3 +virtualenv==20.16.4 # via -r cibuildwheel/resources/constraints.in wheel==0.37.1 # via @@ -24,5 +24,5 @@ wheel==0.37.1 # The following packages are considered to be unsafe in a requirements file: pip==22.2.2 # via -r cibuildwheel/resources/constraints.in -setuptools==65.2.0 +setuptools==65.3.0 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python36.txt b/cibuildwheel/resources/constraints-python36.txt index 51328239..00956c49 100644 --- a/cibuildwheel/resources/constraints-python36.txt +++ b/cibuildwheel/resources/constraints-python36.txt @@ -6,7 +6,7 @@ # delocate==0.10.2 # via -r cibuildwheel/resources/constraints.in -distlib==0.3.5 +distlib==0.3.6 # via virtualenv filelock==3.4.1 # via virtualenv @@ -20,7 +20,7 @@ typing-extensions==4.1.1 # via # delocate # importlib-metadata -virtualenv==20.16.3 +virtualenv==20.16.4 # via -r cibuildwheel/resources/constraints.in wheel==0.37.1 # via diff --git a/cibuildwheel/resources/constraints-python37.txt b/cibuildwheel/resources/constraints-python37.txt index 51c29395..49015dbd 100644 --- a/cibuildwheel/resources/constraints-python37.txt +++ b/cibuildwheel/resources/constraints-python37.txt @@ -6,7 +6,7 @@ # delocate==0.10.2 # via -r cibuildwheel/resources/constraints.in -distlib==0.3.5 +distlib==0.3.6 # via virtualenv filelock==3.8.0 # via virtualenv @@ -18,7 +18,7 @@ typing-extensions==4.3.0 # via # delocate # importlib-metadata -virtualenv==20.16.3 +virtualenv==20.16.4 # via -r cibuildwheel/resources/constraints.in wheel==0.37.1 # via @@ -30,5 +30,5 @@ zipp==3.8.1 # The following packages are considered to be unsafe in a requirements file: pip==22.2.2 # via -r cibuildwheel/resources/constraints.in -setuptools==65.2.0 +setuptools==65.3.0 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python38.txt b/cibuildwheel/resources/constraints-python38.txt index 01256831..0eb2b244 100644 --- a/cibuildwheel/resources/constraints-python38.txt +++ b/cibuildwheel/resources/constraints-python38.txt @@ -6,7 +6,7 @@ # delocate==0.10.2 # via -r cibuildwheel/resources/constraints.in -distlib==0.3.5 +distlib==0.3.6 # via virtualenv filelock==3.8.0 # via virtualenv @@ -14,7 +14,7 @@ platformdirs==2.5.2 # via virtualenv typing-extensions==4.3.0 # via delocate -virtualenv==20.16.3 +virtualenv==20.16.4 # via -r cibuildwheel/resources/constraints.in wheel==0.37.1 # via @@ -24,5 +24,5 @@ wheel==0.37.1 # The following packages are considered to be unsafe in a requirements file: pip==22.2.2 # via -r cibuildwheel/resources/constraints.in -setuptools==65.2.0 +setuptools==65.3.0 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python39.txt b/cibuildwheel/resources/constraints-python39.txt index d0ee57c9..0b7e373d 100644 --- a/cibuildwheel/resources/constraints-python39.txt +++ b/cibuildwheel/resources/constraints-python39.txt @@ -6,7 +6,7 @@ # delocate==0.10.2 # via -r cibuildwheel/resources/constraints.in -distlib==0.3.5 +distlib==0.3.6 # via virtualenv filelock==3.8.0 # via virtualenv @@ -14,7 +14,7 @@ platformdirs==2.5.2 # via virtualenv typing-extensions==4.3.0 # via delocate -virtualenv==20.16.3 +virtualenv==20.16.4 # via -r cibuildwheel/resources/constraints.in wheel==0.37.1 # via @@ -24,5 +24,5 @@ wheel==0.37.1 # The following packages are considered to be unsafe in a requirements file: pip==22.2.2 # via -r cibuildwheel/resources/constraints.in -setuptools==65.2.0 +setuptools==65.3.0 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints.txt b/cibuildwheel/resources/constraints.txt index 05abac2e..51136b16 100644 --- a/cibuildwheel/resources/constraints.txt +++ b/cibuildwheel/resources/constraints.txt @@ -6,7 +6,7 @@ # delocate==0.10.2 # via -r cibuildwheel/resources/constraints.in -distlib==0.3.5 +distlib==0.3.6 # via virtualenv filelock==3.8.0 # via virtualenv @@ -14,7 +14,7 @@ platformdirs==2.5.2 # via virtualenv typing-extensions==4.3.0 # via delocate -virtualenv==20.16.3 +virtualenv==20.16.4 # via -r cibuildwheel/resources/constraints.in wheel==0.37.1 # via @@ -24,5 +24,5 @@ wheel==0.37.1 # The following packages are considered to be unsafe in a requirements file: pip==22.2.2 # via -r cibuildwheel/resources/constraints.in -setuptools==65.2.0 +setuptools==65.3.0 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/defaults.toml b/cibuildwheel/resources/defaults.toml index 5e0bdacf..2b5708c7 100644 --- a/cibuildwheel/resources/defaults.toml +++ b/cibuildwheel/resources/defaults.toml @@ -5,6 +5,7 @@ test-skip = "" archs = ["auto"] build-frontend = "pip" +config-settings = {} dependency-versions = "pinned" environment = {} environment-pass = [] diff --git a/cibuildwheel/resources/pinned_docker_images.cfg b/cibuildwheel/resources/pinned_docker_images.cfg index e6d0de75..f3047929 100644 --- a/cibuildwheel/resources/pinned_docker_images.cfg +++ b/cibuildwheel/resources/pinned_docker_images.cfg @@ -1,48 +1,49 @@ [x86_64] -manylinux1 = quay.io/pypa/manylinux1_x86_64:2022-08-21-7492cb5 +manylinux1 = quay.io/pypa/manylinux1_x86_64:2022-09-04-d1c2903 manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2022-08-05-4535177 -manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2022-08-21-92d3131 -manylinux_2_24 = quay.io/pypa/manylinux_2_24_x86_64:2022-08-21-92d3131 -manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64:2022-08-21-92d3131 -musllinux_1_1 = quay.io/pypa/musllinux_1_1_x86_64:2022-08-21-92d3131 +manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2022-09-04-870f6a2 +manylinux_2_24 = quay.io/pypa/manylinux_2_24_x86_64:2022-09-04-870f6a2 +manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64:2022-09-04-870f6a2 +musllinux_1_1 = quay.io/pypa/musllinux_1_1_x86_64:2022-09-04-870f6a2 [i686] -manylinux1 = quay.io/pypa/manylinux1_i686:2022-08-21-7492cb5 +manylinux1 = quay.io/pypa/manylinux1_i686:2022-09-04-d1c2903 manylinux2010 = quay.io/pypa/manylinux2010_i686:2022-08-05-4535177 -manylinux2014 = quay.io/pypa/manylinux2014_i686:2022-08-21-92d3131 -manylinux_2_24 = quay.io/pypa/manylinux_2_24_i686:2022-08-21-92d3131 -musllinux_1_1 = quay.io/pypa/musllinux_1_1_i686:2022-08-21-92d3131 +manylinux2014 = quay.io/pypa/manylinux2014_i686:2022-09-04-870f6a2 +manylinux_2_24 = quay.io/pypa/manylinux_2_24_i686:2022-09-04-870f6a2 +musllinux_1_1 = quay.io/pypa/musllinux_1_1_i686:2022-09-04-870f6a2 [pypy_x86_64] manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2022-08-05-4535177 -manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2022-08-21-92d3131 -manylinux_2_24 = quay.io/pypa/manylinux_2_24_x86_64:2022-08-21-92d3131 -manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64:2022-08-21-92d3131 +manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2022-09-04-870f6a2 +manylinux_2_24 = quay.io/pypa/manylinux_2_24_x86_64:2022-09-04-870f6a2 +manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64:2022-09-04-870f6a2 [pypy_i686] manylinux2010 = quay.io/pypa/manylinux2010_i686:2022-08-05-4535177 -manylinux2014 = quay.io/pypa/manylinux2014_i686:2022-08-21-92d3131 -manylinux_2_24 = quay.io/pypa/manylinux_2_24_i686:2022-08-21-92d3131 +manylinux2014 = quay.io/pypa/manylinux2014_i686:2022-09-04-870f6a2 +manylinux_2_24 = quay.io/pypa/manylinux_2_24_i686:2022-09-04-870f6a2 [aarch64] -manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2022-08-21-92d3131 -manylinux_2_24 = quay.io/pypa/manylinux_2_24_aarch64:2022-08-21-92d3131 -manylinux_2_28 = quay.io/pypa/manylinux_2_28_aarch64:2022-08-21-92d3131 -musllinux_1_1 = quay.io/pypa/musllinux_1_1_aarch64:2022-08-21-92d3131 +manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2022-09-04-870f6a2 +manylinux_2_24 = quay.io/pypa/manylinux_2_24_aarch64:2022-09-04-870f6a2 +manylinux_2_28 = quay.io/pypa/manylinux_2_28_aarch64:2022-09-04-870f6a2 +musllinux_1_1 = quay.io/pypa/musllinux_1_1_aarch64:2022-09-04-870f6a2 [ppc64le] -manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2022-08-21-92d3131 -manylinux_2_24 = quay.io/pypa/manylinux_2_24_ppc64le:2022-08-21-92d3131 -manylinux_2_28 = quay.io/pypa/manylinux_2_28_ppc64le:2022-08-21-92d3131 -musllinux_1_1 = quay.io/pypa/musllinux_1_1_ppc64le:2022-08-21-92d3131 +manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2022-09-04-870f6a2 +manylinux_2_24 = quay.io/pypa/manylinux_2_24_ppc64le:2022-09-04-870f6a2 +manylinux_2_28 = quay.io/pypa/manylinux_2_28_ppc64le:2022-09-04-870f6a2 +musllinux_1_1 = quay.io/pypa/musllinux_1_1_ppc64le:2022-09-04-870f6a2 [s390x] -manylinux2014 = quay.io/pypa/manylinux2014_s390x:2022-08-21-92d3131 -manylinux_2_24 = quay.io/pypa/manylinux_2_24_s390x:2022-08-21-92d3131 -musllinux_1_1 = quay.io/pypa/musllinux_1_1_s390x:2022-08-21-92d3131 +manylinux2014 = quay.io/pypa/manylinux2014_s390x:2022-09-04-870f6a2 +manylinux_2_24 = quay.io/pypa/manylinux_2_24_s390x:2022-09-04-870f6a2 +manylinux_2_28 = quay.io/pypa/manylinux_2_28_s390x:2022-09-04-870f6a2 +musllinux_1_1 = quay.io/pypa/musllinux_1_1_s390x:2022-09-04-870f6a2 [pypy_aarch64] -manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2022-08-21-92d3131 -manylinux_2_24 = quay.io/pypa/manylinux_2_24_aarch64:2022-08-21-92d3131 -manylinux_2_28 = quay.io/pypa/manylinux_2_28_aarch64:2022-08-21-92d3131 +manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2022-09-04-870f6a2 +manylinux_2_24 = quay.io/pypa/manylinux_2_24_aarch64:2022-09-04-870f6a2 +manylinux_2_28 = quay.io/pypa/manylinux_2_28_aarch64:2022-09-04-870f6a2 diff --git a/cibuildwheel/resources/virtualenv.toml b/cibuildwheel/resources/virtualenv.toml index c14592cd..1c277d73 100644 --- a/cibuildwheel/resources/virtualenv.toml +++ b/cibuildwheel/resources/virtualenv.toml @@ -1,2 +1,2 @@ -version = "20.16.3" -url = "https://github.com/pypa/get-virtualenv/blob/20.16.3/public/virtualenv.pyz?raw=true" +version = "20.16.4" +url = "https://github.com/pypa/get-virtualenv/blob/20.16.4/public/virtualenv.pyz?raw=true" diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 2cf16187..4b3d4fb9 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -60,6 +60,7 @@ __all__ = [ "strtobool", "cached_property", "chdir", + "split_config_settings", ] resources_dir: Final = Path(__file__).parent / "resources" @@ -205,6 +206,11 @@ def get_build_verbosity_extra_flags(level: int) -> list[str]: return [] +def split_config_settings(config_settings: str) -> list[str]: + config_settings_list = shlex.split(config_settings) + return [f"--config-setting={setting}" for setting in config_settings_list] + + def read_python_configs(config: PlatformName) -> list[dict[str, str]]: input_file = resources_dir / "build-platforms.toml" with input_file.open("rb") as f: diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 8afc2ff9..d256c27c 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -34,6 +34,7 @@ from .util import ( prepare_command, read_python_configs, shell, + split_config_settings, unwrap, virtualenv, ) @@ -407,8 +408,10 @@ def build(options: Options, tmp_path: Path) -> None: built_wheel_dir.mkdir() verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity) + extra_flags = split_config_settings(build_options.config_settings) if build_options.build_frontend == "pip": + extra_flags += verbosity_flags # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org # see https://github.com/pypa/cibuildwheel/pull/369 call( @@ -419,11 +422,12 @@ def build(options: Options, tmp_path: Path) -> None: options.globals.package_dir.resolve(), f"--wheel-dir={built_wheel_dir}", "--no-deps", - *get_build_verbosity_extra_flags(build_options.build_verbosity), + *extra_flags, env=env, ) elif build_options.build_frontend == "build": - config_setting = " ".join(verbosity_flags) + verbosity_setting = " ".join(verbosity_flags) + extra_flags += (f"--config-setting={verbosity_setting}",) build_env = env.copy() if build_options.dependency_constraints: constraints_path = ( @@ -450,7 +454,7 @@ def build(options: Options, tmp_path: Path) -> None: build_options.package_dir, "--wheel", f"--outdir={built_wheel_dir}", - f"--config-setting={config_setting}", + *extra_flags, env=build_env, ) else: diff --git a/docs/options.md b/docs/options.md index c4f1f100..cc093079 100644 --- a/docs/options.md +++ b/docs/options.md @@ -529,6 +529,36 @@ Choose which build backend to use. Can either be "pip", which will run build-frontend = "pip" ``` +### `CIBW_CONFIG_SETTINGS` {: #config-settings} +> Specify config-settings for the build backend. + +Specify config settings for the build backend. Each space separated +item will be passed via `--config-setting`. In TOML, you can specify +a table of items, including arrays. + +!!! tip + Currently, "build" supports arrays for options, but "pip" only supports + single values. + +Platform-specific environment variables also available:
+`CIBW_BEFORE_ALL_MACOS` | `CIBW_BEFORE_ALL_WINDOWS` | `CIBW_BEFORE_ALL_LINUX` + + +#### Examples + +!!! tab examples "Environment variables" + + ```yaml + CIBW_CONFIG_SETTINGS: "--build-option=--use-mypyc" + ``` + +!!! tab examples "pyproject.toml" + + ```toml + [tool.cibuildwheel.config-settings] + --build-option = "--use-mypyc" + ``` + ### `CIBW_ENVIRONMENT` {: #environment} > Set environment variables needed during the build @@ -911,7 +941,7 @@ The available options are (default value): Set an alternative Docker image to be used for building [manylinux / musllinux](https://github.com/pypa/manylinux) wheels. For `CIBW_MANYLINUX_*_IMAGE`, the value of this option can either be set to `manylinux1`, `manylinux2010`, `manylinux2014`, `manylinux_2_24` or `manylinux_2_28` to use a pinned version of the [official manylinux images](https://github.com/pypa/manylinux). Alternatively, set these options to any other valid Docker image name. For PyPy, the `manylinux1` image is not available. For architectures other -than x86 (x86\_64 and i686) `manylinux2014`, `manylinux_2_24` or `manylinux_2_28` must be used, because the first version of the manylinux specification that supports additional architectures is `manylinux2014`. `manylinux_2_28` is not supported for `i686` & `s390x` architectures. +than x86 (x86\_64 and i686) `manylinux2014`, `manylinux_2_24` or `manylinux_2_28` must be used, because the first version of the manylinux specification that supports additional architectures is `manylinux2014`. `manylinux_2_28` is not supported for `i686` architecture. For `CIBW_MUSLLINUX_*_IMAGE`, the value of this option can either be set to `musllinux_1_1` to use a pinned version of the [official musllinux images](https://github.com/pypa/musllinux). Alternatively, set these options to any other valid Docker image name. diff --git a/docs/working-examples.md b/docs/working-examples.md index 13ff7323..5bc33c71 100644 --- a/docs/working-examples.md +++ b/docs/working-examples.md @@ -26,9 +26,9 @@ title: Working examples | [duckdb][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | DuckDB is an in-process SQL OLAP Database Management System | | [Apache Beam][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Apache Beam is a unified programming model for Batch and Streaming data processing. | | [asyncpg][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | A fast PostgreSQL Database Client Library for Python/asyncio. | -| [PyGame][] | ![github icon][] | ![apple icon][] ![linux icon][] | pygame (the library) is a Free and Open Source python programming language library for making multimedia applications like games built on top of the excellent SDL library. C, Python, Native, OpenGL. | -| [scikit-image][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Image processing library. Uses cibuildwheel to build and test a project that uses Cython with platform-native code. | +| [PyGame][] | ![github icon][] | ![apple icon][] ![linux icon][] | 🐍🎮 pygame (the library) is a Free and Open Source python programming language library for making multimedia applications like games built on top of the excellent SDL library. C, Python, Native, OpenGL. | | [cmake][] | ![github icon][] ![travisci icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Multitagged binary builds for all supported platforms, using cibw 2 config configuration. | +| [scikit-image][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Image processing library. Uses cibuildwheel to build and test a project that uses Cython with platform-native code. | | [twisted-iocpsupport][] | ![github icon][] | ![windows icon][] | A submodule of Twisted that hooks into native C APIs using Cython. | | [PyOxidizer][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | A modern Python application packaging and distribution tool | | [websockets][] | ![travisci icon][] | ![apple icon][] ![linux icon][] | Library for building WebSocket servers and clients. Mostly written in Python, with a small C 'speedups' extension module. | @@ -59,10 +59,10 @@ title: Working examples | [OpenTimelineIO][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Open Source API and interchange format for editorial timeline information. | | [aioquic][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | QUIC and HTTP/3 implementation in Python | | [ruptures][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Extensive Cython + NumPy [pyproject.toml](https://github.com/deepcharles/ruptures/blob/master/pyproject.toml) example. | -| [DeepForest][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | An Efficient, Scalable and Optimized Python Framework for Deep Forest (2021.2.1) | | [Psycopg 3][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | A modern implementation of a PostgreSQL adapter for Python | -| [google neuroglancer][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | WebGL-based viewer for volumetric data | +| [DeepForest][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | An Efficient, Scalable and Optimized Python Framework for Deep Forest (2021.2.1) | | [envd][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | 🏕️ Development environment for machine learning | +| [google neuroglancer][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | WebGL-based viewer for volumetric data | | [Parselmouth][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | A Python interface to the Praat software package, using pybind11, C++17 and CMake, with the core Praat static library built only once and shared between wheels. | | [AutoPy][] | ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Includes a Windows Travis build. | | [H3-py][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Python bindings for H3, a hierarchical hexagonal geospatial indexing system | @@ -85,8 +85,8 @@ title: Working examples | [iminuit][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Jupyter-friendly Python interface for C++ MINUIT2 | | [Tokenizer][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Fast and customizable text tokenization library with BPE and SentencePiece support | | [PyGLM][] | ![github icon][] | ![apple icon][] ![linux icon][] ![windows icon][] | Fast OpenGL Mathematics (GLM) for Python | -| [boost-histogram][] | ![github icon][] ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Supports full range of wheels, including PyPy and alternate archs. | | [bx-python][] | ![travisci icon][] | ![apple icon][] ![linux icon][] | A library that includes Cython extensions. | +| [boost-histogram][] | ![github icon][] ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Supports full range of wheels, including PyPy and alternate archs. | | [TgCrypto][] | ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Includes a Windows Travis build. | | [iDynTree][] | ![github icon][] | ![linux icon][] | Uses manylinux_2_24 | | [mosec][] | ![github icon][] | ![linux icon][] ![apple icon][] | A high-performance serving framework for ML models, offers dynamic batching and multi-stage pipeline to fully exploit your compute machine | @@ -131,8 +131,8 @@ title: Working examples [Apache Beam]: https://github.com/apache/beam [asyncpg]: https://github.com/MagicStack/asyncpg [PyGame]: https://github.com/pygame/pygame -[scikit-image]: https://github.com/scikit-image/scikit-image [cmake]: https://github.com/scikit-build/cmake-python-distributions +[scikit-image]: https://github.com/scikit-image/scikit-image [twisted-iocpsupport]: https://github.com/twisted/twisted-iocpsupport [PyOxidizer]: https://github.com/indygreg/PyOxidizer [websockets]: https://github.com/aaugustin/websockets @@ -163,10 +163,10 @@ title: Working examples [OpenTimelineIO]: https://github.com/PixarAnimationStudios/OpenTimelineIO [aioquic]: https://github.com/aiortc/aioquic [ruptures]: https://github.com/deepcharles/ruptures -[DeepForest]: https://github.com/LAMDA-NJU/Deep-Forest [Psycopg 3]: https://github.com/psycopg/psycopg -[google neuroglancer]: https://github.com/google/neuroglancer +[DeepForest]: https://github.com/LAMDA-NJU/Deep-Forest [envd]: https://github.com/tensorchord/envd +[google neuroglancer]: https://github.com/google/neuroglancer [Parselmouth]: https://github.com/YannickJadoul/Parselmouth [AutoPy]: https://github.com/autopilot-rs/autopy [H3-py]: https://github.com/uber/h3-py @@ -189,8 +189,8 @@ title: Working examples [iminuit]: https://github.com/scikit-hep/iminuit [Tokenizer]: https://github.com/OpenNMT/Tokenizer [PyGLM]: https://github.com/Zuzu-Typ/PyGLM -[boost-histogram]: https://github.com/scikit-hep/boost-histogram [bx-python]: https://github.com/bxlab/bx-python +[boost-histogram]: https://github.com/scikit-hep/boost-histogram [TgCrypto]: https://github.com/pyrogram/tgcrypto [iDynTree]: https://github.com/robotology/idyntree [mosec]: https://github.com/mosecorg/mosec @@ -222,113 +222,114 @@ title: Working examples [circleci icon]: data/readme_icons/circleci.svg [gitlab icon]: data/readme_icons/gitlab.svg [travisci icon]: data/readme_icons/travisci.svg +[cirrusci icon]: data/readme_icons/cirrusci.svg [windows icon]: data/readme_icons/windows.svg [apple icon]: data/readme_icons/apple.svg [linux icon]: data/readme_icons/linux.svg - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - + + + - - - - - - - + + + + + + + - - - - - - + + + + + + diff --git a/test/test_manylinuxXXXX_only.py b/test/test_manylinuxXXXX_only.py index 6013e856..d2e91b1d 100644 --- a/test/test_manylinuxXXXX_only.py +++ b/test/test_manylinuxXXXX_only.py @@ -59,8 +59,6 @@ def test(manylinux_image, tmp_path): elif platform.machine() not in ["x86_64", "i686"]: if manylinux_image in ["manylinux1", "manylinux2010"]: pytest.skip("manylinux1 and 2010 doesn't exist for non-x86 architectures") - elif manylinux_image == "manylinux_2_28" and platform.machine() == "s390x": - pytest.skip("manylinux_2_28 doesn't exist for s390x architecture") elif manylinux_image == "manylinux_2_28" and platform.machine() == "i686": pytest.skip("manylinux_2_28 doesn't exist for i686 architecture") diff --git a/unit_test/main_tests/main_options_test.py b/unit_test/main_tests/main_options_test.py index 813fd5c5..2ca1b25d 100644 --- a/unit_test/main_tests/main_options_test.py +++ b/unit_test/main_tests/main_options_test.py @@ -14,7 +14,7 @@ else: from cibuildwheel.__main__ import main from cibuildwheel.environment import ParsedEnvironment from cibuildwheel.options import BuildOptions, _get_pinned_container_images -from cibuildwheel.util import BuildSelector, resources_dir +from cibuildwheel.util import BuildSelector, resources_dir, split_config_settings # CIBW_PLATFORM is tested in main_platform_test.py @@ -263,6 +263,27 @@ def test_build_verbosity( assert build_options.build_verbosity == expected_verbosity +@pytest.mark.parametrize("platform_specific", [False, True]) +def test_config_settings(platform_specific, platform, intercepted_build_args, monkeypatch): + config_settings = 'setting=value setting=value2 other="something else"' + if platform_specific: + monkeypatch.setenv("CIBW_CONFIG_SETTINGS_" + platform.upper(), config_settings) + monkeypatch.setenv("CIBW_CONFIG_SETTIGNS", "a=b") + else: + monkeypatch.setenv("CIBW_CONFIG_SETTINGS", config_settings) + + main() + build_options = intercepted_build_args.args[0].build_options(identifier=None) + + assert build_options.config_settings == config_settings + + assert split_config_settings(config_settings) == [ + "--config-setting=setting=value", + "--config-setting=setting=value2", + "--config-setting=other=something else", + ] + + @pytest.mark.parametrize( "selector", [ diff --git a/unit_test/options_test.py b/unit_test/options_test.py index a609bb49..f0fb5f63 100644 --- a/unit_test/options_test.py +++ b/unit_test/options_test.py @@ -1,6 +1,7 @@ from __future__ import annotations import platform as platform_module +import textwrap import pytest @@ -58,7 +59,7 @@ test_command: 'pyproject' default_build_options = options.build_options(identifier=None) - assert default_build_options.environment == parse_environment('FOO="BAR"') + assert default_build_options.environment == parse_environment("FOO=BAR") all_pinned_container_images = _get_pinned_container_images() pinned_x86_64_container_image = all_pinned_container_images["x86_64"] @@ -116,3 +117,32 @@ def test_passthrough_evil(tmp_path, monkeypatch, env_var_value): monkeypatch.setenv("ENV_VAR", env_var_value) parsed_environment = options.build_options(identifier=None).environment assert parsed_environment.as_dictionary(prev_environment={}) == {"ENV_VAR": env_var_value} + + +@pytest.mark.parametrize( + "env_var_value", + [ + "normal value", + '"value wrapped in quotes"', + 'an unclosed double-quote: "', + "string\nwith\ncarriage\nreturns\n", + "a trailing backslash \\", + ], +) +def test_toml_environment_evil(tmp_path, monkeypatch, env_var_value): + args = get_default_command_line_arguments() + args.package_dir = tmp_path + + with tmp_path.joinpath("pyproject.toml").open("w") as f: + f.write( + textwrap.dedent( + f"""\ + [tool.cibuildwheel.environment] + EXAMPLE='''{env_var_value}''' + """ + ) + ) + + options = Options(platform="linux", command_line_arguments=args) + parsed_environment = options.build_options(identifier=None).environment + assert parsed_environment.as_dictionary(prev_environment={}) == {"EXAMPLE": env_var_value} diff --git a/unit_test/options_toml_test.py b/unit_test/options_toml_test.py index a0a1705d..5c01a5fe 100644 --- a/unit_test/options_toml_test.py +++ b/unit_test/options_toml_test.py @@ -331,3 +331,38 @@ test-command = "pyproject-override" with pytest.raises(ConfigOptionError): OptionsReader(config_file_path=pyproject_toml, platform=platform) + + +def test_config_settings(tmp_path): + pyproject_toml: Path = tmp_path / "pyproject.toml" + pyproject_toml.write_text( + """\ +[tool.cibuildwheel.config-settings] +example = "one" +other = ["two", "three"] +""" + ) + + options_reader = OptionsReader(config_file_path=pyproject_toml, platform="linux") + assert ( + options_reader.get("config-settings", table={"item": '{k}="{v}"', "sep": " "}) + == 'example="one" other="two" other="three"' + ) + + +def test_pip_config_settings(tmp_path): + pyproject_toml: Path = tmp_path / "pyproject.toml" + pyproject_toml.write_text( + """\ +[tool.cibuildwheel.config-settings] +--build-option="--use-mypyc" +""" + ) + + options_reader = OptionsReader(config_file_path=pyproject_toml, platform="linux") + assert ( + options_reader.get( + "config-settings", table={"item": "--config-settings='{k}=\"{v}\"'", "sep": " "} + ) + == "--config-settings='--build-option=\"--use-mypyc\"'" + )