From 5b22fc636b9b5b3e4e2b7e01058b703595f5d0da Mon Sep 17 00:00:00 2001 From: Matthieu Darbois Date: Sat, 29 May 2021 00:26:55 +0200 Subject: [PATCH] feat: add CPython 3.10 pre-release support (#675) * Add CPython 3.10 support * Fix logger displaying CPython 3.1 instead of 3.10 * Fix tests failing with CPython 3.10 * Use pytest instead of nose * feat: --pre flag Apply suggestions from code review Co-authored-by: Matthieu Darbois * fix: update Python update script to process beta versions * refactor: prerelease-pythons * Use `strtobool` to parse `CIBW_PRERELEASE_PYTHONS` env var * Update python version filtering for universal2 & arm64 * Filter out CPython 3.10 and above for `test_manylinuxXXXX_only[manylinux1]` test * Use CIBW_BUILD filtering rather than CIBW_SKIP for test_docker_images * Use skip_patterns to filter out pre-releases * Use `prerelease_pythons` instead of `pre` * Reword `CIBW_PRERELEASE_PYTHONS` doc per review. * Use `CIBW_PRERELEASE_PYTHONS: True` for usage example. * Update `cibuildwheel --help` doc * docs: add note on spec.filter * Clean up the BuildSelector __repr__ by refactoring * fix: remove platform variants for CIBW_PRERELEASE_PYTHONS * docs: mention the flag Co-authored-by: Henry Schreiner Co-authored-by: Joe Rickerby --- README.md | 3 ++ bin/update_pythons.py | 33 +++++++++-------- cibuildwheel/__main__.py | 15 +++++++- cibuildwheel/logger.py | 4 +-- cibuildwheel/macos.py | 2 +- cibuildwheel/resources/build-platforms.toml | 10 ++++++ cibuildwheel/util.py | 40 +++++++++++++++++---- docs/options.md | 32 +++++++++++++++-- test/test_0_basic.py | 4 ++- test/test_before_test.py | 6 ++-- test/test_build_skip.py | 2 +- test/test_docker_images.py | 4 +-- test/test_manylinuxXXXX_only.py | 7 ++-- test/test_testing.py | 12 +++---- test/utils.py | 15 +++++--- unit_test/build_selector_test.py | 28 ++++++++++++--- unit_test/main_tests/main_options_test.py | 2 ++ 17 files changed, 166 insertions(+), 53 deletions(-) diff --git a/README.md b/README.md index b39eb2a1..31e6840c 100644 --- a/README.md +++ b/README.md @@ -28,9 +28,11 @@ What does it do? | CPython 3.7 | ✅ | N/A | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | CPython 3.8 | ✅ | N/A¹ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | CPython 3.9 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | +| CPython 3.10² | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | | PyPy 3.7 v7.3 | ✅ | N/A | ✅ | N/A | ✅ | ✅ | ✅ | N/A | N/A | ¹ CPython 3.8's final binary release has experimental Universal2 support, but does not support macOS 10.x, so this is not currently available.
+² Available as a prerelease under a [flag](https://cibuildwheel.readthedocs.io/en/stable/options/#prerelease-pythons)
- Builds manylinux, macOS 10.9+, and Windows wheels for CPython and PyPy - Works on GitHub Actions, Azure Pipelines, Travis CI, AppVeyor, CircleCI, and GitLab CI @@ -108,6 +110,7 @@ Options | | [`CIBW_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip)
[`CIBW_SKIP`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip) | Choose the Python versions to build | | | [`CIBW_ARCHS`](https://cibuildwheel.readthedocs.io/en/stable/options/#archs) | Change the architectures built on your machine by default | | | [`CIBW_PROJECT_REQUIRES_PYTHON`](https://cibuildwheel.readthedocs.io/en/stable/options/#requires-python) | Manually set the Python compatibility of your project | +| | [`CIBW_PRERELEASE_PYTHONS`](https://cibuildwheel.readthedocs.io/en/stable/options/#prerelease-pythons) | Enable building with pre-release versions of Python | | **Build customization** | [`CIBW_ENVIRONMENT`](https://cibuildwheel.readthedocs.io/en/stable/options/#environment) | Set environment variables needed during the build | | | [`CIBW_BEFORE_ALL`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-all) | Execute a shell command on the build system before any wheels are built. | | | [`CIBW_BEFORE_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-build) | Execute a shell command preparing each wheel's build | diff --git a/bin/update_pythons.py b/bin/update_pythons.py index 6b49eaa5..eb813344 100755 --- a/bin/update_pythons.py +++ b/bin/update_pythons.py @@ -6,7 +6,7 @@ import copy import difflib import logging from pathlib import Path -from typing import Any, Union +from typing import Any, Iterable, Union, cast import click import requests @@ -79,23 +79,26 @@ class WindowsVersions: response.raise_for_status() cp_info = response.json() - versions = (Version(v) for v in cp_info["versions"]) - self.versions = sorted(v for v in versions if not v.is_devrelease) + self.version_dict = {Version(v): v for v in cp_info["versions"]} def update_version_windows(self, spec: Specifier) -> ConfigWinCP | None: - versions = sorted(v for v in self.versions if spec.contains(v)) - if not all(v.is_prerelease for v in versions): - versions = [v for v in versions if not v.is_prerelease] + + # Specifier.filter selects all non pre-releases that match the spec, + # unless there are only pre-releases, then it selects pre-releases + # instead (like pip) + unsorted_versions = cast(Iterable[Version], spec.filter(self.version_dict)) + versions = sorted(unsorted_versions, reverse=True) + log.debug(f"Windows {self.arch} {spec} has {', '.join(str(v) for v in versions)}") if not versions: return None - version = versions[-1] + version = versions[0] identifier = f"cp{version.major}{version.minor}-{self.arch}" return ConfigWinCP( identifier=identifier, - version=str(version), + version=self.version_dict[version], arch=self.arch_str, ) @@ -190,21 +193,23 @@ class CPythonVersions: # Removing the prefix, Python 3.9 would use: release["name"].removeprefix("Python ") version = Version(release["name"][7:]) - if not version.is_prerelease and not version.is_devrelease: - uri = int(release["resource_uri"].rstrip("/").split("/")[-1]) - self.versions_dict[version] = uri + uri = int(release["resource_uri"].rstrip("/").split("/")[-1]) + self.versions_dict[version] = uri def update_version_macos( self, identifier: str, version: Version, spec: Specifier ) -> ConfigMacOS | None: - sorted_versions = sorted(v for v in self.versions_dict if spec.contains(v)) + + # see note above on Specifier.filter + unsorted_versions = cast(Iterable[Version], spec.filter(self.versions_dict)) + sorted_versions = sorted(unsorted_versions, reverse=True) if version <= Version("3.8.9999"): file_ident = "macosx10.9.pkg" else: file_ident = "macos11.pkg" - for new_version in reversed(sorted_versions): + for new_version in sorted_versions: # Find the first patch version that contains the requested file uri = self.versions_dict[new_version] response = requests.get( @@ -270,7 +275,7 @@ class AllVersions: @click.command() @click.option("--force", is_flag=True) @click.option( - "--level", default="INFO", type=click.Choice(["INFO", "DEBUG", "TRACE"], case_sensitive=False) + "--level", default="INFO", type=click.Choice(["WARNING", "INFO", "DEBUG"], case_sensitive=False) ) def update_pythons(force: bool, level: str) -> None: diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 6c4ec3cc..3bd68e66 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -132,6 +132,12 @@ def main() -> None: help="Do not report an error code if the build does not match any wheels.", ) + parser.add_argument( + "--prerelease-pythons", + action="store_true", + help="Enable pre-release Python versions if available.", + ) + args = parser.parse_args() detect_obsolete_options() @@ -206,6 +212,9 @@ def main() -> None: build_verbosity_str = get_option_from_environment( "CIBW_BUILD_VERBOSITY", platform=platform, default="" ) + prerelease_pythons = args.prerelease_pythons or cibuildwheel.util.strtobool( + os.environ.get("CIBW_PRERELEASE_PYTHONS", "0") + ) package_files = {"setup.py", "setup.cfg", "pyproject.toml"} @@ -222,8 +231,12 @@ def main() -> None: ) or get_requires_python_str(package_dir) requires_python = None if requires_python_str is None else SpecifierSet(requires_python_str) + # Hardcode pre-releases here, current: Python 3.10 build_selector = BuildSelector( - build_config=build_config, skip_config=skip_config, requires_python=requires_python + build_config=build_config, + skip_config=skip_config, + requires_python=requires_python, + prerelease_pythons=prerelease_pythons, ) test_selector = TestSelector(skip_config=test_skip) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index 654f9faa..0ade9751 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -186,7 +186,7 @@ def build_description_from_identifier(identifier: str) -> str: build_description = "" python_interpreter = python_identifier[0:2] - python_version = python_identifier[2:4] + python_version = python_identifier[2:] if python_interpreter == "cp": build_description += "CPython" @@ -195,7 +195,7 @@ def build_description_from_identifier(identifier: str) -> str: else: raise Exception("unknown python") - build_description += f" {python_version[0]}.{python_version[1]} " + build_description += f" {python_version[0]}.{python_version[1:]} " try: build_description += PLATFORM_IDENTIFIER_DESCIPTIONS[platform_identifier] diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index f3c1f8ea..d494c45f 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -270,7 +270,7 @@ def setup_python( config_is_arm64 = python_configuration.identifier.endswith("arm64") config_is_universal2 = python_configuration.identifier.endswith("universal2") - if python_configuration.version == "3.9": + if python_configuration.version not in {"3.6", "3.7", "3.8"}: if python_configuration.identifier.endswith("x86_64"): # even on the macos11.0 Python installer, on the x86_64 side it's # compatible back to 10.9. diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index 65012101..608598c5 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -4,23 +4,28 @@ python_configurations = [ { identifier = "cp37-manylinux_x86_64", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, { identifier = "cp38-manylinux_x86_64", version = "3.8", path_str = "/opt/python/cp38-cp38" }, { identifier = "cp39-manylinux_x86_64", version = "3.9", path_str = "/opt/python/cp39-cp39" }, + { identifier = "cp310-manylinux_x86_64", version = "3.10", path_str = "/opt/python/cp310-cp310" }, { identifier = "cp36-manylinux_i686", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, { identifier = "cp37-manylinux_i686", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, { identifier = "cp38-manylinux_i686", version = "3.8", path_str = "/opt/python/cp38-cp38" }, { identifier = "cp39-manylinux_i686", version = "3.9", path_str = "/opt/python/cp39-cp39" }, + { identifier = "cp310-manylinux_i686", version = "3.10", path_str = "/opt/python/cp310-cp310" }, { identifier = "pp37-manylinux_x86_64", version = "3.7", path_str = "/opt/python/pp37-pypy37_pp73" }, { identifier = "cp36-manylinux_aarch64", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, { identifier = "cp37-manylinux_aarch64", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, { identifier = "cp38-manylinux_aarch64", version = "3.8", path_str = "/opt/python/cp38-cp38" }, { identifier = "cp39-manylinux_aarch64", version = "3.9", path_str = "/opt/python/cp39-cp39" }, + { identifier = "cp310-manylinux_aarch64", version = "3.10", path_str = "/opt/python/cp310-cp310" }, { identifier = "cp36-manylinux_ppc64le", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, { identifier = "cp37-manylinux_ppc64le", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, { identifier = "cp38-manylinux_ppc64le", version = "3.8", path_str = "/opt/python/cp38-cp38" }, { identifier = "cp39-manylinux_ppc64le", version = "3.9", path_str = "/opt/python/cp39-cp39" }, + { identifier = "cp310-manylinux_ppc64le", version = "3.10", path_str = "/opt/python/cp310-cp310" }, { identifier = "cp36-manylinux_s390x", version = "3.6", path_str = "/opt/python/cp36-cp36m" }, { identifier = "cp37-manylinux_s390x", version = "3.7", path_str = "/opt/python/cp37-cp37m" }, { identifier = "cp38-manylinux_s390x", version = "3.8", path_str = "/opt/python/cp38-cp38" }, { identifier = "cp39-manylinux_s390x", version = "3.9", path_str = "/opt/python/cp39-cp39" }, + { identifier = "cp310-manylinux_s390x", version = "3.10", path_str = "/opt/python/cp310-cp310" }, { identifier = "pp37-manylinux_aarch64", version = "3.7", path_str = "/opt/python/pp37-pypy37_pp73" }, { identifier = "pp37-manylinux_i686", version = "3.7", path_str = "/opt/python/pp37-pypy37_pp73" }, ] @@ -33,6 +38,9 @@ python_configurations = [ { identifier = "cp39-macosx_x86_64", version = "3.9", url = "https://www.python.org/ftp/python/3.9.5/python-3.9.5-macos11.pkg" }, { identifier = "cp39-macosx_arm64", version = "3.9", url = "https://www.python.org/ftp/python/3.9.5/python-3.9.5-macos11.pkg" }, { identifier = "cp39-macosx_universal2", version = "3.9", url = "https://www.python.org/ftp/python/3.9.5/python-3.9.5-macos11.pkg" }, + { identifier = "cp310-macosx_x86_64", version = "3.10", url = "https://www.python.org/ftp/python/3.10.0/python-3.10.0b1-macos11.pkg" }, + { identifier = "cp310-macosx_arm64", version = "3.10", url = "https://www.python.org/ftp/python/3.10.0/python-3.10.0b1-macos11.pkg" }, + { identifier = "cp310-macosx_universal2", version = "3.10", url = "https://www.python.org/ftp/python/3.10.0/python-3.10.0b1-macos11.pkg" }, { identifier = "pp37-macosx_x86_64", version = "3.7", url = "https://downloads.python.org/pypy/pypy3.7-v7.3.5-osx64.tar.bz2" }, ] @@ -46,5 +54,7 @@ python_configurations = [ { identifier = "cp38-win_amd64", version = "3.8.10", arch = "64" }, { identifier = "cp39-win32", version = "3.9.5", arch = "32" }, { identifier = "cp39-win_amd64", version = "3.9.5", arch = "64" }, + { identifier = "cp310-win32", version = "3.10.0-b1", arch = "32" }, + { identifier = "cp310-win_amd64", version = "3.10.0-b1", arch = "64" }, { identifier = "pp37-win_amd64", version = "3.7", arch = "64", url = "https://downloads.python.org/pypy/pypy3.7-v7.3.5-win64.zip" }, ] diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index b61386cc..5d5c4768 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -58,15 +58,25 @@ class IdentifierSelector: This class holds a set of build/skip patterns. You call an instance with a build identifier, and it returns True if that identifier should be included. Only call this on valid identifiers, ones that have at least 2 - numeric digits before the first dash. + numeric digits before the first dash. If a pre-release version X.Y is present, + you can filter it with prerelease="XY". """ + # a pattern that skips prerelease versions, when include_prereleases is False. + PRERELEASE_SKIP = "cp310-*" + def __init__( - self, *, build_config: str, skip_config: str, requires_python: Optional[SpecifierSet] = None + self, + *, + build_config: str, + skip_config: str, + requires_python: Optional[SpecifierSet] = None, + prerelease_pythons: bool = False, ): self.build_patterns = build_config.split() self.skip_patterns = skip_config.split() self.requires_python = requires_python + self.prerelease_pythons = prerelease_pythons def __call__(self, build_id: str) -> bool: # Filter build selectors by python_requires if set @@ -81,17 +91,33 @@ class IdentifierSelector: build_patterns = itertools.chain.from_iterable( bracex.expand(p) for p in self.build_patterns ) - skip_patterns = itertools.chain.from_iterable(bracex.expand(p) for p in self.skip_patterns) + + unexpanded_skip_patterns = self.skip_patterns.copy() + + if not self.prerelease_pythons: + # filter out the prerelease pythons, alongside the user-defined + # skip patterns + unexpanded_skip_patterns += BuildSelector.PRERELEASE_SKIP.split() + + skip_patterns = itertools.chain.from_iterable( + bracex.expand(p) for p in unexpanded_skip_patterns + ) build: bool = any(fnmatch.fnmatch(build_id, pat) for pat in build_patterns) skip: bool = any(fnmatch.fnmatch(build_id, pat) for pat in skip_patterns) return build and not skip def __repr__(self) -> str: - if not self.skip_patterns: - return f'{self.__class__.__name__}({" ".join(self.build_patterns)!r})' - else: - return f'{self.__class__.__name__}({" ".join(self.build_patterns)!r} - {" ".join(self.skip_patterns)!r})' + result = f'{self.__class__.__name__}(build_config={" ".join(self.build_patterns)!r}' + + if self.skip_patterns: + result += f', skip_config={" ".join(self.skip_patterns)!r}' + if self.prerelease_pythons: + result += ", prerelease_pythons=True" + + result += ")" + + return result class BuildSelector(IdentifierSelector): diff --git a/docs/options.md b/docs/options.md index 4332e683..f0afc6e1 100644 --- a/docs/options.md +++ b/docs/options.md @@ -299,6 +299,32 @@ the package is compatible with all versions of Python that it can build. CIBW_PROJECT_REQUIRES_PYTHON: ">=3.6" ``` +### `CIBW_PRERELEASE_PYTHONS` {: #prerelease-pythons} +> Enable building with pre-release versions of Python + +During the beta period, when new versions of Python are being tested, +cibuildwheel will often gain early support for beta releases. If you would +like to test wheel building with these versions, you can enable this flag. + +!!! caution + This option is provided for testing purposes only. It is not + recommended to distribute wheels built when `CIBW_PRERELEASE_PYTHONS` is + set, such as uploading to PyPI. Please _do not_ upload these wheels to + PyPI, as they are not guaranteed to work with the final Python release. + Once Python is ABI stable and enters the release candidate phase, that + version of Python will become available without this flag. + +Default: Off (0) if Python is available in beta phase. No effect otherwise. + +This option can also be set using the [command-line option](#command-line) `--prerelease-pythons`. + +#### Examples + +```yaml +# Include latest Python beta +CIBW_PRERELEASE_PYTHONS: True +``` + ## Build customization ### `CIBW_ENVIRONMENT` {: #environment} @@ -715,6 +741,7 @@ CIBW_BUILD_VERBOSITY: 1 usage: cibuildwheel [-h] [--platform {auto,linux,macos,windows}] [--archs ARCHS] [--output-dir OUTPUT_DIR] [--print-build-identifiers] [--allow-empty] + [--prerelease-pythons] [package_dir] Build wheels for all the platforms. @@ -741,8 +768,8 @@ optional arguments: natively supported on this machine. Set this option to build an architecture via emulation, for example, using binfmt_misc and QEMU. Default: auto. Choices: - auto, native, all, x86_64, i686, aarch64, ppc64le, - s390x, x86, AMD64 + auto, auto64, auto32, native, all, x86_64, i686, + aarch64, ppc64le, s390x, universal2, arm64, x86, AMD64 --output-dir OUTPUT_DIR Destination folder for the wheels. --print-build-identifiers @@ -750,6 +777,7 @@ optional arguments: invocation and exit. --allow-empty Do not report an error code if the build does not match any wheels. + --prerelease-pythons Enable pre-release Python versions if available. ```