From 130fdd2548a297d8118d7d66efaf81087c38aabe Mon Sep 17 00:00:00 2001 From: "cibuildwheel-bot[bot]" <83877280+cibuildwheel-bot[bot]@users.noreply.github.com> Date: Sat, 8 Jun 2024 10:15:04 +0200 Subject: [PATCH] feat: free-threaded Python for macOS, MACOSX_DEPLOYMENT_TARGET updates (#1854) * Update dependencies * feat: macos free-threading Signed-off-by: Henry Schreiner * fix: set min target to 10.13 for 3.13 and floor it always Signed-off-by: Henry Schreiner * fix: tests and corrected pypy mins Signed-off-by: Henry Schreiner --------- Signed-off-by: Henry Schreiner Co-authored-by: cibuildwheel-bot[bot] <83877280+cibuildwheel-bot[bot]@users.noreply.github.com> Co-authored-by: Henry Schreiner Co-authored-by: Matthieu Darbois --- .github/workflows/test.yml | 2 + cibuildwheel/macos.py | 54 +++++++++++++++---- cibuildwheel/resources/build-platforms.toml | 27 +++++----- .../resources/free-threaded-enable-313.xml | 14 +++++ .../resources/pinned_docker_images.cfg | 20 +++---- cibuildwheel/util.py | 2 + test/test_macos_archs.py | 2 +- test/utils.py | 33 ++++++------ 8 files changed, 105 insertions(+), 49 deletions(-) create mode 100644 cibuildwheel/resources/free-threaded-enable-313.xml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 196ebd7a..31c6a4b0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -82,6 +82,8 @@ jobs: output-dir: wheelhouse env: CIBW_ARCHS_MACOS: x86_64 universal2 arm64 + CIBW_FREE_THREADED_SUPPORT: 1 + CIBW_PRERELEASE_PYTHONS: 1 - name: Run a sample build (GitHub Action, only) uses: ./ diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 813531b0..2906b584 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -33,6 +33,7 @@ from .util import ( detect_ci_provider, download, find_compatible_wheel, + free_thread_enable_313, get_build_verbosity_extra_flags, get_pip_version, install_certifi_script, @@ -115,12 +116,13 @@ def get_python_configurations( return python_configurations -def install_cpython(tmp: Path, version: str, url: str) -> Path: - installation_path = Path(f"/Library/Frameworks/Python.framework/Versions/{version}") +def install_cpython(tmp: Path, version: str, url: str, free_threading: bool) -> Path: + ft = "T" if free_threading else "" + installation_path = Path(f"/Library/Frameworks/Python{ft}.framework/Versions/{version}") with FileLock(CIBW_CACHE_PATH / f"cpython{version}.lock"): installed_system_packages = call("pkgutil", "--pkgs", capture_stdout=True).splitlines() # if this version of python isn't installed, get it from python.org and install - python_package_identifier = f"org.python.Python.PythonFramework-{version}" + python_package_identifier = f"org.python.Python.Python{ft}Framework-{version}" if python_package_identifier not in installed_system_packages: if detect_ci_provider() is None: # if running locally, we don't want to install CPython with sudo @@ -137,13 +139,22 @@ def install_cpython(tmp: Path, version: str, url: str) -> Path: # download the pkg download(url, pkg_path) # install - call("sudo", "installer", "-pkg", pkg_path, "-target", "/") + args = [] + if version.startswith("3.13"): + # Python 3.13 is the first version to have a free-threading option + args += ["-applyChoiceChangesXML", str(free_thread_enable_313.resolve())] + call("sudo", "installer", "-pkg", pkg_path, *args, "-target", "/") pkg_path.unlink() env = os.environ.copy() env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1" - call(installation_path / "bin" / "python3", install_certifi_script, env=env) - return installation_path / "bin" / "python3" + if free_threading: + call(installation_path / f"bin/python{version}t", "-m", "ensurepip", env=env) + call(installation_path / f"bin/python{version}t", install_certifi_script, env=env) + else: + call(installation_path / "bin/python3", install_certifi_script, env=env) + + return installation_path / "bin" / (f"python{version}t" if free_threading else "python3") def install_pypy(tmp: Path, url: str) -> Path: @@ -172,13 +183,19 @@ def setup_python( implementation_id = python_configuration.identifier.split("-")[0] log.step(f"Installing Python {implementation_id}...") if implementation_id.startswith("cp"): - base_python = install_cpython(tmp, python_configuration.version, python_configuration.url) + free_threading = "t-macos" in python_configuration.identifier + base_python = install_cpython( + tmp, python_configuration.version, python_configuration.url, free_threading + ) + elif implementation_id.startswith("pp"): base_python = install_pypy(tmp, python_configuration.url) else: msg = "Unknown Python implementation" raise ValueError(msg) - assert base_python.exists() + assert ( + base_python.exists() + ), f"{base_python.name} not found, has {list(base_python.parent.iterdir())}" log.step("Setting up build environment...") venv_path = tmp / "venv" @@ -244,8 +261,25 @@ def setup_python( # Set MACOSX_DEPLOYMENT_TARGET, if the user didn't set it. # For arm64, the minimal deployment target is 11.0. # On x86_64 (or universal2), use 10.9 as a default. - # PyPy defaults to 10.7, causing inconsistencies if it's left unset. - env.setdefault("MACOSX_DEPLOYMENT_TARGET", "11.0" if config_is_arm64 else "10.9") + # CPython 3.13 needs 10.13. + if config_is_arm64: + default_target = "11.0" + elif Version(python_configuration.version) >= Version("3.13"): + default_target = "10.13" + elif python_configuration.identifier.startswith("pp") and Version( + python_configuration.version + ) >= Version("3.9"): + default_target = "10.15" + else: + default_target = "10.9" + env.setdefault("MACOSX_DEPLOYMENT_TARGET", default_target) + + # This is a floor, it can't be set lower than the default_target. + if Version(env["MACOSX_DEPLOYMENT_TARGET"]) < Version(default_target): + log.warning( + f"Bumping MACOSX_DEPLOYMENT_TARGET ({env['MACOSX_DEPLOYMENT_TARGET']}) to the minimum required ({default_target})." + ) + env["MACOSX_DEPLOYMENT_TARGET"] = default_target if python_configuration.version not in {"3.6", "3.7"}: if config_is_arm64: diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index fefdf45a..4871108e 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -123,9 +123,12 @@ python_configurations = [ { identifier = "cp312-macosx_x86_64", version = "3.12", url = "https://www.python.org/ftp/python/3.12.3/python-3.12.3-macos11.pkg" }, { identifier = "cp312-macosx_arm64", version = "3.12", url = "https://www.python.org/ftp/python/3.12.3/python-3.12.3-macos11.pkg" }, { identifier = "cp312-macosx_universal2", version = "3.12", url = "https://www.python.org/ftp/python/3.12.3/python-3.12.3-macos11.pkg" }, - { identifier = "cp313-macosx_x86_64", version = "3.13", url = "https://www.python.org/ftp/python/3.13.0/python-3.13.0b1-macos11.pkg" }, - { identifier = "cp313-macosx_arm64", version = "3.13", url = "https://www.python.org/ftp/python/3.13.0/python-3.13.0b1-macos11.pkg" }, - { identifier = "cp313-macosx_universal2", version = "3.13", url = "https://www.python.org/ftp/python/3.13.0/python-3.13.0b1-macos11.pkg" }, + { identifier = "cp313-macosx_x86_64", version = "3.13", url = "https://www.python.org/ftp/python/3.13.0/python-3.13.0b2-macos11.pkg" }, + { identifier = "cp313-macosx_arm64", version = "3.13", url = "https://www.python.org/ftp/python/3.13.0/python-3.13.0b2-macos11.pkg" }, + { identifier = "cp313-macosx_universal2", version = "3.13", url = "https://www.python.org/ftp/python/3.13.0/python-3.13.0b2-macos11.pkg" }, + { identifier = "cp313t-macosx_x86_64", version = "3.13", url = "https://www.python.org/ftp/python/3.13.0/python-3.13.0b2-macos11.pkg" }, + { identifier = "cp313t-macosx_arm64", version = "3.13", url = "https://www.python.org/ftp/python/3.13.0/python-3.13.0b2-macos11.pkg" }, + { identifier = "cp313t-macosx_universal2", version = "3.13", url = "https://www.python.org/ftp/python/3.13.0/python-3.13.0b2-macos11.pkg" }, { identifier = "pp37-macosx_x86_64", version = "3.7", url = "https://downloads.python.org/pypy/pypy3.7-v7.3.9-osx64.tar.bz2" }, { identifier = "pp38-macosx_x86_64", version = "3.8", url = "https://downloads.python.org/pypy/pypy3.8-v7.3.11-macos_x86_64.tar.bz2" }, { identifier = "pp38-macosx_arm64", version = "3.8", url = "https://downloads.python.org/pypy/pypy3.8-v7.3.11-macos_arm64.tar.bz2" }, @@ -149,18 +152,18 @@ python_configurations = [ { identifier = "cp310-win_amd64", version = "3.10.11", arch = "64" }, { identifier = "cp311-win32", version = "3.11.9", arch = "32" }, { identifier = "cp311-win_amd64", version = "3.11.9", arch = "64" }, - { identifier = "cp312-win32", version = "3.12.3", arch = "32" }, - { identifier = "cp312-win_amd64", version = "3.12.3", arch = "64" }, - { identifier = "cp313-win32", version = "3.13.0-b1", arch = "32" }, - { identifier = "cp313t-win32", version = "3.13.0-b1", arch = "32" }, - { identifier = "cp313-win_amd64", version = "3.13.0-b1", arch = "64" }, - { identifier = "cp313t-win_amd64", version = "3.13.0-b1", arch = "64" }, + { identifier = "cp312-win32", version = "3.12.4", arch = "32" }, + { identifier = "cp312-win_amd64", version = "3.12.4", arch = "64" }, + { identifier = "cp313-win32", version = "3.13.0-b2", arch = "32" }, + { identifier = "cp313t-win32", version = "3.13.0-b2", arch = "32" }, + { identifier = "cp313-win_amd64", version = "3.13.0-b2", arch = "64" }, + { identifier = "cp313t-win_amd64", version = "3.13.0-b2", arch = "64" }, { identifier = "cp39-win_arm64", version = "3.9.10", arch = "ARM64" }, { identifier = "cp310-win_arm64", version = "3.10.11", arch = "ARM64" }, { identifier = "cp311-win_arm64", version = "3.11.9", arch = "ARM64" }, - { identifier = "cp312-win_arm64", version = "3.12.3", arch = "ARM64" }, - { identifier = "cp313-win_arm64", version = "3.13.0-b1", arch = "ARM64" }, - { identifier = "cp313t-win_arm64", version = "3.13.0-b1", arch = "ARM64" }, + { identifier = "cp312-win_arm64", version = "3.12.4", arch = "ARM64" }, + { identifier = "cp313-win_arm64", version = "3.13.0-b2", arch = "ARM64" }, + { identifier = "cp313t-win_arm64", version = "3.13.0-b2", arch = "ARM64" }, { identifier = "pp37-win_amd64", version = "3.7", arch = "64", url = "https://downloads.python.org/pypy/pypy3.7-v7.3.9-win64.zip" }, { identifier = "pp38-win_amd64", version = "3.8", arch = "64", url = "https://downloads.python.org/pypy/pypy3.8-v7.3.11-win64.zip" }, { identifier = "pp39-win_amd64", version = "3.9", arch = "64", url = "https://downloads.python.org/pypy/pypy3.9-v7.3.16-win64.zip" }, diff --git a/cibuildwheel/resources/free-threaded-enable-313.xml b/cibuildwheel/resources/free-threaded-enable-313.xml new file mode 100644 index 00000000..0c1c6c78 --- /dev/null +++ b/cibuildwheel/resources/free-threaded-enable-313.xml @@ -0,0 +1,14 @@ + + + + + + attributeSetting + 1 + choiceAttribute + selected + choiceIdentifier + org.python.Python.PythonTFramework-3.13 + + + diff --git a/cibuildwheel/resources/pinned_docker_images.cfg b/cibuildwheel/resources/pinned_docker_images.cfg index 3a536913..67a49ea2 100644 --- a/cibuildwheel/resources/pinned_docker_images.cfg +++ b/cibuildwheel/resources/pinned_docker_images.cfg @@ -1,29 +1,29 @@ [x86_64] manylinux1 = quay.io/pypa/manylinux1_x86_64:2024-04-29-76807b8 manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2022-08-05-4535177 -manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2024-06-03-e195670 +manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2024-06-06-99f15a7 manylinux_2_24 = quay.io/pypa/manylinux_2_24_x86_64:2022-12-26-0d38463 -manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64:2024-06-03-e195670 -musllinux_1_1 = quay.io/pypa/musllinux_1_1_x86_64:2024-06-03-e195670 -musllinux_1_2 = quay.io/pypa/musllinux_1_2_x86_64:2024-06-03-e195670 +manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64:2024-06-06-99f15a7 +musllinux_1_1 = quay.io/pypa/musllinux_1_1_x86_64:2024-06-06-99f15a7 +musllinux_1_2 = quay.io/pypa/musllinux_1_2_x86_64:2024-06-06-99f15a7 [i686] manylinux1 = quay.io/pypa/manylinux1_i686:2024-04-29-76807b8 manylinux2010 = quay.io/pypa/manylinux2010_i686:2022-08-05-4535177 -manylinux2014 = quay.io/pypa/manylinux2014_i686:2024-06-03-e195670 +manylinux2014 = quay.io/pypa/manylinux2014_i686:2024-06-06-99f15a7 manylinux_2_24 = quay.io/pypa/manylinux_2_24_i686:2022-12-26-0d38463 -musllinux_1_1 = quay.io/pypa/musllinux_1_1_i686:2024-06-03-e195670 -musllinux_1_2 = quay.io/pypa/musllinux_1_2_i686:2024-06-03-e195670 +musllinux_1_1 = quay.io/pypa/musllinux_1_1_i686:2024-06-06-99f15a7 +musllinux_1_2 = quay.io/pypa/musllinux_1_2_i686:2024-06-06-99f15a7 [pypy_x86_64] manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2022-08-05-4535177 -manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2024-06-03-e195670 +manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2024-06-06-99f15a7 manylinux_2_24 = quay.io/pypa/manylinux_2_24_x86_64:2022-12-26-0d38463 -manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64:2024-06-03-e195670 +manylinux_2_28 = quay.io/pypa/manylinux_2_28_x86_64:2024-06-06-99f15a7 [pypy_i686] manylinux2010 = quay.io/pypa/manylinux2010_i686:2022-08-05-4535177 -manylinux2014 = quay.io/pypa/manylinux2014_i686:2024-06-03-e195670 +manylinux2014 = quay.io/pypa/manylinux2014_i686:2024-06-06-99f15a7 manylinux_2_24 = quay.io/pypa/manylinux_2_24_i686:2022-12-26-0d38463 [aarch64] diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 2d127f2b..db6f6aa6 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -60,6 +60,8 @@ resources_dir: Final[Path] = Path(__file__).parent / "resources" install_certifi_script: Final[Path] = resources_dir / "install_certifi.py" +free_thread_enable_313: Final[Path] = resources_dir / "free-threaded-enable-313.xml" + test_fail_cwd_file: Final[Path] = resources_dir / "testing_temp_dir_file.py" diff --git a/test/test_macos_archs.py b/test/test_macos_archs.py index d29118b6..31e57405 100644 --- a/test/test_macos_archs.py +++ b/test/test_macos_archs.py @@ -14,7 +14,7 @@ ALL_MACOS_WHEELS = { *utils.expected_wheels("spam", "0.1.0", machine_arch="arm64", include_universal2=True), } -DEPLOYMENT_TARGET_TOO_LOW_WARNING = "[WARNING] MACOSX_DEPLOYMENT_TARGET is set to a lower value" +DEPLOYMENT_TARGET_TOO_LOW_WARNING = "Bumping MACOSX_DEPLOYMENT_TARGET" def get_xcode_version() -> tuple[int, int]: diff --git a/test/utils.py b/test/utils.py index 779d9a51..eb9f8774 100644 --- a/test/utils.py +++ b/test/utils.py @@ -137,17 +137,11 @@ def cibuildwheel_run( return wheels -def _get_arm64_macosx_deployment_target(macosx_deployment_target: str) -> str: +def _floor_macosx(*args: str) -> str: """ - The first version of macOS that supports arm is 11.0. So the wheel tag - cannot contain an earlier deployment target, even if - MACOSX_DEPLOYMENT_TARGET sets it. + Make sure a deployment target is not less than some value. """ - version_tuple = tuple(map(int, macosx_deployment_target.split("."))) - if version_tuple <= (11, 0): - return "11.0" - else: - return macosx_deployment_target + return max(args, key=lambda x: tuple(map(int, x.split(".")))) def expected_wheels( @@ -202,9 +196,8 @@ def expected_wheels( "cp311-cp311", "cp312-cp312", "cp313-cp313", + "cp313-cp313t", ] - if platform != "macos": - python_abi_tags.append("cp313-cp313t") if machine_arch in ["x86_64", "AMD64", "x86", "aarch64"]: python_abi_tags += [ @@ -223,6 +216,7 @@ def expected_wheels( "cp311-cp311", "cp312-cp312", "cp313-cp313", + "cp313-cp313t", "pp38-pypy38_pp73", "pp39-pypy39_pp73", "pp310-pypy310_pp73", @@ -282,12 +276,19 @@ def expected_wheels( elif platform == "macos": if machine_arch == "arm64": - arm64_macosx_deployment_target = _get_arm64_macosx_deployment_target( - macosx_deployment_target - ) - platform_tags = [f'macosx_{arm64_macosx_deployment_target.replace(".", "_")}_arm64'] + arm64_macosx = _floor_macosx(macosx_deployment_target, "11.0") + platform_tags = [f'macosx_{arm64_macosx.replace(".", "_")}_arm64'] else: - platform_tags = [f'macosx_{macosx_deployment_target.replace(".", "_")}_x86_64'] + if python_abi_tag.startswith("pp") and not python_abi_tag.startswith( + ("pp37", "pp38") + ): + pypy_macosx = _floor_macosx(macosx_deployment_target, "10.15") + platform_tags = [f'macosx_{pypy_macosx.replace(".", "_")}_x86_64'] + elif python_abi_tag.startswith("cp313"): + pypy_macosx = _floor_macosx(macosx_deployment_target, "10.13") + platform_tags = [f'macosx_{pypy_macosx.replace(".", "_")}_x86_64'] + else: + platform_tags = [f'macosx_{macosx_deployment_target.replace(".", "_")}_x86_64'] if include_universal2: platform_tags.append(