From 594d89f6f7cc6f6ef10698edfa00105ce8daba07 Mon Sep 17 00:00:00 2001 From: Niyas Sait Date: Wed, 24 Nov 2021 10:50:49 -0800 Subject: [PATCH] add support for building wheels for windows on arm64 (#920) * add support for building wheels for windows on arm64 * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fix main_platform_test.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Remove unrequired explicit nuget source * add python arm64 3.9.9 version * extend update_pythons.py to add win/arm64 support * update documentation for new win/arm64 platform * style: include ARM64 in literal * Fix python 3.9 identifier for win_arm64 * Fix table layout issues in README and add note about arm64 pip issue Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Henry Schreiner --- README.md | 17 +++++----- bin/update_pythons.py | 10 ++++-- cibuildwheel/architecture.py | 3 +- cibuildwheel/logger.py | 1 + cibuildwheel/resources/build-platforms.toml | 2 ++ cibuildwheel/windows.py | 35 ++++++++++++++++----- docs/options.md | 6 ++-- unit_test/main_tests/main_platform_test.py | 6 +++- 8 files changed, 58 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index 9e129dd9..6cd25e3d 100644 --- a/README.md +++ b/README.md @@ -22,16 +22,17 @@ Python wheels are great. Building them across **Mac, Linux, Windows**, on **mult What does it do? ---------------- -| | macOS Intel | macOS Apple Silicon | Windows 64bit | Windows 32bit | manylinux
musllinux x86_64 | manylinux
musllinux i686 | manylinux
musllinux aarch64 | manylinux
musllinux ppc64le | manylinux
musllinux s390x | -|---------------|----|-----|-----|-----|----|-----|----|-----|-----| -| CPython 3.6 | ✅ | N/A | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| CPython 3.7 | ✅ | N/A | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| CPython 3.8 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| CPython 3.9 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| CPython 3.10 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | -| PyPy 3.7 v7.3 | ✅ | N/A | ✅ | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | +| | macOS Intel | macOS Apple Silicon | Windows 64bit | Windows 32bit | Windows Arm64 | manylinux
musllinux x86_64 | manylinux
musllinux i686 | manylinux
musllinux aarch64 | manylinux
musllinux ppc64le | manylinux
musllinux s390x | +|---------------|----|-----|-----|-----|-----|----|-----|----|-----|-----| +| CPython 3.6 | ✅ | N/A | ✅ | ✅ | N/A | ✅ | ✅ | ✅ | ✅ | ✅ | +| CPython 3.7 | ✅ | N/A | ✅ | ✅ | N/A | ✅ |✅ | ✅ | ✅ | ✅ | +| CPython 3.8 | ✅ | ✅ | ✅ | ✅ | N/A | ✅ | ✅ | ✅ | ✅ | ✅ | +| CPython 3.9 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | +| CPython 3.10 | ✅ | ✅ | ✅ | ✅ | ✅² | ✅ | ✅ | ✅ | ✅ | ✅ | +| PyPy 3.7 v7.3 | ✅ | N/A | ✅ | N/A | N/A | ✅¹ | ✅¹ | ✅¹ | N/A | N/A | ¹ PyPy is only supported for manylinux wheels.
+² Windows arm64 support is experimental.
- Builds manylinux, musllinux, macOS 10.9+, and Windows wheels for CPython and PyPy - Works on GitHub Actions, Azure Pipelines, Travis CI, AppVeyor, CircleCI, and GitLab CI diff --git a/bin/update_pythons.py b/bin/update_pythons.py index 488ccbc1..30859259 100755 --- a/bin/update_pythons.py +++ b/bin/update_pythons.py @@ -28,7 +28,7 @@ DIR: Final[Path] = Path(__file__).parent.parent.resolve() RESOURCES_DIR: Final[Path] = DIR / "cibuildwheel/resources" -ArchStr = Literal["32", "64"] +ArchStr = Literal["32", "64", "ARM64"] class ConfigWinCP(TypedDict): @@ -68,8 +68,8 @@ class WindowsVersions: if resource["@type"] == "PackageBaseAddress/3.0.0": endpoint = resource["@id"] - ARCH_DICT = {"32": "win32", "64": "win_amd64"} - PACKAGE_DICT = {"32": "pythonx86", "64": "python"} + ARCH_DICT = {"32": "win32", "64": "win_amd64", "ARM64": "win_arm64"} + PACKAGE_DICT = {"32": "pythonx86", "64": "python", "ARM64": "pythonarm64"} self.arch_str = arch_str self.arch = ARCH_DICT[arch_str] @@ -237,6 +237,7 @@ class AllVersions: def __init__(self) -> None: self.windows_32 = WindowsVersions("32") self.windows_64 = WindowsVersions("64") + self.windows_arm64 = WindowsVersions("ARM64") self.windows_pypy_64 = PyPyVersions("64") self.macos_cpython = CPythonVersions() @@ -264,6 +265,9 @@ class AllVersions: config_update = self.windows_64.update_version_windows(spec) elif identifier.startswith("pp"): config_update = self.windows_pypy_64.update_version_windows(spec) + elif "win_arm64" in identifier: + if identifier.startswith("cp"): + config_update = self.windows_arm64.update_version_windows(spec) assert config_update is not None, f"{identifier} not found!" config.update(**config_update) diff --git a/cibuildwheel/architecture.py b/cibuildwheel/architecture.py index ec6d144b..9d39d553 100644 --- a/cibuildwheel/architecture.py +++ b/cibuildwheel/architecture.py @@ -29,6 +29,7 @@ class Architecture(Enum): # windows archs x86 = "x86" AMD64 = "AMD64" + ARM64 = "ARM64" # Allow this to be sorted def __lt__(self, other: "Architecture") -> bool: @@ -83,7 +84,7 @@ class Architecture(Enum): elif platform == "macos": return {Architecture.x86_64, Architecture.arm64, Architecture.universal2} elif platform == "windows": - return {Architecture.x86, Architecture.AMD64} + return {Architecture.x86, Architecture.AMD64, Architecture.ARM64} else: assert_never(platform) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index 12c4462a..7f7001b6 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -27,6 +27,7 @@ PLATFORM_IDENTIFIER_DESCIPTIONS = { "musllinux_s390x": "manylinux s390x", "win32": "Windows 32bit", "win_amd64": "Windows 64bit", + "win_arm64": "Windows on ARM 64bit", "macosx_x86_64": "macOS x86_64", "macosx_universal2": "macOS Universal 2 - x86_64 and arm64", "macosx_arm64": "macOS arm64 - Apple Silicon", diff --git a/cibuildwheel/resources/build-platforms.toml b/cibuildwheel/resources/build-platforms.toml index 78efd3f8..c9572029 100644 --- a/cibuildwheel/resources/build-platforms.toml +++ b/cibuildwheel/resources/build-platforms.toml @@ -87,6 +87,8 @@ python_configurations = [ { identifier = "cp39-win_amd64", version = "3.9.8", arch = "64" }, { identifier = "cp310-win32", version = "3.10.0", arch = "32" }, { identifier = "cp310-win_amd64", version = "3.10.0", arch = "64" }, + { identifier = "cp39-win_arm64", version = "3.9.9", arch = "ARM64" }, + { identifier = "cp310-win_arm64", version = "3.10.0", arch = "ARM64" }, { identifier = "pp37-win_amd64", version = "3.7", arch = "64", url = "https://downloads.python.org/pypy/pypy3.7-v7.3.7-win64.zip" }, { identifier = "pp38-win_amd64", version = "3.8", arch = "64", url = "https://downloads.python.org/pypy/pypy3.8-v7.3.7-win64.zip" }, ] diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 2552a961..77c41cd6 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -7,6 +7,8 @@ from pathlib import Path from typing import Dict, List, NamedTuple, Optional, Sequence, Set from zipfile import ZipFile +from packaging.version import Version + from .architecture import Architecture from .environment import ParsedEnvironment from .logger import log @@ -43,9 +45,8 @@ def shell( def get_nuget_args(version: str, arch: str) -> List[str]: - python_name = "python" - if arch == "32": - python_name += "x86" + platform_suffix = {"32": "x86", "64": "", "ARM64": "arm64"} + python_name = "python" + platform_suffix[arch] return [ python_name, "-Version", @@ -73,10 +74,7 @@ def get_python_configurations( python_configurations = [PythonConfiguration(**item) for item in full_python_configs] - map_arch = { - "32": Architecture.x86, - "64": Architecture.AMD64, - } + map_arch = {"32": Architecture.x86, "64": Architecture.AMD64, "ARM64": Architecture.ARM64} # skip builds as required python_configurations = [ @@ -189,10 +187,33 @@ def setup_python( # Install pip requires_reinstall = not (installation_path / "Scripts" / "pip.exe").exists() + if requires_reinstall: # maybe pip isn't installed at all. ensurepip resolves that. call(["python", "-m", "ensurepip"], env=env, cwd=CIBW_INSTALL_PATH) + # pip older than 21.3 builds executables such as pip.exe for x64 platform. + # The first re-install of pip updates pip module but builds pip.exe using + # the old pip which still generates x64 executable. But the second + # re-install uses updated pip and correctly builds pip.exe for the target. + # This can be removed once ARM64 Pythons (currently 3.9 and 3.10) bundle + # pip versions newer than 21.3. + if python_configuration.arch == "ARM64" and Version(get_pip_version(env)) < Version("21.3"): + call( + [ + "python", + "-m", + "pip", + "install", + "--force-reinstall", + "--upgrade", + "pip", + *dependency_constraint_flags, + ], + env=env, + cwd=CIBW_INSTALL_PATH, + ) + # upgrade pip to the version matching our constraints # if necessary, reinstall it to ensure that it's available on PATH as 'pip.exe' call( diff --git a/docs/options.md b/docs/options.md index 70b3f4e0..9dd0f1af 100644 --- a/docs/options.md +++ b/docs/options.md @@ -207,8 +207,8 @@ When setting the options, you can use shell-style globbing syntax, as per [fnmat | Python 3.6 | cp36-macosx_x86_64 | cp36-win_amd64
cp36-win32 | cp36-manylinux_x86_64
cp36-manylinux_i686
cp36-musllinux_x86_64
cp36-musllinux_i686 | cp36-manylinux_aarch64
cp36-manylinux_ppc64le
cp36-manylinux_s390x
cp36-musllinux_aarch64
cp36-musllinux_ppc64le
cp36-musllinux_s390x | | Python 3.7 | cp37-macosx_x86_64 | cp37-win_amd64
cp37-win32 | cp37-manylinux_x86_64
cp37-manylinux_i686
cp37-musllinux_x86_64
cp37-musllinux_i686 | cp37-manylinux_aarch64
cp37-manylinux_ppc64le
cp37-manylinux_s390x
cp37-musllinux_aarch64
cp37-musllinux_ppc64le
cp37-musllinux_s390x | | Python 3.8 | cp38-macosx_x86_64
cp38-macosx_universal2
cp38-macosx_arm64 | cp38-win_amd64
cp38-win32 | cp38-manylinux_x86_64
cp38-manylinux_i686
cp38-musllinux_x86_64
cp38-musllinux_i686 | cp38-manylinux_aarch64
cp38-manylinux_ppc64le
cp38-manylinux_s390x
cp38-musllinux_aarch64
cp38-musllinux_ppc64le
cp38-musllinux_s390x | -| Python 3.9 | cp39-macosx_x86_64
cp39-macosx_universal2
cp39-macosx_arm64 | cp39-win_amd64
cp39-win32 | cp39-manylinux_x86_64
cp39-manylinux_i686
cp39-musllinux_x86_64
cp39-musllinux_i686 | cp39-manylinux_aarch64
cp39-manylinux_ppc64le
cp39-manylinux_s390x
cp39-musllinux_aarch64
cp39-musllinux_ppc64le
cp39-musllinux_s390x | -| Python 3.10 | cp310-macosx_x86_64
cp310-macosx_universal2
cp310-macosx_arm64 | cp310-win_amd64
cp310-win32 | cp310-manylinux_x86_64
cp310-manylinux_i686
cp310-musllinux_x86_64
cp310-musllinux_i686 | cp310-manylinux_aarch64
cp310-manylinux_ppc64le
cp310-manylinux_s390x
cp310-musllinux_aarch64
cp310-musllinux_ppc64le
cp310-musllinux_s390x | +| Python 3.9 | cp39-macosx_x86_64
cp39-macosx_universal2
cp39-macosx_arm64 | cp39-win_amd64
cp39-win32
cp39-win_arm64 | cp39-manylinux_x86_64
cp39-manylinux_i686
cp39-musllinux_x86_64
cp39-musllinux_i686 | cp39-manylinux_aarch64
cp39-manylinux_ppc64le
cp39-manylinux_s390x
cp39-musllinux_aarch64
cp39-musllinux_ppc64le
cp39-musllinux_s390x | +| Python 3.10 | cp310-macosx_x86_64
cp310-macosx_universal2
cp310-macosx_arm64 | cp310-win_amd64
cp310-win32
cp310-win_arm64 | cp310-manylinux_x86_64
cp310-manylinux_i686
cp310-musllinux_x86_64
cp310-musllinux_i686 | cp310-manylinux_aarch64
cp310-manylinux_ppc64le
cp310-manylinux_s390x
cp310-musllinux_aarch64
cp310-musllinux_ppc64le
cp310-musllinux_s390x | | PyPy3.7 v7.3 | pp37-macosx_x86_64 | pp37-win_amd64 | pp37-manylinux_x86_64
pp37-manylinux_i686 | pp37-manylinux_aarch64 | | PyPy3.8 v7.3 | pp38-macosx_x86_64 | pp38-win_amd64 | pp38-manylinux_x86_64
pp38-manylinux_i686 | pp38-manylinux_aarch64 | @@ -217,6 +217,8 @@ The format is `python_tag-platform_tag`, with tags similar to those in [PEP 425] For CPython, the minimally supported macOS version is 10.9; for PyPy 3.7, macOS 10.13 or higher is required. +Windows arm64 platform support is experimental. + See the [cibuildwheel 1 documentation](https://cibuildwheel.readthedocs.io/en/1.x/) for past end of life versions of Python, and PyPy2.7. #### Examples diff --git a/unit_test/main_tests/main_platform_test.py b/unit_test/main_tests/main_platform_test.py index ae676791..09f5b9db 100644 --- a/unit_test/main_tests/main_platform_test.py +++ b/unit_test/main_tests/main_platform_test.py @@ -178,7 +178,11 @@ def test_archs_platform_all(platform, intercepted_build_args, monkeypatch): Architecture.s390x, } elif platform == "windows": - assert options.globals.architectures == {Architecture.x86, Architecture.AMD64} + assert options.globals.architectures == { + Architecture.x86, + Architecture.AMD64, + Architecture.ARM64, + } elif platform == "macos": assert options.globals.architectures == { Architecture.x86_64,