Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: Android improvements needed for building NumPy and related packages (#2695)

Browse Source 
* Use auditwheel on Android

* Add auditwheel command to defaults

* pkgconfig fixes

* Pre-import ctypes before monkey-patching in _cross_venv

* Set PKG_CONFIG and PKG_CONFIG_RELOCATE_PATHS variables

* Initial attempt at using mzakharo/android-gfortran

* Switch to using termux/ndk-toolchain-clang-with-flang

* Move PKG_CONFIG variables from build_env to android_env

* Simplify flang installation

* Add cross build files for NumPy

* Add ldpaths entry for libomp

* Add `--rm` to docker command line

* Make Rust and Fortran shims consistent

* Update documentation

* Default to API level 24 on all Python versions

* Clarify comments

* Cleanups

* Fix tests:
* Set up Android env after installing pkgconf
* Add tests for successfully using an older API level
* Previous commit's auditwheel failure is fixed in the auditwheel PR

* Update how-it-works diagram

* Add tests for repair errors

* Add more repair tests

* Add test for Meson and Fortran

* Add test for cross build files

* Improve test_api_level error message

* Add xbuild-files option

* use pypa/auditwheel@main

* Remove dependencies which are no longer needed

* Fix README

* Update to auditwheel 6.7.0

* Fix compatibility with pkgconf 2.5.1.post2

* Documentation clarifications

Co-authored-by: Joe Rickerby <joerick@mac.com>

* Fortran shim improvements

* Add Jinja variables to new_meson_project

* Update run_example_ci_configs for changed new_meson_project signature

* Add missing dependency to run_example_ci_configs

---------

Co-authored-by: mayeut <mayeut@users.noreply.github.com>
Co-authored-by: Joe Rickerby <joerick@mac.com>
This commit is contained in:
Malcolm Smith
2026-05-29 00:27:28 -04:00
committed by GitHub
co-authored by Joe Rickerby mayeut
parent e3e7cc9e07
commit 1cf64d0d3c
32 changed files with 775 additions and 246 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cibuildwheel/platforms/android.py
+130 -180
View File
@@ -1,16 +1,13 @@
from __future__ import annotations
import csv
import hashlib
import os
import platform
import re
import shlex
import shutil
import subprocess
import sysconfig
import sys
from dataclasses import dataclass
from os.path import relpath
from pathlib import Path
from pprint import pprint
from runpy import run_path
@@ -19,9 +16,8 @@ from typing import Any
from build import ProjectBuilder
from build.env import IsolatedEnv
from elftools.common.exceptions import ELFError
from elftools.elf.elffile import ELFFile
from filelock import FileLock
from packaging.utils import canonicalize_name
from cibuildwheel import errors, platforms # pylint: disable=cyclic-import
from cibuildwheel.architecture import Architecture, arch_synonym
@@ -42,10 +38,9 @@ from cibuildwheel.venv import constraint_flags, find_uv, virtualenv
TYPE_CHECKING = False
if TYPE_CHECKING:
from collections.abc import Iterable, Iterator, MutableMapping
from cibuildwheel.options import BuildOptions, Options
from cibuildwheel.selector import BuildSelector
from cibuildwheel.typing import PathOrStr
RESOURCES_ANDROID = resources.PATH / "android"
ANDROID_TRIPLET = {
@@ -148,6 +143,7 @@ def build(options: Options, tmp_path: Path) -> None:
state = BuildState(
config, build_options, build_path, python_dir, build_env, android_env
)
setup_xbuild_files(state)
compatible_wheel = find_compatible_wheel(built_wheels, config.identifier)
if compatible_wheel:
@@ -162,7 +158,7 @@ def build(options: Options, tmp_path: Path) -> None:
repaired_wheel = repair_wheel(state, built_wheel)
run_audit(tmp_dir=tmp_path, build_options=build_options, wheel=repaired_wheel)
test_wheel(state, repaired_wheel, build_frontend=build_options.build_frontend.name)
test_wheel(state, repaired_wheel)
output_wheel: Path | None = None
if compatible_wheel is None:
@@ -194,6 +190,11 @@ def setup_target_python(config: PythonConfiguration, build_path: Path) -> Path:
# updated to Python versions that include the fix.
call("patch", "-p1", "-i", RESOURCES_ANDROID / "android.patch", cwd=python_dir)
# Work around https://github.com/python/cpython/issues/138800. This can be removed
# once we've updated to Python versions that include the fix.
pc_path = python_dir / f"prefix/lib/pkgconfig/python-{config.version}.pc"
pc_path.write_text(pc_path.read_text().replace("$(BLDLIBRARY)", f"-lpython{config.version}"))
return python_dir
@@ -207,13 +208,7 @@ def setup_env(
* android_env, which uses the environment while simulating running on Android.
"""
log.step("Setting up build environment...")
build_frontend = build_options.build_frontend.name
use_uv = build_frontend in {"build[uv]", "uv"}
uv_path = find_uv()
if use_uv and uv_path is None:
msg = "uv not found"
raise AssertionError(msg)
pip = ["pip"] if not use_uv else [str(uv_path), "pip"]
use_uv, pip = find_pip(build_options)
# Create virtual environment
python_exe = create_python_build_standalone_environment(
@@ -228,6 +223,9 @@ def setup_env(
)
create_cmake_toolchain(config, build_path, python_dir, build_env)
# See platforms.md for the reason why we use this default API level.
build_env.setdefault("ANDROID_API_LEVEL", "24")
# Apply custom environment variables, and check environment is still valid
build_env = build_options.environment.as_dictionary(build_env)
build_env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
@@ -245,14 +243,14 @@ def setup_env(
else:
call(command, "--version", env=build_env)
# Construct an altered environment which simulates running on Android.
android_env = setup_android_env(config, python_dir, venv_dir, build_env)
# Install build tools
if build_frontend not in {"build", "build[uv]", "uv"}:
msg = "Android requires the build frontend to be 'build' or 'uv'"
raise errors.FatalError(msg)
call(*pip, "install", "build", *constraint_flags(dependency_constraint), env=build_env)
tools = ["auditwheel", "patchelf", "pkgconf"]
if build_options.build_frontend.name in {"build", "build[uv]"}:
tools.append("build")
call(*pip, "install", *tools, *constraint_flags(dependency_constraint), env=build_env)
# Construct an altered environment which simulates running on Android.
android_env = setup_android_env(config, python_dir, build_env)
# Build-time requirements must be queried within android_env, because
# `get_requires_for_build` can run arbitrary code in setup.py scripts, which may be
@@ -343,8 +341,7 @@ def localized_vars(
final = final.replace(orig_prefix, str(prefix))
if key == "ANDROID_API_LEVEL":
if api_level := build_env.get(key):
final = int(api_level)
final = int(build_env[key])
# Build systems vary in whether FLAGS variables are read from sysconfig, and if so,
# whether they're replaced by environment variables or combined with them. Even
@@ -367,15 +364,15 @@ def localized_vars(
def setup_android_env(
config: PythonConfiguration, python_dir: Path, venv_dir: Path, build_env: dict[str, str]
config: PythonConfiguration, python_dir: Path, build_env: dict[str, str]
) -> dict[str, str]:
site_packages = next(venv_dir.glob("lib/python*/site-packages"))
site_packages = find_site_packages(build_env)
for suffix in ["pth", "py"]:
shutil.copy(RESOURCES_ANDROID / f"_cross_venv.{suffix}", site_packages)
sysconfigdata_path = Path(
shutil.copy(
next(python_dir.glob("prefix/lib/python*/_sysconfigdata_*.py")),
glob1(python_dir, "prefix/lib/python*/_sysconfigdata_*.py"),
site_packages,
)
)
@@ -414,6 +411,17 @@ def setup_android_env(
# Cargo target linker needs to be specified after CC is set
setup_rust(config, python_dir, android_env)
# Create shims which install additional build tools on first use.
setup_fortran(android_env)
# `android.py env` returns PKG_CONFIG="pkg-config --define-prefix", but some build
# systems can't handle arguments in that variable. Since we have a known version
# of pkgconf, it's safe to use PKG_CONFIG_RELOCATE_PATHS instead.
android_env["PKG_CONFIG"] = call(
"which", "pkgconf-pypi", env=build_env, capture_stdout=True
).strip()
android_env["PKG_CONFIG_RELOCATE_PATHS"] = "1"
# Format the environment so it can be pasted into a shell when debugging.
for key, value in sorted(android_env.items()):
if os.environ.get(key) != value:
@@ -422,11 +430,7 @@ def setup_android_env(
return android_env
def setup_rust(
config: PythonConfiguration,
python_dir: Path,
env: MutableMapping[str, str],
) -> None:
def setup_rust(config: PythonConfiguration, python_dir: Path, env: dict[str, str]) -> None:
cargo_target = android_triplet(config.identifier)
# CARGO_BUILD_TARGET is the variable used by Cargo and setuptools_rust
@@ -450,13 +454,83 @@ def setup_rust(
shim_path.chmod(0o755)
def setup_fortran(env: dict[str, str]) -> None:
# In case there's any autodetection based on the executable name, use the same name
# as the real executable (see fortran_shim.run_flang)
shim_in = RESOURCES_ANDROID / "fortran_shim.py"
shim_out = Path(env["VIRTUAL_ENV"]) / "bin/flang-new"
# The hashbang line runs the shim in cibuildwheel's own virtual environment, so it
# has access to utility functions for downloading and caching files.
shim_out.write_text(f"#!{sys.executable}\n\n" + shim_in.read_text())
shim_out.chmod(0o755)
env["FC"] = str(shim_out)
def setup_xbuild_files(state: BuildState) -> None:
_, pip = find_pip(state.options)
xbf_dir = state.build_path / "xbuild_files"
xbf_dir.mkdir()
for requirement in call(*pip, "freeze", env=state.build_env, capture_stdout=True).splitlines():
name, _, _ = requirement.strip().partition("==")
xbuild_files = state.options.xbuild_files.get(canonicalize_name(name), [])
if xbuild_files:
log.step(f"Installing xbuild-files for {name}...")
pip_install_android(state, xbf_dir, "--no-deps", requirement)
for xbf in xbuild_files:
if (xbf_dir / xbf).exists():
shutil.copy(
xbf_dir / xbf,
find_site_packages(state.build_env) / xbf,
)
else:
log.warning(f"{xbf_dir / xbf} does not exist")
def pip_install_android(state: BuildState, target: Path, *args: PathOrStr) -> None:
use_uv, pip = find_pip(state.options)
call(
*pip,
"install",
"--only-binary=:all:",
*(["--python-platform", android_triplet(state.config.identifier)] if use_uv else []),
"--target",
target,
*args,
env=state.android_env,
)
def find_site_packages(env: dict[str, str]) -> Path:
return glob1(Path(env["VIRTUAL_ENV"]), "lib/python*/site-packages")
def glob1(base: Path, pattern: str) -> Path:
results = list(base.glob(pattern))
if len(results) != 1:
msg = f"{base} contains {len(results)} paths matching '{pattern}'; expected 1"
raise errors.FatalError(msg)
return results[0]
def find_pip(build_options: BuildOptions) -> tuple[bool, list[str]]:
use_uv = build_options.build_frontend.name in {"build[uv]", "uv"}
uv_path = find_uv()
if use_uv and uv_path is None:
msg = "uv not found"
raise AssertionError(msg)
pip = ["pip"] if not use_uv else [str(uv_path), "pip"]
return use_uv, pip
def before_build(state: BuildState) -> None:
if state.options.before_build:
log.step("Running before_build...")
shell_prepared(
state.options.before_build,
build_options=state.options,
env=state.build_env,
env=state.android_env,
)
@@ -507,15 +581,10 @@ def build_wheel(state: BuildState) -> Path:
env=state.android_env,
)
case x:
msg = f"Invalid build backend {x!r}"
raise AssertionError(msg)
built_wheels = list(built_wheel_dir.glob("*.whl"))
if len(built_wheels) != 1:
msg = f"{built_wheel_dir} contains {len(built_wheels)} wheels; expected 1"
raise errors.FatalError(msg)
built_wheel = built_wheels[0]
msg = f"Android requires the build frontend to be 'build' or 'uv', not {x!r}"
raise errors.FatalError(msg)
built_wheel = glob1(built_wheel_dir, "*.whl")
if built_wheel.name.endswith("none-any.whl"):
raise errors.NonPlatformWheelError()
return built_wheel
@@ -527,9 +596,20 @@ def repair_wheel(state: BuildState, built_wheel: Path) -> Path:
repaired_wheel_dir.mkdir()
if state.options.repair_command:
# Tell auditwheel the locations of compiler libraries.
toolchain = Path(state.android_env["CC"]).parent.parent
triplet = android_triplet(state.config.identifier)
ldpaths = ":".join(
str(glob1(toolchain, pattern))
for pattern in [
f"lib/clang/*/lib/linux/{triplet.split('-')[0]}", # libomp
f"sysroot/usr/lib/{triplet}", # libc++_shared
]
)
shell(
prepare_command(
state.options.repair_command,
ldpaths=ldpaths,
wheel=built_wheel,
dest_dir=repaired_wheel_dir,
package=state.options.package_dir,
@@ -538,14 +618,15 @@ def repair_wheel(state: BuildState, built_wheel: Path) -> Path:
env=state.build_env,
)
else:
repair_default(state.android_env, built_wheel, repaired_wheel_dir)
shutil.move(built_wheel, repaired_wheel_dir)
repaired_wheels = list(repaired_wheel_dir.glob("*.whl"))
if len(repaired_wheels) == 0:
raise errors.RepairStepProducedNoWheelError()
if len(repaired_wheels) != 1:
msg = f"{repaired_wheel_dir} contains {len(repaired_wheels)} wheels; expected 1"
raise errors.FatalError(msg)
raise errors.RepairStepProducedMultipleWheelsError(
[rw.name for rw in repaired_wheels],
)
repaired_wheel = repaired_wheels[0]
if repaired_wheel.name.endswith("none-any.whl"):
@@ -553,119 +634,12 @@ def repair_wheel(state: BuildState, built_wheel: Path) -> Path:
return repaired_wheel
def repair_default(
android_env: dict[str, str], built_wheel: Path, repaired_wheel_dir: Path
) -> None:
"""
Adds libc++ to the wheel if anything links against it. In the future this should be
moved to auditwheel and generalized to support more libraries.
"""
if (match := re.search(r"^(.+?)-", built_wheel.name)) is None:
msg = f"Failed to parse wheel filename: {built_wheel.name}"
raise errors.FatalError(msg)
wheel_name = match[1]
unpacked_dir = repaired_wheel_dir / "unpacked"
unpacked_dir.mkdir()
shutil.unpack_archive(built_wheel, unpacked_dir, format="zip")
# Some build systems are inconsistent about name normalization, so don't assume the
# dist-info name is identical to the wheel name.
record_paths = list(unpacked_dir.glob("*.dist-info/RECORD"))
if len(record_paths) != 1:
msg = f"{built_wheel.name} contains {len(record_paths)} dist-info/RECORD files; expected 1"
raise errors.FatalError(msg)
old_soname = "libc++_shared.so"
paths_to_patch = []
for path, elffile in elf_file_filter(
unpacked_dir / filename
for filename, *_ in csv.reader(record_paths[0].read_text().splitlines())
):
if (dynamic := elffile.get_section_by_name(".dynamic")) and any( # type: ignore[no-untyped-call]
tag.entry.d_tag == "DT_NEEDED" and tag.needed == old_soname
for tag in dynamic.iter_tags()
):
paths_to_patch.append(path)
if not paths_to_patch:
shutil.copyfile(built_wheel, repaired_wheel_dir / built_wheel.name)
else:
# Android doesn't support DT_RPATH, but supports DT_RUNPATH since API level 24
# (https://github.com/aosp-mirror/platform_bionic/blob/master/android-changes-for-ndk-developers.md).
if int(sysconfig_print('get_config_vars()["ANDROID_API_LEVEL"]', android_env)) < 24:
msg = f"Adding {old_soname} requires ANDROID_API_LEVEL to be at least 24"
raise errors.FatalError(msg)
toolchain = Path(android_env["CC"]).parent.parent
src_path = toolchain / f"sysroot/usr/lib/{android_env['CIBW_HOST_TRIPLET']}/{old_soname}"
# Use the same library location as auditwheel would.
libs_dir = unpacked_dir / (wheel_name + ".libs")
libs_dir.mkdir()
new_soname = soname_with_hash(src_path)
dst_path = libs_dir / new_soname
shutil.copyfile(src_path, dst_path)
call(which("patchelf"), "--set-soname", new_soname, dst_path)
for path in paths_to_patch:
call(which("patchelf"), "--replace-needed", old_soname, new_soname, path)
call(
which("patchelf"),
"--set-rpath",
f"${{ORIGIN}}/{relpath(libs_dir, path.parent)}",
path,
)
call(which("wheel"), "pack", unpacked_dir, "-d", repaired_wheel_dir)
# If cibuildwheel was called without activating its environment, its scripts directory
# will not be on the PATH.
def which(cmd: str) -> str:
scripts_dir = sysconfig.get_path("scripts")
result = shutil.which(cmd, path=scripts_dir + os.pathsep + os.environ["PATH"])
if result is None:
msg = f"Couldn't find {cmd!r} in {scripts_dir} or on the PATH"
raise errors.FatalError(msg)
return result
def elf_file_filter(paths: Iterable[Path]) -> Iterator[tuple[Path, ELFFile]]:
"""Filter through an iterator of filenames and load up only ELF files"""
for path in paths:
if not path.name.endswith(".py"):
try:
with open(path, "rb") as f:
candidate = ELFFile(f) # type: ignore[no-untyped-call]
yield path, candidate
except ELFError:
pass # Not an ELF file
def soname_with_hash(src_path: Path) -> str:
"""Return the same library filename as auditwheel would"""
shorthash = hashlib.sha256(src_path.read_bytes()).hexdigest()[:8]
src_name = src_path.name
base, ext = src_name.split(".", 1)
if not base.endswith(f"-{shorthash}"):
return f"{base}-{shorthash}.{ext}"
else:
return src_name
def test_wheel(state: BuildState, wheel: Path, *, build_frontend: str) -> None:
def test_wheel(state: BuildState, wheel: Path) -> None:
test_command = state.options.test_command
if not (test_command and state.options.test_selector(state.config.identifier)):
return
log.step("Testing wheel...")
use_uv = build_frontend in {"build[uv]", "uv"}
uv_path = find_uv()
if use_uv and uv_path is None:
msg = "uv not found"
raise AssertionError(msg)
pip = ["pip"] if not use_uv else [str(uv_path), "pip"]
native_arch = arch_synonym(platform.machine(), platforms.native_platform(), "android")
if state.config.arch != native_arch:
log.warning(
@@ -678,31 +652,17 @@ def test_wheel(state: BuildState, wheel: Path, *, build_frontend: str) -> None:
shell_prepared(
state.options.before_test,
build_options=state.options,
env=state.build_env,
env=state.android_env,
)
platform_args = (
["--python-platform", android_triplet(state.config.identifier)]
if use_uv
else [
"--platform",
sysconfig_print("get_platform()", state.android_env).replace("-", "_"),
]
)
# Install the wheel and test-requires.
site_packages_dir = state.build_path / "site-packages"
site_packages_dir.mkdir()
call(
*pip,
"install",
"--only-binary=:all:",
*platform_args,
"--target",
pip_install_android(
state,
site_packages_dir,
f"{wheel}{state.options.test_extras}",
*state.options.test_requires,
env=state.android_env,
)
# Copy test-sources.
@@ -772,13 +732,3 @@ def test_wheel(state: BuildState, wheel: Path, *, build_frontend: str) -> None:
*test_args,
env=state.build_env,
)
def sysconfig_print(method_call: str, env: dict[str, str]) -> str:
return call(
"python",
"-c",
f'import sysconfig; print(sysconfig.{method_call}, end="")',
env=env,
capture_stdout=True,
)