feat: add Pyodide support (#1456)
* feat: add Pyodide support Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> Co-authored-by: Hood Chatham <roberthoodchatham@gmail.com> Co-authored-by: Matthieu Darbois <mayeut@users.noreply.github.com> tests: fix two merge issues Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> fix: include schema Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * Try to fix xbuildenv path [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Try to install pyodide-build from main branch Try again Try again Update constraints file Try again Remove unused variable Drop constraints Remove --download option Fix xbuildenv install Try again Try again [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix: remove pinning on pyodide Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * Update for Pyodide 0.26.0a5 * Install pyodide-build from pypi * Update docs/options.md Co-authored-by: Henry Schreiner <HenrySchreinerIII@gmail.com> * Unxfail things that look like they were just a version mismatch * refactor: add constraints for pyodide Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * chore: minor cleanup Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * Apply suggestions from code review * Apply suggestion from code review * refactor: minor touchup Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * ci: xfail the pyodide test Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * review: use a pinned version of node * fix tests * review: error out on Windows * test: check node & test on macos arm64 * chore: minor cleanup * Add reference to emscripten libc issue * Apply suggestion from code review * review: use a pinned pip in test virtual environment * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * chore: rework test virtual environment seed packages * chore: workaround direct invocation of pytest This allows to still test direct invocation of `pytest` on most platforms (including pyodide on Linux) but falls back to `python -m pytest` when running pyodide on macOS. * Use release version of pyodide * fix: tests for 0.26.0 & parallel initialization of xbuildenv * Debug CI * fix: test/test_build_frontend_args.py * Revert "Debug CI" This reverts commit 917646cffc96dbfc619c47d1c03a828f447bcdb7. --------- Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> Co-authored-by: Henry Schreiner <henryschreineriii@gmail.com> Co-authored-by: Matthieu Darbois <mayeut@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
This commit is contained in:
Hood Chatham
co-authored by
Matthieu Darbois
Henry Schreiner
pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
Matthieu Darbois
Henry Schreiner
pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
parent
687406f664
commit
9f2a3cb67c
@@ -15,6 +15,7 @@ from typing import Protocol
|
||||
import cibuildwheel
|
||||
import cibuildwheel.linux
|
||||
import cibuildwheel.macos
|
||||
import cibuildwheel.pyodide
|
||||
import cibuildwheel.util
|
||||
import cibuildwheel.windows
|
||||
from cibuildwheel._compat.typing import assert_never
|
||||
@@ -45,7 +46,7 @@ def main() -> None:
|
||||
|
||||
parser.add_argument(
|
||||
"--platform",
|
||||
choices=["auto", "linux", "macos", "windows"],
|
||||
choices=["auto", "linux", "macos", "windows", "pyodide"],
|
||||
default=None,
|
||||
help="""
|
||||
Platform to build for. Use this option to override the
|
||||
@@ -176,6 +177,8 @@ def _compute_platform_only(only: str) -> PlatformName:
|
||||
return "macos"
|
||||
if "win_" in only or "win32" in only:
|
||||
return "windows"
|
||||
if "pyodide_" in only:
|
||||
return "pyodide"
|
||||
print(
|
||||
f"Invalid --only='{only}', must be a build selector with a known platform",
|
||||
file=sys.stderr,
|
||||
@@ -246,11 +249,18 @@ def get_platform_module(platform: PlatformName) -> PlatformModule:
|
||||
return cibuildwheel.windows
|
||||
if platform == "macos":
|
||||
return cibuildwheel.macos
|
||||
if platform == "pyodide":
|
||||
return cibuildwheel.pyodide
|
||||
assert_never(platform)
|
||||
|
||||
|
||||
def build_in_directory(args: CommandLineArguments) -> None:
|
||||
platform: PlatformName = _compute_platform(args)
|
||||
if platform == "pyodide" and sys.platform == "win32":
|
||||
msg = "cibuildwheel: Building for pyodide is not supported on Windows"
|
||||
print(msg, file=sys.stderr)
|
||||
sys.exit(2)
|
||||
|
||||
options = compute_options(platform=platform, command_line_arguments=args, env=os.environ)
|
||||
|
||||
package_dir = options.globals.package_dir
|
||||
|
||||
@@ -15,6 +15,7 @@ PRETTY_NAMES: Final[dict[PlatformName, str]] = {
|
||||
"linux": "Linux",
|
||||
"macos": "macOS",
|
||||
"windows": "Windows",
|
||||
"pyodide": "Pyodide",
|
||||
}
|
||||
|
||||
ARCH_SYNONYMS: Final[list[dict[PlatformName, str | None]]] = [
|
||||
@@ -46,6 +47,9 @@ class Architecture(Enum):
|
||||
AMD64 = "AMD64"
|
||||
ARM64 = "ARM64"
|
||||
|
||||
# WebAssembly
|
||||
wasm32 = "wasm32"
|
||||
|
||||
# Allow this to be sorted
|
||||
def __lt__(self, other: Architecture) -> bool:
|
||||
return self.value < other.value
|
||||
@@ -72,8 +76,9 @@ class Architecture(Enum):
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def auto_archs(platform: PlatformName) -> set[Architecture]:
|
||||
native_machine = platform_module.machine()
|
||||
def native_arch(platform: PlatformName) -> Architecture | None:
|
||||
if platform == "pyodide":
|
||||
return Architecture.wasm32
|
||||
|
||||
# Cross-platform support. Used for --print-build-identifiers or docker builds.
|
||||
host_platform: PlatformName = (
|
||||
@@ -82,6 +87,7 @@ class Architecture(Enum):
|
||||
else ("macos" if sys.platform.startswith("darwin") else "linux")
|
||||
)
|
||||
|
||||
native_machine = platform_module.machine()
|
||||
native_architecture = Architecture(native_machine)
|
||||
|
||||
# we might need to rename the native arch to the machine we're running
|
||||
@@ -93,11 +99,18 @@ class Architecture(Enum):
|
||||
|
||||
if synonym is None:
|
||||
# can't build anything on this platform
|
||||
return set()
|
||||
return None
|
||||
|
||||
native_architecture = Architecture(synonym)
|
||||
|
||||
result = {native_architecture}
|
||||
return native_architecture
|
||||
|
||||
@staticmethod
|
||||
def auto_archs(platform: PlatformName) -> set[Architecture]:
|
||||
native_arch = Architecture.native_arch(platform)
|
||||
if native_arch is None:
|
||||
return set() # can't build anything on this platform
|
||||
result = {native_arch}
|
||||
|
||||
if platform == "linux" and Architecture.x86_64 in result:
|
||||
# x86_64 machines can run i686 containers
|
||||
@@ -120,6 +133,7 @@ class Architecture(Enum):
|
||||
},
|
||||
"macos": {Architecture.x86_64, Architecture.arm64, Architecture.universal2},
|
||||
"windows": {Architecture.x86, Architecture.AMD64, Architecture.ARM64},
|
||||
"pyodide": {Architecture.wasm32},
|
||||
}
|
||||
return all_archs_map[platform]
|
||||
|
||||
|
||||
@@ -34,6 +34,7 @@ PLATFORM_IDENTIFIER_DESCRIPTIONS: Final[dict[str, str]] = {
|
||||
"macosx_x86_64": "macOS x86_64",
|
||||
"macosx_universal2": "macOS Universal 2 - x86_64 and arm64",
|
||||
"macosx_arm64": "macOS arm64 - Apple Silicon",
|
||||
"pyodide_wasm32": "Pyodide",
|
||||
}
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,403 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import os
|
||||
import shutil
|
||||
import sys
|
||||
from collections.abc import Sequence, Set
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from filelock import FileLock
|
||||
|
||||
from .architecture import Architecture
|
||||
from .environment import ParsedEnvironment
|
||||
from .logger import log
|
||||
from .options import Options
|
||||
from .typing import PathOrStr
|
||||
from .util import (
|
||||
CIBW_CACHE_PATH,
|
||||
AlreadyBuiltWheelError,
|
||||
BuildFrontendConfig,
|
||||
BuildSelector,
|
||||
NonPlatformWheelError,
|
||||
call,
|
||||
download,
|
||||
ensure_node,
|
||||
extract_zip,
|
||||
find_compatible_wheel,
|
||||
get_pip_version,
|
||||
prepare_command,
|
||||
read_python_configs,
|
||||
shell,
|
||||
split_config_settings,
|
||||
test_fail_cwd_file,
|
||||
virtualenv,
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PythonConfiguration:
|
||||
version: str
|
||||
identifier: str
|
||||
pyodide_version: str
|
||||
emscripten_version: str
|
||||
node_version: str
|
||||
|
||||
|
||||
def install_emscripten(tmp: Path, version: str) -> Path:
|
||||
# We don't need to match the emsdk version to the version we install, but
|
||||
# we do for stability
|
||||
url = f"https://github.com/emscripten-core/emsdk/archive/refs/tags/{version}.zip"
|
||||
installation_path = CIBW_CACHE_PATH / f"emsdk-{version}"
|
||||
emsdk_path = installation_path / f"emsdk-{version}/emsdk"
|
||||
emcc_path = installation_path / f"emsdk-{version}/upstream/emscripten/emcc"
|
||||
with FileLock(f"{installation_path}.lock"):
|
||||
if installation_path.exists():
|
||||
return emcc_path
|
||||
emsdk_zip = tmp / "emsdk.zip"
|
||||
download(url, emsdk_zip)
|
||||
installation_path.mkdir()
|
||||
extract_zip(emsdk_zip, installation_path)
|
||||
call(emsdk_path, "install", version)
|
||||
call(emsdk_path, "activate", version)
|
||||
|
||||
return emcc_path
|
||||
|
||||
|
||||
def install_xbuildenv(env: dict[str, str], pyodide_version: str) -> str:
|
||||
pyodide_root = (
|
||||
CIBW_CACHE_PATH
|
||||
/ f".pyodide-xbuildenv-{pyodide_version}/{pyodide_version}/xbuildenv/pyodide-root"
|
||||
)
|
||||
with FileLock(CIBW_CACHE_PATH / "xbuildenv.lock"):
|
||||
if pyodide_root.exists():
|
||||
return str(pyodide_root)
|
||||
|
||||
# We don't want to mutate env but we need to delete any existing
|
||||
# PYODIDE_ROOT so copy it first.
|
||||
env = dict(env)
|
||||
env.pop("PYODIDE_ROOT", None)
|
||||
call(
|
||||
"pyodide",
|
||||
"xbuildenv",
|
||||
"install",
|
||||
pyodide_version,
|
||||
env=env,
|
||||
cwd=CIBW_CACHE_PATH,
|
||||
)
|
||||
return str(pyodide_root)
|
||||
|
||||
|
||||
def get_base_python(identifier: str) -> Path:
|
||||
implementation_id = identifier.split("-")[0]
|
||||
majorminor = implementation_id[len("cp") :]
|
||||
major_minor = f"{majorminor[0]}.{majorminor[1:]}"
|
||||
python_name = f"python{major_minor}"
|
||||
which_python = shutil.which(python_name)
|
||||
if which_python is None:
|
||||
print(
|
||||
f"Error: CPython {major_minor} is not installed.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
raise SystemExit(1)
|
||||
return Path(which_python)
|
||||
|
||||
|
||||
def setup_python(
|
||||
tmp: Path,
|
||||
python_configuration: PythonConfiguration,
|
||||
dependency_constraint_flags: Sequence[PathOrStr],
|
||||
environment: ParsedEnvironment,
|
||||
) -> dict[str, str]:
|
||||
base_python = get_base_python(python_configuration.identifier)
|
||||
|
||||
log.step("Setting up build environment...")
|
||||
venv_path = tmp / "venv"
|
||||
env = virtualenv(python_configuration.version, base_python, venv_path, [])
|
||||
venv_bin_path = venv_path / "bin"
|
||||
assert venv_bin_path.exists()
|
||||
env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
|
||||
|
||||
# upgrade pip to the version matching our constraints
|
||||
# if necessary, reinstall it to ensure that it's available on PATH as 'pip'
|
||||
call(
|
||||
"python",
|
||||
"-m",
|
||||
"pip",
|
||||
"install",
|
||||
"--upgrade",
|
||||
"pip",
|
||||
*dependency_constraint_flags,
|
||||
env=env,
|
||||
cwd=venv_path,
|
||||
)
|
||||
|
||||
env = environment.as_dictionary(prev_environment=env)
|
||||
|
||||
# check what pip version we're on
|
||||
assert (venv_bin_path / "pip").exists()
|
||||
call("which", "pip", env=env)
|
||||
call("pip", "--version", env=env)
|
||||
which_pip = call("which", "pip", env=env, capture_stdout=True).strip()
|
||||
if which_pip != str(venv_bin_path / "pip"):
|
||||
print(
|
||||
"cibuildwheel: pip available on PATH doesn't match our venv instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
# check what Python version we're on
|
||||
call("which", "python", env=env)
|
||||
call("python", "--version", env=env)
|
||||
which_python = call("which", "python", env=env, capture_stdout=True).strip()
|
||||
if which_python != str(venv_bin_path / "python"):
|
||||
print(
|
||||
"cibuildwheel: python available on PATH doesn't match our venv instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
log.step("Installing build tools...")
|
||||
call(
|
||||
"pip",
|
||||
"install",
|
||||
"--upgrade",
|
||||
"auditwheel-emscripten",
|
||||
"build[virtualenv]",
|
||||
"pyodide-build",
|
||||
*dependency_constraint_flags,
|
||||
env=env,
|
||||
)
|
||||
|
||||
log.step("Installing emscripten...")
|
||||
emcc_path = install_emscripten(tmp, python_configuration.emscripten_version)
|
||||
|
||||
env["PATH"] = os.pathsep.join([str(emcc_path.parent), env["PATH"]])
|
||||
|
||||
log.step("Installing Pyodide xbuildenv...")
|
||||
env["PYODIDE_ROOT"] = install_xbuildenv(env, python_configuration.pyodide_version)
|
||||
|
||||
return env
|
||||
|
||||
|
||||
def get_python_configurations(
|
||||
build_selector: BuildSelector,
|
||||
architectures: Set[Architecture], # noqa: ARG001
|
||||
) -> list[PythonConfiguration]:
|
||||
full_python_configs = read_python_configs("pyodide")
|
||||
|
||||
python_configurations = [PythonConfiguration(**item) for item in full_python_configs]
|
||||
python_configurations = [c for c in python_configurations if build_selector(c.identifier)]
|
||||
return python_configurations
|
||||
|
||||
|
||||
def build(options: Options, tmp_path: Path) -> None:
|
||||
python_configurations = get_python_configurations(
|
||||
options.globals.build_selector, options.globals.architectures
|
||||
)
|
||||
|
||||
if not python_configurations:
|
||||
return
|
||||
|
||||
try:
|
||||
before_all_options_identifier = python_configurations[0].identifier
|
||||
before_all_options = options.build_options(before_all_options_identifier)
|
||||
|
||||
if before_all_options.before_all:
|
||||
log.step("Running before_all...")
|
||||
env = before_all_options.environment.as_dictionary(prev_environment=os.environ)
|
||||
before_all_prepared = prepare_command(
|
||||
before_all_options.before_all, project=".", package=before_all_options.package_dir
|
||||
)
|
||||
shell(before_all_prepared, env=env)
|
||||
|
||||
built_wheels: list[Path] = []
|
||||
|
||||
for config in python_configurations:
|
||||
build_options = options.build_options(config.identifier)
|
||||
build_frontend = build_options.build_frontend or BuildFrontendConfig("build")
|
||||
|
||||
if build_frontend.name == "pip":
|
||||
print("The pyodide platform doesn't support pip frontend", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
log.build_start(config.identifier)
|
||||
|
||||
identifier_tmp_dir = tmp_path / config.identifier
|
||||
built_wheel_dir = identifier_tmp_dir / "built_wheel"
|
||||
repaired_wheel_dir = identifier_tmp_dir / "repaired_wheel"
|
||||
identifier_tmp_dir.mkdir()
|
||||
built_wheel_dir.mkdir()
|
||||
repaired_wheel_dir.mkdir()
|
||||
|
||||
dependency_constraint_flags: Sequence[PathOrStr] = []
|
||||
if build_options.dependency_constraints:
|
||||
constraints_path = build_options.dependency_constraints.get_for_python_version(
|
||||
config.version, variant="pyodide"
|
||||
)
|
||||
dependency_constraint_flags = ["-c", constraints_path]
|
||||
|
||||
env = setup_python(
|
||||
identifier_tmp_dir / "build",
|
||||
config,
|
||||
dependency_constraint_flags,
|
||||
build_options.environment,
|
||||
)
|
||||
pip_version = get_pip_version(env)
|
||||
# The Pyodide command line runner mounts all directories in the host
|
||||
# filesystem into the Pyodide file system, except for the custom
|
||||
# file systems /dev, /lib, /proc, and /tmp. Mounting the mount
|
||||
# points for alternate file systems causes some mysterious failure
|
||||
# of the process (it just quits without any clear error).
|
||||
#
|
||||
# Because of this, by default Pyodide can't see anything under /tmp.
|
||||
# This environment variable tells it also to mount our temp
|
||||
# directory.
|
||||
oldmounts = ""
|
||||
extra_mounts = [str(identifier_tmp_dir)]
|
||||
if str(Path(".").resolve()).startswith("/tmp"):
|
||||
extra_mounts.append(str(Path(".").resolve()))
|
||||
|
||||
if "_PYODIDE_EXTRA_MOUNTS" in env:
|
||||
oldmounts = env["_PYODIDE_EXTRA_MOUNTS"] + ":"
|
||||
env["_PYODIDE_EXTRA_MOUNTS"] = oldmounts + ":".join(extra_mounts)
|
||||
|
||||
compatible_wheel = find_compatible_wheel(built_wheels, config.identifier)
|
||||
if compatible_wheel:
|
||||
log.step_end()
|
||||
print(
|
||||
f"\nFound previously built wheel {compatible_wheel.name}, that's compatible with {config.identifier}. Skipping build step..."
|
||||
)
|
||||
built_wheel = compatible_wheel
|
||||
else:
|
||||
if build_options.before_build:
|
||||
log.step("Running before_build...")
|
||||
before_build_prepared = prepare_command(
|
||||
build_options.before_build, project=".", package=build_options.package_dir
|
||||
)
|
||||
shell(before_build_prepared, env=env)
|
||||
|
||||
log.step("Building wheel...")
|
||||
|
||||
extra_flags = split_config_settings(build_options.config_settings, "build")
|
||||
extra_flags += build_frontend.args
|
||||
|
||||
if not 0 <= build_options.build_verbosity < 2:
|
||||
msg = f"build_verbosity {build_options.build_verbosity} is not supported for build frontend. Ignoring."
|
||||
log.warning(msg)
|
||||
|
||||
build_env = env.copy()
|
||||
if build_options.dependency_constraints:
|
||||
our_constraints = str(constraints_path)
|
||||
user_constraints = build_env.get("PIP_CONSTRAINT")
|
||||
build_env["PIP_CONSTRAINT"] = " ".join(
|
||||
c for c in [our_constraints, user_constraints] if c
|
||||
)
|
||||
build_env["VIRTUALENV_PIP"] = pip_version
|
||||
call(
|
||||
"pyodide",
|
||||
"build",
|
||||
build_options.package_dir,
|
||||
f"--outdir={built_wheel_dir}",
|
||||
*extra_flags,
|
||||
env=build_env,
|
||||
)
|
||||
built_wheel = next(built_wheel_dir.glob("*.whl"))
|
||||
|
||||
if built_wheel.name.endswith("none-any.whl"):
|
||||
raise NonPlatformWheelError()
|
||||
|
||||
if build_options.repair_command:
|
||||
log.step("Repairing wheel...")
|
||||
|
||||
repair_command_prepared = prepare_command(
|
||||
build_options.repair_command,
|
||||
wheel=built_wheel,
|
||||
dest_dir=repaired_wheel_dir,
|
||||
)
|
||||
shell(repair_command_prepared, env=env)
|
||||
else:
|
||||
shutil.move(str(built_wheel), repaired_wheel_dir)
|
||||
|
||||
repaired_wheel = next(repaired_wheel_dir.glob("*.whl"))
|
||||
|
||||
if repaired_wheel.name in {wheel.name for wheel in built_wheels}:
|
||||
raise AlreadyBuiltWheelError(repaired_wheel.name)
|
||||
|
||||
if build_options.test_command and build_options.test_selector(config.identifier):
|
||||
log.step("Testing wheel...")
|
||||
|
||||
venv_dir = identifier_tmp_dir / "venv-test"
|
||||
# set up a virtual environment to install and test from, to make sure
|
||||
# there are no dependencies that were pulled in at build time.
|
||||
|
||||
virtualenv_env = env.copy()
|
||||
virtualenv_env["PATH"] = os.pathsep.join(
|
||||
[
|
||||
str(ensure_node(config.node_version)),
|
||||
virtualenv_env["PATH"],
|
||||
]
|
||||
)
|
||||
|
||||
# pyodide venv uses virtualenv under the hood
|
||||
# use the pip embedded with virtualenv & disable network updates
|
||||
virtualenv_create_env = virtualenv_env.copy()
|
||||
virtualenv_create_env["VIRTUALENV_PIP"] = pip_version
|
||||
virtualenv_create_env["VIRTUALENV_NO_PERIODIC_UPDATE"] = "1"
|
||||
|
||||
call("pyodide", "venv", venv_dir, env=virtualenv_create_env)
|
||||
|
||||
virtualenv_env["PATH"] = os.pathsep.join(
|
||||
[
|
||||
str(venv_dir / "bin"),
|
||||
virtualenv_env["PATH"],
|
||||
]
|
||||
)
|
||||
|
||||
# check that we are using the Python from the virtual environment
|
||||
call("which", "python", env=virtualenv_env)
|
||||
|
||||
if build_options.before_test:
|
||||
before_test_prepared = prepare_command(
|
||||
build_options.before_test,
|
||||
project=".",
|
||||
package=build_options.package_dir,
|
||||
)
|
||||
shell(before_test_prepared, env=virtualenv_env)
|
||||
|
||||
# install the wheel
|
||||
call(
|
||||
"pip",
|
||||
"install",
|
||||
f"{repaired_wheel}{build_options.test_extras}",
|
||||
env=virtualenv_env,
|
||||
)
|
||||
|
||||
# test the wheel
|
||||
if build_options.test_requires:
|
||||
call("pip", "install", *build_options.test_requires, env=virtualenv_env)
|
||||
|
||||
# run the tests from a temp dir, with an absolute path in the command
|
||||
# (this ensures that Python runs the tests against the installed wheel
|
||||
# and not the repo code)
|
||||
test_command_prepared = prepare_command(
|
||||
build_options.test_command,
|
||||
project=Path(".").resolve(),
|
||||
package=build_options.package_dir.resolve(),
|
||||
)
|
||||
|
||||
test_cwd = identifier_tmp_dir / "test_cwd"
|
||||
test_cwd.mkdir(exist_ok=True)
|
||||
(test_cwd / "test_fail.py").write_text(test_fail_cwd_file.read_text())
|
||||
|
||||
shell(test_command_prepared, cwd=test_cwd, env=virtualenv_env)
|
||||
|
||||
# we're all done here; move it to output (overwrite existing)
|
||||
if compatible_wheel is None:
|
||||
with contextlib.suppress(FileNotFoundError):
|
||||
(build_options.output_dir / repaired_wheel.name).unlink()
|
||||
|
||||
shutil.move(str(repaired_wheel), build_options.output_dir)
|
||||
built_wheels.append(build_options.output_dir / repaired_wheel.name)
|
||||
finally:
|
||||
pass
|
||||
@@ -166,3 +166,8 @@ python_configurations = [
|
||||
{ identifier = "pp39-win_amd64", version = "3.9", arch = "64", url = "https://downloads.python.org/pypy/pypy3.9-v7.3.16-win64.zip" },
|
||||
{ identifier = "pp310-win_amd64", version = "3.10", arch = "64", url = "https://downloads.python.org/pypy/pypy3.10-v7.3.16-win64.zip" },
|
||||
]
|
||||
|
||||
[pyodide]
|
||||
python_configurations = [
|
||||
{ identifier = "cp312-pyodide_wasm32", version = "3.12.1", pyodide_version = "0.26.0", emscripten_version = "3.1.58", node_version = "v20" },
|
||||
]
|
||||
|
||||
@@ -765,6 +765,51 @@
|
||||
"$ref": "#/properties/test-requires"
|
||||
}
|
||||
}
|
||||
},
|
||||
"pyodide": {
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"archs": {
|
||||
"$ref": "#/properties/archs"
|
||||
},
|
||||
"before-all": {
|
||||
"$ref": "#/properties/before-all"
|
||||
},
|
||||
"before-build": {
|
||||
"$ref": "#/properties/before-build"
|
||||
},
|
||||
"before-test": {
|
||||
"$ref": "#/properties/before-test"
|
||||
},
|
||||
"build-frontend": {
|
||||
"$ref": "#/properties/build-frontend"
|
||||
},
|
||||
"build-verbosity": {
|
||||
"$ref": "#/properties/build-verbosity"
|
||||
},
|
||||
"config-settings": {
|
||||
"$ref": "#/properties/config-settings"
|
||||
},
|
||||
"dependency-versions": {
|
||||
"$ref": "#/properties/dependency-versions"
|
||||
},
|
||||
"environment": {
|
||||
"$ref": "#/properties/environment"
|
||||
},
|
||||
"repair-wheel-command": {
|
||||
"$ref": "#/properties/repair-wheel-command"
|
||||
},
|
||||
"test-command": {
|
||||
"$ref": "#/properties/test-command"
|
||||
},
|
||||
"test-extras": {
|
||||
"$ref": "#/properties/test-extras"
|
||||
},
|
||||
"test-requires": {
|
||||
"$ref": "#/properties/test-requires"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,123 @@
|
||||
# This file was autogenerated by uv via the following command:
|
||||
# nox -s update_constraints
|
||||
annotated-types==0.7.0
|
||||
# via pydantic
|
||||
anyio==4.4.0
|
||||
# via httpx
|
||||
auditwheel-emscripten==0.0.14
|
||||
# via pyodide-build
|
||||
build==1.2.1
|
||||
# via
|
||||
# -r .nox/update_constraints/tmp/constraints-pyodide.in
|
||||
# pyodide-build
|
||||
certifi==2024.2.2
|
||||
# via
|
||||
# httpcore
|
||||
# httpx
|
||||
# requests
|
||||
charset-normalizer==3.3.2
|
||||
# via requests
|
||||
click==8.1.7
|
||||
# via typer
|
||||
cloudpickle==3.0.0
|
||||
# via loky
|
||||
cmake==3.29.3
|
||||
# via pyodide-build
|
||||
distlib==0.3.8
|
||||
# via virtualenv
|
||||
filelock==3.14.0
|
||||
# via virtualenv
|
||||
h11==0.14.0
|
||||
# via httpcore
|
||||
httpcore==1.0.5
|
||||
# via httpx
|
||||
httpx==0.27.0
|
||||
# via unearth
|
||||
idna==3.7
|
||||
# via
|
||||
# anyio
|
||||
# httpx
|
||||
# requests
|
||||
leb128==1.0.7
|
||||
# via auditwheel-emscripten
|
||||
loky==3.4.1
|
||||
# via pyodide-build
|
||||
markdown-it-py==3.0.0
|
||||
# via rich
|
||||
mdurl==0.1.2
|
||||
# via markdown-it-py
|
||||
packaging==24.0
|
||||
# via
|
||||
# auditwheel-emscripten
|
||||
# build
|
||||
# pyodide-build
|
||||
# unearth
|
||||
pip==24.0
|
||||
# via -r .nox/update_constraints/tmp/constraints-pyodide.in
|
||||
platformdirs==4.2.2
|
||||
# via virtualenv
|
||||
pydantic==2.7.1
|
||||
# via
|
||||
# pyodide-build
|
||||
# pyodide-lock
|
||||
pydantic-core==2.18.2
|
||||
# via pydantic
|
||||
pygments==2.18.0
|
||||
# via rich
|
||||
pyodide-build==0.26.0
|
||||
# via -r .nox/update_constraints/tmp/constraints-pyodide.in
|
||||
pyodide-cli==0.2.3
|
||||
# via
|
||||
# auditwheel-emscripten
|
||||
# pyodide-build
|
||||
pyodide-lock==0.1.0a6
|
||||
# via pyodide-build
|
||||
pyproject-hooks==1.1.0
|
||||
# via build
|
||||
pyyaml==6.0.1
|
||||
# via pyodide-build
|
||||
requests==2.32.2
|
||||
# via pyodide-build
|
||||
resolvelib==1.0.1
|
||||
# via pyodide-build
|
||||
rich==13.7.1
|
||||
# via
|
||||
# pyodide-build
|
||||
# pyodide-cli
|
||||
# typer
|
||||
ruamel-yaml==0.18.6
|
||||
# via pyodide-build
|
||||
ruamel-yaml-clib==0.2.8
|
||||
# via ruamel-yaml
|
||||
shellingham==1.5.4
|
||||
# via typer
|
||||
sniffio==1.3.1
|
||||
# via
|
||||
# anyio
|
||||
# httpx
|
||||
typer==0.12.3
|
||||
# via
|
||||
# auditwheel-emscripten
|
||||
# pyodide-build
|
||||
# pyodide-cli
|
||||
types-requests==2.32.0.20240523
|
||||
# via pyodide-build
|
||||
typing-extensions==4.12.0
|
||||
# via
|
||||
# pydantic
|
||||
# pydantic-core
|
||||
# typer
|
||||
unearth==0.15.3
|
||||
# via pyodide-build
|
||||
urllib3==2.2.1
|
||||
# via
|
||||
# requests
|
||||
# types-requests
|
||||
virtualenv==20.26.2
|
||||
# via
|
||||
# build
|
||||
# pyodide-build
|
||||
wheel==0.43.0
|
||||
# via
|
||||
# auditwheel-emscripten
|
||||
# pyodide-build
|
||||
@@ -46,3 +46,5 @@ repair-wheel-command = "auditwheel repair -w {dest_dir} {wheel}"
|
||||
repair-wheel-command = "delocate-wheel --require-archs {delocate_archs} -w {dest_dir} -v {wheel}"
|
||||
|
||||
[tool.cibuildwheel.windows]
|
||||
|
||||
[tool.cibuildwheel.pyodide]
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
url = "https://nodejs.org/dist/"
|
||||
v20 = "v20.13.1"
|
||||
@@ -22,8 +22,8 @@ else:
|
||||
PathOrStr = Union[str, "os.PathLike[str]"]
|
||||
|
||||
|
||||
PlatformName = Literal["linux", "macos", "windows"]
|
||||
PLATFORMS: Final[set[PlatformName]] = {"linux", "macos", "windows"}
|
||||
PlatformName = Literal["linux", "macos", "windows", "pyodide"]
|
||||
PLATFORMS: Final[set[PlatformName]] = {"linux", "macos", "windows", "pyodide"}
|
||||
|
||||
|
||||
class GenericPythonConfiguration(Protocol):
|
||||
|
||||
+61
-2
@@ -9,6 +9,7 @@ import shlex
|
||||
import ssl
|
||||
import subprocess
|
||||
import sys
|
||||
import tarfile
|
||||
import textwrap
|
||||
import time
|
||||
import typing
|
||||
@@ -19,8 +20,10 @@ from dataclasses import dataclass
|
||||
from enum import Enum
|
||||
from functools import cached_property, lru_cache
|
||||
from pathlib import Path, PurePath
|
||||
from tempfile import TemporaryDirectory
|
||||
from time import sleep
|
||||
from typing import Any, ClassVar, Final, Literal, TextIO, TypeVar
|
||||
from zipfile import ZipFile
|
||||
|
||||
import bracex
|
||||
import certifi
|
||||
@@ -32,6 +35,7 @@ from packaging.version import Version
|
||||
from platformdirs import user_cache_path
|
||||
|
||||
from ._compat import tomllib
|
||||
from .architecture import Architecture
|
||||
from .typing import PathOrStr, PlatformName
|
||||
|
||||
__all__ = [
|
||||
@@ -334,6 +338,27 @@ def download(url: str, dest: Path) -> None:
|
||||
sleep(3)
|
||||
|
||||
|
||||
def extract_zip(zip_src: Path, dest: Path) -> None:
|
||||
with ZipFile(zip_src) as zip_:
|
||||
for zinfo in zip_.filelist:
|
||||
zip_.extract(zinfo, dest)
|
||||
|
||||
# Set permissions to the same values as they were set in the archive
|
||||
# We have to do this manually due to
|
||||
# https://github.com/python/cpython/issues/59999
|
||||
# But some files in the zipfile seem to have external_attr with 0
|
||||
# permissions. In that case just use the default value???
|
||||
permissions = (zinfo.external_attr >> 16) & 0o777
|
||||
if permissions != 0:
|
||||
dest.joinpath(zinfo.filename).chmod(permissions)
|
||||
|
||||
|
||||
def extract_tar(tar_src: Path, dest: Path) -> None:
|
||||
with tarfile.open(tar_src) as tar_:
|
||||
tar_.extraction_filter = getattr(tarfile, "tar_filter", (lambda member, _: member))
|
||||
tar_.extractall(dest)
|
||||
|
||||
|
||||
class DependencyConstraints:
|
||||
def __init__(self, base_file_path: Path):
|
||||
assert base_file_path.exists()
|
||||
@@ -343,12 +368,14 @@ class DependencyConstraints:
|
||||
def with_defaults() -> DependencyConstraints:
|
||||
return DependencyConstraints(base_file_path=resources_dir / "constraints.txt")
|
||||
|
||||
def get_for_python_version(self, version: str) -> Path:
|
||||
def get_for_python_version(
|
||||
self, version: str, *, variant: Literal["python", "pyodide"] = "python"
|
||||
) -> Path:
|
||||
version_parts = version.split(".")
|
||||
|
||||
# try to find a version-specific dependency file e.g. if
|
||||
# ./constraints.txt is the base, look for ./constraints-python36.txt
|
||||
specific_stem = self.base_file_path.stem + f"-python{version_parts[0]}{version_parts[1]}"
|
||||
specific_stem = self.base_file_path.stem + f"-{variant}{version_parts[0]}{version_parts[1]}"
|
||||
specific_name = specific_stem + self.base_file_path.suffix
|
||||
specific_file_path = self.base_file_path.with_name(specific_name)
|
||||
|
||||
@@ -534,6 +561,38 @@ def get_pip_version(env: Mapping[str, str]) -> str:
|
||||
return pip_version
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def ensure_node(major_version: str) -> Path:
|
||||
input_file = resources_dir / "nodejs.toml"
|
||||
with input_file.open("rb") as f:
|
||||
loaded_file = tomllib.load(f)
|
||||
version = str(loaded_file[major_version])
|
||||
base_url = str(loaded_file["url"])
|
||||
ext = "zip" if IS_WIN else "tar.xz"
|
||||
platform = "win" if IS_WIN else ("darwin" if sys.platform.startswith("darwin") else "linux")
|
||||
linux_arch = Architecture.native_arch("linux")
|
||||
assert linux_arch is not None
|
||||
arch = {"x86_64": "x64", "i686": "x86", "aarch64": "arm64"}.get(
|
||||
linux_arch.value, linux_arch.value
|
||||
)
|
||||
name = f"node-{version}-{platform}-{arch}"
|
||||
path = CIBW_CACHE_PATH / name
|
||||
with FileLock(str(path) + ".lock"):
|
||||
if not path.exists():
|
||||
url = f"{base_url}{version}/{name}.{ext}"
|
||||
with TemporaryDirectory() as tmp_path:
|
||||
archive = Path(tmp_path) / f"{name}.{ext}"
|
||||
download(url, archive)
|
||||
if ext == "zip":
|
||||
extract_zip(archive, path.parent)
|
||||
else:
|
||||
extract_tar(archive, path.parent)
|
||||
assert path.exists()
|
||||
if not IS_WIN:
|
||||
return path / "bin"
|
||||
return path
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def _ensure_virtualenv(version: str) -> Path:
|
||||
version_parts = version.split(".")
|
||||
|
||||
@@ -11,7 +11,6 @@ from contextlib import suppress
|
||||
from dataclasses import dataclass
|
||||
from functools import lru_cache
|
||||
from pathlib import Path
|
||||
from zipfile import ZipFile
|
||||
|
||||
from filelock import FileLock
|
||||
from packaging.version import Version
|
||||
@@ -31,6 +30,7 @@ from .util import (
|
||||
NonPlatformWheelError,
|
||||
call,
|
||||
download,
|
||||
extract_zip,
|
||||
find_compatible_wheel,
|
||||
get_build_verbosity_extra_flags,
|
||||
get_pip_version,
|
||||
@@ -96,11 +96,6 @@ def get_python_configurations(
|
||||
return python_configurations
|
||||
|
||||
|
||||
def extract_zip(zip_src: Path, dest: Path) -> None:
|
||||
with ZipFile(zip_src) as zip_:
|
||||
zip_.extractall(dest)
|
||||
|
||||
|
||||
@lru_cache(maxsize=None)
|
||||
def _ensure_nuget() -> Path:
|
||||
nuget = CIBW_CACHE_PATH / "nuget.exe"
|
||||
|
||||
Reference in New Issue
Block a user