feat: add Pyodide support (#1456)

* feat: add Pyodide support Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> Co-authored-by: Hood Chatham <roberthoodchatham@gmail.com> Co-authored-by: Matthieu Darbois <mayeut@users.noreply.github.com> tests: fix two merge issues Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> fix: include schema Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * Try to fix xbuildenv path [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci Try to install pyodide-build from main branch Try again Try again Update constraints file Try again Remove unused variable Drop constraints Remove --download option Fix xbuildenv install Try again Try again [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix: remove pinning on pyodide Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * Update for Pyodide 0.26.0a5 * Install pyodide-build from pypi * Update docs/options.md Co-authored-by: Henry Schreiner <HenrySchreinerIII@gmail.com> * Unxfail things that look like they were just a version mismatch * refactor: add constraints for pyodide Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * chore: minor cleanup Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * Apply suggestions from code review * Apply suggestion from code review * refactor: minor touchup Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * ci: xfail the pyodide test Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> * review: use a pinned version of node * fix tests * review: error out on Windows * test: check node & test on macos arm64 * chore: minor cleanup * Add reference to emscripten libc issue * Apply suggestion from code review * review: use a pinned pip in test virtual environment * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * chore: rework test virtual environment seed packages * chore: workaround direct invocation of pytest This allows to still test direct invocation of `pytest` on most platforms (including pyodide on Linux) but falls back to `python -m pytest` when running pyodide on macOS. * Use release version of pyodide * fix: tests for 0.26.0 & parallel initialization of xbuildenv * Debug CI * fix: test/test_build_frontend_args.py * Revert "Debug CI" This reverts commit 917646cffc96dbfc619c47d1c03a828f447bcdb7. --------- Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com> Co-authored-by: Henry Schreiner <henryschreineriii@gmail.com> Co-authored-by: Matthieu Darbois <mayeut@users.noreply.github.com> Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>
2024-05-28 08:31:36 -04:00
parent 687406f664
commit 9f2a3cb67c
32 changed files with 1154 additions and 87 deletions
@@ -9,6 +9,7 @@ import shlex
 import ssl
 import subprocess
 import sys
+import tarfile
 import textwrap
 import time
 import typing
@@ -19,8 +20,10 @@ from dataclasses import dataclass
 from enum import Enum
 from functools import cached_property, lru_cache
 from pathlib import Path, PurePath
+from tempfile import TemporaryDirectory
 from time import sleep
 from typing import Any, ClassVar, Final, Literal, TextIO, TypeVar
+from zipfile import ZipFile

 import bracex
 import certifi
@@ -32,6 +35,7 @@ from packaging.version import Version
 from platformdirs import user_cache_path

 from ._compat import tomllib
+from .architecture import Architecture
 from .typing import PathOrStr, PlatformName

 __all__ = [
@@ -334,6 +338,27 @@ def download(url: str, dest: Path) -> None:
            sleep(3)


+def extract_zip(zip_src: Path, dest: Path) -> None:
+    with ZipFile(zip_src) as zip_:
+        for zinfo in zip_.filelist:
+            zip_.extract(zinfo, dest)
+
+            # Set permissions to the same values as they were set in the archive
+            # We have to do this manually due to
+            # https://github.com/python/cpython/issues/59999
+            # But some files in the zipfile seem to have external_attr with 0
+            # permissions. In that case just use the default value???
+            permissions = (zinfo.external_attr >> 16) & 0o777
+            if permissions != 0:
+                dest.joinpath(zinfo.filename).chmod(permissions)
+
+
+def extract_tar(tar_src: Path, dest: Path) -> None:
+    with tarfile.open(tar_src) as tar_:
+        tar_.extraction_filter = getattr(tarfile, "tar_filter", (lambda member, _: member))
+        tar_.extractall(dest)
+
+
 class DependencyConstraints:
    def __init__(self, base_file_path: Path):
        assert base_file_path.exists()
@@ -343,12 +368,14 @@ class DependencyConstraints:
    def with_defaults() -> DependencyConstraints:
        return DependencyConstraints(base_file_path=resources_dir / "constraints.txt")

-    def get_for_python_version(self, version: str) -> Path:
+    def get_for_python_version(
+        self, version: str, *, variant: Literal["python", "pyodide"] = "python"
+    ) -> Path:
        version_parts = version.split(".")

        # try to find a version-specific dependency file e.g. if
        # ./constraints.txt is the base, look for ./constraints-python36.txt
-        specific_stem = self.base_file_path.stem + f"-python{version_parts[0]}{version_parts[1]}"
+        specific_stem = self.base_file_path.stem + f"-{variant}{version_parts[0]}{version_parts[1]}"
        specific_name = specific_stem + self.base_file_path.suffix
        specific_file_path = self.base_file_path.with_name(specific_name)

@@ -534,6 +561,38 @@ def get_pip_version(env: Mapping[str, str]) -> str:
    return pip_version


+@lru_cache(maxsize=None)
+def ensure_node(major_version: str) -> Path:
+    input_file = resources_dir / "nodejs.toml"
+    with input_file.open("rb") as f:
+        loaded_file = tomllib.load(f)
+    version = str(loaded_file[major_version])
+    base_url = str(loaded_file["url"])
+    ext = "zip" if IS_WIN else "tar.xz"
+    platform = "win" if IS_WIN else ("darwin" if sys.platform.startswith("darwin") else "linux")
+    linux_arch = Architecture.native_arch("linux")
+    assert linux_arch is not None
+    arch = {"x86_64": "x64", "i686": "x86", "aarch64": "arm64"}.get(
+        linux_arch.value, linux_arch.value
+    )
+    name = f"node-{version}-{platform}-{arch}"
+    path = CIBW_CACHE_PATH / name
+    with FileLock(str(path) + ".lock"):
+        if not path.exists():
+            url = f"{base_url}{version}/{name}.{ext}"
+            with TemporaryDirectory() as tmp_path:
+                archive = Path(tmp_path) / f"{name}.{ext}"
+                download(url, archive)
+                if ext == "zip":
+                    extract_zip(archive, path.parent)
+                else:
+                    extract_tar(archive, path.parent)
+    assert path.exists()
+    if not IS_WIN:
+        return path / "bin"
+    return path
+
+
@lru_cache(maxsize=None)
 def _ensure_virtualenv(version: str) -> Path:
    version_parts = version.split(".")