feat: add Pyodide support (#1456)

* feat: add Pyodide support

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>
Co-authored-by: Hood Chatham <roberthoodchatham@gmail.com>
Co-authored-by: Matthieu Darbois <mayeut@users.noreply.github.com>

tests: fix two merge issues

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

fix: include schema

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* Try to fix xbuildenv path

[pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

Try to install pyodide-build from main branch

Try again

Try again

Update constraints file

Try again

Remove unused variable

Drop constraints

Remove --download option

Fix xbuildenv install

Try again

Try again

[pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* fix: remove pinning on pyodide

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* Update for Pyodide 0.26.0a5

* Install pyodide-build from pypi

* Update docs/options.md

Co-authored-by: Henry Schreiner <HenrySchreinerIII@gmail.com>

* Unxfail things that look like they were just a version mismatch

* refactor: add constraints for pyodide

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* chore: minor cleanup

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* Apply suggestions from code review

* Apply suggestion from code review

* refactor: minor touchup

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* ci: xfail the pyodide test

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>

* review: use a pinned version of node

* fix tests

* review: error out on Windows

* test: check node & test on macos arm64

* chore: minor cleanup

* Add reference to emscripten libc issue

* Apply suggestion from code review

* review: use a pinned pip in test virtual environment

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

* chore: rework test virtual environment seed packages

* chore: workaround direct invocation of pytest

This allows to still test direct invocation of `pytest` on most platforms (including pyodide on Linux) but falls back to `python -m pytest` when running pyodide on macOS.

* Use release version of pyodide

* fix: tests for 0.26.0 & parallel initialization of xbuildenv

* Debug CI

* fix: test/test_build_frontend_args.py

* Revert "Debug CI"

This reverts commit 917646cffc96dbfc619c47d1c03a828f447bcdb7.

---------

Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>
Co-authored-by: Henry Schreiner <henryschreineriii@gmail.com>
Co-authored-by: Matthieu Darbois <mayeut@users.noreply.github.com>
Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

test/conftest.py

@@ -32,6 +32,8 @@ def pytest_addoption(parser) -> None:
     params=[{"CIBW_BUILD_FRONTEND": "pip"}, {"CIBW_BUILD_FRONTEND": "build"}], ids=["pip", "build"]
 )
 def build_frontend_env(request) -> dict[str, str]:
+    if platform == "pyodide":
+        pytest.skip("Can't use pip as build frontend for pyodide platform")
     return request.param  # type: ignore[no-any-return]
 
 

test/test_abi_variants.py

@@ -48,22 +48,32 @@ def test_abi3(tmp_path):
     project_dir = tmp_path / "project"
     limited_api_project.generate(project_dir)
 
+    single_python_tag = "cp{}{}".format(*utils.SINGLE_PYTHON_VERSION)
+
     # build the wheels
     actual_wheels = utils.cibuildwheel_run(
         project_dir,
         add_env={
             # free_threaded and PyPy do not have a Py_LIMITED_API equivalent, just build one of those
             # also limit the number of builds for test performance reasons
-            "CIBW_BUILD": "cp39-* cp310-* pp310-* cp311-* cp313t-*"
+            "CIBW_BUILD": f"cp39-* cp310-* pp310-* {single_python_tag}-* cp313t-*"
         },
     )
 
     # check that the expected wheels are produced
-    expected_wheels = [
-        w.replace("cp310-cp310", "cp310-abi3")
-        for w in utils.expected_wheels("spam", "0.1.0")
-        if "-cp39" in w or "-cp310" in w or "-pp310" in w or "-cp313t" in w
-    ]
+    expected_wheels = utils.expected_wheels("spam", "0.1.0")
+    if utils.platform == "pyodide":
+        # there's only 1 possible configuration for pyodide, the single_python_tag one
+        expected_wheels = [
+            w.replace(f"{single_python_tag}-{single_python_tag}", f"{single_python_tag}-abi3")
+            for w in expected_wheels
+        ]
+    else:
+        expected_wheels = [
+            w.replace("cp310-cp310", "cp310-abi3")
+            for w in expected_wheels
+            if "-cp39" in w or "-cp310" in w or "-pp310" in w or "-cp313t" in w
+        ]
     assert set(actual_wheels) == set(expected_wheels)
 
 
@@ -182,9 +192,9 @@ def test_abi_none(tmp_path, capfd):
         project_dir,
         add_env={
             "CIBW_TEST_REQUIRES": "pytest",
-            "CIBW_TEST_COMMAND": "pytest {project}/test",
+            "CIBW_TEST_COMMAND": f"{utils.invoke_pytest()} {{project}}/test",
             # limit the number of builds for test performance reasons
-            "CIBW_BUILD": "cp38-* cp310-* cp313t-* pp310-*",
+            "CIBW_BUILD": "cp38-* cp{}{}-* cp313t-* pp310-*".format(*utils.SINGLE_PYTHON_VERSION),
         },
     )
 
@@ -195,4 +205,8 @@ def test_abi_none(tmp_path, capfd):
     # check that each wheel was built once, and reused
     captured = capfd.readouterr()
     assert "Building wheel..." in captured.out
-    assert "Found previously built wheel" in captured.out
+    if utils.platform == "pyodide":
+        # there's only 1 possible configuration for pyodide, we won't see the message expected on following builds
+        assert "Found previously built wheel" not in captured.out
+    else:
+        assert "Found previously built wheel" in captured.out

test/test_before_build.py

@@ -7,9 +7,13 @@ import pytest
 
 from . import test_projects, utils
 
+# pyodide does not support building without isolation, need to check the base_prefix
+SYS_PREFIX = f"sys.{'base_' if utils.platform == 'pyodide' else ''}prefix"
+
+
 project_with_before_build_asserts = test_projects.new_c_project(
     setup_py_add=textwrap.dedent(
-        r"""
+        rf"""
         import os
 
         # assert that the Python version as written to pythonversion_bb.txt in the CIBW_BEFORE_BUILD step
@@ -24,11 +28,11 @@ project_with_before_build_asserts = test_projects.new_c_project(
         with open('pythonprefix_bb.txt') as f:
             stored_prefix = f.read()
         print('stored_prefix', stored_prefix)
-        print('sys.prefix', sys.prefix)
+        print('{SYS_PREFIX}', {SYS_PREFIX})
         #  Works around path-comparison bugs caused by short-paths on Windows e.g.
         #  vssadm~1 instead of vssadministrator
 
-        assert os.path.samefile(stored_prefix, sys.prefix)
+        assert os.path.samefile(stored_prefix, {SYS_PREFIX})
         """
     )
 )
@@ -40,7 +44,7 @@ def test(tmp_path):
 
     before_build = (
         """python -c "import sys; open('{project}/pythonversion_bb.txt', 'w').write(sys.version)" && """
-        '''python -c "import sys; open('{project}/pythonprefix_bb.txt', 'w').write(sys.prefix)"'''
+        f'''python -c "import sys; open('{{project}}/pythonprefix_bb.txt', 'w').write({SYS_PREFIX})"'''
     )
 
     # build the wheels

test/test_before_test.py

@@ -40,11 +40,19 @@ def test(tmp_path):
     test_project_dir = project_dir / "dependency"
     test_projects.new_c_project().generate(test_project_dir)
 
-    before_test = (
-        """python -c "import os, sys; open('{project}/pythonversion_bt.txt', 'w').write(sys.version)" && """
-        """python -c "import os, sys; open('{project}/pythonprefix_bt.txt', 'w').write(sys.prefix)" && """
-        """python -m pip install {project}/dependency"""
-    )
+    before_test_steps = [
+        '''python -c "import os, sys; open('{project}/pythonversion_bt.txt', 'w').write(sys.version)"''',
+        '''python -c "import os, sys; open('{project}/pythonprefix_bt.txt', 'w').write(sys.prefix)"''',
+    ]
+
+    if utils.platform == "pyodide":
+        before_test_steps.extend(
+            ["pyodide build {project}/dependency", "pip install --find-links dist/ spam"]
+        )
+    else:
+        before_test_steps.append("python -m pip install {project}/dependency")
+
+    before_test = " && ".join(before_test_steps)
 
     # build the wheels
     actual_wheels = utils.cibuildwheel_run(
@@ -56,7 +64,7 @@ def test(tmp_path):
             "CIBW_TEST_REQUIRES": "pytest",
             # the 'false ||' bit is to ensure this command runs in a shell on
             # mac/linux.
-            "CIBW_TEST_COMMAND": "false || pytest {project}/test",
+            "CIBW_TEST_COMMAND": f"false || {utils.invoke_pytest()} {{project}}/test",
             "CIBW_TEST_COMMAND_WINDOWS": "pytest {project}/test",
         },
     )

test/test_build_frontend_args.py

@@ -6,19 +6,24 @@ from . import utils
 from .test_projects.c import new_c_project
 
 
-@pytest.mark.parametrize("frontend_name", ["pip", "build"])
+@pytest.mark.parametrize(
+    "frontend_name",
+    [
+        pytest.param("pip", marks=utils.skip_if_pyodide("No pip for pyodide")),
+        "build",
+    ],
+)
 def test_build_frontend_args(tmp_path, capfd, frontend_name):
     project = new_c_project()
     project_dir = tmp_path / "project"
     project.generate(project_dir)
 
     # the build will fail because the frontend is called with '-h' - it prints the help message
+    add_env = {"CIBW_BUILD_FRONTEND": f"{frontend_name}; args: -h"}
+    if utils.platform == "pyodide":
+        add_env["TERM"] = "dumb"  # disable color / style
     with pytest.raises(subprocess.CalledProcessError):
-        utils.cibuildwheel_run(
-            project_dir,
-            add_env={"CIBW_BUILD_FRONTEND": f"{frontend_name}; args: -h"},
-            single_python=True,
-        )
+        utils.cibuildwheel_run(project_dir, add_env=add_env, single_python=True)
 
     captured = capfd.readouterr()
     print(captured.out)
@@ -27,6 +32,8 @@ def test_build_frontend_args(tmp_path, capfd, frontend_name):
     if frontend_name == "pip":
         assert "Usage:" in captured.out
         assert "Wheel Options:" in captured.out
+    elif utils.platform == "pyodide":
+        assert "Usage: pyodide build" in captured.out
     else:
         assert "usage:" in captured.out
         assert "A simple, correct Python build frontend." in captured.out

test/test_custom_repair_wheel.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import subprocess
+from contextlib import nullcontext as does_not_raise
 
 import pytest
 
@@ -31,8 +32,14 @@ def test(tmp_path, capfd):
     project_dir = tmp_path / "project"
     basic_project.generate(project_dir)
 
-    with pytest.raises(subprocess.CalledProcessError):
-        utils.cibuildwheel_run(
+    num_builds = len(utils.cibuildwheel_get_build_identifiers(project_dir))
+    if num_builds > 1:
+        expectation = pytest.raises(subprocess.CalledProcessError)
+    else:
+        expectation = does_not_raise()
+
+    with expectation:
+        result = utils.cibuildwheel_run(
             project_dir,
             add_env={
                 "CIBW_REPAIR_WHEEL_COMMAND": "python repair.py {wheel} {dest_dir}",
@@ -40,4 +47,14 @@ def test(tmp_path, capfd):
         )
 
     captured = capfd.readouterr()
-    assert "Build failed because a wheel named" in captured.err
+    if num_builds > 1:
+        assert "Build failed because a wheel named" in captured.err
+    else:
+        # We only produced one wheel (currently Pyodide)
+        # check that it has the right name
+        #
+        # As far as I can tell, this is the only full test coverage for
+        # CIBW_REPAIR_WHEEL_COMMAND so this is useful even in the case when no
+        # error is raised
+        assert "spam-0.1.0-py2-none-pyodide" in captured.out
+        assert result[0].startswith("spam-0.1.0-py2-none-")

test/test_emscripten.py

@@ -0,0 +1,84 @@
+from __future__ import annotations
+
+import shutil
+import sys
+import textwrap
+
+import pytest
+
+from cibuildwheel.util import CIBW_CACHE_PATH
+
+from . import test_projects, utils
+
+basic_project = test_projects.new_c_project()
+basic_project.files["check_node.py"] = r"""
+import sys
+import shutil
+from pathlib import Path
+from pyodide.code import run_js
+
+
+def check_node():
+    # cibuildwheel adds a pinned node version to the PATH
+    # check it's in the PATH then, check it's the one that runs pyoodide
+    cibw_cache_path = Path(sys.argv[1]).resolve(strict=True)
+    # find the node executable in PATH
+    node = shutil.which("node")
+    assert node is not None, "node is None"
+    node_path = Path(node).resolve(strict=True)
+    # it shall be in cibuildwheel cache
+    assert cibw_cache_path in node_path.parents, f"{cibw_cache_path} not a parent of {node_path}"
+    # find the path to the node executable that runs pyodide
+    node_js = run_js("globalThis.process.execPath")
+    assert node_js is not None, "node_js is None"
+    node_js_path = Path(node_js).resolve(strict=True)
+    # it shall be the one pinned by cibuildwheel
+    assert node_js_path == node_path, f"{node_js_path} != {node_path}"
+
+
+if __name__ == "__main__":
+    check_node()
+"""
+
+
+@pytest.mark.parametrize("use_pyproject_toml", [True, False])
+def test_pyodide_build(tmp_path, use_pyproject_toml):
+    if sys.platform == "win32":
+        pytest.skip("emsdk doesn't work correctly on Windows")
+
+    if not shutil.which("python3.12"):
+        pytest.skip("Python 3.12 not installed")
+
+    if use_pyproject_toml:
+        basic_project.files["pyproject.toml"] = textwrap.dedent(
+            """
+            [build-system]
+            requires = ["setuptools"]
+            build-backend = "setuptools.build_meta"
+            """
+        )
+
+    project_dir = tmp_path / "project"
+    basic_project.generate(project_dir)
+
+    # check for node in 1 case only to reduce CI load
+    add_env = {}
+    if use_pyproject_toml:
+        add_env["CIBW_TEST_COMMAND"] = f"python {{project}}/check_node.py {CIBW_CACHE_PATH}"
+
+    # build the wheels
+    actual_wheels = utils.cibuildwheel_run(
+        project_dir,
+        add_args=["--platform", "pyodide"],
+        add_env=add_env,
+    )
+
+    # check that the expected wheels are produced
+    expected_wheels = [
+        "spam-0.1.0-cp312-cp312-pyodide_2024_0_wasm32.whl",
+    ]
+
+    print("actual_wheels", actual_wheels)
+    print("expected_wheels", expected_wheels)
+
+    assert set(actual_wheels) == set(expected_wheels)

test/test_environment.py

@@ -93,10 +93,18 @@ def test_overridden_path(tmp_path, capfd):
 
     assert len(os.listdir(output_dir)) == 0
     captured = capfd.readouterr()
-    assert "python available on PATH doesn't match our installed instance" in captured.err
+    assert "python available on PATH doesn't match our installed instance" in captured.err.replace(
+        "venv", "installed"
+    )
 
 
-@pytest.mark.parametrize("build_frontend", ["pip", "build"])
+@pytest.mark.parametrize(
+    "build_frontend",
+    [
+        pytest.param("pip", marks=utils.skip_if_pyodide("No pip for pyodide")),
+        "build",
+    ],
+)
 def test_overridden_pip_constraint(tmp_path, build_frontend):
     """
     Verify that users can use PIP_CONSTRAINT to specify a specific version of
@@ -109,7 +117,7 @@ def test_overridden_pip_constraint(tmp_path, build_frontend):
         setup_py_add=textwrap.dedent(
             """
             import pytz
-            assert pytz.__version__ == "2022.4"
+            assert pytz.__version__ == "2022.4", f"{pytz.__version__!r} != '2022.4'"
         """
         )
     )

test/test_projects/c.py

@@ -42,15 +42,20 @@ PyMODINIT_FUNC PyInit_spam(void)
 """
 
 SETUP_PY_TEMPLATE = r"""
+import os
 import sys
+
 from setuptools import setup, Extension
 
 {{ setup_py_add }}
 
 libraries = []
-if sys.platform.startswith('linux'):
+# Emscripten fails if you pass -lc...
+# See: https://github.com/emscripten-core/emscripten/issues/16680
+if sys.platform.startswith('linux') and "emscripten" not in os.environ.get("_PYTHON_HOST_PLATFORM", ""):
     libraries.extend(['m', 'c'])
 
+
 setup(
     ext_modules=[Extension(
         'spam',

test/test_testing.py

@@ -66,7 +66,7 @@ class TestSpam(TestCase):
         # See #336 for more info.
         bits = struct.calcsize("P") * 8
         if bits == 32:
-            self.assertEqual(platform.machine(), "i686")
+            self.assertIn(platform.machine(), ["i686", "wasm32"])
 '''
 
 
@@ -81,7 +81,7 @@ def test(tmp_path):
             "CIBW_TEST_REQUIRES": "pytest",
             # the 'false ||' bit is to ensure this command runs in a shell on
             # mac/linux.
-            "CIBW_TEST_COMMAND": "false || pytest {project}/test",
+            "CIBW_TEST_COMMAND": f"false || {utils.invoke_pytest()} {{project}}/test",
             "CIBW_TEST_COMMAND_WINDOWS": "COLOR 00 || pytest {project}/test",
         },
     )
@@ -102,7 +102,7 @@ def test_extras_require(tmp_path):
             "CIBW_TEST_EXTRAS": "test",
             # the 'false ||' bit is to ensure this command runs in a shell on
             # mac/linux.
-            "CIBW_TEST_COMMAND": "false || pytest {project}/test",
+            "CIBW_TEST_COMMAND": f"false || {utils.invoke_pytest()} {{project}}/test",
             "CIBW_TEST_COMMAND_WINDOWS": "COLOR 00 || pytest {project}/test",
         },
         single_python=True,

test/utils.py

@@ -13,6 +13,8 @@ import sys
 from tempfile import TemporaryDirectory
 from typing import Final
 
+import pytest
+
 from cibuildwheel.architecture import Architecture
 from cibuildwheel.util import CIBW_CACHE_PATH
 
@@ -188,6 +190,8 @@ def expected_wheels(
     if musllinux_versions is None:
         musllinux_versions = ["musllinux_1_2"]
 
+    if platform == "pyodide" and python_abi_tags is None:
+        python_abi_tags = ["cp312-cp312"]
     if python_abi_tags is None:
         python_abi_tags = [
             "cp36-cp36m",
@@ -236,6 +240,12 @@ def expected_wheels(
 
     wheels = []
 
+    if platform == "pyodide":
+        assert len(python_abi_tags) == 1
+        python_abi_tag = python_abi_tags[0]
+        platform_tag = "pyodide_2024_0_wasm32"
+        return [f"{package_name}-{package_version}-{python_abi_tag}-{platform_tag}.whl"]
+
     for python_abi_tag in python_abi_tags:
         platform_tags = []
 
@@ -283,7 +293,6 @@ def expected_wheels(
                 platform_tags.append(
                     f'macosx_{macosx_deployment_target.replace(".", "_")}_universal2',
                 )
-
         else:
             msg = f"Unsupported platform {platform!r}"
             raise Exception(msg)
@@ -306,6 +315,17 @@ def get_macos_version():
     return tuple(map(int, version_str.split(".")[:2]))
 
 
+def skip_if_pyodide(reason: str):
+    return pytest.mark.skipif(platform == "pyodide", reason=reason)
+
+
+def invoke_pytest() -> str:
+    # see https://github.com/pyodide/pyodide/issues/4802
+    if platform == "pyodide" and sys.platform.startswith("darwin"):
+        return "python -m pytest"
+    return "pytest"
+
+
 def arch_name_for_linux(arch: str):
     """
     Archs have different names on different platforms, but it's useful to be