Merge remote-tracking branch 'upstream/main' into arm64

This commit is contained in:
Steve Dower
2022-08-01 19:54:50 +01:00
118 changed files with 1960 additions and 1141 deletions
+3 -16
View File
@@ -1,4 +1,4 @@
from typing import Dict
from __future__ import annotations
import pytest
@@ -7,24 +7,11 @@ def pytest_addoption(parser) -> None:
parser.addoption(
"--run-emulation", action="store_true", default=False, help="run emulation tests"
)
def pytest_configure(config):
config.addinivalue_line("markers", "emulation: mark test requiring qemu binfmt_misc to run")
def pytest_collection_modifyitems(config, items) -> None:
if config.getoption("--run-emulation"):
# --run-emulation given in cli: do not skip emulation tests
return
skip_emulation = pytest.mark.skip(reason="need --run-emulation option to run")
for item in items:
if "emulation" in item.keywords:
item.add_marker(skip_emulation)
parser.addoption("--run-podman", action="store_true", default=False, help="run podman tests")
@pytest.fixture(
params=[{"CIBW_BUILD_FRONTEND": "pip"}, {"CIBW_BUILD_FRONTEND": "build"}], ids=["pip", "build"]
)
def build_frontend_env(request) -> Dict[str, str]:
def build_frontend_env(request) -> dict[str, str]:
return request.param # type: ignore[no-any-return]
+2
View File
@@ -1,3 +1,5 @@
from __future__ import annotations
import textwrap
import pytest
+181
View File
@@ -0,0 +1,181 @@
from __future__ import annotations
import textwrap
from . import test_projects, utils
limited_api_project = test_projects.new_c_project(
setup_py_add=textwrap.dedent(
r"""
cmdclass = {}
extension_kwargs = {}
if sys.version_info[:2] >= (3, 8):
from wheel.bdist_wheel import bdist_wheel as _bdist_wheel
class bdist_wheel_abi3(_bdist_wheel):
def finalize_options(self):
_bdist_wheel.finalize_options(self)
self.root_is_pure = False
def get_tag(self):
python, abi, plat = _bdist_wheel.get_tag(self)
return python, "abi3", plat
cmdclass["bdist_wheel"] = bdist_wheel_abi3
extension_kwargs["define_macros"] = [("Py_LIMITED_API", "0x03080000")]
extension_kwargs["py_limited_api"] = True
"""
),
setup_py_extension_args_add="**extension_kwargs",
setup_py_setup_args_add="cmdclass=cmdclass",
)
def test_abi3(tmp_path):
project_dir = tmp_path / "project"
limited_api_project.generate(project_dir)
# build the wheels
actual_wheels = utils.cibuildwheel_run(
project_dir,
add_env={
"CIBW_SKIP": "pp* ", # PyPy does not have a Py_LIMITED_API equivalent
},
)
# check that the expected wheels are produced
expected_wheels = [
w.replace("cp38-cp38", "cp38-abi3")
for w in utils.expected_wheels("spam", "0.1.0")
if "-pp" not in w and "-cp39" not in w and "-cp310" not in w and "-cp311" not in w
]
assert set(actual_wheels) == set(expected_wheels)
ctypes_project = test_projects.TestProject()
ctypes_project.files["setup.py"] = textwrap.dedent(
"""
from setuptools import setup, Extension
from distutils.command.build_ext import build_ext as _build_ext
class CTypesExtension(Extension): pass
class build_ext(_build_ext):
def build_extension(self, ext):
self._ctypes = isinstance(ext, CTypesExtension)
return super().build_extension(ext)
def get_export_symbols(self, ext):
if self._ctypes:
return ext.export_symbols
return super().get_export_symbols(ext)
def get_ext_filename(self, ext_name):
if self._ctypes:
return ext_name + '.so'
return super().get_ext_filename(ext_name)
from wheel.bdist_wheel import bdist_wheel as _bdist_wheel
class bdist_wheel_abi_none(_bdist_wheel):
def finalize_options(self):
_bdist_wheel.finalize_options(self)
self.root_is_pure = False
def get_tag(self):
python, abi, plat = _bdist_wheel.get_tag(self)
return "py3", "none", plat
setup(
name="ctypesexample",
version="1.0.0",
py_modules = ["ctypesexample.summing"],
ext_modules=[
CTypesExtension(
"ctypesexample.csumlib",
["ctypesexample/csumlib.c"],
),
],
cmdclass={'build_ext': build_ext, 'bdist_wheel': bdist_wheel_abi_none},
)
"""
)
ctypes_project.files["ctypesexample/csumlib.c"] = textwrap.dedent(
"""
#ifdef _WIN32
#define LIBRARY_API __declspec(dllexport)
#else
#define LIBRARY_API
#endif
#include <stdlib.h>
LIBRARY_API double *add_vec3(double *a, double *b)
{
double *res = malloc(sizeof(double) * 3);
for (int i = 0; i < 3; ++i)
{
res[i] = a[i] + b[i];
}
return res;
}
"""
)
ctypes_project.files["ctypesexample/summing.py"] = textwrap.dedent(
"""
import ctypes
import pathlib
# path of the shared library
libfile = pathlib.Path(__file__).parent / "csumlib.so"
csumlib = ctypes.CDLL(str(libfile))
type_vec3 = ctypes.POINTER(ctypes.c_double * 3)
csumlib.add_vec3.restype = type_vec3
csumlib.add_vec3.argtypes = [type_vec3, type_vec3]
def add(a: list, b: list) -> list:
a_p = (ctypes.c_double * 3)(*a)
b_p = (ctypes.c_double * 3)(*b)
r_p = csumlib.add_vec3(a_p,b_p)
return [l for l in r_p.contents]
"""
)
ctypes_project.files["test/add_test.py"] = textwrap.dedent(
"""
import ctypesexample.summing
def test():
a = [1, 2, 3]
b = [4, 5, 6]
assert ctypesexample.summing.add(a, b) == [5, 7, 9]
"""
)
def test_abi_none(tmp_path, capfd):
project_dir = tmp_path / "project"
ctypes_project.generate(project_dir)
# build the wheels
actual_wheels = utils.cibuildwheel_run(
project_dir,
add_env={
"CIBW_TEST_REQUIRES": "pytest",
"CIBW_TEST_COMMAND": "pytest {project}/test",
# limit the number of builds for test performance reasons
"CIBW_BUILD": "cp38-* cp310-* pp39-*",
},
)
# check that the expected wheels are produced
expected_wheels = utils.expected_wheels("ctypesexample", "1.0.0", python_abi_tags=["py3-none"])
assert set(actual_wheels) == set(expected_wheels)
# check that each wheel was built once, and reused
captured = capfd.readouterr()
assert "Building wheel..." in captured.out
assert "Found previously built wheel" in captured.out
+2
View File
@@ -1,3 +1,5 @@
from __future__ import annotations
import subprocess
import textwrap
+2
View File
@@ -1,3 +1,5 @@
from __future__ import annotations
import subprocess
import textwrap
+2
View File
@@ -1,3 +1,5 @@
from __future__ import annotations
from . import test_projects, utils
before_test_project = test_projects.new_c_project()
+2
View File
@@ -1,3 +1,5 @@
from __future__ import annotations
import textwrap
from . import test_projects, utils
@@ -1,3 +1,5 @@
from __future__ import annotations
import platform
import textwrap
+2
View File
@@ -1,3 +1,5 @@
from __future__ import annotations
import os
import jinja2
+2
View File
@@ -1,3 +1,5 @@
from __future__ import annotations
import re
import textwrap
+6 -2
View File
@@ -1,3 +1,5 @@
from __future__ import annotations
import subprocess
import pytest
@@ -17,8 +19,10 @@ def test_spam():
"""
@pytest.mark.emulation
def test(tmp_path):
def test(tmp_path, request):
if not request.config.getoption("--run-emulation"):
pytest.skip("needs --run-emulation option to run")
project_dir = tmp_path / "project"
project_with_a_test.generate(project_dir)
+5 -1
View File
@@ -1,5 +1,8 @@
from __future__ import annotations
import os
import subprocess
import sys
import textwrap
import pytest
@@ -33,6 +36,7 @@ project_with_environment_asserts = test_projects.new_c_project(
def test(tmp_path):
python_echo = f"'{sys.executable}' -c \"import sys; print(*sys.argv[1:])\""
project_dir = tmp_path / "project"
project_with_environment_asserts.generate(project_dir)
@@ -43,7 +47,7 @@ def test(tmp_path):
project_dir,
add_env={
"CIBW_ENVIRONMENT": """CIBW_TEST_VAR="a b c" CIBW_TEST_VAR_2=1 CIBW_TEST_VAR_3="$(echo 'test string 3')" PATH=$PATH:/opt/cibw_test_path""",
"CIBW_ENVIRONMENT_WINDOWS": '''CIBW_TEST_VAR="a b c" CIBW_TEST_VAR_2=1 CIBW_TEST_VAR_3="$(echo 'test string 3')" PATH="$PATH;/opt/cibw_test_path"''',
"CIBW_ENVIRONMENT_WINDOWS": f'''CIBW_TEST_VAR="a b c" CIBW_TEST_VAR_2=1 CIBW_TEST_VAR_3="$({python_echo} 'test string 3')" PATH="$PATH;/opt/cibw_test_path"''',
},
)
+2
View File
@@ -1,3 +1,5 @@
from __future__ import annotations
import os
import subprocess
import sys
-50
View File
@@ -1,50 +0,0 @@
import textwrap
from . import test_projects, utils
limited_api_project = test_projects.new_c_project(
setup_py_add=textwrap.dedent(
r"""
cmdclass = {}
extension_kwargs = {}
if sys.version_info[:2] >= (3, 8):
from wheel.bdist_wheel import bdist_wheel as _bdist_wheel
class bdist_wheel_abi3(_bdist_wheel):
def finalize_options(self):
_bdist_wheel.finalize_options(self)
self.root_is_pure = False
def get_tag(self):
python, abi, plat = _bdist_wheel.get_tag(self)
return python, "abi3", plat
cmdclass["bdist_wheel"] = bdist_wheel_abi3
extension_kwargs["define_macros"] = [("Py_LIMITED_API", "0x03080000")]
extension_kwargs["py_limited_api"] = True
"""
),
setup_py_extension_args_add="**extension_kwargs",
setup_py_setup_args_add="cmdclass=cmdclass",
)
def test(tmp_path):
project_dir = tmp_path / "project"
limited_api_project.generate(project_dir)
# build the wheels
actual_wheels = utils.cibuildwheel_run(
project_dir,
add_env={
"CIBW_SKIP": "pp* ", # PyPy does not have a Py_LIMITED_API equivalent
},
)
# check that the expected wheels are produced
expected_wheels = [
w.replace("cp38-cp38", "cp38-abi3")
for w in utils.expected_wheels("spam", "0.1.0")
if "-pp" not in w and "-cp39" not in w and "-cp310" not in w and "-cp311" not in w
]
assert set(actual_wheels) == set(expected_wheels)
+37 -3
View File
@@ -1,6 +1,7 @@
from __future__ import annotations
import platform
import subprocess
from typing import Tuple
import pytest
@@ -14,10 +15,10 @@ ALL_MACOS_WHEELS = {
}
def get_xcode_version() -> Tuple[int, int]:
def get_xcode_version() -> tuple[int, int]:
output = subprocess.run(
["xcodebuild", "-version"],
universal_newlines=True,
text=True,
check=True,
stdout=subprocess.PIPE,
).stdout
@@ -133,3 +134,36 @@ def test_universal2_testing(tmp_path, capfd, skip_arm64_test):
expected_wheels = [w for w in ALL_MACOS_WHEELS if "cp39" in w and "universal2" in w]
assert set(actual_wheels) == set(expected_wheels)
def test_cp38_arm64_testing(tmp_path, capfd):
if utils.platform != "macos":
pytest.skip("this test is only relevant to macos")
if get_xcode_version() < (12, 2):
pytest.skip("this test only works with Xcode 12.2 or greater")
if platform.machine() != "arm64":
pytest.skip("this test only works on arm64")
project_dir = tmp_path / "project"
basic_project.generate(project_dir)
actual_wheels = utils.cibuildwheel_run(
project_dir,
add_env={
"CIBW_BUILD": "cp38-*",
"CIBW_TEST_COMMAND": '''python -c "import platform; print('running tests on ' + platform.machine())"''',
"CIBW_ARCHS": "x86_64,universal2,arm64",
},
)
captured = capfd.readouterr()
assert "running tests on x86_64" in captured.out
assert "running tests on arm64" not in captured.out
warning_message = "While cibuildwheel can build CPython 3.8 universal2/arm64 wheels, we cannot test the arm64 part of them"
assert warning_message in captured.err
expected_wheels = [w for w in ALL_MACOS_WHEELS if "cp38" in w]
assert set(actual_wheels) == set(expected_wheels)
+28 -5
View File
@@ -1,3 +1,5 @@
from __future__ import annotations
import platform
import textwrap
@@ -21,17 +23,24 @@ project_with_manylinux_symbols = test_projects.new_c_project(
#if !__GLIBC_PREREQ(2, 5) /* manylinux1 is glibc 2.5 */
#error "Must run on a glibc >= 2.5 linux environment"
#endif
#if __GLIBC_PREREQ(2, 28)
#include <threads.h>
#endif
"""
),
spam_c_function_add=textwrap.dedent(
r"""
#if defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 24)
#if __GLIBC_PREREQ(2, 28)
// thrd_equal & thrd_current are only available in manylinux_2_28+
sts = thrd_equal(thrd_current(), thrd_current()) ? 0 : 1;;
#elif __GLIBC_PREREQ(2, 24)
// nextupf is only available in manylinux_2_24+
sts = (int)nextupf(0.0F);
#elif defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 17) /* manylinux2014 is glibc 2.17 */
#elif __GLIBC_PREREQ(2, 17) /* manylinux2014 is glibc 2.17 */
// secure_getenv is only available in manylinux2014+
sts = (int)(intptr_t)secure_getenv("NON_EXISTING_ENV_VARIABLE");
#elif defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 10) /* manylinux2010 is glibc 2.12 */
#elif __GLIBC_PREREQ(2, 10) /* manylinux2010 is glibc 2.12 */
// malloc_info is only available on manylinux2010+
sts = malloc_info(0, stdout);
#endif
@@ -41,14 +50,19 @@ project_with_manylinux_symbols = test_projects.new_c_project(
@pytest.mark.parametrize(
"manylinux_image", ["manylinux1", "manylinux2010", "manylinux2014", "manylinux_2_24"]
"manylinux_image",
["manylinux1", "manylinux2010", "manylinux2014", "manylinux_2_24", "manylinux_2_28"],
)
def test(manylinux_image, tmp_path):
if utils.platform != "linux":
pytest.skip("the docker test is only relevant to the linux build")
pytest.skip("the container image test is only relevant to the linux build")
elif platform.machine() not in ["x86_64", "i686"]:
if manylinux_image in ["manylinux1", "manylinux2010"]:
pytest.skip("manylinux1 and 2010 doesn't exist for non-x86 architectures")
elif manylinux_image == "manylinux_2_28" and platform.machine() == "s390x":
pytest.skip("manylinux_2_28 doesn't exist for s390x architecture")
elif manylinux_image == "manylinux_2_28" and platform.machine() == "i686":
pytest.skip("manylinux_2_28 doesn't exist for i686 architecture")
project_dir = tmp_path / "project"
project_with_manylinux_symbols.generate(project_dir)
@@ -74,6 +88,9 @@ def test(manylinux_image, tmp_path):
if manylinux_image in {"manylinux2010"}:
# We don't have a manylinux2010 image for PyPy 3.9, CPython 3.11
add_env["CIBW_SKIP"] = "pp39* cp311*"
if manylinux_image == "manylinux_2_28" and platform.machine() == "x86_64":
# We don't have a manylinux_2_28 image for i686
add_env["CIBW_ARCHS"] = "x86_64"
actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env)
@@ -91,7 +108,13 @@ def test(manylinux_image, tmp_path):
if manylinux_image in {"manylinux1"}:
# remove PyPy & CPython 3.10 and above
expected_wheels = [w for w in expected_wheels if "-pp" not in w and "-cp31" not in w]
if manylinux_image in {"manylinux2010"}:
# remove PyPy 3.9 & CPython 3.11
expected_wheels = [w for w in expected_wheels if "-pp39" not in w and "-cp311" not in w]
if manylinux_image == "manylinux_2_28" and platform.machine() == "x86_64":
# We don't have a manylinux_2_28 image for i686
expected_wheels = [w for w in expected_wheels if "i686" not in w]
assert set(actual_wheels) == set(expected_wheels)
+2
View File
@@ -1,3 +1,5 @@
from __future__ import annotations
import textwrap
from . import test_projects, utils
+40
View File
@@ -0,0 +1,40 @@
from __future__ import annotations
import pytest
from . import test_projects, utils
basic_project = test_projects.new_c_project()
def test(tmp_path, capfd, request):
if utils.platform != "linux":
pytest.skip("the test is only relevant to the linux build")
if not request.config.getoption("--run-podman"):
pytest.skip("needs --run-podman option to run")
project_dir = tmp_path / "project"
basic_project.generate(project_dir)
# build some musllinux and manylinux wheels (ensuring that we use two containers)
actual_wheels = utils.cibuildwheel_run(
project_dir,
add_env={
"CIBW_BUILD": "cp310-*{manylinux,musllinux}_x86_64",
"CIBW_BEFORE_ALL": "echo 'test log statement from before-all'",
"CIBW_CONTAINER_ENGINE": "podman",
},
)
# check that the expected wheels are produced
expected_wheels = [
w
for w in utils.expected_wheels("spam", "0.1.0")
if ("-cp310-" in w) and ("x86_64" in w) and ("manylinux" in w or "musllinux" in w)
]
assert set(actual_wheels) == set(expected_wheels)
# check that stdout is bring passed-though from container correctly
captured = capfd.readouterr()
assert "test log statement from before-all" in captured.out
+2
View File
@@ -1,3 +1,5 @@
from __future__ import annotations
from .base import TestProject
from .c import new_c_project
+2
View File
@@ -1,3 +1,5 @@
from __future__ import annotations
import importlib
import subprocess
import sys
+2
View File
@@ -1,3 +1,5 @@
from __future__ import annotations
from pathlib import Path
from typing import Any, Dict, Union
+2
View File
@@ -1,3 +1,5 @@
from __future__ import annotations
import jinja2
from .base import TestProject
+2
View File
@@ -1,3 +1,5 @@
from __future__ import annotations
import subprocess
from test import test_projects
+44
View File
@@ -0,0 +1,44 @@
from __future__ import annotations
import subprocess
from test import test_projects
import pytest
from . import utils
basic_project = test_projects.new_c_project()
basic_project.files[
"repair.py"
] = """
import shutil
import sys
from pathlib import Path
wheel = Path(sys.argv[1])
dest_dir = Path(sys.argv[2])
platform = wheel.stem.split("-")[-1]
name = f"spam-0.1.0-py2-none-{platform}.whl"
dest = dest_dir / name
dest_dir.mkdir(parents=True, exist_ok=True)
if dest.exists():
dest.unlink()
shutil.copy(wheel, dest)
"""
def test(tmp_path, capfd):
# this test checks that a generated wheel name shall be unique in a given cibuildwheel run
project_dir = tmp_path / "project"
basic_project.generate(project_dir)
with pytest.raises(subprocess.CalledProcessError):
utils.cibuildwheel_run(
project_dir,
add_env={
"CIBW_REPAIR_WHEEL_COMMAND": "python repair.py {wheel} {dest_dir}",
},
)
captured = capfd.readouterr()
assert "Build failed because a wheel named" in captured.err
+2
View File
@@ -1,3 +1,5 @@
from __future__ import annotations
import textwrap
from . import test_projects, utils
+2
View File
@@ -1,3 +1,5 @@
from __future__ import annotations
from pathlib import Path
import jinja2
+2
View File
@@ -1,3 +1,5 @@
from __future__ import annotations
import os
import subprocess
import textwrap
+2
View File
@@ -1,3 +1,5 @@
from __future__ import annotations
import subprocess
import pytest
+2
View File
@@ -1,3 +1,5 @@
from __future__ import annotations
import pytest
from . import test_projects, utils
+22 -18
View File
@@ -4,6 +4,8 @@ Utility functions used by the cibuildwheel tests.
This file is added to the PYTHONPATH in the test runner at bin/run_test.py.
"""
from __future__ import annotations
import os
import platform as pm
import subprocess
@@ -35,7 +37,7 @@ def cibuildwheel_get_build_identifiers(project_path, env=None, *, prerelease_pyt
cmd_output = subprocess.run(
cmd,
universal_newlines=True,
text=True,
env=env,
check=True,
stdout=subprocess.PIPE,
@@ -113,6 +115,7 @@ def expected_wheels(
musllinux_versions=None,
macosx_deployment_target="10.9",
machine_arch=None,
python_abi_tags=None,
):
"""
Returns a list of expected wheels from a run of cibuildwheel.
@@ -139,21 +142,22 @@ def expected_wheels(
if musllinux_versions is None:
musllinux_versions = ["musllinux_1_1"]
python_abi_tags = [
"cp36-cp36m",
"cp37-cp37m",
"cp38-cp38",
"cp39-cp39",
"cp310-cp310",
"cp311-cp311",
]
if python_abi_tags is None:
python_abi_tags = [
"cp36-cp36m",
"cp37-cp37m",
"cp38-cp38",
"cp39-cp39",
"cp310-cp310",
"cp311-cp311",
]
if machine_arch in ["x86_64", "AMD64", "x86", "aarch64"]:
python_abi_tags += ["pp37-pypy37_pp73", "pp38-pypy38_pp73", "pp39-pypy39_pp73"]
if machine_arch in ["x86_64", "AMD64", "x86", "aarch64"]:
python_abi_tags += ["pp37-pypy37_pp73", "pp38-pypy38_pp73", "pp39-pypy39_pp73"]
if platform == "macos" and machine_arch == "arm64":
# currently, arm64 macs are only supported by cp39, cp310 & cp311
python_abi_tags = ["cp39-cp39", "cp310-cp310", "cp311-cp311"]
if platform == "macos" and machine_arch == "arm64":
# arm64 macs are only supported by cp38+
python_abi_tags = ["cp38-cp38", "cp39-cp39", "cp310-cp310", "cp311-cp311"]
wheels = []
@@ -185,13 +189,13 @@ def expected_wheels(
)
elif platform == "windows":
if python_abi_tag.startswith("cp"):
platform_tags = ["win32", "win_amd64"]
else:
if python_abi_tag.startswith("pp"):
platform_tags = ["win_amd64"]
else:
platform_tags = ["win32", "win_amd64"]
elif platform == "macos":
if python_abi_tag == "cp39-cp39" and machine_arch == "arm64":
if machine_arch == "arm64":
arm64_macosx_deployment_target = _get_arm64_macosx_deployment_target(
macosx_deployment_target
)