fix: patching pypy venv (#502)

Potentially revert / remove on release of PyPy 7.3.4.

* patching linux and macos

* try test

* add patch files for pypy2.7

* fix installing

* Combine patch files into one

* Simplify further, and add test to remind ourselves to remove the PyPy patch

* ignore fail of patch on linux

make new test failing only on linux

* fix test

* add timeout

* fix patch on linux

* change `-N` to `--force`

Co-authored-by: Yannick Jadoul <yannick.jadoul@belgacom.net>

bin/run_tests.py

@@ -17,4 +17,4 @@ if __name__ == '__main__':
     subprocess.check_call(unit_test_args)
 
     # run the integration tests
-    subprocess.check_call([sys.executable, '-m', 'pytest', '-x', '--durations', '0', 'test'])
+    subprocess.check_call([sys.executable, '-m', 'pytest', '-x', '--durations', '0', '--timeout=1200', 'test'])

cibuildwheel/linux.py

@@ -138,6 +138,16 @@ def build(options: BuildOptions) -> None:
                     log.build_start(config.identifier)
 
                     dependency_constraint_flags: List[Union[str, PathLike]] = []
+                    if config.identifier.startswith("pp"):
+                        # Patch PyPy to make sure headers get installed into a venv
+                        patch_version = '_27' if config.version == '2.7' else ''
+                        patch_path = Path(__file__).absolute().parent / 'resources' / f'pypy_venv{patch_version}.patch'
+                        patch_docker_path = PurePath('/pypy_venv.patch')
+                        docker.copy_into(patch_path, patch_docker_path)
+                        try:
+                            docker.call(['patch', '--force', '-p1', '-d', config.path, '-i', patch_docker_path])
+                        except subprocess.CalledProcessError:
+                            print("PyPy patch not applied", file=sys.stderr)
 
                     if options.dependency_constraints:
                         constraints_file = options.dependency_constraints.get_for_python_version(config.version)

cibuildwheel/macos.py

@@ -105,6 +105,10 @@ def install_pypy(version: str, url: str) -> Path:
         downloaded_tar_bz2 = Path("/tmp") / pypy_tar_bz2
         download(url, downloaded_tar_bz2)
         call(['tar', '-C', '/tmp', '-xf', downloaded_tar_bz2])
+        # Patch PyPy to make sure headers get installed into a venv
+        patch_version = '_27' if version == '2.7' else ''
+        patch_path = Path(__file__).absolute().parent / 'resources' / f'pypy_venv{patch_version}.patch'
+        call(['patch', '--force', '-d', installation_path, patch_path])
 
     installation_bin_path = installation_path / 'bin'
     python_executable = 'pypy3' if version[0] == '3' else 'pypy'

cibuildwheel/resources/pypy_venv.patch

@@ -0,0 +1,36 @@
+diff --git a/lib-python/3/sysconfig.py b/lib-python/3/sysconfig.py
+index 0d08cde66b4150e1772c13fdc6fe34d079c2b10d..30c77d165a2505d98bc7660ff0cc3b728f56c682 100644
+--- a/lib-python/3/sysconfig.py
++++ b/lib-python/3/sysconfig.py
+@@ -618,9 +618,6 @@ def get_config_vars(*args):
+             import _osx_support
+             _osx_support.customize_config_vars(_CONFIG_VARS)
+
+-        _CONFIG_VARS['INCLUDEPY'] = os.path.join(_CONFIG_VARS['prefix'],
+-                                                 'include')
+-
+     if args:
+         vals = []
+         for name in args:
+diff --git a/lib_pypy/_sysconfigdata.py b/lib_pypy/_sysconfigdata.py
+index f1cca41a4b7553819bb1a2a63af1c1a9b8591ce9..a34216e8e12b82abd72fabdd7793b3e8bb19bbcf 100644
+--- a/lib_pypy/_sysconfigdata.py
++++ b/lib_pypy/_sysconfigdata.py
+@@ -5,6 +5,7 @@ from distutils.spawn import find_executable
+
+ so_ext = _imp.extension_suffixes()[0]
+
++mybase = os.path.dirname(os.path.dirname(__file__))
+
+ build_time_vars = {
+     # SOABI is PEP 3149 compliant, but CPython3 has so_ext.split('.')[1]
+@@ -25,7 +26,8 @@ build_time_vars = {
+     'AR': "ar",
+     'ARFLAGS': "rc",
+     'EXE': "",
+-    'LIBDIR': os.path.join(sys.prefix, 'bin'),
++    'LIBDIR': os.path.join(mybase, 'lib'),
++    'INCLUDEPY': os.path.join(mybase, 'include'),
+ }
+
+ if find_executable("gcc"):

cibuildwheel/resources/pypy_venv_27.patch

@@ -0,0 +1,27 @@
+--- a/lib-python/2.7/sysconfig.py
++++ b/lib-python/2.7/sysconfig.py
+@@ -553,9 +553,6 @@ def get_config_vars(*args):
+             if type_ == imp.C_EXTENSION:
+                 _CONFIG_VARS['SOABI'] = suffix.split('.')[1]
+                 break
+-        _CONFIG_VARS['INCLUDEPY'] = os.path.join(_CONFIG_VARS['prefix'],
+-                                                 'include')
+-
+     if args:
+         vals = []
+         for name in args:
+--- a/lib_pypy/_sysconfigdata.py
++++ b/lib_pypy/_sysconfigdata.py
+@@ -1,5 +1,11 @@
+-import imp
++import imp, os
++
++mybase = os.path.dirname(os.path.dirname(__file__))
+
+ build_time_vars = {
+     "SO": [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0]
+ }
++
++build_time_vars['LIBDIR'] = os.path.join(mybase, 'lib')
++build_time_vars['INCLUDEPY'] = os.path.join(mybase, 'include')
++

requirements-dev.txt

@@ -1,5 +1,6 @@
 -e .
 pytest
+pytest-timeout
 mkdocs==1.0.4
 mkdocs-include-markdown-plugin==2.1.1
 pymdown-extensions

test/test_before_test.py

@@ -37,13 +37,15 @@ class TestBeforeTest(TestCase):
 def test(tmp_path):
     project_dir = tmp_path / 'project'
     before_test_project.generate(project_dir)
+    test_project_dir = project_dir / 'dependency'
+    test_projects.new_c_project().generate(test_project_dir)
 
     # build the wheels
     actual_wheels = utils.cibuildwheel_run(project_dir, add_env={
         # write python version information to a temporary file, this is
         # checked in setup.py
-        'CIBW_BEFORE_TEST': '''python -c "import sys; open('/tmp/pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('/tmp/pythonprefix.txt', 'w').write(sys.prefix)"''',
+        'CIBW_BEFORE_TEST': '''python -c "import sys; open('/tmp/pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('/tmp/pythonprefix.txt', 'w').write(sys.prefix)" && python -m pip install {project}/dependency''',
-        'CIBW_BEFORE_TEST_WINDOWS': '''python -c "import sys; open('c:\\pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('c:\\pythonprefix.txt', 'w').write(sys.prefix)"''',
+        'CIBW_BEFORE_TEST_WINDOWS': '''python -c "import sys; open('c:\\pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('c:\\pythonprefix.txt', 'w').write(sys.prefix)" && python -m pip install {project}/dependency''',
         'CIBW_TEST_REQUIRES': 'nose',
         # the 'false ||' bit is to ensure this command runs in a shell on
         # mac/linux.

test/test_testing.py

@@ -64,6 +64,10 @@ class TestSpam(TestCase):
         bits = struct.calcsize("P") * 8
         if bits == 32:
             self.assertEqual(platform.machine(), "i686")
+
+    def test_time_to_remove_the_pypy_venv_patch(self):
+        if sys.platform == "darwin":
+            assert not hasattr(sys, "pypy_version_info") or sys.pypy_version_info < (7,3,4)
 '''