Converting tests to the new-style

test/11_before_test/cibuildwheel_test.py

@@ -1,23 +0,0 @@
-import os
-import utils
-
-
-def test():
-    project_dir = os.path.dirname(__file__)
-
-    # build the wheels
-    actual_wheels = utils.cibuildwheel_run(project_dir, add_env={
-        # write python version information to a temporary file, this is
-        # checked in setup.py
-        'CIBW_BEFORE_TEST': '''python -c "import sys; open('/tmp/pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('/tmp/pythonprefix.txt', 'w').write(sys.prefix)"''',
-        'CIBW_BEFORE_TEST_WINDOWS': '''python -c "import sys; open('c:\\pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('c:\\pythonprefix.txt', 'w').write(sys.prefix)"''',
-        'CIBW_TEST_REQUIRES': 'nose',
-        # the 'false ||' bit is to ensure this command runs in a shell on
-        # mac/linux.
-        'CIBW_TEST_COMMAND': 'false || nosetests {project}/test',
-        'CIBW_TEST_COMMAND_WINDOWS': 'nosetests {project}/test',
-    })
-
-    # also check that we got the right wheels
-    expected_wheels = utils.expected_wheels('spam', '0.1.0')
-    assert set(actual_wheels) == set(expected_wheels)

test/conftest.py

@@ -1,4 +0,0 @@
-import os
-import sys
-
-sys.path.append(os.path.join(os.path.dirname(__file__), 'shared'))

test/old/01_basic/cibuildwheel_test.py

@@ -1,10 +1,19 @@
 import os
 import platform
+import textwrap
 
 import utils
 
 project_dir = os.path.dirname(__file__)
 
+project_spec = TemplateProjectC(
+    setup_py_add=textwrap.dedent('''
+        import os
+        if os.environ.get("CIBUILDWHEEL", "0") != "1":
+            raise Exception("CIBUILDWHEEL environment variable is not set to 1")
+    ''')
+)
+
 
 def test():
     # build the wheels

test/template_projects/__init__.py

@@ -0,0 +1,3 @@
+from .base import TemplateProject  # noqa
+from .c import CTemplateProject  # noqa
+from .setuptools import SetuptoolsTemplateProject  # noqa

test/template_projects/base.py

@@ -1,29 +1,28 @@
 import os
 import io
 import jinja2
-from typing import Union, Dict, Any
+from typing import Union, Dict, Any, Optional
 
 
 FilesDict = Dict[str, Union[str, jinja2.Template]]
+TemplateContext = Dict[str, Any]
 
 
 class TemplateProject:
-    default_files: FilesDict = {}
     files: FilesDict
-    context: Dict[str, Any]
+    template_context: TemplateContext
 
-    def __init__(self, *, extra_files: FilesDict):
-        self.files = self.default_files.copy()
-        self.files.update(extra_files)
-        self.context = {}
+    def __init__(self):
+        self.files = {}
+        self.template_context = {}
 
-    def generate(self, path):
+    def generate(self, path: str):
         for filename, content in self.files.items():
             file_path = os.path.join(path, filename)
             os.makedirs(os.path.dirname(file_path), exist_ok=True)
 
             with io.open(file_path, 'w', encoding='utf8') as f:
                 if isinstance(content, jinja2.Template):
-                    content = content.render(self.context)
+                    content = content.render(self.template_context)
 
                 f.write(content)

test/template_projects/c.py

@@ -1,81 +1,83 @@
-import textwrap
 import jinja2
-from .base import TemplateProject
+from .setuptools import SetuptoolsTemplateProject
 
 
-class CTemplateProject(TemplateProject):
-    default_files = {
-        'spam.c': jinja2.Template(textwrap.dedent(r'''
-            #include <Python.h>
+spam_c_template = r'''
+#include <Python.h>
 
-            {{ spam_c_top_level_add }}
+{{ spam_c_top_level_add }}
 
-            static PyObject *
-            spam_system(PyObject *self, PyObject *args)
-            {
-                const char *command;
-                int sts;
+static PyObject *
+spam_system(PyObject *self, PyObject *args)
+{
+    const char *command;
+    int sts;
 
-                if (!PyArg_ParseTuple(args, "s", &command))
-                    return NULL;
+    if (!PyArg_ParseTuple(args, "s", &command))
+        return NULL;
 
-                sts = system(command);
+    sts = system(command);
 
-                {{ spam_c_function_add | indent(4) }}
+    {{ spam_c_function_add | indent(4) }}
 
-                return PyLong_FromLong(sts);
-            }
+    return PyLong_FromLong(sts);
+}
 
-            /* Module initialization */
+/* Module initialization */
 
-            #if PY_MAJOR_VERSION >= 3
-                #define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void)
-                #define MOD_DEF(m, name, doc, methods, module_state_size) \
-                    static struct PyModuleDef moduledef = { \
-                        PyModuleDef_HEAD_INIT, name, doc, module_state_size, methods, }; \
-                    m = PyModule_Create(&moduledef);
-                #define MOD_RETURN(m) return m;
-            #else
-                #define MOD_INIT(name) PyMODINIT_FUNC init##name(void)
-                #define MOD_DEF(m, name, doc, methods, module_state_size) \
-                    m = Py_InitModule3(name, methods, doc);
-                #define MOD_RETURN(m) return;
-            #endif
+#if PY_MAJOR_VERSION >= 3
+    #define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void)
+    #define MOD_DEF(m, name, doc, methods, module_state_size) \
+        static struct PyModuleDef moduledef = { \
+            PyModuleDef_HEAD_INIT, name, doc, module_state_size, methods, }; \
+        m = PyModule_Create(&moduledef);
+    #define MOD_RETURN(m) return m;
+#else
+    #define MOD_INIT(name) PyMODINIT_FUNC init##name(void)
+    #define MOD_DEF(m, name, doc, methods, module_state_size) \
+        m = Py_InitModule3(name, methods, doc);
+    #define MOD_RETURN(m) return;
+#endif
 
-            static PyMethodDef module_methods[] = {
-                {"system", (PyCFunction)spam_system, METH_VARARGS,
-                 "Execute a shell command."},
-                {NULL}  /* Sentinel */
-            };
+static PyMethodDef module_methods[] = {
+    {"system", (PyCFunction)spam_system, METH_VARARGS,
+     "Execute a shell command."},
+    {NULL}  /* Sentinel */
+};
 
-            MOD_INIT(spam)
-            {
-                PyObject* m;
+MOD_INIT(spam)
+{
+    PyObject* m;
 
-                MOD_DEF(m,
-                        "spam",
-                        "Example module",
-                        module_methods,
-                        -1)
+    MOD_DEF(m,
+            "spam",
+            "Example module",
+            module_methods,
+            -1)
 
-                MOD_RETURN(m)
-            }
-        ''')),
-        'setup.py': jinja2.Template(textwrap.dedent(r'''
-            from setuptools import setup, Extension
+    MOD_RETURN(m)
+}
+'''
 
-            {{ setup_py_add }}
 
-            setup(
-                name="spam",
-                ext_modules=[Extension("spam", sources=["spam.c"])],
-                version="0.1.0",
-                {{ setup_py_setup_args_add | indent(4) }}
-            )
-        '''))
-    }
+class CTemplateProject(SetuptoolsTemplateProject):
+    def __init__(self, *, spam_c_top_level_add='', spam_c_function_add='', setup_py_add='',
+                 setup_py_setup_args_add='', setup_cfg_add=''):
+        setup_py_setup_args_add += '''
+            ext_modules=[Extension('spam', sources=['spam.c'])],
+        '''
 
-    def __init__(self, spam_c_top_level_add, spam_c_function_add, setup_py_add,
-                 setup_py_setup_args_add, extra_files):
-        super().__init__(extra_files)
-        self.context = {}
+        super().__init__(
+            setup_py_add=setup_py_add,
+            setup_py_setup_args_add=setup_py_setup_args_add,
+            setup_cfg_add=setup_cfg_add
+        )
+
+        self.files.update({
+            'spam.c': jinja2.Template(spam_c_template),
+        })
+
+        self.template_context.update({
+            'spam_c_top_level_add': spam_c_top_level_add,
+            'spam_c_function_add': spam_c_function_add,
+        })

test/template_projects/setuptools.py

@@ -0,0 +1,38 @@
+
+import jinja2
+from .base import TemplateProject
+
+
+setup_py_template = r'''
+from setuptools import setup, Extension
+
+{{ setup_py_add }}
+
+setup(
+    {{ setup_py_setup_args_add | indent(4) }}
+)
+'''
+
+setup_cfg_template = r'''
+[metadata]
+name = spam
+version = 0.1.0
+
+{{ setup_cfg_add }}
+'''
+
+
+class SetuptoolsTemplateProject(TemplateProject):
+    def __init__(self, *, setup_py_add='', setup_py_setup_args_add='', setup_cfg_add=''):
+        super().__init__()
+
+        self.files.update({
+            'setup.py': jinja2.Template(setup_py_template),
+            'setup.cfg': jinja2.Template(setup_cfg_template),
+        })
+
+        self.template_context.update({
+            'setup_py_add': setup_py_add,
+            'setup_py_setup_args_add': setup_py_setup_args_add,
+            'setup_cfg_add': setup_cfg_add,
+        })

test/test_01_basic.py

@@ -0,0 +1,42 @@
+import os
+import textwrap
+from .template_projects import CTemplateProject
+from . import utils
+
+basic_project = CTemplateProject(
+    setup_py_add=textwrap.dedent('''
+        import os
+
+        if os.environ.get("CIBUILDWHEEL", "0") != "1":
+            raise Exception("CIBUILDWHEEL environment variable is not set to 1")
+    ''')
+)
+
+def test(tmpdir):
+    project_dir = str(tmpdir)
+    basic_project.generate(project_dir)
+
+    # build the wheels
+    actual_wheels = utils.cibuildwheel_run(project_dir)
+
+    # check that the expected wheels are produced
+    expected_wheels = utils.expected_wheels("spam", "0.1.0")
+    assert set(actual_wheels) == set(expected_wheels)
+
+
+def test_build_identifiers(tmpdir):
+    project_dir = str(tmpdir)
+    basic_project.generate(project_dir)
+
+    # check that the number of expected wheels matches the number of build
+    # identifiers
+    # after adding CIBW_MANYLINUX_IMAGE to support manylinux2010, there
+    # can be multiple wheels for each wheel, though, so we need to limit
+    # the expected wheels
+    expected_wheels = [
+        w
+        for w in utils.expected_wheels("spam", "0.1.0")
+        if not "-manylinux" in w or "-manylinux1" in w
+    ]
+    build_identifiers = utils.cibuildwheel_get_build_identifiers(project_dir)
+    assert len(expected_wheels) == len(build_identifiers)

test/test_02_testing.py

@@ -0,0 +1,75 @@
+import os, subprocess
+import pytest, textwrap
+from . import utils
+from .template_projects import CTemplateProject
+
+project_with_a_test = CTemplateProject()
+
+project_with_a_test.files['test/spam_test.py'] = r'''
+from unittest import TestCase
+import spam
+
+class TestSpam(TestCase):
+    def test_system(self):
+        self.assertEqual(0, spam.system('python -c "exit(0)"'))
+        self.assertNotEqual(0, spam.system('python -c "exit(1)"'))
+'''
+
+
+def test(tmpdir):
+    project_dir = str(tmpdir)
+    project_with_a_test.generate(project_dir)
+
+    # build and test the wheels
+    actual_wheels = utils.cibuildwheel_run(
+        project_dir,
+        add_env={
+            "CIBW_TEST_REQUIRES": "nose",
+            # the 'false ||' bit is to ensure this command runs in a shell on
+            # mac/linux.
+            "CIBW_TEST_COMMAND": "false || nosetests {project}/test",
+            "CIBW_TEST_COMMAND_WINDOWS": "nosetests {project}/test",
+        },
+    )
+
+    # also check that we got the right wheels
+    expected_wheels = utils.expected_wheels("spam", "0.1.0")
+    assert set(actual_wheels) == set(expected_wheels)
+
+
+def test_extras_require(tmpdir):
+    project_dir = str(tmpdir)
+    project_with_a_test.generate(project_dir)
+
+    # build and test the wheels
+    actual_wheels = utils.cibuildwheel_run(
+        project_dir,
+        add_env={
+            "CIBW_TEST_EXTRAS": "test",
+            "CIBW_TEST_COMMAND": "nosetests {project}/test",
+        },
+    )
+
+    # also check that we got the right wheels
+    expected_wheels = utils.expected_wheels("spam", "0.1.0")
+    assert set(actual_wheels) == set(expected_wheels)
+
+
+def test_failing_test(tmpdir):
+    """Ensure a failing test causes cibuildwheel to error out and exit"""
+    project_dir = str(tmpdir)
+    project_with_a_test.generate(project_dir)
+
+    with pytest.raises(subprocess.CalledProcessError):
+        utils.cibuildwheel_run(
+            project_dir,
+            add_env={
+                "CIBW_TEST_COMMAND": "false",
+                # manylinux1 has a version of bash that's been shown to have
+                # problems with this, so let's check that.
+                "CIBW_MANYLINUX_I686_IMAGE": "manylinux1",
+                "CIBW_MANYLINUX_X86_64_IMAGE": "manylinux1",
+            },
+        )
+
+    assert len(os.listdir("wheelhouse"))

test/test_03_before_build.py

@@ -0,0 +1,55 @@
+import os, textwrap
+from . import utils
+from .template_projects import CTemplateProject
+
+
+project_with_before_build_asserts = CTemplateProject(
+    setup_py_add=textwrap.dedent(r'''
+        import sys, os
+
+        # assert that the Python version as written to pythonversion.txt in the CIBW_BEFORE_BUILD step
+        # is the same one as is currently running.
+        version_file = (
+            "c:\\pythonversion.txt" if sys.platform == "win32" else "/tmp/pythonversion.txt"
+        )
+        with open(version_file) as f:
+            stored_version = f.read()
+        print("stored_version", stored_version)
+        print("sys.version", sys.version)
+        assert stored_version == sys.version
+
+        # check that the executable also was written
+        executable_file = (
+            "c:\\pythonexecutable.txt" if sys.platform == "win32" else "/tmp/pythonexecutable.txt"
+        )
+        with open(executable_file) as f:
+            stored_executable = f.read()
+        print("stored_executable", stored_executable)
+        print("sys.executable", sys.executable)
+        # windows/mac are case insensitive
+        assert (
+            os.path.realpath(stored_executable).lower()
+            == os.path.realpath(sys.executable).lower()
+        )
+    ''')
+)
+
+
+def test(tmpdir):
+    project_dir = str(tmpdir)
+    project_with_before_build_asserts.generate(project_dir)
+
+    # build the wheels
+    actual_wheels = utils.cibuildwheel_run(
+        project_dir,
+        add_env={
+            # write python version information to a temporary file, this is
+            # checked in setup.py
+            "CIBW_BEFORE_BUILD": '''python -c "import sys; open('/tmp/pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('/tmp/pythonexecutable.txt', 'w').write(sys.executable)"''',
+            "CIBW_BEFORE_BUILD_WINDOWS": '''python -c "import sys; open('c:\\pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('c:\\pythonexecutable.txt', 'w').write(sys.executable)"''',
+        },
+    )
+
+    # also check that we got the right wheels
+    expected_wheels = utils.expected_wheels("spam", "0.1.0")
+    assert set(actual_wheels) == set(expected_wheels)

test/test_04_build_skip.py

@@ -0,0 +1,31 @@
+import os, textwrap
+from . import utils
+from .template_projects import CTemplateProject
+
+
+project_with_skip_asserts = CTemplateProject(
+    setup_py_add=textwrap.dedent(r'''
+        # explode if run on Python 2.7 or Python 3.4 (these should be skipped)
+        if sys.version_info[0:2] == (2, 7):
+            raise Exception("Python 2.7 should not be built")
+        if sys.version_info[0:2] == (3, 4):
+            raise Exception("Python 3.4 should be skipped")
+    ''')
+)
+
+def test(tmpdir):
+    project_dir = str(tmpdir)
+    project_with_skip_asserts.generate(project_dir)
+
+    # build the wheels
+    actual_wheels = utils.cibuildwheel_run(
+        project_dir, add_env={"CIBW_BUILD": "cp3?-*", "CIBW_SKIP": "cp37-*",}
+    )
+
+    # check that we got the right wheels. There should be no 2.7 or 3.7.
+    expected_wheels = [
+        w
+        for w in utils.expected_wheels("spam", "0.1.0")
+        if ("-cp3" in w) and ("-cp37" not in w)
+    ]
+    assert set(actual_wheels) == set(expected_wheels)

test/test_05_environment.py

@@ -0,0 +1,44 @@
+import os, inspect, textwrap
+from . import utils
+from .template_projects import CTemplateProject
+
+
+project_with_environment_asserts = CTemplateProject(
+    setup_py_add=textwrap.dedent(r'''
+        import os
+
+        # explode if environment isn't correct, as set in CIBW_ENVIRONMENT
+        CIBW_TEST_VAR = os.environ.get("CIBW_TEST_VAR")
+        CIBW_TEST_VAR_2 = os.environ.get("CIBW_TEST_VAR_2")
+        PATH = os.environ.get("PATH")
+
+        if CIBW_TEST_VAR != "a b c":
+            raise Exception('CIBW_TEST_VAR should equal "a b c". It was "%s"' % CIBW_TEST_VAR)
+        if CIBW_TEST_VAR_2 != "1":
+            raise Exception('CIBW_TEST_VAR_2 should equal "1". It was "%s"' % CIBW_TEST_VAR_2)
+        if "/opt/cibw_test_path" not in PATH:
+            raise Exception('PATH should contain "/opt/cibw_test_path". It was "%s"' % PATH)
+        if "$PATH" in PATH:
+            raise Exception('$PATH should be expanded in PATH. It was "%s"' % PATH)
+    ''')
+)
+
+
+def test(tmpdir):
+    project_dir = str(tmpdir)
+    project_with_environment_asserts.generate(project_dir)
+
+    # write some information into the CIBW_ENVIRONMENT, for expansion and
+    # insertion into the environment by cibuildwheel. This is checked
+    # in setup_py_add
+    actual_wheels = utils.cibuildwheel_run(
+        project_dir,
+        add_env={
+            "CIBW_ENVIRONMENT": """CIBW_TEST_VAR="a b c" CIBW_TEST_VAR_2=1 CIBW_TEST_VAR_3="$(echo 'test string 3')" PATH=$PATH:/opt/cibw_test_path""",
+            "CIBW_ENVIRONMENT_WINDOWS": '''CIBW_TEST_VAR="a b c" CIBW_TEST_VAR_2=1 CIBW_TEST_VAR_3="$(echo 'test string 3')" PATH="$PATH;/opt/cibw_test_path"''',
+        },
+    )
+
+    # also check that we got the right wheels built
+    expected_wheels = utils.expected_wheels("spam", "0.1.0")
+    assert set(actual_wheels) == set(expected_wheels)

test/test_06_docker_images.py

@@ -0,0 +1,42 @@
+import os, pytest, textwrap
+from . import utils
+from .template_projects import CTemplateProject
+
+
+dockcross_only_project = CTemplateProject(
+    setup_py_add=textwrap.dedent(r'''
+        import os, sys
+
+        # check that we're running in the correct docker image as specified in the
+        # environment options CIBW_MANYLINUX1_*_IMAGE
+        if "linux" in sys.platform and not os.path.exists("/dockcross"):
+            raise Exception(
+                "/dockcross directory not found. Is this test running in the correct docker image?"
+            )
+    ''')
+)
+
+def test(tmpdir):
+    if utils.platform != "linux":
+        pytest.skip("the test is only relevant to the linux build")
+
+    project_dir = str(tmpdir)
+    dockcross_only_project.generate(project_dir)
+
+    actual_wheels = utils.cibuildwheel_run(
+        project_dir,
+        add_env={
+            "CIBW_MANYLINUX_X86_64_IMAGE": "dockcross/manylinux2010-x64",
+            "CIBW_MANYLINUX_I686_IMAGE": "dockcross/manylinux1-x86",
+            "CIBW_BEFORE_BUILD": "/opt/python/cp36-cp36m/bin/pip install -U auditwheel",  # Currently necessary on dockcross images to get auditwheel 2.1 supporting AUDITWHEEL_PLAT
+            "CIBW_ENVIRONMENT": 'AUDITWHEEL_PLAT=`if [ $(uname -i) == "x86_64" ]; then echo "manylinux2010_x86_64"; else echo "manylinux1_i686"; fi`',
+        },
+    )
+
+    # also check that we got the right wheels built
+    expected_wheels = [
+        w
+        for w in utils.expected_wheels("spam", "0.1.0")
+        if "-manylinux2010_i686" not in w
+    ]
+    assert set(actual_wheels) == set(expected_wheels)

test/test_07_ssl.py

@@ -0,0 +1,30 @@
+import os, textwrap
+from . import utils
+from .template_projects import CTemplateProject
+
+
+project_with_ssl_tests = CTemplateProject(
+    setup_py_add=textwrap.dedent(r'''
+        import ssl
+        import sys
+
+        if sys.version_info[0] == 2:
+            from urllib2 import urlopen
+        else:
+            from urllib.request import urlopen
+
+        if sys.version_info[0:2] == (3, 3):
+            data = urlopen("https://www.nist.gov")
+        else:
+            context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
+            data = urlopen("https://www.nist.gov", context=context)
+    ''')
+)
+
+def test(tmpdir):
+    # this test checks that SSL is working in the build environment using
+    # some checks in setup.py.
+    project_dir = str(tmpdir)
+    project_with_ssl_tests.generate(project_dir)
+
+    utils.cibuildwheel_run(project_dir)

test/test_08_manylinuxXXXX_only.py

@@ -0,0 +1,67 @@
+import os, pytest, textwrap, platform
+from . import utils
+from .template_projects import CTemplateProject
+
+# TODO: specify these at runtime according to manylinux_image
+project_with_manylinux_symbols = CTemplateProject(
+    spam_c_top_level_add=textwrap.dedent(r'''
+        #include <malloc.h>
+
+        #if !defined(__GLIBC_PREREQ)
+        #error "Must run on a glibc linux environment"
+        #endif
+
+        #if !__GLIBC_PREREQ(2, 5)  /* manylinux1 is glibc 2.5 */
+        #error "Must run on a glibc >= 2.5 linux environment"
+        #endif
+    '''),
+    spam_c_function_add=textwrap.dedent(r'''
+        #if defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 17)  /* manylinux2014 is glibc 2.17 */
+            // secure_getenv is only available in manylinux2014, ensuring
+            // that only a manylinux2014 wheel is produced
+            secure_getenv("NON_EXISTING_ENV_VARIABLE");
+        #elif defined(__GLIBC_PREREQ) && __GLIBC_PREREQ(2, 10)  /* manylinux2010 is glibc 2.12 */
+            // malloc_info is only available on manylinux2010+
+            malloc_info(0, stdout);
+        #endif
+    '''),
+)
+
+
+@pytest.mark.parametrize('manylinux_image', ['manylinux1', 'manylinux2010', 'manylinux2014'])
+def test(manylinux_image, tmpdir):
+    if utils.platform != 'linux':
+        pytest.skip('the docker test is only relevant to the linux build')
+    elif platform.machine() not in ['x86_64', 'i686']:
+        if manylinux_image in ['manylinux1', 'manylinux2010']:
+            pytest.skip("manylinux1 and 2010 doesn't exist for non-x86 architectures")
+
+    project_dir = str(tmpdir)
+    project_with_manylinux_symbols.generate(project_dir)
+
+    # build the wheels
+    # CFLAGS environment variable is necessary to fail on 'malloc_info' (on manylinux1) during compilation/linking,
+    # rather than when dynamically loading the Python
+    add_env = {
+        'CIBW_ENVIRONMENT': 'CFLAGS="$CFLAGS -Werror=implicit-function-declaration"',
+        'CIBW_MANYLINUX_X86_64_IMAGE': manylinux_image,
+        'CIBW_MANYLINUX_I686_IMAGE': manylinux_image,
+        'CIBW_MANYLINUX_PYPY_X86_64_IMAGE': manylinux_image,
+        'CIBW_MANYLINUX_AARCH64_IMAGE': manylinux_image,
+        'CIBW_MANYLINUX_PPC64LE_IMAGE': manylinux_image,
+        'CIBW_MANYLINUX_S390X_IMAGE': manylinux_image,
+    }
+    if manylinux_image == 'manylinux1':
+        # We don't have a manylinux1 image for PyPy
+        add_env['CIBW_SKIP'] = 'pp*'
+    elif manylinux_image == 'manylinux2014':
+        # We don't have a manylinux2014 image for PyPy (yet?)
+        add_env['CIBW_SKIP'] = 'cp27* pp*'  # Python 2.7 not available on manylinux2014
+    actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env)
+
+    expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0', manylinux_versions=[manylinux_image])]
+    if manylinux_image == 'manylinux2014':
+        expected_wheels = [w for w in expected_wheels if '-cp27' not in w]
+    if manylinux_image in ['manylinux1', 'manylinux2014']:
+        expected_wheels = [w for w in expected_wheels if '-pp' not in w]
+    assert set(actual_wheels) == set(expected_wheels)

test/test_09_cpp_standards.py

@@ -0,0 +1,150 @@
+import os
+import textwrap
+
+import pytest
+
+from . import utils
+from .template_projects import SetuptoolsTemplateProject
+import jinja2
+
+# TODO: specify these at runtime according to manylinux_image
+cpp_project = SetuptoolsTemplateProject(
+    setup_py_add='''ext_modules=[Extension('spam', sources=['spam.cpp'])],'''
+)
+cpp_project.files['spam.cpp'] = jinja2.Template(r'''
+#include <Python.h>
+
+{{ spam_cpp_add }}
+
+static PyObject *
+spam_system(PyObject *self, PyObject *args)
+{
+    const char *command;
+    int sts;
+
+    if (!PyArg_ParseTuple(args, "s", &command))
+        return NULL;
+    sts = system(command);
+    return PyLong_FromLong(sts);
+}
+
+/* Module initialization */
+
+#if PY_MAJOR_VERSION >= 3
+    #define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void)
+    #define MOD_DEF(m, name, doc, methods, module_state_size) \
+        static struct PyModuleDef moduledef = { \
+            PyModuleDef_HEAD_INIT, name, doc, module_state_size, methods, }; \
+        m = PyModule_Create(&moduledef);
+    #define MOD_RETURN(m) return m;
+#else
+    #define MOD_INIT(name) PyMODINIT_FUNC init##name(void)
+    #define MOD_DEF(m, name, doc, methods, module_state_size) \
+        m = Py_InitModule3(name, methods, doc);
+    #define MOD_RETURN(m) return;
+#endif
+
+static PyMethodDef module_methods[] = {
+    {"system", (PyCFunction)spam_system, METH_VARARGS,
+     "Execute a shell command."},
+    {NULL}  /* Sentinel */
+};
+
+MOD_INIT(spam)
+{
+    PyObject* m;
+
+    MOD_DEF(m,
+            "spam",
+            "Example module",
+            module_methods,
+            -1)
+
+    MOD_RETURN(m)
+}
+''')
+
+spam_cpp_top_level_add = '''
+// Depending on the requested standard, use a modern C++ feature
+// that was introduced in that standard.
+#if STANDARD == 11
+    #include <array>
+#elif STANDARD == 14
+    int a = 100'000;
+#elif STANDARD == 17
+    #include <utility>
+    auto a = std::pair(5.0, false);
+#else
+    #error Standard needed
+#endif
+'''
+
+project_dir = os.path.dirname(__file__)
+
+def test_cpp11(tmpdir):
+    # This test checks that the C++11 standard is supported
+    project_dir = str(tmpdir)
+
+    cpp_project.template_context['spam_cpp_add'] = '#include <array>'
+    cpp_project.generate(project_dir)
+
+    # VC++ for Python 2.7 does not support modern standards
+    add_env = {'CIBW_SKIP': 'cp27-win* pp27-win32'}
+
+    actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env)
+    expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0')
+                       if 'cp27-cp27m-win' not in w and 'pp27-pypy_73-win32' not in w]
+
+    assert set(actual_wheels) == set(expected_wheels)
+
+
+def test_cpp14(tmpdir):
+    # This test checks that the C++14 standard is supported
+    project_dir = str(tmpdir)
+
+    cpp_project.template_context['spam_cpp_add'] = "int a = 100'000;"
+    cpp_project.generate(project_dir)
+
+    # VC++ for Python 2.7 does not support modern standards
+    # The manylinux1 docker image does not have a compiler which supports C++11
+    # Python 3.4 and 3.5 are compiled with MSVC 10, which does not support C++14
+    add_env = {'CIBW_SKIP': 'cp27-win* pp27-win32 cp35-win*'}
+
+    actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env)
+    expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0')
+                       if 'cp27-cp27m-win' not in w
+                       and 'pp27-pypy_73-win32' not in w
+                       and 'cp35-cp35m-win' not in w]
+
+    assert set(actual_wheels) == set(expected_wheels)
+
+
+def test_cpp17(tmpdir):
+    # This test checks that the C++17 standard is supported
+    project_dir = str(tmpdir)
+
+    cpp_project.template_context['spam_cpp_add'] = textwrap.dedent('''
+            #include <utility>
+            auto a = std::pair(5.0, false);
+    ''')
+    cpp_project.generate(project_dir)
+
+    # Python and PyPy 2.7 use the `register` keyword which is forbidden in the C++17 standard
+    # The manylinux1 docker image does not have a compiler which supports C++11
+    # Python 3.5 and PyPy 3.6 are compiled with MSVC 10, which does not support C++17
+    if os.environ.get('APPVEYOR_BUILD_WORKER_IMAGE', '') == 'Visual Studio 2015':
+        pytest.skip('Visual Studio 2015 does not support C++17')
+
+    add_env = {'CIBW_SKIP': 'cp27-win* pp27-win32 cp35-win* pp36-win32'}
+
+    if utils.platform == 'macos':
+        add_env['MACOSX_DEPLOYMENT_TARGET'] = '10.13'
+
+    actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env)
+    expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0', macosx_deployment_target='10.13')
+                       if 'cp27-cp27m-win' not in w
+                       and 'pp27-pypy_73-win32' not in w
+                       and 'cp35-cp35m-win' not in w
+                       and 'pp36-pypy36_pp73-win32' not in w]
+
+    assert set(actual_wheels) == set(expected_wheels)

test/test_10_before_test.py

@@ -0,0 +1,56 @@
+import os
+import utils
+from test.template_projects.c import CTemplateProject
+
+before_test_project = CTemplateProject()
+before_test_project.files['test/spam_test.py'] = r'''
+import sys
+import os
+from unittest import TestCase
+
+
+class TestBeforeTest(TestCase):
+    def test_version(self):
+        # assert that the Python version as written to pythonversion.txt in the CIBW_BEFORE_TEST step
+        # is the same one as is currently running.
+        # because of use symlinks in MacOS run this test is also need
+        version_file = 'c:\\pythonversion.txt' if sys.platform == 'win32' else '/tmp/pythonversion.txt'
+        with open(version_file) as f:
+            stored_version = f.read()
+        print('stored_version', stored_version)
+        print('sys.version', sys.version)
+        assert stored_version == sys.version
+
+    def test_prefix(self):
+        # check that the prefix also was written
+        prefix_file = 'c:\\pythonprefix.txt' if sys.platform == 'win32' else '/tmp/pythonprefix.txt'
+        with open(prefix_file) as f:
+            stored_prefix = f.read()
+        print('stored_prefix', stored_prefix)
+        print('sys.prefix', sys.prefix)
+        #  Works around path-comparison bugs caused by short-paths on Windows e.g.
+        #  vssadm~1 instead of vssadministrator
+
+        assert os.stat(stored_prefix) == os.stat(sys.prefix)
+'''
+
+def test(tmpdir):
+    project_dir = str(tmpdir)
+    before_test_project.generate(project_dir)
+
+    # build the wheels
+    actual_wheels = utils.cibuildwheel_run(project_dir, add_env={
+        # write python version information to a temporary file, this is
+        # checked in setup.py
+        'CIBW_BEFORE_TEST': '''python -c "import sys; open('/tmp/pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('/tmp/pythonprefix.txt', 'w').write(sys.prefix)"''',
+        'CIBW_BEFORE_TEST_WINDOWS': '''python -c "import sys; open('c:\\pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('c:\\pythonprefix.txt', 'w').write(sys.prefix)"''',
+        'CIBW_TEST_REQUIRES': 'nose',
+        # the 'false ||' bit is to ensure this command runs in a shell on
+        # mac/linux.
+        'CIBW_TEST_COMMAND': 'false || nosetests {project}/test',
+        'CIBW_TEST_COMMAND_WINDOWS': 'nosetests {project}/test',
+    })
+
+    # also check that we got the right wheels
+    expected_wheels = utils.expected_wheels('spam', '0.1.0')
+    assert set(actual_wheels) == set(expected_wheels)

test/test_11_dependency_versions.py

@@ -4,7 +4,36 @@ import pytest
 import textwrap
 import cibuildwheel.util
 
-import utils
+from . import utils
+from .template_projects import CTemplateProject
+
+
+project_with_expected_version_checks = CTemplateProject(
+    setup_py_add=textwrap.dedent(r'''
+        import subprocess
+        import os
+
+        versions_output_text = subprocess.check_output(
+            ['pip', 'freeze', '--all', '-qq'],
+            universal_newlines=True,
+        )
+        versions = versions_output_text.strip().splitlines()
+
+        # `versions` now looks like:
+        # ['pip==x.x.x', 'setuptools==x.x.x', 'wheel==x.x.x']
+
+        print('Gathered versions', versions)
+
+        for package_name in ['pip', 'setuptools', 'wheel']:
+            env_name = 'EXPECTED_{}_VERSION'.format(package_name.upper())
+            expected_version = os.environ[env_name]
+
+            assert '{}=={}'.format(package_name, expected_version) in versions, (
+                'error: {} version should equal {}'.format(package_name, expected_version)
+            )
+    ''')
+)
+
 
 VERSION_REGEX = r'([\w-]+)==([^\s]+)'
 
@@ -22,11 +51,12 @@ def get_versions_from_constraint_file(constraint_file):
 
 
 @pytest.mark.parametrize('python_version', ['2.7', '3.5', '3.8'])
-def test_pinned_versions(python_version):
+def test_pinned_versions(tmpdir, python_version):
     if utils.platform == 'linux':
         pytest.skip('linux doesn\'t pin individual tool versions, it pins manylinux images instead')
 
-    project_dir = os.path.dirname(__file__)
+    project_dir = str(tmpdir)
+    project_with_expected_version_checks.generate(project_dir)
 
     build_environment = {}
 
@@ -78,7 +108,8 @@ def test_dependency_constraints_file(tmp_path, python_version):
     if utils.platform == 'linux':
         pytest.skip('linux doesn\'t pin individual tool versions, it pins manylinux images instead')
 
-    project_dir = os.path.dirname(__file__)
+    project_dir = str(tmp_path / 'project')
+    project_with_expected_version_checks.generate(project_dir)
 
     tool_versions = {
         'pip': '20.0.2',