diff --git a/bin/run_test.py b/bin/run_test.py
index af361a65..bb5cd6db 100755
--- a/bin/run_test.py
+++ b/bin/run_test.py
@@ -1,28 +1,13 @@
 #!/usr/bin/python
 
 from __future__ import print_function
-import os, sys, subprocess, shutil, json
-from glob import glob
+import os, sys, subprocess, shutil
 
 def single_run(test_project):
-    # load project settings into environment
-    env_file = os.path.join(test_project, 'environment.json')
-    project_env = {}
-    if os.path.exists(env_file):
-        with open(env_file) as f:
-            project_env = json.load(f)
-
-    # run the build
-    env = os.environ.copy()
-    project_env = {str(k): str(v) for k, v in project_env.items()} # unicode not allowed in env
-    env.update(project_env)
-    print('Building %s with environment %s' % (test_project, project_env))
-    subprocess.check_call([sys.executable, '-m', 'cibuildwheel', test_project], env=env)
-    wheels = glob('wheelhouse/*.whl')
-    print('%s built successfully. %i wheels built.' % (test_project, len(wheels)))
-
-    # check some wheels were actually built
-    subprocess.check_call([sys.executable, os.path.join(test_project, 'check.py')])
+    # run the test
+    subprocess.check_call(
+        [sys.executable, '-m', 'pytest', '-v', os.path.join(test_project, 'cibuildwheel_test.py')]
+    )
 
     # clean up
     shutil.rmtree('wheelhouse')
@@ -41,5 +26,3 @@ if __name__ == '__main__':
         exit(2)
 
     single_run(project_path)
-
-    print('Project built successfully.')
diff --git a/test/01_basic/check.py b/test/01_basic/check.py
deleted file mode 100644
index bec98563..00000000
--- a/test/01_basic/check.py
+++ /dev/null
@@ -1,9 +0,0 @@
-import glob
-import subprocess
-import sys
-
-build_identifiers = subprocess.check_output([sys.executable, '-m', 'cibuildwheel', '--print-build-identifiers'], universal_newlines=True).strip().split('\n')
-expected_identifiers = build_identifiers
-built_wheels = glob.glob('wheelhouse/*.whl')
-
-assert len(built_wheels) == len(expected_identifiers)
diff --git a/test/01_basic/cibuildwheel_test.py b/test/01_basic/cibuildwheel_test.py
new file mode 100644
index 00000000..28b83184
--- /dev/null
+++ b/test/01_basic/cibuildwheel_test.py
@@ -0,0 +1,50 @@
+import subprocess, sys, os
+from glob import glob
+project_dir = os.path.dirname(__file__)
+sys.path.append(os.path.join(os.path.dirname(project_dir), 'shared'))
+
+import utils
+
+def test():
+    # build the wheels
+    subprocess.check_call([sys.executable, '-m', 'cibuildwheel', project_dir]) 
+    
+    # check that every wheel is produced
+    if utils.platform == 'linux':
+        assert glob('wheelhouse/*-cp27-cp27m-manylinux1_x86_64.whl')
+        assert glob('wheelhouse/*-cp27-cp27mu-manylinux1_x86_64.whl')
+        assert glob('wheelhouse/*-cp34-cp34m-manylinux1_x86_64.whl')
+        assert glob('wheelhouse/*-cp35-cp35m-manylinux1_x86_64.whl')
+        assert glob('wheelhouse/*-cp36-cp36m-manylinux1_x86_64.whl')
+        assert glob('wheelhouse/*-cp37-cp37m-manylinux1_x86_64.whl')
+        assert glob('wheelhouse/*-cp27-cp27m-manylinux1_i686.whl')
+        assert glob('wheelhouse/*-cp27-cp27mu-manylinux1_i686.whl')
+        assert glob('wheelhouse/*-cp34-cp34m-manylinux1_i686.whl')
+        assert glob('wheelhouse/*-cp35-cp35m-manylinux1_i686.whl')
+        assert glob('wheelhouse/*-cp36-cp36m-manylinux1_i686.whl')
+        assert glob('wheelhouse/*-cp37-cp37m-manylinux1_i686.whl')
+
+    if utils.platform == 'windows':
+        assert glob('wheelhouse/*-cp27-*-win32.whl')
+        assert glob('wheelhouse/*-cp34-*-win32.whl')
+        assert glob('wheelhouse/*-cp35-*-win32.whl')
+        assert glob('wheelhouse/*-cp36-*-win32.whl')
+        assert glob('wheelhouse/*-cp37-*-win32.whl')
+        assert glob('wheelhouse/*-cp27-*-win_amd64.whl')
+        assert glob('wheelhouse/*-cp34-*-win_amd64.whl')
+        assert glob('wheelhouse/*-cp35-*-win_amd64.whl')
+        assert glob('wheelhouse/*-cp36-*-win_amd64.whl')
+        assert glob('wheelhouse/*-cp37-*-win_amd64.whl')
+
+    if utils.platform == 'macos':
+        assert glob('wheelhouse/*-cp27-*-macosx_10_6_intel.whl')
+        assert glob('wheelhouse/*-cp34-*-macosx_10_6_intel.whl')
+        assert glob('wheelhouse/*-cp35-*-macosx_10_6_intel.whl')
+        assert glob('wheelhouse/*-cp36-*-macosx_10_6_intel.whl')
+        assert glob('wheelhouse/*-cp37-*-macosx_10_6_intel.whl')
+
+    # also check that the number of built wheels matches the number of build
+    # identifiers
+    expected_identifiers = utils.cibuildwheel_get_build_identifiers(project_dir)
+    built_wheels = glob('wheelhouse/*.whl')
+    assert len(built_wheels) == len(expected_identifiers)
diff --git a/test/02_test/check.py b/test/02_test/check.py
deleted file mode 100644
index bec98563..00000000
--- a/test/02_test/check.py
+++ /dev/null
@@ -1,9 +0,0 @@
-import glob
-import subprocess
-import sys
-
-build_identifiers = subprocess.check_output([sys.executable, '-m', 'cibuildwheel', '--print-build-identifiers'], universal_newlines=True).strip().split('\n')
-expected_identifiers = build_identifiers
-built_wheels = glob.glob('wheelhouse/*.whl')
-
-assert len(built_wheels) == len(expected_identifiers)
diff --git a/test/02_test/cibuildwheel_test.py b/test/02_test/cibuildwheel_test.py
new file mode 100644
index 00000000..82f314c8
--- /dev/null
+++ b/test/02_test/cibuildwheel_test.py
@@ -0,0 +1,23 @@
+import subprocess, sys, os
+from glob import glob
+project_dir = os.path.dirname(__file__)
+sys.path.append(os.path.join(os.path.dirname(project_dir), 'shared'))
+
+import utils
+
+def test():
+    # set up the environment
+    env = os.environ.copy()
+    env["CIBW_TEST_REQUIRES"] = "nose",
+    # the 'false ||' bit is to ensure this command runs in a shell on
+    # mac/linux.
+    env["CIBW_TEST_COMMAND"] = "false || nosetests {project}/test",
+    env["CIBW_TEST_COMMAND_WINDOWS"] = "nosetests {project}/test"
+
+    # build & test the wheels
+    subprocess.check_call([sys.executable, '-m', 'cibuildwheel', project_dir], env=env)
+    
+    # also check that we got the right number of built wheels
+    expected_identifiers = utils.cibuildwheel_get_build_identifiers(project_dir)
+    built_wheels = glob('wheelhouse/*.whl')
+    assert len(built_wheels) == len(expected_identifiers)
diff --git a/test/02_test/environment.json b/test/02_test/environment.json
deleted file mode 100644
index 3a541dd4..00000000
--- a/test/02_test/environment.json
+++ /dev/null
@@ -1,6 +0,0 @@
-{
-    "CIBW_TEST_REQUIRES": "nose",
-    "CIBW_TEST_COMMAND": "false || nosetests {project}/test",
-    "comment": "The 'false ||' bit is to ensure this command runs in a shell on Mac and Linux",
-    "CIBW_TEST_COMMAND_WINDOWS": "nosetests {project}/test"
-}
diff --git a/test/03_before_build/check.py b/test/03_before_build/check.py
deleted file mode 100644
index bec98563..00000000
--- a/test/03_before_build/check.py
+++ /dev/null
@@ -1,9 +0,0 @@
-import glob
-import subprocess
-import sys
-
-build_identifiers = subprocess.check_output([sys.executable, '-m', 'cibuildwheel', '--print-build-identifiers'], universal_newlines=True).strip().split('\n')
-expected_identifiers = build_identifiers
-built_wheels = glob.glob('wheelhouse/*.whl')
-
-assert len(built_wheels) == len(expected_identifiers)
diff --git a/test/03_before_build/cibuildwheel_test.py b/test/03_before_build/cibuildwheel_test.py
new file mode 100644
index 00000000..92013a16
--- /dev/null
+++ b/test/03_before_build/cibuildwheel_test.py
@@ -0,0 +1,22 @@
+import subprocess, sys, os
+from glob import glob
+project_dir = os.path.dirname(__file__)
+sys.path.append(os.path.join(os.path.dirname(project_dir), 'shared'))
+
+import utils
+
+def test():
+    # set up the environment
+    env = os.environ.copy()
+    # write python version information to a temporary file, this is checked
+    # in setup.py
+    env["CIBW_BEFORE_BUILD"] = "python -c \"import sys; open('/tmp/pythonversion.txt', 'w').write(sys.version)\" && python -c \"import sys; open('/tmp/pythonexecutable.txt', 'w').write(sys.executable)\"",
+    env["CIBW_BEFORE_BUILD_WINDOWS"] = "python -c \"import sys; open('c:\\pythonversion.txt', 'w').write(sys.version)\" && python -c \"import sys; open('c:\\pythonexecutable.txt', 'w').write(sys.executable)\""
+
+    # build the wheels
+    subprocess.check_call([sys.executable, '-m', 'cibuildwheel', project_dir], env=env)
+    
+    # check that we got the right number of built wheels
+    expected_identifiers = utils.cibuildwheel_get_build_identifiers(project_dir)
+    built_wheels = glob('wheelhouse/*.whl')
+    assert len(built_wheels) == len(expected_identifiers)
diff --git a/test/03_before_build/environment.json b/test/03_before_build/environment.json
deleted file mode 100644
index 3e9108ea..00000000
--- a/test/03_before_build/environment.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
-    "CIBW_BEFORE_BUILD": "python -c \"import sys; open('/tmp/pythonversion.txt', 'w').write(sys.version)\" && python -c \"import sys; open('/tmp/pythonexecutable.txt', 'w').write(sys.executable)\"",
-    "CIBW_BEFORE_BUILD_WINDOWS": "python -c \"import sys; open('c:\\pythonversion.txt', 'w').write(sys.version)\" && python -c \"import sys; open('c:\\pythonexecutable.txt', 'w').write(sys.executable)\""
-}
diff --git a/test/03_before_build/version.txt b/test/03_before_build/version.txt
deleted file mode 100644
index 3e4354cb..00000000
--- a/test/03_before_build/version.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-3.6.0 (default, Feb  7 2017, 23:55:32) 
-[GCC 4.8.2 20140120 (Red Hat 4.8.2-15)]
\ No newline at end of file
diff --git a/test/04_build_skip/check.py b/test/04_build_skip/check.py
deleted file mode 100644
index 6f199b0e..00000000
--- a/test/04_build_skip/check.py
+++ /dev/null
@@ -1,9 +0,0 @@
-import glob
-import subprocess
-import sys
-
-build_identifiers = subprocess.check_output([sys.executable, '-m', 'cibuildwheel', '--print-build-identifiers'], universal_newlines=True).strip().split('\n')
-expected_identifiers = [identifier for identifier in build_identifiers if "cp3" in identifier and "cp34" not in identifier]
-built_wheels = glob.glob('wheelhouse/*.whl')
-
-assert len(built_wheels) == len(expected_identifiers)
diff --git a/test/04_build_skip/cibuildwheel_test.py b/test/04_build_skip/cibuildwheel_test.py
new file mode 100644
index 00000000..570ceec7
--- /dev/null
+++ b/test/04_build_skip/cibuildwheel_test.py
@@ -0,0 +1,49 @@
+import subprocess, sys, os
+from glob import glob
+project_dir = os.path.dirname(__file__)
+sys.path.append(os.path.join(os.path.dirname(project_dir), 'shared'))
+
+import utils
+
+def test():
+    # set up the environment
+    env = os.environ.copy()
+    env["CIBW_BUILD"] = "cp3?-*",
+    env["CIBW_SKIP"] = "cp34-*"
+
+    # build the wheels
+    subprocess.check_call([sys.executable, '-m', 'cibuildwheel', project_dir], env=env)
+
+    # check that we got the right wheels. There should be no 2.7 or 3.4.
+    if utils.platform == 'linux':
+        assert not glob('wheelhouse/*-cp27-cp27m-manylinux1_x86_64.whl')
+        assert not glob('wheelhouse/*-cp27-cp27mu-manylinux1_x86_64.whl')
+        assert not glob('wheelhouse/*-cp34-cp34m-manylinux1_x86_64.whl')
+        assert glob('wheelhouse/*-cp35-cp35m-manylinux1_x86_64.whl')
+        assert glob('wheelhouse/*-cp36-cp36m-manylinux1_x86_64.whl')
+        assert glob('wheelhouse/*-cp37-cp37m-manylinux1_x86_64.whl')
+        assert not glob('wheelhouse/*-cp27-cp27m-manylinux1_i686.whl')
+        assert not glob('wheelhouse/*-cp27-cp27mu-manylinux1_i686.whl')
+        assert not glob('wheelhouse/*-cp34-cp34m-manylinux1_i686.whl')
+        assert glob('wheelhouse/*-cp35-cp35m-manylinux1_i686.whl')
+        assert glob('wheelhouse/*-cp36-cp36m-manylinux1_i686.whl')
+        assert glob('wheelhouse/*-cp37-cp37m-manylinux1_i686.whl')
+
+    if utils.platform == 'windows':
+        assert not glob('wheelhouse/*-cp27-*-win32.whl')
+        assert not glob('wheelhouse/*-cp34-*-win32.whl')
+        assert glob('wheelhouse/*-cp35-*-win32.whl')
+        assert glob('wheelhouse/*-cp36-*-win32.whl')
+        assert glob('wheelhouse/*-cp37-*-win32.whl')
+        assert not glob('wheelhouse/*-cp27-*-win_amd64.whl')
+        assert not glob('wheelhouse/*-cp34-*-win_amd64.whl')
+        assert glob('wheelhouse/*-cp35-*-win_amd64.whl')
+        assert glob('wheelhouse/*-cp36-*-win_amd64.whl')
+        assert glob('wheelhouse/*-cp37-*-win_amd64.whl')
+
+    if utils.platform == 'macos':
+        assert not glob('wheelhouse/*-cp27-*-macosx_10_6_intel.whl')
+        assert not glob('wheelhouse/*-cp34-*-macosx_10_6_intel.whl')
+        assert glob('wheelhouse/*-cp35-*-macosx_10_6_intel.whl')
+        assert glob('wheelhouse/*-cp36-*-macosx_10_6_intel.whl')
+        assert glob('wheelhouse/*-cp37-*-macosx_10_6_intel.whl')
diff --git a/test/04_build_skip/environment.json b/test/04_build_skip/environment.json
deleted file mode 100644
index a47c85d2..00000000
--- a/test/04_build_skip/environment.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
-    "CIBW_BUILD": "cp3?-*",
-    "CIBW_SKIP": "cp34-*"
-}
diff --git a/test/04_build_skip/version.txt b/test/04_build_skip/version.txt
deleted file mode 100644
index 3e4354cb..00000000
--- a/test/04_build_skip/version.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-3.6.0 (default, Feb  7 2017, 23:55:32) 
-[GCC 4.8.2 20140120 (Red Hat 4.8.2-15)]
\ No newline at end of file
diff --git a/test/05_environment/check.py b/test/05_environment/check.py
deleted file mode 100644
index bec98563..00000000
--- a/test/05_environment/check.py
+++ /dev/null
@@ -1,9 +0,0 @@
-import glob
-import subprocess
-import sys
-
-build_identifiers = subprocess.check_output([sys.executable, '-m', 'cibuildwheel', '--print-build-identifiers'], universal_newlines=True).strip().split('\n')
-expected_identifiers = build_identifiers
-built_wheels = glob.glob('wheelhouse/*.whl')
-
-assert len(built_wheels) == len(expected_identifiers)
diff --git a/test/05_environment/cibuildwheel_test.py b/test/05_environment/cibuildwheel_test.py
new file mode 100644
index 00000000..90d66288
--- /dev/null
+++ b/test/05_environment/cibuildwheel_test.py
@@ -0,0 +1,22 @@
+import subprocess, sys, os
+from glob import glob
+project_dir = os.path.dirname(__file__)
+sys.path.append(os.path.join(os.path.dirname(project_dir), 'shared'))
+import utils
+
+def test():
+    # set up the environment
+    env = os.environ.copy()
+    # write some information into the CIBW_ENVIRONMENT, for expansion and
+    # insertion into the environment by cibuildwheel. This is checked
+    # in setup.py
+    env['CIBW_ENVIRONMENT'] = 'CIBW_TEST_VAR="a b c" CIBW_TEST_VAR_2=1 CIBW_TEST_VAR_3="$(echo \'test string 3\')" PATH=$PATH:/opt/cibw_test_path',
+    env['CIBW_ENVIRONMENT_WINDOWS'] = 'CIBW_TEST_VAR="a b c" CIBW_TEST_VAR_2=1 CIBW_TEST_VAR_3="$(echo \'test string 3\')" PATH="$PATH;/opt/cibw_test_path"'
+
+    # build the wheels
+    subprocess.check_call([sys.executable, '-m', 'cibuildwheel', project_dir], env=env)
+    
+    # check that we got the right number of built wheels
+    expected_identifiers = utils.cibuildwheel_get_build_identifiers(project_dir)
+    built_wheels = glob('wheelhouse/*.whl')
+    assert len(built_wheels) == len(expected_identifiers)
diff --git a/test/05_environment/environment.json b/test/05_environment/environment.json
deleted file mode 100644
index 9ae227bd..00000000
--- a/test/05_environment/environment.json
+++ /dev/null
@@ -1,4 +0,0 @@
-{
-    "CIBW_ENVIRONMENT": "CIBW_TEST_VAR=\"a b c\" CIBW_TEST_VAR_2=1 CIBW_TEST_VAR_3=\"$(echo 'test string 3')\" PATH=$PATH:/opt/cibw_test_path",
-    "CIBW_ENVIRONMENT_WINDOWS": "CIBW_TEST_VAR=\"a b c\" CIBW_TEST_VAR_2=1 CIBW_TEST_VAR_3=\"$(echo 'test string 3')\" PATH=\"$PATH;/opt/cibw_test_path\""
-}
diff --git a/test/05_environment/version.txt b/test/05_environment/version.txt
deleted file mode 100644
index 3e4354cb..00000000
--- a/test/05_environment/version.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-3.6.0 (default, Feb  7 2017, 23:55:32) 
-[GCC 4.8.2 20140120 (Red Hat 4.8.2-15)]
\ No newline at end of file
diff --git a/test/06_docker_images/check.py b/test/06_docker_images/check.py
deleted file mode 100644
index d2061288..00000000
--- a/test/06_docker_images/check.py
+++ /dev/null
@@ -1,9 +0,0 @@
-import glob
-import subprocess
-import sys
-
-build_identifiers = subprocess.check_output([sys.executable, '-m', 'cibuildwheel', '--print-build-identifiers'], universal_newlines=True).strip().split('\n')
-expected_identifiers = [identifier for identifier in build_identifiers if 'macos' not in identifier and 'win' not in identifier]
-built_wheels = glob.glob('wheelhouse/*.whl')
-
-assert len(built_wheels) == len(expected_identifiers)
diff --git a/test/06_docker_images/cibuildwheel_test.py b/test/06_docker_images/cibuildwheel_test.py
new file mode 100644
index 00000000..cc7ad968
--- /dev/null
+++ b/test/06_docker_images/cibuildwheel_test.py
@@ -0,0 +1,24 @@
+import subprocess, sys, os, pytest
+from glob import glob
+project_dir = os.path.dirname(__file__)
+sys.path.append(os.path.join(os.path.dirname(project_dir), 'shared'))
+import utils
+
+def test():
+    if not sys.platform.startswith('linux'):
+        pytest.skip('the docker test is only relevant to the linux build')
+
+    # set up the environment
+    env = os.environ.copy()
+    # change the docker images to use. The docker image is tested in setup.py 
+    # during the build
+    env["CIBW_MANYLINUX1_X86_64_IMAGE"] = "dockcross/manylinux-x64"
+    env["CIBW_MANYLINUX1_I686_IMAGE"] = "dockcross/manylinux-x86"
+
+    # build the wheels
+    subprocess.check_call([sys.executable, '-m', 'cibuildwheel', project_dir], env=env)
+    
+    # check that we got the right number of built wheels
+    expected_identifiers = utils.cibuildwheel_get_build_identifiers(project_dir)
+    built_wheels = glob('wheelhouse/*.whl')
+    assert len(built_wheels) == len(expected_identifiers)
diff --git a/test/06_docker_images/environment.json b/test/06_docker_images/environment.json
deleted file mode 100644
index e5e73cc8..00000000
--- a/test/06_docker_images/environment.json
+++ /dev/null
@@ -1,5 +0,0 @@
-{
-    "CIBW_MANYLINUX1_X86_64_IMAGE": "dockcross/manylinux-x64",
-    "CIBW_MANYLINUX1_I686_IMAGE": "dockcross/manylinux-x86",
-    "CIBW_SKIP": "*macos* *win*"
-}
diff --git a/test/07_ssl/check.py b/test/07_ssl/check.py
deleted file mode 100644
index bec98563..00000000
--- a/test/07_ssl/check.py
+++ /dev/null
@@ -1,9 +0,0 @@
-import glob
-import subprocess
-import sys
-
-build_identifiers = subprocess.check_output([sys.executable, '-m', 'cibuildwheel', '--print-build-identifiers'], universal_newlines=True).strip().split('\n')
-expected_identifiers = build_identifiers
-built_wheels = glob.glob('wheelhouse/*.whl')
-
-assert len(built_wheels) == len(expected_identifiers)
diff --git a/test/07_ssl/cibuildwheel_test.py b/test/07_ssl/cibuildwheel_test.py
new file mode 100644
index 00000000..ec3cc1aa
--- /dev/null
+++ b/test/07_ssl/cibuildwheel_test.py
@@ -0,0 +1,17 @@
+import subprocess, sys, os
+from glob import glob
+project_dir = os.path.dirname(__file__)
+sys.path.append(os.path.join(os.path.dirname(project_dir), 'shared'))
+import utils
+
+def test():
+    # this test checks that SSL is working in the build environment using
+    # some checks in setup.py.
+
+    # build the wheels
+    subprocess.check_call([sys.executable, '-m', 'cibuildwheel', project_dir])
+    
+    # check that we got the right number of built wheels
+    expected_identifiers = utils.cibuildwheel_get_build_identifiers(project_dir)
+    built_wheels = glob('wheelhouse/*.whl')
+    assert len(built_wheels) == len(expected_identifiers)
diff --git a/test/shared/utils.py b/test/shared/utils.py
new file mode 100644
index 00000000..e4eca4cb
--- /dev/null
+++ b/test/shared/utils.py
@@ -0,0 +1,25 @@
+import subprocess, sys
+
+def cibuildwheel_get_build_identifiers(project_path, env=None):
+    '''
+    Returns the list of build identifiers that cibuildwheel will try to build
+    for the current platform.
+    '''
+    cmd_output = subprocess.check_output(
+        [sys.executable, '-m', 'cibuildwheel', '--print-build-identifiers', project_path],
+        universal_newlines=True,
+        env=env,
+    )
+
+    return cmd_output.strip().split('\n')
+
+platform = None
+
+if sys.platform.startswith('linux'):
+    platform = 'linux'
+elif sys.platform.startswith('darwin'):
+    platform = 'macos'
+elif sys.platform in ['win32', 'cygwin']:
+    platform = 'windows'
+else:
+    raise Exception('Unsupported platform')