Run each test using a temporary directory (#217)

bin/run_test.py

@@ -3,27 +3,13 @@
 from __future__ import print_function
 import os, sys, subprocess, shutil
 
-project_root = os.path.dirname(os.path.dirname(__file__))
-test_utils_dir = os.path.join(project_root, 'test', 'shared')
 
 def single_run(test_project):
-    # set up an environment that gives access to the test utils
-    env = os.environ.copy()
-
-    if 'PYTHONPATH' in env:
-        env['PYTHONPATH'] += os.pathsep + test_utils_dir
-    else:
-        env['PYTHONPATH'] = test_utils_dir
-
     # run the test
     subprocess.check_call(
         [sys.executable, '-m', 'pytest', '-vv', os.path.join(test_project, 'cibuildwheel_test.py')],
-        env=env,
     )
 
-    # clean up
-    if os.path.exists('wheelhouse'):
-        shutil.rmtree('wheelhouse')
 
 if __name__ == '__main__':
     import argparse
@@ -33,7 +19,7 @@ if __name__ == '__main__':
     args = parser.parse_args()
 
     project_path = os.path.abspath(args.test_project_dir)
-    
+
     if not os.path.exists(project_path):
         print('No test project not found.', file=sys.stderr)
         exit(2)

test/01_basic/cibuildwheel_test.py

@@ -6,11 +6,10 @@ project_dir = os.path.dirname(__file__)
 
 def test():
     # build the wheels
-    utils.cibuildwheel_run(project_dir)
+    actual_wheels = utils.cibuildwheel_run(project_dir)
 
     # check that the expected wheels are produced
     expected_wheels = utils.expected_wheels('spam', '0.1.0')
-    actual_wheels = os.listdir('wheelhouse')
     assert set(actual_wheels) == set(expected_wheels)
 
 

test/02_test/cibuildwheel_test.py

@@ -6,7 +6,7 @@ def test():
     project_dir = os.path.dirname(__file__)
 
     # build and test the wheels
-    utils.cibuildwheel_run(project_dir, add_env={
+    actual_wheels = utils.cibuildwheel_run(project_dir, add_env={
         'CIBW_TEST_REQUIRES': 'nose',
         # the 'false ||' bit is to ensure this command runs in a shell on
         # mac/linux.
@@ -16,7 +16,6 @@ def test():
 
     # also check that we got the right wheels
     expected_wheels = utils.expected_wheels('spam', '0.1.0')
-    actual_wheels = os.listdir('wheelhouse')
     assert set(actual_wheels) == set(expected_wheels)
 
 
@@ -24,7 +23,7 @@ def test_extras_require():
     project_dir = os.path.dirname(__file__)
 
     # build and test the wheels
-    utils.cibuildwheel_run(project_dir, add_env={
+    actual_wheels = utils.cibuildwheel_run(project_dir, add_env={
         'CIBW_TEST_EXTRAS': 'test',
         # the 'false ||' bit is to ensure this command runs in a shell on
         # mac/linux.
@@ -34,22 +33,19 @@ def test_extras_require():
 
     # also check that we got the right wheels
     expected_wheels = utils.expected_wheels('spam', '0.1.0')
-    actual_wheels = os.listdir('wheelhouse')
     assert set(actual_wheels) == set(expected_wheels)
 
 
-def test_failing_test():
+def test_failing_test(tmp_path):
     '''Ensure a failing test causes cibuildwheel to error out and exit'''
     project_dir = os.path.dirname(__file__)
 
     with pytest.raises(subprocess.CalledProcessError):
-        utils.cibuildwheel_run(project_dir, add_env={
+        utils.cibuildwheel_run(project_dir, output_dir=tmp_path, add_env={
             'CIBW_TEST_COMMAND': 'false',
-            # manylinux1 has a version of bash that's been shown to have 
+            # manylinux1 has a version of bash that's been shown to have
             # problems with this, so let's check that.
             'CIBW_MANYLINUX_I686_IMAGE': 'manylinux1',
             'CIBW_MANYLINUX_X86_64_IMAGE': 'manylinux1',
         })
-
-    assert len(os.listdir('wheelhouse'))
-
+    assert len(os.listdir(str(tmp_path))) == 0

test/03_before_build/cibuildwheel_test.py

@@ -5,14 +5,13 @@ def test():
     project_dir = os.path.dirname(__file__)
 
     # build the wheels
-    utils.cibuildwheel_run(project_dir, add_env={
+    actual_wheels = utils.cibuildwheel_run(project_dir, add_env={
         # write python version information to a temporary file, this is
         # checked in setup.py
         'CIBW_BEFORE_BUILD': '''python -c "import sys; open('/tmp/pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('/tmp/pythonexecutable.txt', 'w').write(sys.executable)"''',
         'CIBW_BEFORE_BUILD_WINDOWS': '''python -c "import sys; open('c:\\pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('c:\\pythonexecutable.txt', 'w').write(sys.executable)"''',
     })
-    
+
     # also check that we got the right wheels
     expected_wheels = utils.expected_wheels('spam', '0.1.0')
-    actual_wheels = os.listdir('wheelhouse')
     assert set(actual_wheels) == set(expected_wheels)

test/04_build_skip/cibuildwheel_test.py

@@ -5,7 +5,7 @@ def test():
     project_dir = os.path.dirname(__file__)
 
     # build the wheels
-    utils.cibuildwheel_run(project_dir, add_env={
+    actual_wheels = utils.cibuildwheel_run(project_dir, add_env={
         'CIBW_BUILD': 'cp3?-*',
         'CIBW_SKIP': 'cp37-*',
     })
@@ -13,5 +13,4 @@ def test():
     # check that we got the right wheels. There should be no 2.7 or 3.7.
     expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0')
                        if ('-cp3' in w) and ('-cp37' not in w)]
-    actual_wheels = os.listdir('wheelhouse')
     assert set(actual_wheels) == set(expected_wheels)

test/05_environment/cibuildwheel_test.py

@@ -7,12 +7,11 @@ def test():
     # write some information into the CIBW_ENVIRONMENT, for expansion and
     # insertion into the environment by cibuildwheel. This is checked
     # in setup.py
-    utils.cibuildwheel_run(project_dir, add_env={
+    actual_wheels = utils.cibuildwheel_run(project_dir, add_env={
         'CIBW_ENVIRONMENT': '''CIBW_TEST_VAR="a b c" CIBW_TEST_VAR_2=1 CIBW_TEST_VAR_3="$(echo 'test string 3')" PATH=$PATH:/opt/cibw_test_path''',
         'CIBW_ENVIRONMENT_WINDOWS': '''CIBW_TEST_VAR="a b c" CIBW_TEST_VAR_2=1 CIBW_TEST_VAR_3="$(echo 'test string 3')" PATH="$PATH;/opt/cibw_test_path"''',
     })
 
     # also check that we got the right wheels built
     expected_wheels = utils.expected_wheels('spam', '0.1.0')
-    actual_wheels = os.listdir('wheelhouse')
     assert set(actual_wheels) == set(expected_wheels)

test/06_docker_images/cibuildwheel_test.py

@@ -7,7 +7,7 @@ def test():
     if utils.platform != 'linux':
         pytest.skip('the test is only relevant to the linux build')
 
-    utils.cibuildwheel_run(project_dir, add_env={
+    actual_wheels = utils.cibuildwheel_run(project_dir, add_env={
         'CIBW_MANYLINUX_X86_64_IMAGE': 'dockcross/manylinux2010-x64',
         'CIBW_MANYLINUX_I686_IMAGE': 'dockcross/manylinux1-x86',
         'CIBW_BEFORE_BUILD': '/opt/python/cp36-cp36m/bin/pip install -U auditwheel',  # Currently necessary on dockcross images to get auditwheel 2.1 supporting AUDITWHEEL_PLAT
@@ -17,5 +17,4 @@ def test():
     # also check that we got the right wheels built
     expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0')
                        if '-manylinux2010_i686' not in w]
-    actual_wheels = os.listdir('wheelhouse')
     assert set(actual_wheels) == set(expected_wheels)

test/08_manylinux2010_only/cibuildwheel_test.py

@@ -9,13 +9,12 @@ def test():
 
     # build the wheels
     # CFLAGS environment veriable is ecessary to fail on 'malloc_info' (on manylinux1) during compilation/linking,
-    # rather than when dynamically loading the Python 
-    utils.cibuildwheel_run(project_dir, add_env={
+    # rather than when dynamically loading the Python
+    actual_wheels = utils.cibuildwheel_run(project_dir, add_env={
         'CIBW_ENVIRONMENT': 'CFLAGS="$CFLAGS -Werror=implicit-function-declaration"',
     })
-    
+
     # also check that we got the right wheels
     expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0')
                        if not '-manylinux' in w or '-manylinux2010' in w]
-    actual_wheels = os.listdir('wheelhouse')
     assert set(actual_wheels) == set(expected_wheels)

test/conftest.py

@@ -0,0 +1,4 @@
+import os
+import sys
+
+sys.path.append(os.path.join(os.path.dirname(__file__), 'shared'))

test/shared/utils.py

@@ -4,12 +4,26 @@ Utility functions used by the cibuildwheel tests.
 This file is added to the PYTHONPATH in the test runner at bin/run_test.py.
 '''
 
-import subprocess, sys, os
+import subprocess, sys, os, shutil
+from tempfile import mkdtemp
+from contextlib import contextmanager
+
 
 IS_WINDOWS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache')
 IS_WINDOWS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows'
 
 
+# Python 2 does not have a tempfile.TemporaryDirectory context manager
+@contextmanager
+def TemporaryDirectoryIfNone(path):
+    _path = path or mkdtemp()
+    try:
+        yield _path
+    finally:
+        if path is None:
+            shutil.rmtree(_path)
+
+
 def cibuildwheel_get_build_identifiers(project_path, env=None):
     '''
     Returns the list of build identifiers that cibuildwheel will try to build
@@ -24,12 +38,18 @@ def cibuildwheel_get_build_identifiers(project_path, env=None):
     return cmd_output.strip().split('\n')
 
 
-def cibuildwheel_run(project_path, env=None, add_env=None):
+def cibuildwheel_run(project_path, env=None, add_env=None, output_dir=None):
     '''
     Runs cibuildwheel as a subprocess, building the project at project_path.
 
     Uses the current Python interpreter.
-    Configure settings using env.
+
+    :param project_path: path of the project to be built.
+    :param env: full environment to be used, os.environ if None
+    :param add_env: environment used to update env
+    :param output_dir: directory where wheels are saved. If None, a temporary
+    directory will be used for the duration of the command.
+    :return: list of built wheels (file names).
     '''
     if env is None:
         env = os.environ.copy()
@@ -37,10 +57,13 @@ def cibuildwheel_run(project_path, env=None, add_env=None):
     if add_env is not None:
         env.update(add_env)
 
-    subprocess.check_call(
-        [sys.executable, '-m', 'cibuildwheel', project_path],
-        env=env,
-    )
+    with TemporaryDirectoryIfNone(output_dir) as _output_dir:
+        subprocess.check_call(
+            [sys.executable, '-m', 'cibuildwheel', '--output-dir', str(_output_dir), project_path],
+            env=env,
+        )
+        wheels = os.listdir(_output_dir)
+    return wheels
 
 
 def expected_wheels(package_name, package_version):