From e3416abeb9538ec0a8f351cbe8d41013cc7b9e35 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Sat, 7 Sep 2019 20:20:31 +0100 Subject: [PATCH 01/13] Use virtual environment for testing on MacOS X --- cibuildwheel/macos.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index adfc27f4..ceef69ad 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -1,4 +1,5 @@ from __future__ import print_function +import tempfile import os, subprocess, shlex, sys, shutil from collections import namedtuple from glob import glob @@ -120,6 +121,12 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef call(['delocate-wheel', '-w', '/tmp/delocated_wheel', built_wheel], env=env) delocated_wheel = glob('/tmp/delocated_wheel/*.whl')[0] + # set up a virtual environment to install and test from + call(['pip', 'install', 'virtualenv', env=env) + venv_dir = tempfile.mkdtemp() + call(['virtualenv', venv_dir, env=env) + call(['source', os.path.join(venv_dir, 'bin', 'activate')]) + # install the wheel call(['pip', 'install', delocated_wheel + test_extras], env=env) From 3ff59eefc6299f9f428bef959c1581a3aefc1a5a Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Sat, 7 Sep 2019 20:24:38 +0100 Subject: [PATCH 02/13] Fix syntax --- cibuildwheel/macos.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index ceef69ad..8b26c341 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -122,10 +122,10 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef delocated_wheel = glob('/tmp/delocated_wheel/*.whl')[0] # set up a virtual environment to install and test from - call(['pip', 'install', 'virtualenv', env=env) + call(['pip', 'install', 'virtualenv'], env=env) venv_dir = tempfile.mkdtemp() - call(['virtualenv', venv_dir, env=env) - call(['source', os.path.join(venv_dir, 'bin', 'activate')]) + call(['virtualenv', venv_dir], env=env) + call(['source', os.path.join(venv_dir, 'bin', 'activate')], env=env) # install the wheel call(['pip', 'install', delocated_wheel + test_extras], env=env) From fdc2ba8715f59bab09641672d3740bad57287523 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Sat, 7 Sep 2019 20:34:29 +0100 Subject: [PATCH 03/13] Try and work around the fact we can't source activate --- cibuildwheel/macos.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 8b26c341..3111e26f 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -125,7 +125,8 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef call(['pip', 'install', 'virtualenv'], env=env) venv_dir = tempfile.mkdtemp() call(['virtualenv', venv_dir], env=env) - call(['source', os.path.join(venv_dir, 'bin', 'activate')], env=env) + env['PATH'] = os.pathsep.join([os.path.join(venv_dir, 'bin'), env['PATH']]) + call(['which', 'python'], env=env) # install the wheel call(['pip', 'install', delocated_wheel + test_extras], env=env) From 40968897a8e7c7fcdc5a5ff0972894e605dc7585 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Sat, 7 Sep 2019 20:45:09 +0100 Subject: [PATCH 04/13] Test package in a virtualenv on Linux and Windows --- cibuildwheel/linux.py | 9 +++++++++ cibuildwheel/macos.py | 5 ++++- cibuildwheel/windows.py | 10 ++++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 473838b0..2df50c2e 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -85,6 +85,15 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef fi delocated_wheel=(/tmp/delocated_wheel/*.whl) + # Set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + pip install virtualenv + virtualenv tmp-test-env + source tmp-test-env/bin/activate + + # Check that we are using the Python from the virtual environment + which python + # Install the wheel we just built "$PYBIN/pip" install "$delocated_wheel"{test_extras} diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 3111e26f..cba86b31 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -121,11 +121,14 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef call(['delocate-wheel', '-w', '/tmp/delocated_wheel', built_wheel], env=env) delocated_wheel = glob('/tmp/delocated_wheel/*.whl')[0] - # set up a virtual environment to install and test from + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. call(['pip', 'install', 'virtualenv'], env=env) venv_dir = tempfile.mkdtemp() call(['virtualenv', venv_dir], env=env) env['PATH'] = os.pathsep.join([os.path.join(venv_dir, 'bin'), env['PATH']]) + + # check that we are using the Python from the virtual environment call(['which', 'python'], env=env) # install the wheel diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 13fd4f54..034879e9 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -121,6 +121,16 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef shell(['pip', 'wheel', abs_project_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(build_verbosity), env=env) built_wheel = glob(built_wheel_dir+'/*.whl')[0] + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + shell(['pip', 'install', 'virtualenv'], env=env) + venv_dir = tempfile.mkdtemp() + shell(['virtualenv', venv_dir], env=env) + env['PATH'] = os.pathsep.join([os.path.join(venv_dir, 'Scripts'), env['PATH']]) + + # check that we are using the Python from the virtual environment + shell(['which', 'python'], env=env) + # install the wheel shell(['pip', 'install', built_wheel + test_extras], env=env) From 853699c2654b7dcb2d66053d4d02c09748d65269 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Sat, 7 Sep 2019 20:48:39 +0100 Subject: [PATCH 05/13] Fix calls to pip and virtualenv --- cibuildwheel/linux.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 2df50c2e..49fe2afc 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -87,8 +87,8 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef # Set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. - pip install virtualenv - virtualenv tmp-test-env + "$PYBIN/pip" install virtualenv + "$PYBIN/virtualenv" tmp-test-env source tmp-test-env/bin/activate # Check that we are using the Python from the virtual environment From d4732130a7793946ad61393ccc91ef621a7ad994 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Sat, 7 Sep 2019 21:15:52 +0100 Subject: [PATCH 06/13] Use unique temporary directory for virtual environment on linux --- cibuildwheel/linux.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 49fe2afc..e934ef73 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -88,8 +88,9 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef # Set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. "$PYBIN/pip" install virtualenv - "$PYBIN/virtualenv" tmp-test-env - source tmp-test-env/bin/activate + venv_dir=`mktemp -d`/venv + "$PYBIN/virtualenv" $venv_dir + source $venv_dir/bin/activate # Check that we are using the Python from the virtual environment which python From 2e3bc016b43752b8ab20db52827855a8c68efeaa Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Sat, 7 Sep 2019 21:47:15 +0100 Subject: [PATCH 07/13] Try and avoid relying on global virtualenv command --- cibuildwheel/macos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index cba86b31..1a9b90de 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -125,7 +125,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef # there are no dependencies that were pulled in at build time. call(['pip', 'install', 'virtualenv'], env=env) venv_dir = tempfile.mkdtemp() - call(['virtualenv', venv_dir], env=env) + call(['python', '-m', 'virtualenv', venv_dir], env=env) env['PATH'] = os.pathsep.join([os.path.join(venv_dir, 'bin'), env['PATH']]) # check that we are using the Python from the virtual environment From 9470ca623fb1d6ed00a8bf8d80b7603f60e10c8a Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Sat, 7 Sep 2019 21:52:49 +0100 Subject: [PATCH 08/13] Use python -m virtualenv on Windows and Linux --- cibuildwheel/linux.py | 2 +- cibuildwheel/windows.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index e934ef73..978d3010 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -89,7 +89,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef # there are no dependencies that were pulled in at build time. "$PYBIN/pip" install virtualenv venv_dir=`mktemp -d`/venv - "$PYBIN/virtualenv" $venv_dir + "$PYBIN/python" -m virtualenv $venv_dir source $venv_dir/bin/activate # Check that we are using the Python from the virtual environment diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 034879e9..8f2d90b5 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -125,7 +125,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef # there are no dependencies that were pulled in at build time. shell(['pip', 'install', 'virtualenv'], env=env) venv_dir = tempfile.mkdtemp() - shell(['virtualenv', venv_dir], env=env) + shell(['python', '-m', 'virtualenv', venv_dir], env=env) env['PATH'] = os.pathsep.join([os.path.join(venv_dir, 'Scripts'), env['PATH']]) # check that we are using the Python from the virtual environment From c963013858dd63837d5cc78e61662e57f269a3c4 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Wed, 11 Sep 2019 23:56:27 +0200 Subject: [PATCH 09/13] Fixing weird issue with the shebang of pip installed scripts and __PYVENV_LAUNCHER__ --- cibuildwheel/macos.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 1a9b90de..9026fab4 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -127,6 +127,9 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef venv_dir = tempfile.mkdtemp() call(['python', '-m', 'virtualenv', venv_dir], env=env) env['PATH'] = os.pathsep.join([os.path.join(venv_dir, 'bin'), env['PATH']]) + # Some weird issue with the shebang of installed scripts + # See https://github.com/theacodes/nox/issues/44 and https://github.com/pypa/virtualenv/issues/620 + env.pop('__PYVENV_LAUNCHER__', None) # check that we are using the Python from the virtual environment call(['which', 'python'], env=env) From cf90846161964d93f41107e4706d1f21430055d3 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 12 Oct 2019 10:17:28 +0100 Subject: [PATCH 10/13] linux: run tests in a subshell, only create env if test defined --- cibuildwheel/linux.py | 51 ++++++++++++++++++++++++------------------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 978d3010..ddcec627 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -85,29 +85,36 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef fi delocated_wheel=(/tmp/delocated_wheel/*.whl) - # Set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - "$PYBIN/pip" install virtualenv - venv_dir=`mktemp -d`/venv - "$PYBIN/python" -m virtualenv $venv_dir - source $venv_dir/bin/activate - - # Check that we are using the Python from the virtual environment - which python - - # Install the wheel we just built - "$PYBIN/pip" install "$delocated_wheel"{test_extras} - - # Install any requirements to run the tests - if [ ! -z "{test_requires}" ]; then - "$PYBIN/pip" install {test_requires} - fi - - # Run the tests from a different directory if [ ! -z {test_command} ]; then - pushd $HOME - PATH="$PYBIN:$PATH" sh -c {test_command} - popd + # Set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + "$PYBIN/pip" install virtualenv + venv_dir=`mktemp -d`/venv + "$PYBIN/python" -m virtualenv "$venv_dir" + + # run the tests in a subshell to keep that `activate` + # script from polluting the env + ( + source "$venv_dir/bin/activate" + + echo "Running tests using `which python`" + + # Install the wheel we just built + pip install "$delocated_wheel"{test_extras} + + # Install any requirements to run the tests + if [ ! -z "{test_requires}" ]; then + pip install {test_requires} + fi + + # Run the tests from a different directory + pushd $HOME + sh -c {test_command} + popd + ) + + # clean up + rm -rf "$venv_dir" fi # we're all done here; move it to output From b0c2fd8af20c00b2dd269746287c1befdc625f68 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 12 Oct 2019 10:35:45 +0100 Subject: [PATCH 11/13] macos: only make venv when test is defined and isolate env vars --- cibuildwheel/macos.py | 57 +++++++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 24 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 9026fab4..b4bf8157 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -121,31 +121,40 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef call(['delocate-wheel', '-w', '/tmp/delocated_wheel', built_wheel], env=env) delocated_wheel = glob('/tmp/delocated_wheel/*.whl')[0] - # set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - call(['pip', 'install', 'virtualenv'], env=env) - venv_dir = tempfile.mkdtemp() - call(['python', '-m', 'virtualenv', venv_dir], env=env) - env['PATH'] = os.pathsep.join([os.path.join(venv_dir, 'bin'), env['PATH']]) - # Some weird issue with the shebang of installed scripts - # See https://github.com/theacodes/nox/issues/44 and https://github.com/pypa/virtualenv/issues/620 - env.pop('__PYVENV_LAUNCHER__', None) - - # check that we are using the Python from the virtual environment - call(['which', 'python'], env=env) - - # install the wheel - call(['pip', 'install', delocated_wheel + test_extras], env=env) - - # test the wheel - if test_requires: - call(['pip', 'install'] + test_requires, env=env) if test_command: - # run the tests from $HOME, with an absolute path in the command - # (this ensures that Python runs the tests against the installed wheel - # and not the repo code) - test_command_prepared = prepare_command(test_command, project=abs_project_dir) - call(test_command_prepared, cwd=os.environ['HOME'], env=env, shell=True) + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + call(['pip', 'install', 'virtualenv'], env=env) + venv_dir = tempfile.mkdtemp() + call(['python', '-m', 'virtualenv', venv_dir], env=env) + + virtualenv_env = env.copy() + virtualenv_env['PATH'] = os.pathsep.join([ + os.path.join(venv_dir, 'bin'), + virtualenv_env['PATH'], + ]) + # Fix some weird issue with the shebang of installed scripts + # See https://github.com/theacodes/nox/issues/44 and https://github.com/pypa/virtualenv/issues/620 + virtualenv_env.pop('__PYVENV_LAUNCHER__', None) + + # check that we are using the Python from the virtual environment + call(['which', 'python'], env=virtualenv_env) + + # install the wheel + call(['pip', 'install', delocated_wheel + test_extras], env=virtualenv_env) + + # test the wheel + if test_requires: + call(['pip', 'install'] + test_requires, env=virtualenv_env) + if test_command: + # run the tests from $HOME, with an absolute path in the command + # (this ensures that Python runs the tests against the installed wheel + # and not the repo code) + test_command_prepared = prepare_command(test_command, project=abs_project_dir) + call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True) + + # clean up + shutil.rmtree(venv_dir) # we're all done here; move it to output (overwrite existing) dst = os.path.join(output_dir, os.path.basename(delocated_wheel)) From bfa3eebb7f048ded987856526c36e0d707f4534e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 12 Oct 2019 10:44:40 +0100 Subject: [PATCH 12/13] Remove redundant if clause --- cibuildwheel/macos.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index b4bf8157..c62eafc0 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -146,12 +146,12 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef # test the wheel if test_requires: call(['pip', 'install'] + test_requires, env=virtualenv_env) - if test_command: - # run the tests from $HOME, with an absolute path in the command - # (this ensures that Python runs the tests against the installed wheel - # and not the repo code) - test_command_prepared = prepare_command(test_command, project=abs_project_dir) - call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True) + + # run the tests from $HOME, with an absolute path in the command + # (this ensures that Python runs the tests against the installed wheel + # and not the repo code) + test_command_prepared = prepare_command(test_command, project=abs_project_dir) + call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True) # clean up shutil.rmtree(venv_dir) From 2a244616ad86862dbc94c38c92deb4d4185ff70e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 12 Oct 2019 10:47:00 +0100 Subject: [PATCH 13/13] windows: only make venv when test is defined and isolate env vars --- cibuildwheel/windows.py | 43 +++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 17 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 8f2d90b5..a25dab69 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -121,28 +121,37 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef shell(['pip', 'wheel', abs_project_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(build_verbosity), env=env) built_wheel = glob(built_wheel_dir+'/*.whl')[0] - # set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - shell(['pip', 'install', 'virtualenv'], env=env) - venv_dir = tempfile.mkdtemp() - shell(['python', '-m', 'virtualenv', venv_dir], env=env) - env['PATH'] = os.pathsep.join([os.path.join(venv_dir, 'Scripts'), env['PATH']]) - - # check that we are using the Python from the virtual environment - shell(['which', 'python'], env=env) - - # install the wheel - shell(['pip', 'install', built_wheel + test_extras], env=env) - - # test the wheel - if test_requires: - shell(['pip', 'install'] + test_requires, env=env) if test_command: + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + shell(['pip', 'install', 'virtualenv'], env=env) + venv_dir = tempfile.mkdtemp() + shell(['python', '-m', 'virtualenv', venv_dir], env=env) + + virtualenv_env = env.copy() + virtualenv_env['PATH'] = os.pathsep.join([ + os.path.join(venv_dir, 'Scripts'), + virtualenv_env['PATH'], + ]) + + # check that we are using the Python from the virtual environment + shell(['which', 'python'], env=virtualenv_env) + + # install the wheel + shell(['pip', 'install', built_wheel + test_extras], env=virtualenv_env) + + # test the wheel + if test_requires: + shell(['pip', 'install'] + test_requires, env=virtualenv_env) + # run the tests from c:\, with an absolute path in the command # (this ensures that Python runs the tests against the installed wheel # and not the repo code) test_command_prepared = prepare_command(test_command, project=abs_project_dir) - shell([test_command_prepared], cwd='c:\\', env=env) + shell([test_command_prepared], cwd='c:\\', env=virtualenv_env) + + # clean up + shutil.rmtree(venv_dir) # we're all done here; move it to output (remove if already exists) dst = os.path.join(output_dir, os.path.basename(built_wheel))