From 42a231f830453d99f9e8dbef5048d7ee353fb94e Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Mon, 3 Feb 2020 14:11:53 +0100 Subject: [PATCH 1/3] Making sure that python and pip are the ones we expect --- cibuildwheel/linux.py | 24 ++++++++++++++++-------- cibuildwheel/macos.py | 4 ++++ cibuildwheel/windows.py | 4 ++++ 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 3497d4ff..e3cd9fa5 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -63,17 +63,25 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef mkdir /output cd /project - {environment_exports} + for PYBIN in {pybin_paths}; do ( + # Temporary hack/workaround, putting loop body in subshell; fixed in PR #256 + + export PATH="$PYBIN:$PATH" + {environment_exports} + + # check the active python and pip are in PYBIN + # if `test` returns false, the script will exit due to errexit + test "$(which pip)" = "$PYBIN/pip" + test "$(which python)" = "$PYBIN/python" - for PYBIN in {pybin_paths}; do if [ ! -z {before_build} ]; then - PATH="$PYBIN:$PATH" sh -c {before_build} + sh -c {before_build} fi # Build the wheel rm -rf /tmp/built_wheel mkdir /tmp/built_wheel - PATH="$PYBIN:$PATH" "$PYBIN/pip" wheel . -w /tmp/built_wheel --no-deps {build_verbosity_flag} + pip wheel . -w /tmp/built_wheel --no-deps {build_verbosity_flag} built_wheel=(/tmp/built_wheel/*.whl) # repair the wheel @@ -92,9 +100,9 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef if [ ! -z {test_command} ]; then # Set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. - "$PYBIN/pip" install virtualenv + pip install virtualenv venv_dir=`mktemp -d`/venv - "$PYBIN/python" -m virtualenv "$venv_dir" + python -m virtualenv "$venv_dir" # run the tests in a subshell to keep that `activate` # script from polluting the env @@ -123,7 +131,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef ) # exit if tests failed (needed for older bash versions) if [ $? -ne 0 ]; then - exit 1; + exit 1; fi # clean up @@ -133,7 +141,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef # we're all done here; move it to output mv "${{repaired_wheels[@]}}" /output for repaired_wheel in "${{repaired_wheels[@]}}"; do chown {uid}:{gid} "/output/$(basename "$repaired_wheel")"; done - done + ) done '''.format( pybin_paths=' '.join(c.path + '/bin' for c in platform_configs), test_requires=' '.join(test_requires), diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index fa56a66c..b0bbc18d 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -137,11 +137,15 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef # check what version we're on call(['which', 'python'], env=env) call(['python', '--version'], env=env) + # TODO Cleanup/merge with above `call` once we have `subprocess.run` after dropping Python 2 support? + assert subprocess.check_output(['which', 'python'], env=env, universal_newlines=True).strip() == '/tmp/cibw_bin/python' # install pip & wheel call(['python', get_pip_script], env=env, cwd="/tmp") assert os.path.exists(os.path.join(installation_bin_path, 'pip')) call(['pip', '--version'], env=env) + # TODO Cleanup/merge with above `call` once we have `subprocess.run` after dropping Python 2 support? + assert subprocess.check_output(['which', 'pip'], env=env, universal_newlines=True).strip() == '/tmp/cibw_bin/pip' call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate'], env=env) # setup target platform, only required for python 3.5 diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 4263d759..8a8aff97 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -141,11 +141,15 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef simple_shell(['where', 'python'], env=env) simple_shell(['python', '--version'], env=env) simple_shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)"'], env=env) + # TODO Cleanup/merge with above `simple_shell` once we have `subprocess.run` after dropping Python 2 support? + assert subprocess.check_output(['where', 'python'], env=env, universal_newlines=True).splitlines()[0] == os.path.join(installation_path, 'python.exe') # make sure pip is installed if not os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe')): simple_shell(['python', get_pip_script], env=env, cwd="C:\\cibw") assert os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe')) + # TODO Cleanup/merge with above `simple_shell` once we have `subprocess.run` after dropping Python 2 support? + assert subprocess.check_output(['where', 'pip'], env=env, universal_newlines=True).splitlines()[0] == os.path.join(installation_path, 'Scripts', 'pip.exe') # prepare the Python environment simple_shell(['python', '-m', 'pip', 'install', '--upgrade', 'pip'], env=env) From 2ce304d5d9c8ac14b550e9a5e50ba25a86a6045f Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Mon, 3 Feb 2020 14:26:53 +0100 Subject: [PATCH 2/3] Add test checking cibuildwheel detects when PATH is messed up by CIBW_ENVIRONMENT --- test/05_environment/cibuildwheel_test.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/test/05_environment/cibuildwheel_test.py b/test/05_environment/cibuildwheel_test.py index 4f0b922d..ab467111 100644 --- a/test/05_environment/cibuildwheel_test.py +++ b/test/05_environment/cibuildwheel_test.py @@ -1,5 +1,6 @@ import os - +import pytest +import subprocess import utils @@ -17,3 +18,15 @@ def test(): # also check that we got the right wheels built expected_wheels = utils.expected_wheels('spam', '0.1.0') assert set(actual_wheels) == set(expected_wheels) + + +def test_overridden_path(tmp_path): + project_dir = os.path.dirname(__file__) + + # mess up PATH, somehow + with pytest.raises(subprocess.CalledProcessError): + utils.cibuildwheel_run(project_dir, output_dir=tmp_path, add_env={ + 'CIBW_ENVIRONMENT': '''SOMETHING="$(mkdir new_path && touch new_path/python)" PATH="$(realpath new_path):$PATH"''', + 'CIBW_ENVIRONMENT_WINDOWS': '''SOMETHING="$(mkdir new_path && type nul > new_path/python.exe)" PATH="$CD\\new_path;$PATH"''', + }) + assert len(os.listdir(str(tmp_path))) == 0 From d14427d781e20d4ac9407739dc4e90a99206c311 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Wed, 19 Feb 2020 17:56:11 +0100 Subject: [PATCH 3/3] Adding more informative error messages --- cibuildwheel/linux.py | 11 ++++++++--- cibuildwheel/macos.py | 14 ++++++++++---- cibuildwheel/windows.py | 13 +++++++++---- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index e3cd9fa5..b2569692 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -70,9 +70,14 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef {environment_exports} # check the active python and pip are in PYBIN - # if `test` returns false, the script will exit due to errexit - test "$(which pip)" = "$PYBIN/pip" - test "$(which python)" = "$PYBIN/python" + if [ "$(which pip)" != "$PYBIN/pip" ]; then + echo "cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it." + exit 1 + fi + if [ "$(which python)" != "$PYBIN/python" ]; then + echo "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it." + exit 1 + fi if [ ! -z {before_build} ]; then sh -c {before_build} diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index b0bbc18d..0e95ff28 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -2,6 +2,7 @@ import os import shlex import shutil import subprocess +import sys import tempfile from collections import namedtuple from glob import glob @@ -137,15 +138,20 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef # check what version we're on call(['which', 'python'], env=env) call(['python', '--version'], env=env) - # TODO Cleanup/merge with above `call` once we have `subprocess.run` after dropping Python 2 support? - assert subprocess.check_output(['which', 'python'], env=env, universal_newlines=True).strip() == '/tmp/cibw_bin/python' + which_python = subprocess.check_output(['which', 'python'], env=env, universal_newlines=True).strip() + if which_python != '/tmp/cibw_bin/python': + print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr) + exit(1) # install pip & wheel call(['python', get_pip_script], env=env, cwd="/tmp") assert os.path.exists(os.path.join(installation_bin_path, 'pip')) + call(['which', 'pip'], env=env) call(['pip', '--version'], env=env) - # TODO Cleanup/merge with above `call` once we have `subprocess.run` after dropping Python 2 support? - assert subprocess.check_output(['which', 'pip'], env=env, universal_newlines=True).strip() == '/tmp/cibw_bin/pip' + which_pip = subprocess.check_output(['which', 'pip'], env=env, universal_newlines=True).strip() + if which_pip != '/tmp/cibw_bin/pip': + print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr) + exit(1) call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate'], env=env) # setup target platform, only required for python 3.5 diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 8a8aff97..bd93e1db 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -1,6 +1,7 @@ import os import shutil import subprocess +import sys import tempfile from collections import namedtuple from glob import glob @@ -141,15 +142,19 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef simple_shell(['where', 'python'], env=env) simple_shell(['python', '--version'], env=env) simple_shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)"'], env=env) - # TODO Cleanup/merge with above `simple_shell` once we have `subprocess.run` after dropping Python 2 support? - assert subprocess.check_output(['where', 'python'], env=env, universal_newlines=True).splitlines()[0] == os.path.join(installation_path, 'python.exe') + where_python = subprocess.check_output(['where', 'python'], env=env, universal_newlines=True).splitlines()[0].strip() + if where_python != os.path.join(installation_path, 'python.exe'): + print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr) + exit(1) # make sure pip is installed if not os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe')): simple_shell(['python', get_pip_script], env=env, cwd="C:\\cibw") assert os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe')) - # TODO Cleanup/merge with above `simple_shell` once we have `subprocess.run` after dropping Python 2 support? - assert subprocess.check_output(['where', 'pip'], env=env, universal_newlines=True).splitlines()[0] == os.path.join(installation_path, 'Scripts', 'pip.exe') + where_pip = subprocess.check_output(['where', 'pip'], env=env, universal_newlines=True).splitlines()[0].strip() + if where_pip.strip() != os.path.join(installation_path, 'Scripts', 'pip.exe'): + print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr) + exit(1) # prepare the Python environment simple_shell(['python', '-m', 'pip', 'install', '--upgrade', 'pip'], env=env)