From 06696c46a082d6e6c13505f9e90411b8e0134e07 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 28 Jan 2020 22:04:37 +0000 Subject: [PATCH] Pin tool dependencies whenever we pip install using a constraints file --- bin/update_constraints | 12 +++++++++++ cibuildwheel/__main__.py | 10 +++++++++ cibuildwheel/linux.py | 7 +++++-- cibuildwheel/macos.py | 10 +++++---- cibuildwheel/resources/constraints.in | 6 ++++++ cibuildwheel/resources/constraints.txt | 29 ++++++++++++++++++++++++++ cibuildwheel/windows.py | 12 ++++++----- requirements-dev.txt | 2 ++ 8 files changed, 77 insertions(+), 11 deletions(-) create mode 100755 bin/update_constraints create mode 100644 cibuildwheel/resources/constraints.in create mode 100644 cibuildwheel/resources/constraints.txt diff --git a/bin/update_constraints b/bin/update_constraints new file mode 100755 index 00000000..76952e46 --- /dev/null +++ b/bin/update_constraints @@ -0,0 +1,12 @@ +#!/bin/bash + +cd "$(dirname "$0")" +cd .. + +export CUSTOM_COMPILE_COMMAND="bin/update_constraints" + +pip-compile \ + --allow-unsafe \ + --generate-hashes \ + --upgrade \ + cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 9b3db905..6a495951 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -102,6 +102,15 @@ def main(): repair_command_default = '' repair_command = get_option_from_environment('CIBW_REPAIR_WHEEL_COMMAND', platform=platform, default=repair_command_default) environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform, default='') + dependency_versions = get_option_from_environment('CIBW_DEPENDENCY_VERSIONS', platform=platform, default='pinned') + if dependency_versions == 'pinned': + dependency_constraints = os.path.join( + os.path.dirname(__file__), 'resources', 'constraints.txt' + ) + elif dependency_versions == 'latest': + dependency_constraints = '' + else: + dependency_constraints = dependency_versions if test_extras: test_extras = '[{0}]'.format(test_extras) @@ -144,6 +153,7 @@ def main(): build_selector=build_selector, repair_command=repair_command, environment=environment, + dependency_constraints=dependency_constraints, ) if platform == 'linux': diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index eb420516..b9b16e55 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -30,7 +30,7 @@ def get_python_configurations(build_selector): return [c for c in python_configurations if build_selector(c.identifier)] -def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, manylinux_images): +def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, manylinux_images, dependency_constraints): try: subprocess.check_call(['docker', '--version']) except: @@ -86,7 +86,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef if [ ! -z {test_command} ]; then # Set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. - "$PYBIN/pip" install virtualenv + "$PYBIN/pip" install {dependency_install_flags} virtualenv venv_dir=`mktemp -d`/venv "$PYBIN/python" -m virtualenv "$venv_dir" @@ -145,6 +145,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef environment_exports='\n'.join(environment.as_shell_commands()), uid=os.getuid(), gid=os.getgid(), + dependency_install_flags='-c /constraints.txt' if dependency_constraints else '', ) def run_docker(command, stdin_str=None): @@ -171,6 +172,8 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef '-v', '/:/host', # ignored on Circle docker_image, '/bin/bash']) run_docker(['cp', os.path.abspath(project_dir) + '/.', container_name + ':/project']) + if dependency_constraints: + run_docker(['cp', os.path.abspath(dependency_constraints) + '/.', container_name + ':/constraints.txt']) run_docker(['start', '-i', '-a', container_name], stdin_str=bash_script) run_docker(['cp', container_name + ':/output/.', os.path.abspath(output_dir)]) except subprocess.CalledProcessError: diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index d7a50d0c..1cc00eaa 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -25,7 +25,7 @@ def get_python_configurations(build_selector): return [c for c in python_configurations if build_selector(c.identifier)] -def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment): +def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, dependency_constraints): abs_project_dir = os.path.abspath(project_dir) temp_dir = tempfile.mkdtemp(prefix='cibuildwheel') built_wheel_dir = os.path.join(temp_dir, 'built_wheel') @@ -92,11 +92,13 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef call(['which', 'python'], env=env) call(['python', '--version'], env=env) + dependency_constraint_flags = ['-c', dependency_constraints] if dependency_constraints else [] + # install pip & wheel - call(['python', get_pip_script, '--no-setuptools', '--no-wheel'], env=env, cwd="/tmp") + call(['python', get_pip_script, '--no-setuptools', '--no-wheel'] + dependency_constraint_flags, env=env, cwd="/tmp") assert os.path.exists(os.path.join(installation_bin_path, 'pip')) call(['pip', '--version'], env=env) - call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate'], env=env) + call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate'] + dependency_constraint_flags, env=env) # run the before_build command if before_build: @@ -125,7 +127,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef if test_command: # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. - call(['pip', 'install', 'virtualenv'], env=env) + call(['pip', 'install', 'virtualenv'] + dependency_constraint_flags, env=env) venv_dir = tempfile.mkdtemp() call(['python', '-m', 'virtualenv', venv_dir], env=env) diff --git a/cibuildwheel/resources/constraints.in b/cibuildwheel/resources/constraints.in new file mode 100644 index 00000000..d07485ae --- /dev/null +++ b/cibuildwheel/resources/constraints.in @@ -0,0 +1,6 @@ +pip +setuptools +wheel +auditwheel +delocate +virtualenv diff --git a/cibuildwheel/resources/constraints.txt b/cibuildwheel/resources/constraints.txt new file mode 100644 index 00000000..19b904ee --- /dev/null +++ b/cibuildwheel/resources/constraints.txt @@ -0,0 +1,29 @@ +# +# This file is autogenerated by pip-compile +# To update, run: +# +# bin/update_constraints +# +auditwheel==2.1.1 \ + --hash=sha256:d9f0f47c0ecdf4f72316eabfb19223b81aec95f0deba97729e9c09ea4f2c166e \ + --hash=sha256:f60e265dc168ae7c9d65a814410881c5ca170d88644c178f8e3aa20eecd0fae1 +delocate==0.8.0 \ + --hash=sha256:ac9f4d8d28b2582d94f764db0d72a1fe3c42ab8212e9b780168455de385d5b95 +pyelftools==0.26 \ + --hash=sha256:86ac6cee19f6c945e8dedf78c6ee74f1112bd14da5a658d8c9d4103aed5756a2 \ + --hash=sha256:cc0ea0de82b240a73ef4056fce44acbb4727dca7d66759371aff2bad457ed711 \ + # via auditwheel +virtualenv==16.7.9 \ + --hash=sha256:0d62c70883c0342d59c11d0ddac0d954d0431321a41ab20851facf2b222598f3 \ + --hash=sha256:55059a7a676e4e19498f1aad09b8313a38fcc0cdbe4fdddc0e9b06946d21b4bb +wheel==0.31.1 \ + --hash=sha256:0a2e54558a0628f2145d2fc822137e322412115173e8a2ddbe1c9024338ae83c \ + --hash=sha256:80044e51ec5bbf6c894ba0bc48d26a8c20a9ba629f4ca19ea26ecfcf87685f5f + +# The following packages are considered to be unsafe in a requirements file: +pip==20.0.2 \ + --hash=sha256:4ae14a42d8adba3205ebeb38aa68cfc0b6c346e1ae2e699a0b3bad4da19cef5c \ + --hash=sha256:7db0c8ea4c7ea51c8049640e8e6e7fde949de672bfa4949920675563a5a6967f +setuptools==44.0.0 \ + --hash=sha256:180081a244d0888b0065e18206950d603f6550721bd6f8c0a10221ed467dd78e \ + --hash=sha256:e5baf7723e5bb8382fc146e33032b241efc63314211a3a120aaa55d62d2bb008 diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index c0923934..df2f8f4a 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -58,7 +58,7 @@ def get_python_configurations(build_selector): return python_configurations -def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment): +def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, dependency_constraints): def simple_shell(args, env=None, cwd=None): print('+ ' + ' '.join(args)) args = ['cmd', '/E:ON', '/V:ON', '/C'] + args @@ -135,15 +135,17 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef simple_shell(['python', '--version'], env=env) simple_shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)\"'], env=env) + dependency_constraint_flags = ['-c', dependency_constraints] if dependency_constraints else [] + # make sure pip is installed if not os.path.exists(os.path.join(config_python_path, 'Scripts', 'pip.exe')): - simple_shell(['python', get_pip_script], env=env, cwd="C:\\cibw") + simple_shell(['python', get_pip_script] + dependency_constraint_flags, env=env, cwd="C:\\cibw") assert os.path.exists(os.path.join(config_python_path, 'Scripts', 'pip.exe')) # prepare the Python environment - simple_shell(['python', '-m', 'pip', 'install', '--upgrade', 'pip'], env=env) + simple_shell(['python', '-m', 'pip', 'install', '--upgrade', 'pip'] + dependency_constraint_flags, env=env) simple_shell(['pip', '--version'], env=env) - simple_shell(['pip', 'install', '--upgrade', 'setuptools', 'wheel'], env=env) + simple_shell(['pip', 'install', '--upgrade', 'setuptools', 'wheel'] + dependency_constraint_flags, env=env) # run the before_build command if before_build: @@ -172,7 +174,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef if test_command: # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. - shell(['pip', 'install', 'virtualenv'], env=env) + shell(['pip', 'install', 'virtualenv'] + dependency_constraint_flags, env=env) venv_dir = tempfile.mkdtemp() shell(['python', '-m', 'virtualenv', venv_dir], env=env) diff --git a/requirements-dev.txt b/requirements-dev.txt index 1ecc8596..31c8f086 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,3 +2,5 @@ -e ./docs/mkdocs_include_markdown_plugin pytest mkdocs==1.0.4 +pip-tools +