Add a workflow to update dependencies (#490)

* Add a workflow to update dependencies

* Use '--no-docker' to run dependency update using local pythons

Default method is to use docker

* Only run the dependency update workflow manually

PR are only created for master

* Create constraints for all python versions

* Use manylinux2010_x86_64:latest to update dependencies

Co-authored-by: Joe Rickerby <joerick@mac.com>

.github/workflows/update-dependencies.yml

@@ -0,0 +1,31 @@
+name: Update dependencies
+
+on: workflow_dispatch
+
+jobs:
+  update-dependencies:
+    name: Update dependencies
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: actions/setup-python@v2
+      name: Install Python 3.9
+      with:
+        python-version: 3.9
+        architecture: x64
+    - name: Install dependencies
+      run: python -m pip install requests pip-tools
+    - name: Run update
+      run: python ./bin/update_dependencies.py
+    - name: Create Pull Request
+      if: github.ref == 'refs/heads/master'
+      uses: peter-evans/create-pull-request@v3
+      with:
+        commit-message: Update dependencies
+        title: '[Bot] Update dependencies'
+        body: |
+          Update the versions of our dependencies.
+
+          PR generated by "Update dependencies" [workflow](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}).
+        branch: update-dependencies-pr
+        delete-branch: true

bin/update_dependencies.py

@@ -2,7 +2,9 @@
 
 import configparser
 import os
+import shutil
 import subprocess
+import sys
 from collections import namedtuple
 
 import requests
@@ -12,21 +14,39 @@ os.chdir('..')
 
 # CUSTOM_COMPILE_COMMAND is a pip-compile option that tells users how to
 # regenerate the constraints files
-os.environ['CUSTOM_COMPILE_COMMAND'] = "bin/update_constraints.py"
+os.environ['CUSTOM_COMPILE_COMMAND'] = "bin/update_dependencies.py"
-subprocess.check_call([
+
-    'pip-compile',
+PYTHON_VERSIONS = ['27', '35', '36', '37', '38', '39']
-    '--allow-unsafe',
+
-    '--upgrade',
+if '--no-docker' in sys.argv:
-    'cibuildwheel/resources/constraints.in',
+    for python_version in PYTHON_VERSIONS:
-])
+        subprocess.check_call([
-for python_version in ['27', '35', '36', '37']:
+            f'./env{python_version}/bin/pip-compile',
-    subprocess.check_call([
+            '--allow-unsafe',
-        f'./env{python_version}/bin/pip-compile',
+            '--upgrade',
-        '--allow-unsafe',
+            'cibuildwheel/resources/constraints.in',
-        '--upgrade',
+            '--output-file', f'cibuildwheel/resources/constraints-python{python_version}.txt'
-        'cibuildwheel/resources/constraints.in',
+        ])
-        '--output-file', f'cibuildwheel/resources/constraints-python{python_version}.txt'
+else:
-    ])
+    image = 'quay.io/pypa/manylinux2010_x86_64:latest'
+    subprocess.check_call(['docker', 'pull', image])
+    for python_version in PYTHON_VERSIONS:
+        abi_flags = '' if int(python_version) >= 38 else 'm'
+        python_path = f'/opt/python/cp{python_version}-cp{python_version}{abi_flags}/bin/'
+        subprocess.check_call([
+            'docker', 'run', '--rm',
+            '-e', 'CUSTOM_COMPILE_COMMAND',
+            '-v', f'{os.getcwd()}:/volume',
+            '--workdir', '/volume', image,
+            'bash', '-c',
+            f'{python_path}pip install pip-tools &&'
+            f'{python_path}pip-compile --allow-unsafe --upgrade '
+            'cibuildwheel/resources/constraints.in '
+            f'--output-file cibuildwheel/resources/constraints-python{python_version}.txt'
+        ])
+
+# default constraints.txt
+shutil.copyfile(f'cibuildwheel/resources/constraints-python{PYTHON_VERSIONS[-1]}.txt', 'cibuildwheel/resources/constraints.txt')
 
 Image = namedtuple('Image', [
     'manylinux_version',

cibuildwheel/resources/constraints-python27.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    bin/update_constraints.py
+#    bin/update_dependencies.py
 #
 appdirs==1.4.4            # via virtualenv
 configparser==4.0.2       # via importlib-metadata

cibuildwheel/resources/constraints-python35.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    bin/update_constraints.py
+#    bin/update_dependencies.py
 #
 appdirs==1.4.4            # via virtualenv
 delocate==0.8.2           # via -r cibuildwheel/resources/constraints.in

cibuildwheel/resources/constraints-python36.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    bin/update_constraints.py
+#    bin/update_dependencies.py
 #
 appdirs==1.4.4            # via virtualenv
 delocate==0.8.2           # via -r cibuildwheel/resources/constraints.in

cibuildwheel/resources/constraints-python37.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    bin/update_constraints.py
+#    bin/update_dependencies.py
 #
 appdirs==1.4.4            # via virtualenv
 delocate==0.8.2           # via -r cibuildwheel/resources/constraints.in

cibuildwheel/resources/constraints-python38.txt

@@ -0,0 +1,17 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    bin/update_dependencies.py
+#
+appdirs==1.4.4            # via virtualenv
+delocate==0.8.2           # via -r cibuildwheel/resources/constraints.in
+distlib==0.3.1            # via virtualenv
+filelock==3.0.12          # via virtualenv
+six==1.15.0               # via virtualenv
+virtualenv==20.2.2        # via -r cibuildwheel/resources/constraints.in
+wheel==0.36.2             # via -r cibuildwheel/resources/constraints.in, delocate
+
+# The following packages are considered to be unsafe in a requirements file:
+pip==20.3.3               # via -r cibuildwheel/resources/constraints.in
+setuptools==51.0.0        # via -r cibuildwheel/resources/constraints.in

cibuildwheel/resources/constraints-python39.txt

@@ -0,0 +1,17 @@
+#
+# This file is autogenerated by pip-compile
+# To update, run:
+#
+#    bin/update_dependencies.py
+#
+appdirs==1.4.4            # via virtualenv
+delocate==0.8.2           # via -r cibuildwheel/resources/constraints.in
+distlib==0.3.1            # via virtualenv
+filelock==3.0.12          # via virtualenv
+six==1.15.0               # via virtualenv
+virtualenv==20.2.2        # via -r cibuildwheel/resources/constraints.in
+wheel==0.36.2             # via -r cibuildwheel/resources/constraints.in, delocate
+
+# The following packages are considered to be unsafe in a requirements file:
+pip==20.3.3               # via -r cibuildwheel/resources/constraints.in
+setuptools==51.0.0        # via -r cibuildwheel/resources/constraints.in

cibuildwheel/resources/constraints.txt

@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile
 # To update, run:
 #
-#    bin/update_constraints.py
+#    bin/update_dependencies.py
 #
 appdirs==1.4.4            # via virtualenv
 delocate==0.8.2           # via -r cibuildwheel/resources/constraints.in

unit_test/dependency_constraints_test.py

@@ -10,7 +10,8 @@ def test_defaults():
     resources_dir = project_root / 'cibuildwheel' / 'resources'
 
     assert dependency_constraints.base_file_path.samefile(resources_dir / 'constraints.txt')
-    assert dependency_constraints.get_for_python_version('3.8').samefile(resources_dir / 'constraints.txt')
+    assert dependency_constraints.get_for_python_version('3.99').samefile(resources_dir / 'constraints.txt')
+    assert dependency_constraints.get_for_python_version('3.9').samefile(resources_dir / 'constraints-python39.txt')
     assert dependency_constraints.get_for_python_version('3.6').samefile(resources_dir / 'constraints-python36.txt')
     assert dependency_constraints.get_for_python_version('3.5').samefile(resources_dir / 'constraints-python35.txt')
     assert dependency_constraints.get_for_python_version('2.7').samefile(resources_dir / 'constraints-python27.txt')