Bump version: v1.6.4

* Use certifi provided certificates when downloading files

* fix typo

* Add comment about certifi usage

Co-authored-by: Joe Rickerby <joerick@mac.com>

Co-authored-by: Joe Rickerby <joerick@mac.com>

README.md

@@ -85,7 +85,7 @@ env:
     # Note: TWINE_PASSWORD is set to a PyPI API token in Travis settings
 
 install:
-  - python3 -m pip install cibuildwheel==1.6.2
+  - python3 -m pip install cibuildwheel==1.6.4
 
 script:
   # build the wheels, put them into './wheelhouse'
@@ -145,7 +145,7 @@ Here are some repos that use cibuildwheel.
 - [pikepdf](https://github.com/pikepdf/pikepdf)
 - [fathon](https://github.com/stfbnc/fathon)
 - [etebase-py](https://github.com/etesync/etebase-py) - Python bindings to a Rust library using `setuptools-rust`, and `sccache` for improved speed, built on Travis CI.
-
+- [xmlstarlet](https://github.com/dimitern/xmlstarlet) - Python 3.6+ CFFI bindings with true MSVC build and GitHub Actions.
 > Add your repo here! Send a PR.
 >
 > <sup>I'd like to include notes here to indicate why an example might be interesting to cibuildwheel users - the styles/technologies/techniques used in each. Please include that in future additions!</sup>
@@ -162,6 +162,20 @@ This is similar to static linking, so it might have some licence implications. C
 Changelog
 =========
 
+### 1.6.4
+
+_31 October 2020_
+
+- 🐛 Fix crash on Appveyor during nuget install due to old system CA
+  certificates. We now use certifi's CA certs to download files. (#455)
+
+### 1.6.3
+
+_12 October 2020_
+
+- 🐛 Fix missing SSL certificates on macOS (#447)
+- 🛠 Update OpenSSL Python 3.5 patch to 1.1.1h on macOS (#449)
+
 ### 1.6.2
 
 _9 October 2020_

bin/bump_version.py

@@ -7,6 +7,7 @@ import cibuildwheel
 from packaging.version import Version, InvalidVersion
 import subprocess
 import glob
+import urllib.parse
 
 config = [
     # file path, version find/replace format
@@ -87,6 +88,24 @@ def bump_version():
 
     print('Files updated. If you want to update the changelog as part of this')
     print('commit, do that now.')
+    print()
+
+    try:
+        commit_date_str = subprocess.run([
+            'git',
+            'show', '-s', '--pretty=format:%ci',
+            f'v{current_version}^{{commit}}'
+        ], check=True, capture_output=True, encoding='utf8').stdout
+        commit_date_parts = commit_date_str.split(' ')
+
+        url = 'https://github.com/joerick/cibuildwheel/pulls?' + urllib.parse.urlencode({
+            'q': f'is:pr merged:>{commit_date_parts[0]}T{commit_date_parts[1]}{commit_date_parts[2]}',
+        })
+        print(f'PRs merged since last release:\n  {url}')
+        print()
+    except subprocess.CalledProcessError as e:
+        print(e)
+        print('Failed to get previous version tag information.')
 
     while input('Type "done" to continue: ').strip().lower() != 'done':
         pass

cibuildwheel/__init__.py

@@ -1 +1 @@
-__version__ = '1.6.2'
+__version__ = '1.6.4'

cibuildwheel/macos.py

@@ -11,7 +11,7 @@ from typing import Dict, List, NamedTuple, Optional, Sequence, Union
 from .environment import ParsedEnvironment
 from .util import (BuildOptions, BuildSelector, NonPlatformWheelError, download,
                    get_build_verbosity_extra_flags, get_pip_script,
-                   prepare_command)
+                   prepare_command, install_certifi_script)
 
 
 def call(args: Union[str, Sequence[Union[str, PathLike]]], env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None, shell: bool = False) -> int:
@@ -72,6 +72,9 @@ def install_cpython(version: str, url: str) -> Path:
 
     # if this version of python isn't installed, get it from python.org and install
     python_package_identifier = f'org.python.Python.PythonFramework-{version}'
+    python_executable = 'python3' if version[0] == '3' else 'python'
+    installation_bin_path = Path(f'/Library/Frameworks/Python.framework/Versions/{version}/bin')
+
     if python_package_identifier not in installed_system_packages:
         # download the pkg
         download(url, Path('/tmp/Python.pkg'))
@@ -79,12 +82,12 @@ def install_cpython(version: str, url: str) -> Path:
         call(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/'])
         # patch open ssl
         if version == '3.5':
-            open_ssl_patch_url = f'https://github.com/mayeut/patch-macos-python-openssl/releases/download/v1.0.2u/patch-macos-python-{version}-openssl-v1.0.2u.tar.gz'
+            open_ssl_patch_url = f'https://github.com/mayeut/patch-macos-python-openssl/releases/download/v1.1.1h/patch-macos-python-{version}-openssl-v1.1.1h.tar.gz'
             download(open_ssl_patch_url, Path('/tmp/python-patch.tar.gz'))
             call(['sudo', 'tar', '-C', f'/Library/Frameworks/Python.framework/Versions/{version}/', '-xmf', '/tmp/python-patch.tar.gz'])
 
-    installation_bin_path = Path(f'/Library/Frameworks/Python.framework/Versions/{version}/bin')
+        call(["sudo", str(installation_bin_path/python_executable), str(install_certifi_script)])
-    python_executable = 'python3' if version[0] == '3' else 'python'
+
     pip_executable = 'pip3' if version[0] == '3' else 'pip'
     make_symlinks(installation_bin_path, python_executable, pip_executable)
 

cibuildwheel/resources/install_certifi.py

@@ -0,0 +1,50 @@
+# Based on: https://github.com/python/cpython/blob/master/Mac/BuildScript/resources/install_certificates.command
+
+# install_certifi.py
+#
+# sample script to install or update a set of default Root Certificates
+# for the ssl module.  Uses the certificates provided by the certifi package:
+#       https://pypi.org/project/certifi/
+
+import os
+import os.path
+import ssl
+import stat
+import subprocess
+import sys
+
+STAT_0o775 = (stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR
+              | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP
+              | stat.S_IROTH | stat.S_IXOTH)
+
+if sys.version_info[0] == 2:
+    FileNotFoundError = OSError
+
+
+def main():
+    openssl_dir, openssl_cafile = os.path.split(
+        ssl.get_default_verify_paths().openssl_cafile)
+    print(" -- pip install --upgrade certifi")
+    subprocess.check_call([sys.executable,
+                           "-E", "-s", "-m", "pip", "install", "--upgrade", "certifi"])
+
+    import certifi
+    # change working directory to the default SSL directory
+    os.chdir(openssl_dir)
+    relpath_to_certifi_cafile = os.path.relpath(certifi.where())
+
+    print(" -- removing any existing file or link")
+    try:
+        os.remove(openssl_cafile)
+    except FileNotFoundError:
+        pass
+    print(" -- creating symlink to certifi certificate bundle")
+    os.symlink(relpath_to_certifi_cafile, openssl_cafile)
+
+    print(" -- setting permissions")
+    os.chmod(openssl_cafile, STAT_0o775)
+    print(" -- update complete")
+
+
+if __name__ == '__main__':
+    main()

cibuildwheel/util.py

@@ -1,6 +1,8 @@
 import os
 import textwrap
+import certifi
 import urllib.request
+import ssl
 from fnmatch import fnmatch
 from pathlib import Path
 from time import sleep
@@ -66,10 +68,14 @@ def download(url: str, dest: Path) -> None:
     if not dest_dir.exists():
         dest_dir.mkdir(parents=True)
 
+    # we've had issues when relying on the host OS' CA certificates on Windows,
+    # so we use certifi (this sounds odd but requests also does this by default)
+    cafile = os.environ.get('SSL_CERT_FILE', certifi.where())
+    context = ssl.create_default_context(cafile=cafile)
     repeat_num = 3
     for i in range(repeat_num):
         try:
-            response = urllib.request.urlopen(url)
+            response = urllib.request.urlopen(url, context=context)
         except Exception:
             if i == repeat_num - 1:
                 raise
@@ -107,6 +113,9 @@ class DependencyConstraints:
         else:
             return self.base_file_path
 
+    def __str__(self):
+        return f"File '{self.base_file_path}'"
+
 
 class BuildOptions(NamedTuple):
     package_dir: Path
@@ -127,6 +136,7 @@ class BuildOptions(NamedTuple):
 
 resources_dir = Path(__file__).resolve().parent / 'resources'
 get_pip_script = resources_dir / 'get-pip.py'
+install_certifi_script = resources_dir / "install_certifi.py"
 
 
 class NonPlatformWheelError(Exception):

examples/appveyor-minimal.yml

@@ -12,7 +12,7 @@ stack: python 3.7
 init:
 - cmd: set PATH=C:\Python37;C:\Python37\Scripts;%PATH%
 
-install: python -m pip install cibuildwheel==1.6.2
+install: python -m pip install cibuildwheel==1.6.4
 
 build_script: python -m cibuildwheel --output-dir wheelhouse
 

examples/azure-pipelines-minimal.yml

@@ -5,7 +5,7 @@ jobs:
     - task: UsePythonVersion@0
     - bash: |
         python3 -m pip install --upgrade pip
-        pip3 install cibuildwheel==1.6.2
+        pip3 install cibuildwheel==1.6.4
         cibuildwheel --output-dir wheelhouse .
     - task: PublishBuildArtifacts@1
       inputs: {pathtoPublish: 'wheelhouse'}
@@ -16,7 +16,7 @@ jobs:
     - task: UsePythonVersion@0
     - bash: |
         python3 -m pip install --upgrade pip
-        pip3 install cibuildwheel==1.6.2
+        pip3 install cibuildwheel==1.6.4
         cibuildwheel --output-dir wheelhouse .
     - task: PublishBuildArtifacts@1
       inputs: {pathtoPublish: 'wheelhouse'}
@@ -29,7 +29,7 @@ jobs:
       displayName: Install Visual C++ for Python 2.7
     - bash: |
         python -m pip install --upgrade pip
-        pip install cibuildwheel==1.6.2
+        pip install cibuildwheel==1.6.4
         cibuildwheel --output-dir wheelhouse .
     - task: PublishBuildArtifacts@1
       inputs: {pathtoPublish: 'wheelhouse'}

examples/circleci-minimal.yml

@@ -11,7 +11,7 @@ jobs:
       - run:
           name: Build the Linux wheels.
           command: |
-            pip3 install --user cibuildwheel==1.6.2
+            pip3 install --user cibuildwheel==1.6.4
             cibuildwheel --output-dir wheelhouse
       - store_artifacts:
           path: wheelhouse/
@@ -26,7 +26,7 @@ jobs:
       - run:
           name: Build the OS X wheels.
           command: |
-            pip3 install cibuildwheel==1.6.2
+            pip3 install cibuildwheel==1.6.4
             cibuildwheel --output-dir wheelhouse
       - store_artifacts:
           path: wheelhouse/

examples/github-deploy.yml

@@ -28,7 +28,7 @@ jobs:
 
       - name: Install cibuildwheel
         run: |
-          python -m pip install cibuildwheel==1.6.2
+          python -m pip install cibuildwheel==1.6.4
 
       - name: Install Visual C++ for Python 2.7
         if: runner.os == 'Windows'

examples/github-minimal.yml

@@ -20,7 +20,7 @@ jobs:
 
       - name: Install cibuildwheel
         run: |
-          python -m pip install cibuildwheel==1.6.2
+          python -m pip install cibuildwheel==1.6.4
 
       - name: Install Visual C++ for Python 2.7
         if: runner.os == 'Windows'

examples/travis-ci-deploy.yml

@@ -27,7 +27,7 @@ env:
     # Note: TWINE_PASSWORD is set to a PyPI API token in Travis settings
 
 install:
-  - python3 -m pip install cibuildwheel==1.6.2
+  - python3 -m pip install cibuildwheel==1.6.4
 
 script:
   # build the wheels, put them into './wheelhouse'

examples/travis-ci-minimal.yml

@@ -28,7 +28,7 @@ jobs:
         - ln -s /c/Python38/python.exe /c/Python38/python3.exe
 
 install:
-  - python3 -m pip install cibuildwheel==1.6.2
+  - python3 -m pip install cibuildwheel==1.6.4
 
 script:
   # build the wheels, put them into './wheelhouse'

examples/travis-ci-test-and-deploy.yml

@@ -55,7 +55,7 @@ jobs:
     - stage: deploy
       name: Build and deploy Linux wheels
       services: docker
-      install: python3 -m pip install cibuildwheel==1.6.2
+      install: python3 -m pip install cibuildwheel==1.6.4
       script: python3 -m cibuildwheel --output-dir wheelhouse
       after_success: |
         python3 -m pip install twine
@@ -67,7 +67,7 @@ jobs:
       # PyPy 7.3.2 needs macOS >= 10.14
       osx_image: xcode10.2
       language: shell
-      install: python3 -m pip install cibuildwheel==1.6.2
+      install: python3 -m pip install cibuildwheel==1.6.4
       script: python3 -m cibuildwheel --output-dir wheelhouse
       after_success: |
         python3 -m pip install twine
@@ -77,7 +77,7 @@ jobs:
       name: Build and deploy Windows wheels
       os: windows
       language: shell
-      install: python3 -m pip install cibuildwheel==1.6.2
+      install: python3 -m pip install cibuildwheel==1.6.4
       script: python3 -m cibuildwheel --output-dir wheelhouse
       after_success: |
         python3 -m pip install twine

setup.py

@@ -11,8 +11,8 @@ long_description = (this_directory / 'README.md').read_text(encoding='utf-8')
 
 setup(
     name='cibuildwheel',
-    version='1.6.2',
+    version='1.6.4',
-    install_requires=['bashlex!=0.13', 'toml'],
+    install_requires=['bashlex!=0.13', 'toml', 'certifi'],
     description="Build Python wheels on CI with minimal configuration.",
     long_description=long_description,
     long_description_content_type='text/markdown',

test/test_ssl.py

@@ -13,11 +13,10 @@ project_with_ssl_tests = test_projects.new_c_project(
         else:
             from urllib.request import urlopen
 
-        if sys.version_info[0:2] == (3, 3):
+        context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
-            data = urlopen("https://www.nist.gov")
+        data = urlopen("https://www.nist.gov", context=context)
-        else:
+        data = urlopen("https://raw.githubusercontent.com/joerick/cibuildwheel/master/CI.md", context=context)
-            context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
+        data = urlopen("https://raw.githubusercontent.com/joerick/cibuildwheel/master/CI.md")
-            data = urlopen("https://www.nist.gov", context=context)
     ''')
 )
 

unit_test/download_test.py

@@ -0,0 +1,31 @@
+import certifi
+import pytest
+import ssl
+
+from cibuildwheel.util import download
+
+
+DOWNLOAD_URL = 'https://raw.githubusercontent.com/joerick/cibuildwheel/v1.6.3/requirements-dev.txt'
+
+
+def test_download(monkeypatch, tmp_path):
+    monkeypatch.delenv('SSL_CERT_FILE', raising=False)
+    dest = tmp_path / 'file.txt'
+    download(DOWNLOAD_URL, dest)
+    assert len(dest.read_bytes()) == 134
+
+
+def test_download_good_ssl_cert_file(monkeypatch, tmp_path):
+    monkeypatch.setenv('SSL_CERT_FILE', certifi.where())
+    dest = tmp_path / 'file.txt'
+    download(DOWNLOAD_URL, dest)
+    assert len(dest.read_bytes()) == 134
+
+
+def test_download_bad_ssl_cert_file(monkeypatch, tmp_path):
+    bad_cafile = tmp_path / 'ca.pem'
+    bad_cafile.write_text('bad certificates')
+    monkeypatch.setenv('SSL_CERT_FILE', str(bad_cafile))
+    dest = tmp_path / 'file.txt'
+    with pytest.raises(ssl.SSLError):
+        download(DOWNLOAD_URL, dest)