Merge remote-tracking branch 'origin/master' into deterministic-builds

.travis.yml

@@ -1,50 +1,54 @@
-language: generic
+language: minimal
 
-matrix:
+jobs:
   include:
-    # Linux Python 3
+    - name: Linux | x86_64 + i686 | Python 3.5
-    - sudo: required
       language: python
       python: 3.5
       services: docker
       env: PYTHON=python
 
-    # Linux Python 3
+    - name: Linux | arm64 | Python 3.5
-    - sudo: required
       language: python
       python: 3.5
       services: docker
       arch: arm64
       env: PYTHON=python
 
-    # Linux Python 3
+    - name: Linux | ppc64le | Python 3.5
-    - sudo: required
       language: python
       python: 3.5
       services: docker
       arch: ppc64le
       env: PYTHON=python
 
-    # Linux Python 3
+    - name: macOS | x86_64 | Python 3.x
-    - sudo: required
+      os: osx
+      env: PYTHON=python3
+
+    - name: Windows | x86_64 | Python 3.5
+      os: windows
+      language: shell
+      before_install:
+       - choco install python3 --version 3.5.4 --no-progress -y
+      env:
+        - PYTHON=C:\\Python35\\python
+
+    - name: Linux | s390x | Python 3.5
       language: python
       python: 3.5
       services: docker
       arch: s390x
       env: PYTHON=python
 
-    # macOS Python 3
+  allow_failures:
-    - os: osx
+    # must repeat the s390x job above exactly to match
-      env: PYTHON=python3
+    - name: Linux | s390x | Python 3.5
-
+      language: python
-    - os: windows
+      python: 3.5
-      language: shell
+      services: docker
-      before_install:
+      arch: s390x
-       - choco install python3 --version 3.5.4 --no-progress -y
+      env: PYTHON=python
-      install:
-       - C:\\Python35\\python -m pip install -r requirements-dev.txt
-      script:
-       - C:\\Python35\\python ./bin/run_tests.py
 
 install: $PYTHON -m pip install -r requirements-dev.txt
 

README.md

@@ -34,13 +34,13 @@ What does it do?
 Usage
 -----
 
-`cibuildwheel` currently works on **Travis CI** and **Azure Pipelines** to build wheels for all three supported platforms (Linux, macOS, Windows). On **CircleCI** Linux and macOS wheels can be built, and on **AppVeyor** Linux and Windows are supported.
+`cibuildwheel` currently works on **Travis CI**, **Azure Pipelines** and **AppVeyor** to build wheels for all three supported platforms (Linux, macOS, Windows). On **CircleCI** Linux and macOS wheels can be built.
 
 |                 | Linux | macOS | Windows |
 |-----------------|-------|-------|---------|
 | Azure Pipelines | ✅    | ✅    | ✅      |
 | Travis CI       | ✅    | ✅    | ✅      |
-| AppVeyor        | ✅    |       | ✅      |
+| AppVeyor        | ✅    | ✅    | ✅      |
 | CircleCI        | ✅    | ✅    |         |
 
 `cibuildwheel` is not intended to run on your development machine. Because it uses system Python from Python.org it will try to install packages globally - not what you expect from a build tool! Instead, isolated CI services like Travis CI, CircleCI, Azure Pipelines and AppVeyor are ideal.

appveyor.yml

@@ -1,11 +1,21 @@
-image:
+environment:
-  - Ubuntu
+  matrix:
-  - Visual Studio 2015
+    - APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu
+      APPVEYOR_JOB_NAME: "python37-x64-ubuntu"
+    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
+      APPVEYOR_JOB_NAME: "python37-x64-vs2015"
+    - APPVEYOR_BUILD_WORKER_IMAGE: macos-mojave
+      APPVEYOR_JOB_NAME: "python37-x64-macos-mojave"
 
-build_script:
+stack: python 3.7
-  - cmd: "C:\\Python37\\python.exe -m pip install -r requirements-dev.txt"
+
-  - sh: "${HOME}/.localpython3.7.4/bin/python3 -m pip install -r requirements-dev.txt"
+build: off
-  # the '-u' flag is required so the output is in the correct order.
+
-  # See https://github.com/joerick/cibuildwheel/pull/24 for more info.
+init:
-  - cmd: "C:\\Python37\\python.exe -u ./bin/run_tests.py"
+- cmd: set PATH=C:\Python37;C:\Python37\Scripts;%PATH%
-  - sh: "${HOME}/.localpython3.7.4/bin/python3 ./bin/run_tests.py"
+
+install: python -m pip install -r requirements-dev.txt
+
+# the '-u' flag is required so the output is in the correct order.
+# See https://github.com/joerick/cibuildwheel/pull/24 for more info.
+test_script: python -u ./bin/run_tests.py

cibuildwheel/linux.py

@@ -166,6 +166,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
                             pip install {dependency_install_flags} virtualenv
                             venv_dir=`mktemp -d`/venv
                             python -m virtualenv --no-download "$venv_dir"
+                            export __CIBW_VIRTUALENV_PATH__=$venv_dir
 
                             # run the tests in a subshell to keep that `activate`
                             # script from polluting the env

cibuildwheel/macos.py

@@ -32,7 +32,7 @@ def get_python_configurations(build_selector):
         PythonConfiguration(version='3.5', identifier='cp35-macosx_x86_64', url='https://www.python.org/ftp/python/3.5.4/python-3.5.4-macosx10.6.pkg'),
         PythonConfiguration(version='3.6', identifier='cp36-macosx_x86_64', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg'),
         PythonConfiguration(version='3.7', identifier='cp37-macosx_x86_64', url='https://www.python.org/ftp/python/3.7.6/python-3.7.6-macosx10.9.pkg'),
-        PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.1/python-3.8.1-macosx10.9.pkg'),
+        PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.2/python-3.8.2-macosx10.9.pkg'),
         PythonConfiguration(version='2.7-v7.3.0', identifier='pp27-macosx_x86_64', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.0-osx64.tar.bz2'),
         PythonConfiguration(version='3.6-v7.3.0', identifier='pp36-macosx_x86_64', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.3.0-osx64.tar.bz2'),
     ]
@@ -214,6 +214,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
                 os.path.join(venv_dir, 'bin'),
                 virtualenv_env['PATH'],
             ])
+            virtualenv_env["__CIBW_VIRTUALENV_PATH__"] = venv_dir
 
             # check that we are using the Python from the virtual environment
             call(['which', 'python'], env=virtualenv_env)

cibuildwheel/resources/appveyor_run_with_env.cmd

@@ -1,88 +0,0 @@
-:: To build extensions for 64 bit Python 3, we need to configure environment
-:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of:
-:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1)
-::
-:: To build extensions for 64 bit Python 2, we need to configure environment
-:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of:
-:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0)
-::
-:: 32 bit builds, and 64-bit builds for 3.5 and beyond, do not require specific
-:: environment configurations.
-::
-:: Note: this script needs to be run with the /E:ON and /V:ON flags for the
-:: cmd interpreter, at least for (SDK v7.0)
-::
-:: More details at:
-:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows
-:: http://stackoverflow.com/a/13751649/163740
-::
-:: Author: Olivier Grisel
-:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/
-::
-:: Notes about batch files for Python people:
-::
-:: Quotes in values are literally part of the values:
-::      SET FOO="bar"
-:: FOO is now five characters long: " b a r "
-:: If you don't want quotes, don't include them on the right-hand side.
-::
-:: The CALL lines at the end of this file look redundant, but if you move them
-:: outside of the IF clauses, they do not run properly in the SET_SDK_64==Y
-:: case, I don't know why.
-@ECHO OFF
-
-SET COMMAND_TO_RUN=%*
-SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows
-SET WIN_WDK=c:\Program Files (x86)\Windows Kits\10\Include\wdf
-
-:: Extract the major and minor versions, and allow for the minor version to be
-:: more than 9.  This requires the version number to have two dots in it.
-SET MAJOR_PYTHON_VERSION=%PYTHON_VERSION:~0,1%
-IF "%PYTHON_VERSION:~3,1%" == "." (
-    SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,1%
-) ELSE (
-    SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,2%
-)
-
-:: Based on the Python version, determine what SDK version to use, and whether
-:: to set the SDK for 64-bit.
-IF %MAJOR_PYTHON_VERSION% == 2 (
-    SET WINDOWS_SDK_VERSION="v7.0"
-    SET SET_SDK_64=Y
-) ELSE (
-    IF %MAJOR_PYTHON_VERSION% == 3 (
-        SET WINDOWS_SDK_VERSION="v7.1"
-        IF %MINOR_PYTHON_VERSION% LEQ 4 (
-            SET SET_SDK_64=Y
-        ) ELSE (
-            SET SET_SDK_64=N
-            IF EXIST "%WIN_WDK%" (
-                :: See: https://connect.microsoft.com/VisualStudio/feedback/details/1610302/
-                REN "%WIN_WDK%" 0wdf
-            )
-        )
-    ) ELSE (
-        ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%"
-        EXIT 1
-    )
-)
-
-IF %PYTHON_ARCH% == 64 (
-    IF %SET_SDK_64% == Y (
-        ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture
-        SET DISTUTILS_USE_SDK=1
-        SET MSSdk=1
-        "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION%
-        "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release
-        ECHO Executing: %COMMAND_TO_RUN%
-        call %COMMAND_TO_RUN% || EXIT 1
-    ) ELSE (
-        ECHO Using default MSVC build environment for 64 bit architecture
-        ECHO Executing: %COMMAND_TO_RUN%
-        call %COMMAND_TO_RUN% || EXIT 1
-    )
-) ELSE (
-    ECHO Using default MSVC build environment for 32 bit architecture
-    ECHO Executing: %COMMAND_TO_RUN%
-    call %COMMAND_TO_RUN% || EXIT 1
-)

cibuildwheel/windows.py

@@ -19,26 +19,12 @@ IS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache')
 IS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows'
 
 
-def simple_shell(args, env=None, cwd=None):
+def shell(args, env=None, cwd=None):
     print('+ ' + ' '.join(args))
     args = ['cmd', '/E:ON', '/V:ON', '/C'] + args
     return subprocess.check_call(' '.join(args), env=env, cwd=cwd)
 
 
-if IS_RUNNING_ON_AZURE or IS_RUNNING_ON_TRAVIS:
-    shell = simple_shell
-else:
-    run_with_env = os.path.abspath(os.path.join(os.path.dirname(__file__), 'resources', 'appveyor_run_with_env.cmd'))
-
-    # run_with_env is a cmd file that sets the right environment variables
-    # to build on AppVeyor.
-    def shell(args, env=None, cwd=None):
-        # print the command executing for the logs
-        print('+ ' + ' '.join(args))
-        args = ['cmd', '/E:ON', '/V:ON', '/C', run_with_env] + args
-        return subprocess.check_call(' '.join(args), env=env, cwd=cwd)
-
-
 def get_nuget_args(version, arch):
     python_name = 'python' if version[0] == '3' else 'python2'
     if arch == '32':
@@ -57,8 +43,8 @@ def get_python_configurations(build_selector):
         PythonConfiguration(version='3.6.8', arch='64', identifier='cp36-win_amd64', url=None),
         PythonConfiguration(version='3.7.6', arch='32', identifier='cp37-win32', url=None),
         PythonConfiguration(version='3.7.6', arch='64', identifier='cp37-win_amd64', url=None),
-        PythonConfiguration(version='3.8.1', arch='32', identifier='cp38-win32', url=None),
+        PythonConfiguration(version='3.8.2', arch='32', identifier='cp38-win32', url=None),
-        PythonConfiguration(version='3.8.1', arch='64', identifier='cp38-win_amd64', url=None),
+        PythonConfiguration(version='3.8.2', arch='64', identifier='cp38-win_amd64', url=None),
         PythonConfiguration(version='2.7-v7.3.0', arch='32', identifier='pp27-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.0-win32.zip'),
         PythonConfiguration(version='3.6-v7.3.0', arch='32', identifier='pp36-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.3.0-win32.zip'),
     ]
@@ -82,7 +68,7 @@ def extract_zip(zip_src, dest):
 def install_cpython(version, arch, nuget):
     nuget_args = get_nuget_args(version, arch)
     installation_path = os.path.join(nuget_args[-1], nuget_args[0] + '.' + version, 'tools')
-    simple_shell([nuget, 'install'] + nuget_args)
+    shell([nuget, 'install'] + nuget_args)
     return installation_path
 
 
@@ -97,8 +83,8 @@ def install_pypy(version, arch, url):
         # Extract to the parent directory because the zip file still contains a directory
         extract_zip(pypy_zip, os.path.dirname(installation_path))
         pypy_exe = 'pypy3.exe' if version[0] == '3' else 'pypy.exe'
-        simple_shell(['mklink', os.path.join(installation_path, 'python.exe'), os.path.join(installation_path, pypy_exe)])
+        shell(['mklink', os.path.join(installation_path, 'python.exe'), os.path.join(installation_path, pypy_exe)])
-        simple_shell(['mklink', '/d', os.path.join(installation_path, 'Scripts'), os.path.join(installation_path, 'bin')])
+        shell(['mklink', '/d', os.path.join(installation_path, 'Scripts'), os.path.join(installation_path, 'bin')])
     return installation_path
 
 
@@ -137,9 +123,9 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
         env = environment.as_dictionary(prev_environment=env)
 
         # for the logs - check we're running the right version of python
-        simple_shell(['where', 'python'], env=env)
+        shell(['where', 'python'], env=env)
-        simple_shell(['python', '--version'], env=env)
+        shell(['python', '--version'], env=env)
-        simple_shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)"'], env=env)
+        shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)"'], env=env)
         where_python = subprocess.check_output(['where', 'python'], env=env, universal_newlines=True).splitlines()[0].strip()
         if where_python != os.path.join(installation_path, 'python.exe'):
             print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr)
@@ -153,7 +139,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
 
         # make sure pip is installed
         if not os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe')):
-            simple_shell(['python', get_pip_script], env=env, cwd="C:\\cibw")
+            shell(['python', get_pip_script], env=env, cwd="C:\\cibw")
         assert os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe'))
         where_pip = subprocess.check_output(['where', 'pip'], env=env, universal_newlines=True).splitlines()[0].strip()
         if where_pip.strip() != os.path.join(installation_path, 'Scripts', 'pip.exe'):
@@ -161,9 +147,9 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
             exit(1)
 
         # prepare the Python environment
-        simple_shell(['python', '-m', 'pip', 'install', '--upgrade', 'pip'] + dependency_constraint_flags, env=env)
+        shell(['python', '-m', 'pip', 'install', '--upgrade', 'pip'] + dependency_constraint_flags, env=env)
-        simple_shell(['pip', '--version'], env=env)
+        shell(['pip', '--version'], env=env)
-        simple_shell(['pip', 'install', '--upgrade', 'setuptools', 'wheel'] + dependency_constraint_flags, env=env)
+        shell(['pip', 'install', '--upgrade', 'setuptools', 'wheel'] + dependency_constraint_flags, env=env)
 
         # run the before_build command
         if before_build:
@@ -200,10 +186,16 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
             shell(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env)
 
             virtualenv_env = env.copy()
+
+            venv_script_path = os.path.join(venv_dir, 'Scripts')
+            if os.path.exists(os.path.join(venv_dir, 'bin')):
+                # pypy2.7 bugfix
+                venv_script_path = os.pathsep.join([venv_script_path, os.path.join(venv_dir, 'bin')])
             virtualenv_env['PATH'] = os.pathsep.join([
-                os.path.join(venv_dir, 'Scripts'),
+                venv_script_path,
                 virtualenv_env['PATH'],
             ])
+            virtualenv_env["__CIBW_VIRTUALENV_PATH__"] = venv_dir
 
             # check that we are using the Python from the virtual environment
             shell(['which', 'python'], env=virtualenv_env)

docs/setup.md

@@ -49,9 +49,9 @@ Commit this file, enable building of your repo on CircleCI, and push.
 
 CircleCI will store the built wheels for you - you can access them from the project console. Check out the CircleCI [docs](https://circleci.com/docs/2.0/configuration-reference/#section=configuration) for more info on this config file.
 
-# AppVeyor [linux/windows] {: #appveyor}
+# AppVeyor [linux/mac/windows] {: #appveyor}
 
-To build Linux and Windows wheels on AppVeyor, create an `appveyor.yml` file in your repo.
+To build Linux, Mac, and Windows wheels on AppVeyor, create an `appveyor.yml` file in your repo.
 
 > appveyor.yml
 

examples/appveyor-minimal.yml

@@ -1,14 +1,20 @@
-image:
+environment:
-  - Ubuntu
+  matrix:
-  - Visual Studio 2015
+    - APPVEYOR_BUILD_WORKER_IMAGE: Ubuntu
+      APPVEYOR_JOB_NAME: "python37-x64-ubuntu"
+    - APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2015
+      APPVEYOR_JOB_NAME: "python37-x64-vs2015"
+    - APPVEYOR_BUILD_WORKER_IMAGE: macos-mojave
+      APPVEYOR_JOB_NAME: "python37-x64-macos-mojave"
 
-build_script:
+stack: python 3.7
-  # windows
+
-  - cmd: pip3 install cibuildwheel==1.1.0
+init:
-  - cmd: cibuildwheel --output-dir wheelhouse
+- cmd: set PATH=C:\Python37;C:\Python37\Scripts;%PATH%
-  # linux
+
-  - sh: "${HOME}/.localpython3.7.4/bin/python3 -m pip install cibuildwheel==1.1.0"
+install: python -m pip install cibuildwheel==1.1.0
-  - sh: "${HOME}/.localpython3.7.4/bin/python3 -m cibuildwheel --output-dir wheelhouse"
+
+build_script: python -m cibuildwheel --output-dir wheelhouse
 
 artifacts:
   - path: "wheelhouse\\*.whl"

examples/azure-pipelines-minimal.yml

@@ -28,8 +28,8 @@ jobs:
     - script: choco install vcpython27 -f -y
       displayName: Install Visual C++ for Python 2.7
     - bash: |
-        python3 -m pip install --upgrade pip
+        python -m pip install --upgrade pip
-        pip3 install cibuildwheel==1.1.0
+        pip install cibuildwheel==1.1.0
         cibuildwheel --output-dir wheelhouse .
     - task: PublishBuildArtifacts@1
       inputs: {pathtoPublish: 'wheelhouse'}

examples/travis-ci-deploy-only.yml

@@ -16,6 +16,8 @@ jobs:
       before_install:
         - choco install python --version 3.8.0
         - export PATH="/c/Python38:/c/Python38/Scripts:$PATH"
+        # make sure it's on PATH as 'python3'
+        - ln -s /c/Python38/python.exe /c/Python38/python3.exe
 
 env:
   global:

examples/travis-ci-minimal.yml

@@ -22,6 +22,8 @@ jobs:
       before_install:
         - choco install python --version 3.8.0
         - export PATH="/c/Python38:/c/Python38/Scripts:$PATH"
+        # make sure it's on PATH as 'python3'
+        - ln -s /c/Python38/python.exe /c/Python38/python3.exe
 
 install:
   - python3 -m pip install cibuildwheel==1.1.0

examples/travis-ci-test-and-deploy.yml

@@ -17,6 +17,8 @@ before_install:
     if [[ "$TRAVIS_OS_NAME" = windows ]]; then
         choco install python --version 3.8.0
         export PATH="/c/Python38:/c/Python38/Scripts:$PATH"
+        # make sure it's on PATH as 'python3'
+        ln -s /c/Python38/python.exe /c/Python38/python3.exe
     fi
 
 install:

test/02_test/test/spam_test.py

@@ -1,9 +1,45 @@
+from __future__ import print_function
+import os
+import sys
 from unittest import TestCase
 
 import spam
 
 
+def path_contains(parent, child):
+    ''' returns True if `child` is inside `parent`.
+
+    Works around path-comparison bugs caused by short-paths on Windows e.g.
+    vssadm~1 instead of vssadministrator
+    '''
+    parent = os.path.abspath(parent)
+    child = os.path.abspath(child)
+
+    while child != os.path.dirname(child):
+        child = os.path.dirname(child)
+        if os.stat(parent) == os.stat(child):
+            # parent and child refer to the same directory on the filesystem
+            return True
+    return False
+
+
 class TestSpam(TestCase):
     def test_system(self):
         self.assertEqual(0, spam.system('python -c "exit(0)"'))
         self.assertNotEqual(0, spam.system('python -c "exit(1)"'))
+
+    def test_virtualenv(self):
+        virtualenv_path = os.environ.get("__CIBW_VIRTUALENV_PATH__")
+        if not virtualenv_path:
+            self.fail("No virtualenv path defined in environment variable __CIBW_VIRTUALENV_PATH__")
+
+        print("=[executable]", sys.executable)
+        print("=[spam location]", spam.__file__)
+        print("=[virtualenv path]", virtualenv_path)
+        print("=[listdir]", os.listdir(virtualenv_path))
+        if os.path.exists(os.path.join(virtualenv_path, 'Scripts')):
+            print("=[listdir]2", os.listdir(os.path.join(virtualenv_path, 'Scripts')))
+        if os.path.exists(os.path.join(virtualenv_path, 'bin')):
+            print("=[listdir]2", os.listdir(os.path.join(virtualenv_path, 'bin')))
+        self.assertTrue(path_contains(virtualenv_path, sys.executable))
+        self.assertTrue(path_contains(virtualenv_path, spam.__file__))