diff --git a/README.md b/README.md index fec41842..e1c96a3f 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ cibuildwheel Python wheels are great. Building them across **Mac, Linux, Windows**, on **multiple versions of Python**, is not. -`cibuildwheel` is here to help. `cibuildwheel` runs on your CI server - currently it supports Travis CI, Appveyor, and CircleCI - and it builds and tests your wheels across all of your platforms. +`cibuildwheel` is here to help. `cibuildwheel` runs on your CI server - currently it supports Azure Pipelines, Travis CI, Appveyor, and CircleCI - and it builds and tests your wheels across all of your platforms. **`cibuildwheel` is in beta**. It's brand new - I'd love for you to try it and help make it better! @@ -14,25 +14,95 @@ What does it do? | | macOS 10.6+ | manylinux i686 | manylinux x86_64 | Windows 32bit | Windows 64bit | |---|---|---|---|---|---| -| Python 2.7 | ✅ | ✅ | ✅ | ✅ | ✅ | -| Python 3.4 | ✅ | ✅ | ✅ | ✅ | ✅ | -| Python 3.5 | ✅ | ✅ | ✅ | ✅ | ✅ | -| Python 3.6 | ✅ | ✅ | ✅ | ✅ | ✅ | -| Python 3.7 | ✅ | ✅ | ✅ | ✅ | ✅ | +| Python 2.7 | ✅ | ✅ | ✅ | ✅ | ✅ | +| Python 3.4 | ✅ | ✅ | ✅ | ✅* | ✅* | +| Python 3.5 | ✅ | ✅ | ✅ | ✅ | ✅ | +| Python 3.6 | ✅ | ✅ | ✅ | ✅ | ✅ | +| Python 3.7 | ✅ | ✅ | ✅ | ✅ | ✅ | -- Builds manylinux, macOS and Windows (32 and 64bit) wheels using Travis CI, Appveyor, and CircleCI +> \* Not supported on Azure Pipelines + +- Builds manylinux, macOS and Windows (32 and 64bit) wheels using Azure Pipelines, Travis CI, Appveyor, and CircleCI - Bundles shared library dependencies on Linux and macOS through [auditwheel](https://github.com/pypa/auditwheel) and [delocate](https://github.com/matthew-brett/delocate) - Runs the library test suite against the wheel-installed version of your library Usage ----- -`cibuildwheel` currently works on **Travis CI** and **CircleCI** to build Linux and Mac wheels, and **Appveyor** to build Windows wheels. +`cibuildwheel` currently works **Travis CI** and **CircleCI** to build Linux and Mac wheels, and **Appveyor** to build Windows wheels. **Azure Pipelines** supports all three. + +| | Linux | macOS | Windows | +|-----------------|-------|-------|---------| +| Azure Pipelines | ✅ | ✅ | ✅ | +| Travis CI | ✅ | ✅ | | +| Appveyor | | | ✅ | +| CircleCI | ✅ | ✅ | | `cibuildwheel` is not intended to run on your development machine. It will try to install packages globally; this is no good. Travis CI, CircleCI, and Appveyor run their builds in isolated environments, so are ideal for this kind of script. ### Minimal setup +
+ Azure Pipelines + + + + + +- Using Azure pipelines, you can build all three platforms on the same service. Create a `azure-pipelines.yml` file in your repo. + +**azure-pipelines.yml** +```yaml +jobs: +- job: linux + pool: {vmImage: 'Ubuntu-16.04'} + steps: + - task: UsePythonVersion@0 + - bash: | + python -m pip install --upgrade pip + pip install cibuildwheel==0.10.1 + cibuildwheel --output-dir wheelhouse . + - task: PublishBuildArtifacts@1 + inputs: {pathtoPublish: 'wheelhouse'} +- job: macos + pool: {vmImage: 'macOS-10.13'} + steps: + - task: UsePythonVersion@0 + - bash: | + python -m pip install --upgrade pip + pip install cibuildwheel==0.10.1 + cibuildwheel --output-dir wheelhouse . + - task: PublishBuildArtifacts@1 + inputs: {pathtoPublish: 'wheelhouse'} +- job: windows + pool: {vmImage: 'vs2017-win2016'} + steps: + - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x86}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.5', architecture: x86}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.5', architecture: x64}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x86}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.7', architecture: x86}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.7', architecture: x64}} + - script: choco install vcpython27 -f -y + displayName: Install Visual C++ for Python 2.7 + - bash: | + python -m pip install --upgrade pip + pip install cibuildwheel==0.10.1 + cibuildwheel --output-dir wheelhouse . + - task: PublishBuildArtifacts@1 + inputs: {pathtoPublish: 'wheelhouse'} +``` + +
+ +
+ Travis CI + + + + - To build Linux and Mac wheels on Travis CI, create a `.travis.yml` file in your repo. ``` @@ -55,6 +125,14 @@ Usage Then setup a deployment method by following the [Travis CI deployment docs](https://docs.travis-ci.com/user/deployment/), or see [Delivering to PyPI](#delivering-to-pypi) below. +
+ +
+ CircleCI + + + + - To build Linux and Mac wheels on CircleCI, create a `.circleci/config.yml` file in your repo, ``` @@ -102,6 +180,14 @@ Usage CircleCI will store the built wheels for you - you can access them from the project console. +
+ + +
+ Appveyor + + + - To build Windows wheels on Appveyor, create an `appveyor.yml` file in your repo. ``` @@ -115,6 +201,8 @@ Usage Appveyor will store the built wheels for you - you can access them from the project console. Alternatively, you may want to store them in the same place as the Travis CI build. See [Appveyor deployment docs](https://www.appveyor.com/docs/deployment/) for more info, or see [Delivering to PyPI](#delivering-to-pypi) below. +
+ - Commit those files, enable building of your repo on Travis CI and Appveyor, and push. All being well, you should get wheels delivered to you in a few minutes. @@ -224,7 +312,7 @@ Examples: - Skip Python 3.6 on Linux: `CIBW_SKIP`:`cp36-manylinux*` - Only build on Python 3 and skip 32-bit builds: `CIBW_BUILD`:`cp3?-*` and `CIBW_SKIP`:`*-win32 *-manylinux1_i686` -** +*** | Environment variable: `CIBW_BUILD_VERBOSITY` | --- diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 00000000..5a9e501c --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,34 @@ +jobs: +- job: linux + pool: {vmImage: 'Ubuntu-16.04'} + steps: + - task: UsePythonVersion@0 + - bash: | + python -m pip install -r requirements-dev.txt + python ./bin/run_tests.py + +- job: macos + pool: {vmImage: 'macOS-10.13'} + steps: + - task: UsePythonVersion@0 + - bash: | + python -m pip install -r requirements-dev.txt + python ./bin/run_tests.py + +- job: windows + pool: {vmImage: 'vs2017-win2016'} + steps: + - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x86}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.5', architecture: x86}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.5', architecture: x64}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x86}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.7', architecture: x86}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.7', architecture: x64}} + - script: choco install vcpython27 -f -y + displayName: Install Visual C++ for Python 2.7 + - bash: | + python -m pip install -r requirements-dev.txt + python ./bin/run_tests.py + diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 8a926c60..d27c09dd 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -72,6 +72,13 @@ def main(): platform = 'linux' elif sys.platform.startswith('darwin'): platform = 'macos' + elif 'AZURE_HTTP_USER_AGENT' in os.environ: + if os.environ['AGENT_OS'] == 'Linux': + platform = 'linux' + elif os.environ['AGENT_OS'] == 'Darwin': + platform = 'macos' + elif os.environ['AGENT_OS'] == 'Windows_NT': + platform = 'windows' if platform is None: print('cibuildwheel: Unable to detect platform. cibuildwheel should run on your CI server, ' diff --git a/cibuildwheel/resources/appveyor_run_with_env.cmd b/cibuildwheel/resources/appveyor_run_with_env.cmd new file mode 100644 index 00000000..d549afe5 --- /dev/null +++ b/cibuildwheel/resources/appveyor_run_with_env.cmd @@ -0,0 +1,88 @@ +:: To build extensions for 64 bit Python 3, we need to configure environment +:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of: +:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1) +:: +:: To build extensions for 64 bit Python 2, we need to configure environment +:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of: +:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0) +:: +:: 32 bit builds, and 64-bit builds for 3.5 and beyond, do not require specific +:: environment configurations. +:: +:: Note: this script needs to be run with the /E:ON and /V:ON flags for the +:: cmd interpreter, at least for (SDK v7.0) +:: +:: More details at: +:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows +:: http://stackoverflow.com/a/13751649/163740 +:: +:: Author: Olivier Grisel +:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ +:: +:: Notes about batch files for Python people: +:: +:: Quotes in values are literally part of the values: +:: SET FOO="bar" +:: FOO is now five characters long: " b a r " +:: If you don't want quotes, don't include them on the right-hand side. +:: +:: The CALL lines at the end of this file look redundant, but if you move them +:: outside of the IF clauses, they do not run properly in the SET_SDK_64==Y +:: case, I don't know why. +@ECHO OFF + +SET COMMAND_TO_RUN=%* +SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows +SET WIN_WDK=c:\Program Files (x86)\Windows Kits\10\Include\wdf + +:: Extract the major and minor versions, and allow for the minor version to be +:: more than 9. This requires the version number to have two dots in it. +SET MAJOR_PYTHON_VERSION=%PYTHON_VERSION:~0,1% +IF "%PYTHON_VERSION:~3,1%" == "." ( + SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,1% +) ELSE ( + SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,2% +) + +:: Based on the Python version, determine what SDK version to use, and whether +:: to set the SDK for 64-bit. +IF %MAJOR_PYTHON_VERSION% == 2 ( + SET WINDOWS_SDK_VERSION="v7.0" + SET SET_SDK_64=Y +) ELSE ( + IF %MAJOR_PYTHON_VERSION% == 3 ( + SET WINDOWS_SDK_VERSION="v7.1" + IF %MINOR_PYTHON_VERSION% LEQ 4 ( + SET SET_SDK_64=Y + ) ELSE ( + SET SET_SDK_64=N + IF EXIST "%WIN_WDK%" ( + :: See: https://connect.microsoft.com/VisualStudio/feedback/details/1610302/ + REN "%WIN_WDK%" 0wdf + ) + ) + ) ELSE ( + ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%" + EXIT 1 + ) +) + +IF %PYTHON_ARCH% == 64 ( + IF %SET_SDK_64% == Y ( + ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture + SET DISTUTILS_USE_SDK=1 + SET MSSdk=1 + "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION% + "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 + ) ELSE ( + ECHO Using default MSVC build environment for 64 bit architecture + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 + ) +) ELSE ( + ECHO Using default MSVC build environment for 32 bit architecture + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 +) \ No newline at end of file diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 0748b278..eeb5ca39 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -1,15 +1,37 @@ from __future__ import print_function -import os, tempfile, subprocess, sys, shutil -try: - from urllib2 import urlopen -except ImportError: - from urllib.request import urlopen +import os, tempfile, subprocess, shutil from collections import namedtuple from glob import glob from .util import prepare_command, get_build_verbosity_extra_flags +IS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache') + +def get_python_path(config): + if IS_RUNNING_ON_AZURE: + # We can't hard-code the paths because on Azure, we don't know which + # bugfix release of Python we are getting so we need to check which + # ones exist. We just use the first one that is found since there should + # only be one. + path_pattern = 'C:\\hostedtoolcache\\windows\\Python\\{version}\\{arch}'.format( + version=config.version.replace('x', '*'), + arch='x86' if config.arch == '32' else 'x64' + ) + try: + return glob(path_pattern)[0] + except IndexError: + raise Exception('Could not find a Python install at ' + path_pattern) + else: + # Assume we're running on Appveyor + major, minor = config.version.split('.')[:2] + return 'C:\\Python{major}{minor}{arch}'.format( + major=major, + minor=minor, + arch = '-x64' if config.arch == '64' else '' + ) + + def get_python_configurations(build_selector): PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier', 'path']) python_configurations = [ @@ -25,34 +47,44 @@ def get_python_configurations(build_selector): PythonConfiguration(version='3.7.x', arch="64", identifier='cp37-win_amd64', path='C:\Python37-x64'), ] + if IS_RUNNING_ON_AZURE: + # Python 3.4 isn't supported on Azure. + # See https://github.com/Microsoft/azure-pipelines-tasks/issues/9674 + python_configurations = [c for c in python_configurations if c.version != '3.4.x'] + # skip builds as required return [c for c in python_configurations if build_selector(c.identifier)] def build(project_dir, output_dir, test_command, test_requires, before_build, build_verbosity, build_selector, environment): - # run_with_env is a cmd file that sets the right environment variables to - run_with_env = os.path.join(tempfile.gettempdir(), 'appveyor_run_with_env.cmd') - if not os.path.exists(run_with_env): - with open(run_with_env, 'wb') as f: - request = urlopen('https://github.com/ogrisel/python-appveyor-demo/raw/09a1c8672e5015a74d8f69d07add6ee803c176ec/appveyor/run_with_env.cmd') - f.write(request.read()) + if IS_RUNNING_ON_AZURE: + def shell(args, env=None, cwd=None): + print('+ ' + ' '.join(args)) + args = ['cmd', '/E:ON', '/V:ON', '/C'] + args + return subprocess.check_call(' '.join(args), env=env, cwd=cwd) + else: + run_with_env = os.path.abspath(os.path.join(os.path.dirname(__file__), 'resources', 'appveyor_run_with_env.cmd')) - def shell(args, env=None, cwd=None): - # print the command executing for the logs - print('+ ' + ' '.join(args)) - args = ['cmd', '/E:ON', '/V:ON', '/C', run_with_env] + args - return subprocess.check_call(' '.join(args), env=env, cwd=cwd) - - python_configurations = get_python_configurations(build_selector) + # run_with_env is a cmd file that sets the right environment variables + # to build on Appveyor. + def shell(args, env=None, cwd=None): + # print the command executing for the logs + print('+ ' + ' '.join(args)) + args = ['cmd', '/E:ON', '/V:ON', '/C', run_with_env] + args + return subprocess.check_call(' '.join(args), env=env, cwd=cwd) abs_project_dir = os.path.abspath(project_dir) temp_dir = tempfile.mkdtemp(prefix='cibuildwheel') built_wheel_dir = os.path.join(temp_dir, 'built_wheel') + python_configurations = get_python_configurations(build_selector) + for config in python_configurations: + config_python_path = get_python_path(config) + # check python & pip exist for this configuration - assert os.path.exists(os.path.join(config.path, 'python.exe')) - assert os.path.exists(os.path.join(config.path, 'Scripts', 'pip.exe')) + assert os.path.exists(os.path.join(config_python_path, 'python.exe')) + assert os.path.exists(os.path.join(config_python_path, 'Scripts', 'pip.exe')) # setup dirs if os.path.exists(built_wheel_dir): @@ -64,8 +96,8 @@ def build(project_dir, output_dir, test_command, test_requires, before_build, bu env['PYTHON_VERSION'] = config.version env['PYTHON_ARCH'] = config.arch env['PATH'] = os.pathsep.join([ - config.path, - os.path.join(config.path, 'Scripts'), + config_python_path, + os.path.join(config_python_path, 'Scripts'), env['PATH'] ]) env = environment.as_dictionary(prev_environment=env) diff --git a/test/shared/utils.py b/test/shared/utils.py index b6a720f5..0284dd4a 100644 --- a/test/shared/utils.py +++ b/test/shared/utils.py @@ -6,6 +6,8 @@ This file is added to the PYTHONPATH in the test runner at bin/run_test.py. import subprocess, sys, os +IS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache') + def cibuildwheel_get_build_identifiers(project_path, env=None): ''' @@ -83,6 +85,10 @@ def expected_wheels(package_name, package_version): else: raise Exception('unsupported platform') + if IS_RUNNING_ON_AZURE: + # Python 3.4 isn't supported on Azure. + templates = [t for t in templates if '-cp34-' not in t] + return [filename.format(package_name=package_name, package_version=package_version) for filename in templates]