From ac76af4f755cda808e2bfd6e0f35a9dc3209d383 Mon Sep 17 00:00:00 2001 From: Thomas Robitaille Date: Fri, 22 Feb 2019 21:03:04 +0000 Subject: [PATCH 01/15] Support Python location on Windows in Azure Pipelines --- cibuildwheel/windows.py | 50 ++++++++++++++++++++++++++++++++--------- 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index f5a4ffb8..a9b4862f 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -25,17 +25,45 @@ def build(project_dir, output_dir, test_command, test_requires, before_build, bu return subprocess.check_call(' '.join(args), env=env, cwd=cwd) PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier', 'path']) + + # At this point, we need to check if we are running on Azure, because if + # so Python is not located in the usual place. We recognize Azure by + # checking for a C:\hostedtoolcache directory - there aren't any nice + # environment variables we can use as on some other CI frameworks. + + if os.path.exists('C:\hostedtoolcache'): + + # We can't hard-code the paths because on Azure, we don't know which + # bugfix release of Python we are getting so we need to check which + # ones exist. We just use the first one that is found since there should + # only be one. + + def python_path(version, arch): + major, minor = version.split('.')[:2] + suffix = 'x86' if arch == '32' else 'x64' + path = glob("C:\\hostedtoolcache\\windows\\Python\\" + version.replace('x', '*') + "\\" + suffix)[0] + return path + + else: + + def python_path(version, arch): + major, minor = version.split('.')[:2] + path = 'C:\\Python' + major + minor + if arch == '64': + path += '-x64' + return path + python_configurations = [ - PythonConfiguration(version='2.7.x', arch="32", identifier='cp27-win32', path='C:\Python27'), - PythonConfiguration(version='2.7.x', arch="64", identifier='cp27-win_amd64', path='C:\Python27-x64'), - PythonConfiguration(version='3.4.x', arch="32", identifier='cp34-win32', path='C:\Python34'), - PythonConfiguration(version='3.4.x', arch="64", identifier='cp34-win_amd64', path='C:\Python34-x64'), - PythonConfiguration(version='3.5.x', arch="32", identifier='cp35-win32', path='C:\Python35'), - PythonConfiguration(version='3.5.x', arch="64", identifier='cp35-win_amd64', path='C:\Python35-x64'), - PythonConfiguration(version='3.6.x', arch="32", identifier='cp36-win32', path='C:\Python36'), - PythonConfiguration(version='3.6.x', arch="64", identifier='cp36-win_amd64', path='C:\Python36-x64'), - PythonConfiguration(version='3.7.x', arch="32", identifier='cp37-win32', path='C:\Python37'), - PythonConfiguration(version='3.7.x', arch="64", identifier='cp37-win_amd64', path='C:\Python37-x64'), + PythonConfiguration(version='2.7.x', arch="32", identifier='cp27-win32', path=python_path('2.7.x', '32')), + PythonConfiguration(version='2.7.x', arch="64", identifier='cp27-win_amd64', path=python_path('2.7.x', '64')), + PythonConfiguration(version='3.4.x', arch="32", identifier='cp34-win32', path=python_path('3.4.x', '32')), + PythonConfiguration(version='3.4.x', arch="64", identifier='cp34-win_amd64', path=python_path('3.4.x', '64')), + PythonConfiguration(version='3.5.x', arch="32", identifier='cp35-win32', path=python_path('3.5.x', '32')), + PythonConfiguration(version='3.5.x', arch="64", identifier='cp35-win_amd64', path=python_path('3.5.x', '64')), + PythonConfiguration(version='3.6.x', arch="32", identifier='cp36-win32', path=python_path('3.6.x', '32')), + PythonConfiguration(version='3.6.x', arch="64", identifier='cp36-win_amd64', path=python_path('3.6.x', '64')), + PythonConfiguration(version='3.7.x', arch="32", identifier='cp37-win32', path=python_path('3.7.x', '32')), + PythonConfiguration(version='3.7.x', arch="64", identifier='cp37-win_amd64', path=python_path('3.7.x', '64')), ] abs_project_dir = os.path.abspath(project_dir) @@ -46,7 +74,7 @@ def build(project_dir, output_dir, test_command, test_requires, before_build, bu if not build_selector(config.identifier): print('cibuildwheel: Skipping build %s' % config.identifier, file=sys.stderr) continue - + # check python & pip exist for this configuration assert os.path.exists(os.path.join(config.path, 'python.exe')) assert os.path.exists(os.path.join(config.path, 'Scripts', 'pip.exe')) From dcce949d3c724a19d9ad502a3f1bd1dd2764ebb1 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 10 Mar 2019 22:03:31 +0000 Subject: [PATCH 02/15] Vendor appveyor_run_with_cmd, but don't use it on azure pipelines --- .../resources/appveyor_run_with_env.cmd | 88 +++++++++++++++++++ cibuildwheel/windows.py | 44 +++++----- 2 files changed, 111 insertions(+), 21 deletions(-) create mode 100644 cibuildwheel/resources/appveyor_run_with_env.cmd diff --git a/cibuildwheel/resources/appveyor_run_with_env.cmd b/cibuildwheel/resources/appveyor_run_with_env.cmd new file mode 100644 index 00000000..d549afe5 --- /dev/null +++ b/cibuildwheel/resources/appveyor_run_with_env.cmd @@ -0,0 +1,88 @@ +:: To build extensions for 64 bit Python 3, we need to configure environment +:: variables to use the MSVC 2010 C++ compilers from GRMSDKX_EN_DVD.iso of: +:: MS Windows SDK for Windows 7 and .NET Framework 4 (SDK v7.1) +:: +:: To build extensions for 64 bit Python 2, we need to configure environment +:: variables to use the MSVC 2008 C++ compilers from GRMSDKX_EN_DVD.iso of: +:: MS Windows SDK for Windows 7 and .NET Framework 3.5 (SDK v7.0) +:: +:: 32 bit builds, and 64-bit builds for 3.5 and beyond, do not require specific +:: environment configurations. +:: +:: Note: this script needs to be run with the /E:ON and /V:ON flags for the +:: cmd interpreter, at least for (SDK v7.0) +:: +:: More details at: +:: https://github.com/cython/cython/wiki/64BitCythonExtensionsOnWindows +:: http://stackoverflow.com/a/13751649/163740 +:: +:: Author: Olivier Grisel +:: License: CC0 1.0 Universal: http://creativecommons.org/publicdomain/zero/1.0/ +:: +:: Notes about batch files for Python people: +:: +:: Quotes in values are literally part of the values: +:: SET FOO="bar" +:: FOO is now five characters long: " b a r " +:: If you don't want quotes, don't include them on the right-hand side. +:: +:: The CALL lines at the end of this file look redundant, but if you move them +:: outside of the IF clauses, they do not run properly in the SET_SDK_64==Y +:: case, I don't know why. +@ECHO OFF + +SET COMMAND_TO_RUN=%* +SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows +SET WIN_WDK=c:\Program Files (x86)\Windows Kits\10\Include\wdf + +:: Extract the major and minor versions, and allow for the minor version to be +:: more than 9. This requires the version number to have two dots in it. +SET MAJOR_PYTHON_VERSION=%PYTHON_VERSION:~0,1% +IF "%PYTHON_VERSION:~3,1%" == "." ( + SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,1% +) ELSE ( + SET MINOR_PYTHON_VERSION=%PYTHON_VERSION:~2,2% +) + +:: Based on the Python version, determine what SDK version to use, and whether +:: to set the SDK for 64-bit. +IF %MAJOR_PYTHON_VERSION% == 2 ( + SET WINDOWS_SDK_VERSION="v7.0" + SET SET_SDK_64=Y +) ELSE ( + IF %MAJOR_PYTHON_VERSION% == 3 ( + SET WINDOWS_SDK_VERSION="v7.1" + IF %MINOR_PYTHON_VERSION% LEQ 4 ( + SET SET_SDK_64=Y + ) ELSE ( + SET SET_SDK_64=N + IF EXIST "%WIN_WDK%" ( + :: See: https://connect.microsoft.com/VisualStudio/feedback/details/1610302/ + REN "%WIN_WDK%" 0wdf + ) + ) + ) ELSE ( + ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%" + EXIT 1 + ) +) + +IF %PYTHON_ARCH% == 64 ( + IF %SET_SDK_64% == Y ( + ECHO Configuring Windows SDK %WINDOWS_SDK_VERSION% for Python %MAJOR_PYTHON_VERSION% on a 64 bit architecture + SET DISTUTILS_USE_SDK=1 + SET MSSdk=1 + "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Setup\WindowsSdkVer.exe" -q -version:%WINDOWS_SDK_VERSION% + "%WIN_SDK_ROOT%\%WINDOWS_SDK_VERSION%\Bin\SetEnv.cmd" /x64 /release + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 + ) ELSE ( + ECHO Using default MSVC build environment for 64 bit architecture + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 + ) +) ELSE ( + ECHO Using default MSVC build environment for 32 bit architecture + ECHO Executing: %COMMAND_TO_RUN% + call %COMMAND_TO_RUN% || EXIT 1 +) \ No newline at end of file diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index a9b4862f..efc499df 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -11,27 +11,7 @@ from .util import prepare_command, get_build_verbosity_extra_flags def build(project_dir, output_dir, test_command, test_requires, before_build, build_verbosity, build_selector, environment): - # run_with_env is a cmd file that sets the right environment variables to - run_with_env = os.path.join(tempfile.gettempdir(), 'appveyor_run_with_env.cmd') - if not os.path.exists(run_with_env): - with open(run_with_env, 'wb') as f: - request = urlopen('https://github.com/ogrisel/python-appveyor-demo/raw/09a1c8672e5015a74d8f69d07add6ee803c176ec/appveyor/run_with_env.cmd') - f.write(request.read()) - - def shell(args, env=None, cwd=None): - # print the command executing for the logs - print('+ ' + ' '.join(args)) - args = ['cmd', '/E:ON', '/V:ON', '/C', run_with_env] + args - return subprocess.check_call(' '.join(args), env=env, cwd=cwd) - - PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier', 'path']) - - # At this point, we need to check if we are running on Azure, because if - # so Python is not located in the usual place. We recognize Azure by - # checking for a C:\hostedtoolcache directory - there aren't any nice - # environment variables we can use as on some other CI frameworks. - - if os.path.exists('C:\hostedtoolcache'): + if os.path.exists('C:\\hostedtoolcache'): # We can't hard-code the paths because on Azure, we don't know which # bugfix release of Python we are getting so we need to check which @@ -44,6 +24,10 @@ def build(project_dir, output_dir, test_command, test_requires, before_build, bu path = glob("C:\\hostedtoolcache\\windows\\Python\\" + version.replace('x', '*') + "\\" + suffix)[0] return path + def shell(args, env=None, cwd=None): + print('+ ' + ' '.join(args)) + args = ['cmd', '/E:ON', '/V:ON', '/C'] + args + return subprocess.check_call(' '.join(args), env=env, cwd=cwd) else: def python_path(version, arch): @@ -53,6 +37,24 @@ def build(project_dir, output_dir, test_command, test_requires, before_build, bu path += '-x64' return path + run_with_env = os.path.join(os.path.dirname(__file__), 'resources', 'appveyor_run_with_env.cmd') + + # run_with_env is a cmd file that sets the right environment variables to + + def shell(args, env=None, cwd=None): + # print the command executing for the logs + print('+ ' + ' '.join(args)) + args = ['cmd', '/E:ON', '/V:ON', '/C', run_with_env] + args + return subprocess.check_call(' '.join(args), env=env, cwd=cwd) + + PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier', 'path']) + + # At this point, we need to check if we are running on Azure, because if + # so Python is not located in the usual place. We recognize Azure by + # checking for a C:\hostedtoolcache directory - there aren't any nice + # environment variables we can use as on some other CI frameworks. + + python_configurations = [ PythonConfiguration(version='2.7.x', arch="32", identifier='cp27-win32', path=python_path('2.7.x', '32')), PythonConfiguration(version='2.7.x', arch="64", identifier='cp27-win_amd64', path=python_path('2.7.x', '64')), From d46fb62e954177405f8b1b880b8474e173a15f20 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 10 Mar 2019 22:41:55 +0000 Subject: [PATCH 03/15] Refactor for code style --- cibuildwheel/windows.py | 79 +++++++++++++++++++++-------------------- 1 file changed, 40 insertions(+), 39 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index efc499df..c564fe6e 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -10,62 +10,61 @@ from glob import glob from .util import prepare_command, get_build_verbosity_extra_flags -def build(project_dir, output_dir, test_command, test_requires, before_build, build_verbosity, build_selector, environment): - if os.path.exists('C:\\hostedtoolcache'): +IS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache') +def get_python_path(config): + if IS_RUNNING_ON_AZURE: # We can't hard-code the paths because on Azure, we don't know which # bugfix release of Python we are getting so we need to check which # ones exist. We just use the first one that is found since there should # only be one. + path_pattern = 'C:\\hostedtoolcache\\windows\\Python\\{version}\\{arch}'.format( + version=config.version.replace('x', '*'), + arch='x86' if config.arch == '32' else 'x64' + ) + try: + return glob(path_pattern)[0] + except IndexError: + raise Exception('Could not find a Python install at ' + path_pattern) + else: + major, minor = config.version.split('.')[:2] + return 'C:\\Python{major}{minor}{arch}'.format( + major=major, + minor=minor, + arch = '-x64' if config.arch == '64' else '' + ) - def python_path(version, arch): - major, minor = version.split('.')[:2] - suffix = 'x86' if arch == '32' else 'x64' - path = glob("C:\\hostedtoolcache\\windows\\Python\\" + version.replace('x', '*') + "\\" + suffix)[0] - return path +def build(project_dir, output_dir, test_command, test_requires, before_build, build_verbosity, build_selector, environment): + if IS_RUNNING_ON_AZURE: def shell(args, env=None, cwd=None): print('+ ' + ' '.join(args)) args = ['cmd', '/E:ON', '/V:ON', '/C'] + args return subprocess.check_call(' '.join(args), env=env, cwd=cwd) else: + run_with_env = os.path.abspath(os.path.join(os.path.dirname(__file__), 'resources', 'appveyor_run_with_env.cmd')) - def python_path(version, arch): - major, minor = version.split('.')[:2] - path = 'C:\\Python' + major + minor - if arch == '64': - path += '-x64' - return path - - run_with_env = os.path.join(os.path.dirname(__file__), 'resources', 'appveyor_run_with_env.cmd') - - # run_with_env is a cmd file that sets the right environment variables to - + # run_with_env is a cmd file that sets the right environment variables + # to build on Appveyor. def shell(args, env=None, cwd=None): # print the command executing for the logs print('+ ' + ' '.join(args)) args = ['cmd', '/E:ON', '/V:ON', '/C', run_with_env] + args return subprocess.check_call(' '.join(args), env=env, cwd=cwd) - PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier', 'path']) - - # At this point, we need to check if we are running on Azure, because if - # so Python is not located in the usual place. We recognize Azure by - # checking for a C:\hostedtoolcache directory - there aren't any nice - # environment variables we can use as on some other CI frameworks. - + PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier']) python_configurations = [ - PythonConfiguration(version='2.7.x', arch="32", identifier='cp27-win32', path=python_path('2.7.x', '32')), - PythonConfiguration(version='2.7.x', arch="64", identifier='cp27-win_amd64', path=python_path('2.7.x', '64')), - PythonConfiguration(version='3.4.x', arch="32", identifier='cp34-win32', path=python_path('3.4.x', '32')), - PythonConfiguration(version='3.4.x', arch="64", identifier='cp34-win_amd64', path=python_path('3.4.x', '64')), - PythonConfiguration(version='3.5.x', arch="32", identifier='cp35-win32', path=python_path('3.5.x', '32')), - PythonConfiguration(version='3.5.x', arch="64", identifier='cp35-win_amd64', path=python_path('3.5.x', '64')), - PythonConfiguration(version='3.6.x', arch="32", identifier='cp36-win32', path=python_path('3.6.x', '32')), - PythonConfiguration(version='3.6.x', arch="64", identifier='cp36-win_amd64', path=python_path('3.6.x', '64')), - PythonConfiguration(version='3.7.x', arch="32", identifier='cp37-win32', path=python_path('3.7.x', '32')), - PythonConfiguration(version='3.7.x', arch="64", identifier='cp37-win_amd64', path=python_path('3.7.x', '64')), + PythonConfiguration(version='2.7.x', arch="32", identifier='cp27-win32'), + PythonConfiguration(version='2.7.x', arch="64", identifier='cp27-win_amd64'), + PythonConfiguration(version='3.4.x', arch="32", identifier='cp34-win32'), + PythonConfiguration(version='3.4.x', arch="64", identifier='cp34-win_amd64'), + PythonConfiguration(version='3.5.x', arch="32", identifier='cp35-win32'), + PythonConfiguration(version='3.5.x', arch="64", identifier='cp35-win_amd64'), + PythonConfiguration(version='3.6.x', arch="32", identifier='cp36-win32'), + PythonConfiguration(version='3.6.x', arch="64", identifier='cp36-win_amd64'), + PythonConfiguration(version='3.7.x', arch="32", identifier='cp37-win32'), + PythonConfiguration(version='3.7.x', arch="64", identifier='cp37-win_amd64'), ] abs_project_dir = os.path.abspath(project_dir) @@ -76,10 +75,12 @@ def build(project_dir, output_dir, test_command, test_requires, before_build, bu if not build_selector(config.identifier): print('cibuildwheel: Skipping build %s' % config.identifier, file=sys.stderr) continue + + config_python_path = get_python_path(config) # check python & pip exist for this configuration - assert os.path.exists(os.path.join(config.path, 'python.exe')) - assert os.path.exists(os.path.join(config.path, 'Scripts', 'pip.exe')) + assert os.path.exists(os.path.join(config_python_path, 'python.exe')) + assert os.path.exists(os.path.join(config_python_path, 'Scripts', 'pip.exe')) # setup dirs if os.path.exists(built_wheel_dir): @@ -91,8 +92,8 @@ def build(project_dir, output_dir, test_command, test_requires, before_build, bu env['PYTHON_VERSION'] = config.version env['PYTHON_ARCH'] = config.arch env['PATH'] = os.pathsep.join([ - config.path, - os.path.join(config.path, 'Scripts'), + config_python_path, + os.path.join(config_python_path, 'Scripts'), env['PATH'] ]) env = environment.as_dictionary(prev_environment=env) From aefde9535b743a764847381b56c51130baba83d0 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 10 Mar 2019 22:50:07 +0000 Subject: [PATCH 04/15] Add azure-pipelines test run config --- azure-pipelines.yml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 azure-pipelines.yml diff --git a/azure-pipelines.yml b/azure-pipelines.yml new file mode 100644 index 00000000..ca4f038e --- /dev/null +++ b/azure-pipelines.yml @@ -0,0 +1,36 @@ +jobs: +- job: linux + pool: {vmImage: 'Ubuntu-16.04'} + steps: + - task: UsePythonVersion@0 + - bash: | + python -m pip install -r requirements-dev.txt + python ./bin/run_tests.py + +- job: macos + pool: {vmImage: 'macOS-10.13'} + steps: + - task: UsePythonVersion@0 + - bash: | + python -m pip install -r requirements-dev.txt + python ./bin/run_tests.py + +- job: windows + pool: {vmImage: 'vs2017-win2016'} + steps: + - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x86}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.4', architecture: x86}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.4', architecture: x64}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.5', architecture: x86}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.5', architecture: x64}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x86}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.7', architecture: x86}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.7', architecture: x64}} + - script: choco install vcpython27 -f -y + displayName: Install Visual C++ for Python 2.7 + - bash: | + python -m pip install -r requirements-dev.txt + python ./bin/run_tests.py + From 1bfe6d45bd6e6cd849ff411b67a86a58a76cd78c Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 10 Mar 2019 23:07:14 +0000 Subject: [PATCH 05/15] Add auto-detect platform for Azure --- cibuildwheel/__main__.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 6d0b1c94..f5b3341b 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -68,6 +68,13 @@ def main(): platform = 'linux' elif sys.platform.startswith('darwin'): platform = 'macos' + elif 'AZURE_HTTP_USER_AGENT' in os.environ: + if os.environ['AGENT_OS'] == 'Linux': + platform = 'linux' + elif os.environ['AGENT_OS'] == 'Darwin': + platform = 'macos' + elif os.environ['AGENT_OD'] == 'Windows_NT': + platform = 'windows' if platform is None: print('cibuildwheel: Unable to detect platform. cibuildwheel should run on your CI server, ' From 652d094e46ccea4e34f691e04202a9798739adcd Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 10 Mar 2019 23:19:22 +0000 Subject: [PATCH 06/15] Fix typo --- cibuildwheel/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index f5b3341b..38008cbf 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -73,7 +73,7 @@ def main(): platform = 'linux' elif os.environ['AGENT_OS'] == 'Darwin': platform = 'macos' - elif os.environ['AGENT_OD'] == 'Windows_NT': + elif os.environ['AGENT_OS'] == 'Windows_NT': platform = 'windows' if platform is None: From a8f6684bfd6b9ea383f2af2048729d1021677438 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 11 Mar 2019 19:59:31 +0000 Subject: [PATCH 07/15] Skip Python 3.4 builds on Azure --- cibuildwheel/windows.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index c564fe6e..8332a3f8 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -27,6 +27,7 @@ def get_python_path(config): except IndexError: raise Exception('Could not find a Python install at ' + path_pattern) else: + # Assume we're running on Appveyor major, minor = config.version.split('.')[:2] return 'C:\\Python{major}{minor}{arch}'.format( major=major, @@ -67,6 +68,11 @@ def build(project_dir, output_dir, test_command, test_requires, before_build, bu PythonConfiguration(version='3.7.x', arch="64", identifier='cp37-win_amd64'), ] + if IS_RUNNING_ON_AZURE: + # Python 3.4 isn't supported on Azure. + # See https://github.com/Microsoft/azure-pipelines-tasks/issues/9674 + python_configurations = [c for c in python_configurations if c.version != '3.4.x'] + abs_project_dir = os.path.abspath(project_dir) temp_dir = tempfile.mkdtemp(prefix='cibuildwheel') built_wheel_dir = os.path.join(temp_dir, 'built_wheel') From 9116217b607f20e3c1fe30fa16f09ddfb52ae04f Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 11 Mar 2019 22:37:59 +0000 Subject: [PATCH 08/15] Update README.md Readme improvements, adding Azure config and putting services behind disclosure triangles --- README.md | 81 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) diff --git a/README.md b/README.md index f4445d13..ac055833 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,69 @@ Usage ### Minimal setup +
+ Azure Pipelines + + + + + +- Using Azure pipelines, you can build all three platforms on the same service. Create a `azure-pipelines.yml` file in your repo. + +**azure-pipelines.yml** +```yaml +jobs: +- job: linux + pool: {vmImage: 'Ubuntu-16.04'} + steps: + - task: UsePythonVersion@0 + - bash: | + python -m pip install --upgrade pip + pip install cibuildwheel==0.10.1 + cibuildwheel --output-dir wheelhouse . + - task: PublishBuildArtifacts@1 + inputs: {pathtoPublish: 'wheelhouse'} +- job: macos + pool: {vmImage: 'macOS-10.13'} + steps: + - task: UsePythonVersion@0 + - bash: | + python -m pip install --upgrade pip + pip install cibuildwheel==0.10.1 + cibuildwheel --output-dir wheelhouse . + - task: PublishBuildArtifacts@1 + inputs: {pathtoPublish: 'wheelhouse'} +- job: windows + pool: {vmImage: 'vs2017-win2016'} + steps: + - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x86}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.4', architecture: x86}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.4', architecture: x64}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.5', architecture: x86}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.5', architecture: x64}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x86}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.7', architecture: x86}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.7', architecture: x64}} + - script: choco install vcpython27 -f -y + displayName: Install Visual C++ for Python 2.7 + - bash: | + python -m pip install --upgrade pip + pip install cibuildwheel==0.10.1 + cibuildwheel --output-dir wheelhouse . + - task: PublishBuildArtifacts@1 + inputs: {pathtoPublish: 'wheelhouse'} +``` + +
+ +
+ Travis CI + + + + - To build Linux and Mac wheels on Travis CI, create a `.travis.yml` file in your repo. ``` @@ -55,6 +118,14 @@ Usage Then setup a deployment method by following the [Travis CI deployment docs](https://docs.travis-ci.com/user/deployment/), or see [Delivering to PyPI](#delivering-to-pypi) below. +
+ +
+ CircleCI + + + + - To build Linux and Mac wheels on CircleCI, create a `.circleci/config.yml` file in your repo, ``` @@ -102,6 +173,14 @@ Usage CircleCI will store the built wheels for you - you can access them from the project console. +
+ + +
+ Appveyor + + + - To build Windows wheels on Appveyor, create an `appveyor.yml` file in your repo. ``` @@ -115,6 +194,8 @@ Usage Appveyor will store the built wheels for you - you can access them from the project console. Alternatively, you may want to store them in the same place as the Travis CI build. See [Appveyor deployment docs](https://www.appveyor.com/docs/deployment/) for more info, or see [Delivering to PyPI](#delivering-to-pypi) below. +
+ - Commit those files, enable building of your repo on Travis CI and Appveyor, and push. All being well, you should get wheels delivered to you in a few minutes. From d5856884a36326d5c8b28597c7e9968bbade5527 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 6 Apr 2019 16:18:17 +0100 Subject: [PATCH 09/15] README updates --- README.md | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index ac055833..8da0a608 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ cibuildwheel Python wheels are great. Building them across **Mac, Linux, Windows**, on **multiple versions of Python**, is not. -`cibuildwheel` is here to help. `cibuildwheel` runs on your CI server - currently it supports Travis CI, Appveyor, and CircleCI - and it builds and tests your wheels across all of your platforms. +`cibuildwheel` is here to help. `cibuildwheel` runs on your CI server - currently it supports Azure Pipelines, Travis CI, Appveyor, and CircleCI - and it builds and tests your wheels across all of your platforms. **`cibuildwheel` is in beta**. It's brand new - I'd love for you to try it and help make it better! @@ -14,20 +14,29 @@ What does it do? | | macOS 10.6+ | manylinux i686 | manylinux x86_64 | Windows 32bit | Windows 64bit | |---|---|---|---|---|---| -| Python 2.7 | ✅ | ✅ | ✅ | ✅ | ✅ | -| Python 3.4 | ✅ | ✅ | ✅ | ✅ | ✅ | -| Python 3.5 | ✅ | ✅ | ✅ | ✅ | ✅ | -| Python 3.6 | ✅ | ✅ | ✅ | ✅ | ✅ | -| Python 3.7 | ✅ | ✅ | ✅ | ✅ | ✅ | +| Python 2.7 | ✅ | ✅ | ✅ | ✅ | ✅ | +| Python 3.4 | ✅ | ✅ | ✅ | ✅* | ✅* | +| Python 3.5 | ✅ | ✅ | ✅ | ✅ | ✅ | +| Python 3.6 | ✅ | ✅ | ✅ | ✅ | ✅ | +| Python 3.7 | ✅ | ✅ | ✅ | ✅ | ✅ | -- Builds manylinux, macOS and Windows (32 and 64bit) wheels using Travis CI, Appveyor, and CircleCI +> * Not supported on Azure Pipelines + +- Builds manylinux, macOS and Windows (32 and 64bit) wheels using Azure Pipelines, Travis CI, Appveyor, and CircleCI - Bundles shared library dependencies on Linux and macOS through [auditwheel](https://github.com/pypa/auditwheel) and [delocate](https://github.com/matthew-brett/delocate) - Runs the library test suite against the wheel-installed version of your library Usage ----- -`cibuildwheel` currently works on **Travis CI** and **CircleCI** to build Linux and Mac wheels, and **Appveyor** to build Windows wheels. +`cibuildwheel` currently works **Travis CI** and **CircleCI** to build Linux and Mac wheels, and **Appveyor** to build Windows wheels. **Azure Pipelines** supports all three. + +| | Linux | macOS | Windows | +|-----------------|-------|-------|---------| +| Azure Pipelines | ✅ | ✅ | ✅ | +| Travis CI | ✅ | ✅ | | +| Appveyor | | | ✅ | +| CircleCI | ✅ | ✅ | | `cibuildwheel` is not intended to run on your development machine. It will try to install packages globally; this is no good. Travis CI, CircleCI, and Appveyor run their builds in isolated environments, so are ideal for this kind of script. From be57269d5d8b924d0aac3d3c72fefdc5649dcded Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 6 Apr 2019 16:19:05 +0100 Subject: [PATCH 10/15] Remove 3.4 from the pipelines config because 3.4 isn't supported --- README.md | 2 -- azure-pipelines.yml | 2 -- 2 files changed, 4 deletions(-) diff --git a/README.md b/README.md index 8da0a608..8bfa69ab 100644 --- a/README.md +++ b/README.md @@ -79,8 +79,6 @@ jobs: steps: - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x86}} - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} - - {task: UsePythonVersion@0, inputs: {versionSpec: '3.4', architecture: x86}} - - {task: UsePythonVersion@0, inputs: {versionSpec: '3.4', architecture: x64}} - {task: UsePythonVersion@0, inputs: {versionSpec: '3.5', architecture: x86}} - {task: UsePythonVersion@0, inputs: {versionSpec: '3.5', architecture: x64}} - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x86}} diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ca4f038e..5a9e501c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -20,8 +20,6 @@ jobs: steps: - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x86}} - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} - - {task: UsePythonVersion@0, inputs: {versionSpec: '3.4', architecture: x86}} - - {task: UsePythonVersion@0, inputs: {versionSpec: '3.4', architecture: x64}} - {task: UsePythonVersion@0, inputs: {versionSpec: '3.5', architecture: x86}} - {task: UsePythonVersion@0, inputs: {versionSpec: '3.5', architecture: x64}} - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x86}} From 63778545453fd13b6ff7fc4621e8c238ebebcf16 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 6 Apr 2019 17:34:36 +0100 Subject: [PATCH 11/15] Add azure-template.yml for more compact config --- azure-template.yml | 44 ++++++++++++++++++++++++++++++++++++++++++++ setup.cfg | 4 ++++ 2 files changed, 48 insertions(+) create mode 100644 azure-template.yml diff --git a/azure-template.yml b/azure-template.yml new file mode 100644 index 00000000..620126a0 --- /dev/null +++ b/azure-template.yml @@ -0,0 +1,44 @@ +# Template config to build a Python package. You can either copy/paste this +# into your azure-pipelines.yml or reference it using the `template` syntax. +# See: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/templates?view=azure-devops#using-other-repositories + +jobs: +- job: linux + pool: {vmImage: 'Ubuntu-16.04'} + steps: + - task: UsePythonVersion@0 + - bash: | + python -m pip install --upgrade pip + pip install cibuildwheel==0.10.1 + cibuildwheel --output-dir wheelhouse . + - task: PublishBuildArtifacts@1 + inputs: {pathtoPublish: 'wheelhouse'} +- job: macos + pool: {vmImage: 'macOS-10.13'} + steps: + - task: UsePythonVersion@0 + - bash: | + python -m pip install --upgrade pip + pip install cibuildwheel==0.10.1 + cibuildwheel --output-dir wheelhouse . + - task: PublishBuildArtifacts@1 + inputs: {pathtoPublish: 'wheelhouse'} +- job: windows + pool: {vmImage: 'vs2017-win2016'} + steps: + - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x86}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.5', architecture: x86}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.5', architecture: x64}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x86}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.7', architecture: x86}} + - {task: UsePythonVersion@0, inputs: {versionSpec: '3.7', architecture: x64}} + - script: choco install vcpython27 -f -y + displayName: Install Visual C++ for Python 2.7 + - bash: | + python -m pip install --upgrade pip + pip install cibuildwheel==0.10.1 + cibuildwheel --output-dir wheelhouse . + - task: PublishBuildArtifacts@1 + inputs: {pathtoPublish: 'wheelhouse'} diff --git a/setup.cfg b/setup.cfg index 80361271..4ae17bd1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,3 +12,7 @@ message = Bump version search = cibuildwheel=={current_version} replace = cibuildwheel=={new_version} +[bumpversion:file:azure-template.yml] +search = cibuildwheel=={current_version} +replace = cibuildwheel=={new_version} + From 1e2a2e7d4de6c953646a3897ea78dc0e89dba1b3 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 6 Apr 2019 18:03:05 +0100 Subject: [PATCH 12/15] Revert "Add azure-template.yml for more compact config" This reverts commit 63778545453fd13b6ff7fc4621e8c238ebebcf16. --- azure-template.yml | 44 -------------------------------------------- setup.cfg | 4 ---- 2 files changed, 48 deletions(-) delete mode 100644 azure-template.yml diff --git a/azure-template.yml b/azure-template.yml deleted file mode 100644 index 620126a0..00000000 --- a/azure-template.yml +++ /dev/null @@ -1,44 +0,0 @@ -# Template config to build a Python package. You can either copy/paste this -# into your azure-pipelines.yml or reference it using the `template` syntax. -# See: https://docs.microsoft.com/en-us/azure/devops/pipelines/process/templates?view=azure-devops#using-other-repositories - -jobs: -- job: linux - pool: {vmImage: 'Ubuntu-16.04'} - steps: - - task: UsePythonVersion@0 - - bash: | - python -m pip install --upgrade pip - pip install cibuildwheel==0.10.1 - cibuildwheel --output-dir wheelhouse . - - task: PublishBuildArtifacts@1 - inputs: {pathtoPublish: 'wheelhouse'} -- job: macos - pool: {vmImage: 'macOS-10.13'} - steps: - - task: UsePythonVersion@0 - - bash: | - python -m pip install --upgrade pip - pip install cibuildwheel==0.10.1 - cibuildwheel --output-dir wheelhouse . - - task: PublishBuildArtifacts@1 - inputs: {pathtoPublish: 'wheelhouse'} -- job: windows - pool: {vmImage: 'vs2017-win2016'} - steps: - - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x86}} - - {task: UsePythonVersion@0, inputs: {versionSpec: '2.7', architecture: x64}} - - {task: UsePythonVersion@0, inputs: {versionSpec: '3.5', architecture: x86}} - - {task: UsePythonVersion@0, inputs: {versionSpec: '3.5', architecture: x64}} - - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x86}} - - {task: UsePythonVersion@0, inputs: {versionSpec: '3.6', architecture: x64}} - - {task: UsePythonVersion@0, inputs: {versionSpec: '3.7', architecture: x86}} - - {task: UsePythonVersion@0, inputs: {versionSpec: '3.7', architecture: x64}} - - script: choco install vcpython27 -f -y - displayName: Install Visual C++ for Python 2.7 - - bash: | - python -m pip install --upgrade pip - pip install cibuildwheel==0.10.1 - cibuildwheel --output-dir wheelhouse . - - task: PublishBuildArtifacts@1 - inputs: {pathtoPublish: 'wheelhouse'} diff --git a/setup.cfg b/setup.cfg index 4ae17bd1..80361271 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,7 +12,3 @@ message = Bump version search = cibuildwheel=={current_version} replace = cibuildwheel=={new_version} -[bumpversion:file:azure-template.yml] -search = cibuildwheel=={current_version} -replace = cibuildwheel=={new_version} - From 0cd3e426bc3a6e64f3a1ba3ab636c4a5b8abe36f Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 22 Apr 2019 19:05:34 +0100 Subject: [PATCH 13/15] Minor formatting fixes --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8bfa69ab..ddcf342c 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ What does it do? | Python 3.6 | ✅ | ✅ | ✅ | ✅ | ✅ | | Python 3.7 | ✅ | ✅ | ✅ | ✅ | ✅ | -> * Not supported on Azure Pipelines +> \* Not supported on Azure Pipelines - Builds manylinux, macOS and Windows (32 and 64bit) wheels using Azure Pipelines, Travis CI, Appveyor, and CircleCI - Bundles shared library dependencies on Linux and macOS through [auditwheel](https://github.com/pypa/auditwheel) and [delocate](https://github.com/matthew-brett/delocate) @@ -310,7 +310,7 @@ Examples: - Skip Python 3.6 on Linux: `CIBW_SKIP`:`cp36-manylinux*` - Only build on Python 3 and skip 32-bit builds: `CIBW_BUILD`:`cp3?-*` and `CIBW_SKIP`:`*-win32 *-manylinux1_i686` -** +*** | Environment variable: `CIBW_BUILD_VERBOSITY` | --- From 49bc742add98c950cbf718ac22c338e1b16968ed Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 27 Apr 2019 10:23:07 +0100 Subject: [PATCH 14/15] Remove unused imports --- cibuildwheel/windows.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 5ac16817..eeb5ca39 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -1,9 +1,5 @@ from __future__ import print_function -import os, tempfile, subprocess, sys, shutil -try: - from urllib2 import urlopen -except ImportError: - from urllib.request import urlopen +import os, tempfile, subprocess, shutil from collections import namedtuple from glob import glob From 17566cdee7dc504a8febd255bb187e0ca44d8f26 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 27 Apr 2019 12:12:02 +0100 Subject: [PATCH 15/15] Don't expect Python 3.4 wheels on Azure --- test/shared/utils.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/shared/utils.py b/test/shared/utils.py index b6a720f5..0284dd4a 100644 --- a/test/shared/utils.py +++ b/test/shared/utils.py @@ -6,6 +6,8 @@ This file is added to the PYTHONPATH in the test runner at bin/run_test.py. import subprocess, sys, os +IS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache') + def cibuildwheel_get_build_identifiers(project_path, env=None): ''' @@ -83,6 +85,10 @@ def expected_wheels(package_name, package_version): else: raise Exception('unsupported platform') + if IS_RUNNING_ON_AZURE: + # Python 3.4 isn't supported on Azure. + templates = [t for t in templates if '-cp34-' not in t] + return [filename.format(package_name=package_name, package_version=package_version) for filename in templates]