Merge remote-tracking branch 'origin/master' into azure-windows

This commit is contained in:
Joe Rickerby
2019-04-27 10:22:00 +01:00
23 changed files with 320 additions and 97 deletions
+7 -4
View File
@@ -240,9 +240,8 @@ Options
-------
```
usage: cibuildwheel [-h]
[--output-dir OUTPUT_DIR]
[--platform PLATFORM]
usage: cibuildwheel [-h] [--platform {auto,linux,macos,windows}]
[--output-dir OUTPUT_DIR] [--print-build-identifiers]
[project_dir]
Build wheels for all the platforms.
@@ -262,7 +261,10 @@ optional arguments:
you need to run in Windows, and it will build and test
for all versions of Python at C:\PythonXX[-x64].
--output-dir OUTPUT_DIR
Destination folder for the wheels.
Destination folder for the wheels.
--print-build-identifiers
Print the build identifiers matched by the current
invocation and exit.
```
@@ -469,6 +471,7 @@ If your wheel didn't compile, check the list below for some debugging tips.
- A mistake in your config. To quickly test your config without doing a git push and waiting for your code to build on CI, you can run the Linux build in a Docker container. On Mac or Linux, with Docker running, try `cibuildwheel --platform linux`. You'll have to bring your config into the current environment first.
- Missing dependency. You might need to install something on the build machine. You can do this in `.travis.yml`, `appveyor.yml`, or `.circleci/config.yml`, with apt-get, brew or whatever Windows uses :P . Given how the Linux build works, we'll probably have to build something into `cibuildwheel`. Let's chat about that over in the issues!
- Windows: missing C feature. The Windows C compiler doesn't support C language features invented after 1990, so you'll have to backport your C code to C90. For me, this mostly involved putting my variable declarations at the top of the function like an animal.
- MacOS: calling cibuildwheel from a python3 script and getting a `ModuleNotFoundError`? Due to a [bug](https://bugs.python.org/issue22490) in CPython, you'll need to [unset the `__PYVENV_LAUNCHER__` variable](https://github.com/joerick/cibuildwheel/issues/133#issuecomment-478288597) before activating a venv.
Working examples
----------------
+18 -22
View File
@@ -1,31 +1,29 @@
#!/usr/bin/python
#!/usr/bin/env python
from __future__ import print_function
import os, sys, subprocess, shutil, json
from glob import glob
import os, sys, subprocess, shutil
project_root = os.path.dirname(os.path.dirname(__file__))
test_utils_dir = os.path.join(project_root, 'test', 'shared')
def single_run(test_project):
# load project settings into environment
env_file = os.path.join(test_project, 'environment.json')
project_env = {}
if os.path.exists(env_file):
with open(env_file) as f:
project_env = json.load(f)
# run the build
# set up an environment that gives access to the test utils
env = os.environ.copy()
project_env = {str(k): str(v) for k, v in project_env.items()} # unicode not allowed in env
env.update(project_env)
print('Building %s with environment %s' % (test_project, project_env))
subprocess.check_call([sys.executable, '-m', 'cibuildwheel', test_project], env=env)
wheels = glob('wheelhouse/*.whl')
print('%s built successfully. %i wheels built.' % (test_project, len(wheels)))
# check some wheels were actually built
assert len(wheels) >= 3
if 'PYTHONPATH' in env:
env['PYTHONPATH'] += os.pathsep + test_utils_dir
else:
env['PYTHONPATH'] = test_utils_dir
# run the test
subprocess.check_call(
[sys.executable, '-m', 'pytest', '-vv', os.path.join(test_project, 'cibuildwheel_test.py')],
env=env,
)
# clean up
shutil.rmtree('wheelhouse')
if os.path.exists('wheelhouse'):
shutil.rmtree('wheelhouse')
if __name__ == '__main__':
import argparse
@@ -41,5 +39,3 @@ if __name__ == '__main__':
exit(2)
single_run(project_path)
print('Project built successfully.')
+2 -2
View File
@@ -1,4 +1,4 @@
#!/usr/bin/python
#!/usr/bin/env python
from __future__ import print_function
import os, sys, subprocess, shutil, json
@@ -14,7 +14,7 @@ if __name__ == '__main__':
### run the integration tests
test_projects = glob('test/??_*')
test_projects = sorted(glob('test/??_*'))
if len(test_projects) == 0:
print('No test projects found. Aborting.', file=sys.stderr)
+24
View File
@@ -48,6 +48,10 @@ def main():
help=('Path to the project that you want wheels for. Default: the current '
'directory.'))
parser.add_argument('--print-build-identifiers',
action='store_true',
help='Print the build identifiers matched by the current invocation and exit.')
args = parser.parse_args()
if args.platform != 'auto':
@@ -116,6 +120,10 @@ def main():
print('cibuildwheel: Could not find setup.py at root of project', file=sys.stderr)
exit(2)
if args.print_build_identifiers:
print_build_identifiers(platform, build_selector)
exit(0)
build_options = dict(
project_dir=project_dir,
output_dir=output_dir,
@@ -156,6 +164,7 @@ def main():
else:
raise Exception('Unsupported platform')
def print_preamble(platform, build_options):
print(textwrap.dedent('''
_ _ _ _ _ _ _
@@ -180,6 +189,21 @@ def print_preamble(platform, build_options):
print('\nHere we go!\n')
def print_build_identifiers(platform, build_selector):
if platform == 'linux':
python_configurations = cibuildwheel.linux.get_python_configurations(build_selector)
elif platform == 'windows':
python_configurations = cibuildwheel.windows.get_python_configurations(build_selector)
elif platform == 'macos':
python_configurations = cibuildwheel.macos.get_python_configurations(build_selector)
else:
python_configurations = []
for config in python_configurations:
print(config.identifier)
def detect_warnings(platform, build_options):
warnings = []
+14 -11
View File
@@ -9,16 +9,7 @@ except ImportError:
from pipes import quote as shlex_quote
def build(project_dir, output_dir, test_command, test_requires, before_build, build_verbosity, build_selector, environment, manylinux1_images):
try:
subprocess.check_call(['docker', '--version'])
except:
print('cibuildwheel: Docker not found. Docker is required to run Linux builds. '
'If you\'re building on Travis CI, add `services: [docker]` to your .travis.yml.'
'If you\'re building on Circle CI in Linux, add a `setup_remote_docker` step to your .circleci/config.yml',
file=sys.stderr)
exit(2)
def get_python_configurations(build_selector):
PythonConfiguration = namedtuple('PythonConfiguration', ['identifier', 'path'])
python_configurations = [
PythonConfiguration(identifier='cp27-manylinux1_x86_64', path='/opt/python/cp27-cp27m'),
@@ -36,8 +27,20 @@ def build(project_dir, output_dir, test_command, test_requires, before_build, bu
]
# skip builds as required
python_configurations = [c for c in python_configurations if build_selector(c.identifier)]
return [c for c in python_configurations if build_selector(c.identifier)]
def build(project_dir, output_dir, test_command, test_requires, before_build, build_verbosity, build_selector, environment, manylinux1_images):
try:
subprocess.check_call(['docker', '--version'])
except:
print('cibuildwheel: Docker not found. Docker is required to run Linux builds. '
'If you\'re building on Travis CI, add `services: [docker]` to your .travis.yml.'
'If you\'re building on Circle CI in Linux, add a `setup_remote_docker` step to your .circleci/config.yml',
file=sys.stderr)
exit(2)
python_configurations = get_python_configurations(build_selector)
platforms = [
('manylinux1_x86_64', manylinux1_images.get('x86_64') or 'quay.io/pypa/manylinux1_x86_64'),
('manylinux1_i686', manylinux1_images.get('i686') or 'quay.io/pypa/manylinux1_i686'),
+8 -5
View File
@@ -10,7 +10,7 @@ except ImportError:
from .util import prepare_command, get_build_verbosity_extra_flags
def build(project_dir, output_dir, test_command, test_requires, before_build, build_verbosity, build_selector, environment):
def get_python_configurations(build_selector):
PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'url'])
python_configurations = [
PythonConfiguration(version='2.7', identifier='cp27-macosx_10_6_intel', url='https://www.python.org/ftp/python/2.7.16/python-2.7.16-macosx10.6.pkg'),
@@ -19,6 +19,13 @@ def build(project_dir, output_dir, test_command, test_requires, before_build, bu
PythonConfiguration(version='3.6', identifier='cp36-macosx_10_6_intel', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.6.pkg'),
PythonConfiguration(version='3.7', identifier='cp37-macosx_10_6_intel', url='https://www.python.org/ftp/python/3.7.2/python-3.7.2-macosx10.6.pkg'),
]
# skip builds as required
return [c for c in python_configurations if build_selector(c.identifier)]
def build(project_dir, output_dir, test_command, test_requires, before_build, build_verbosity, build_selector, environment):
python_configurations = get_python_configurations(build_selector)
get_pip_url = 'https://bootstrap.pypa.io/get-pip.py'
get_pip_script = '/tmp/get-pip.py'
@@ -42,10 +49,6 @@ def build(project_dir, output_dir, test_command, test_requires, before_build, bu
call(['curl', '-L', '-o', get_pip_script, get_pip_url])
for config in python_configurations:
if not build_selector(config.identifier):
print('cibuildwheel: Skipping build %s' % config.identifier, file=sys.stderr)
continue
# if this version of python isn't installed, get it from python.org and install
python_package_identifier = 'org.python.Python.PythonFramework-%s' % config.version
if python_package_identifier not in installed_system_packages:
+26 -24
View File
@@ -36,6 +36,30 @@ def get_python_path(config):
)
def get_python_configurations(build_selector):
PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier', 'path'])
python_configurations = [
PythonConfiguration(version='2.7.x', arch="32", identifier='cp27-win32', path='C:\Python27'),
PythonConfiguration(version='2.7.x', arch="64", identifier='cp27-win_amd64', path='C:\Python27-x64'),
PythonConfiguration(version='3.4.x', arch="32", identifier='cp34-win32', path='C:\Python34'),
PythonConfiguration(version='3.4.x', arch="64", identifier='cp34-win_amd64', path='C:\Python34-x64'),
PythonConfiguration(version='3.5.x', arch="32", identifier='cp35-win32', path='C:\Python35'),
PythonConfiguration(version='3.5.x', arch="64", identifier='cp35-win_amd64', path='C:\Python35-x64'),
PythonConfiguration(version='3.6.x', arch="32", identifier='cp36-win32', path='C:\Python36'),
PythonConfiguration(version='3.6.x', arch="64", identifier='cp36-win_amd64', path='C:\Python36-x64'),
PythonConfiguration(version='3.7.x', arch="32", identifier='cp37-win32', path='C:\Python37'),
PythonConfiguration(version='3.7.x', arch="64", identifier='cp37-win_amd64', path='C:\Python37-x64'),
]
if IS_RUNNING_ON_AZURE:
# Python 3.4 isn't supported on Azure.
# See https://github.com/Microsoft/azure-pipelines-tasks/issues/9674
python_configurations = [c for c in python_configurations if c.version != '3.4.x']
# skip builds as required
return [c for c in python_configurations if build_selector(c.identifier)]
def build(project_dir, output_dir, test_command, test_requires, before_build, build_verbosity, build_selector, environment):
if IS_RUNNING_ON_AZURE:
def shell(args, env=None, cwd=None):
@@ -53,35 +77,13 @@ def build(project_dir, output_dir, test_command, test_requires, before_build, bu
args = ['cmd', '/E:ON', '/V:ON', '/C', run_with_env] + args
return subprocess.check_call(' '.join(args), env=env, cwd=cwd)
PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier'])
python_configurations = [
PythonConfiguration(version='2.7.x', arch="32", identifier='cp27-win32'),
PythonConfiguration(version='2.7.x', arch="64", identifier='cp27-win_amd64'),
PythonConfiguration(version='3.4.x', arch="32", identifier='cp34-win32'),
PythonConfiguration(version='3.4.x', arch="64", identifier='cp34-win_amd64'),
PythonConfiguration(version='3.5.x', arch="32", identifier='cp35-win32'),
PythonConfiguration(version='3.5.x', arch="64", identifier='cp35-win_amd64'),
PythonConfiguration(version='3.6.x', arch="32", identifier='cp36-win32'),
PythonConfiguration(version='3.6.x', arch="64", identifier='cp36-win_amd64'),
PythonConfiguration(version='3.7.x', arch="32", identifier='cp37-win32'),
PythonConfiguration(version='3.7.x', arch="64", identifier='cp37-win_amd64'),
]
if IS_RUNNING_ON_AZURE:
# Python 3.4 isn't supported on Azure.
# See https://github.com/Microsoft/azure-pipelines-tasks/issues/9674
python_configurations = [c for c in python_configurations if c.version != '3.4.x']
abs_project_dir = os.path.abspath(project_dir)
temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
built_wheel_dir = os.path.join(temp_dir, 'built_wheel')
python_configurations = get_python_configurations(build_selector)
for config in python_configurations:
if not build_selector(config.identifier):
print('cibuildwheel: Skipping build %s' % config.identifier, file=sys.stderr)
continue
config_python_path = get_python_path(config)
# check python & pip exist for this configuration
+22
View File
@@ -0,0 +1,22 @@
import os
import utils
project_dir = os.path.dirname(__file__)
def test():
# build the wheels
utils.cibuildwheel_run(project_dir)
# check that the expected wheels are produced
expected_wheels = utils.expected_wheels('spam', '0.1.0')
actual_wheels = os.listdir('wheelhouse')
assert set(actual_wheels) == set(expected_wheels)
def test_build_identifiers():
# check that the number of expected wheels matches the number of build
# identifiers
expected_wheels = utils.expected_wheels('spam', '0.1.0')
build_identifiers = utils.cibuildwheel_get_build_identifiers(project_dir)
assert len(expected_wheels) == len(build_identifiers)
+19
View File
@@ -0,0 +1,19 @@
import os
import utils
def test():
project_dir = os.path.dirname(__file__)
# build and test the wheels
utils.cibuildwheel_run(project_dir, add_env={
'CIBW_TEST_REQUIRES': 'nose',
# the 'false ||' bit is to ensure this command runs in a shell on
# mac/linux.
'CIBW_TEST_COMMAND': 'false || nosetests {project}/test',
'CIBW_TEST_COMMAND_WINDOWS': 'nosetests {project}/test',
})
# also check that we got the right wheels
expected_wheels = utils.expected_wheels('spam', '0.1.0')
actual_wheels = os.listdir('wheelhouse')
assert set(actual_wheels) == set(expected_wheels)
-6
View File
@@ -1,6 +0,0 @@
{
"CIBW_TEST_REQUIRES": "nose",
"CIBW_TEST_COMMAND": "false || nosetests {project}/test",
"comment": "The 'false ||' bit is to ensure this command runs in a shell on Mac and Linux",
"CIBW_TEST_COMMAND_WINDOWS": "nosetests {project}/test"
}
+18
View File
@@ -0,0 +1,18 @@
import os
import utils
def test():
project_dir = os.path.dirname(__file__)
# build the wheels
utils.cibuildwheel_run(project_dir, add_env={
# write python version information to a temporary file, this is
# checked in setup.py
'CIBW_BEFORE_BUILD': '''python -c "import sys; open('/tmp/pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('/tmp/pythonexecutable.txt', 'w').write(sys.executable)"''',
'CIBW_BEFORE_BUILD_WINDOWS': '''python -c "import sys; open('c:\\pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('c:\\pythonexecutable.txt', 'w').write(sys.executable)"''',
})
# also check that we got the right wheels
expected_wheels = utils.expected_wheels('spam', '0.1.0')
actual_wheels = os.listdir('wheelhouse')
assert set(actual_wheels) == set(expected_wheels)
-4
View File
@@ -1,4 +0,0 @@
{
"CIBW_BEFORE_BUILD": "python -c \"import sys; open('/tmp/pythonversion.txt', 'w').write(sys.version)\" && python -c \"import sys; open('/tmp/pythonexecutable.txt', 'w').write(sys.executable)\"",
"CIBW_BEFORE_BUILD_WINDOWS": "python -c \"import sys; open('c:\\pythonversion.txt', 'w').write(sys.version)\" && python -c \"import sys; open('c:\\pythonexecutable.txt', 'w').write(sys.executable)\""
}
-2
View File
@@ -1,2 +0,0 @@
3.6.0 (default, Feb 7 2017, 23:55:32)
[GCC 4.8.2 20140120 (Red Hat 4.8.2-15)]
+17
View File
@@ -0,0 +1,17 @@
import os
import utils
def test():
project_dir = os.path.dirname(__file__)
# build the wheels
utils.cibuildwheel_run(project_dir, add_env={
'CIBW_BUILD': 'cp3?-*',
'CIBW_SKIP': 'cp34-*',
})
# check that we got the right wheels. There should be no 2.7 or 3.4.
expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0')
if ('-cp3' in w) and ('-cp34' not in w)]
actual_wheels = os.listdir('wheelhouse')
assert set(actual_wheels) == set(expected_wheels)
-4
View File
@@ -1,4 +0,0 @@
{
"CIBW_BUILD": "cp3?-*",
"CIBW_SKIP": "cp34-*"
}
-2
View File
@@ -1,2 +0,0 @@
3.6.0 (default, Feb 7 2017, 23:55:32)
[GCC 4.8.2 20140120 (Red Hat 4.8.2-15)]
+18
View File
@@ -0,0 +1,18 @@
import os
import utils
def test():
project_dir = os.path.dirname(__file__)
# write some information into the CIBW_ENVIRONMENT, for expansion and
# insertion into the environment by cibuildwheel. This is checked
# in setup.py
utils.cibuildwheel_run(project_dir, add_env={
'CIBW_ENVIRONMENT': '''CIBW_TEST_VAR="a b c" CIBW_TEST_VAR_2=1 CIBW_TEST_VAR_3="$(echo 'test string 3')" PATH=$PATH:/opt/cibw_test_path''',
'CIBW_ENVIRONMENT_WINDOWS': '''CIBW_TEST_VAR="a b c" CIBW_TEST_VAR_2=1 CIBW_TEST_VAR_3="$(echo 'test string 3')" PATH="$PATH;/opt/cibw_test_path"''',
})
# also check that we got the right wheels built
expected_wheels = utils.expected_wheels('spam', '0.1.0')
actual_wheels = os.listdir('wheelhouse')
assert set(actual_wheels) == set(expected_wheels)
-4
View File
@@ -1,4 +0,0 @@
{
"CIBW_ENVIRONMENT": "CIBW_TEST_VAR=\"a b c\" CIBW_TEST_VAR_2=1 CIBW_TEST_VAR_3=\"$(echo 'test string 3')\" PATH=$PATH:/opt/cibw_test_path",
"CIBW_ENVIRONMENT_WINDOWS": "CIBW_TEST_VAR=\"a b c\" CIBW_TEST_VAR_2=1 CIBW_TEST_VAR_3=\"$(echo 'test string 3')\" PATH=\"$PATH;/opt/cibw_test_path\""
}
-2
View File
@@ -1,2 +0,0 @@
3.6.0 (default, Feb 7 2017, 23:55:32)
[GCC 4.8.2 20140120 (Red Hat 4.8.2-15)]
@@ -0,0 +1,18 @@
import os, pytest
import utils
def test():
project_dir = os.path.dirname(__file__)
if utils.platform != 'linux':
pytest.skip('the docker test is only relevant to the linux build')
utils.cibuildwheel_run(project_dir, add_env={
'CIBW_MANYLINUX1_X86_64_IMAGE': 'dockcross/manylinux-x64',
'CIBW_MANYLINUX1_I686_IMAGE': 'dockcross/manylinux-x86',
})
# also check that we got the right wheels built
expected_wheels = utils.expected_wheels('spam', '0.1.0')
actual_wheels = os.listdir('wheelhouse')
assert set(actual_wheels) == set(expected_wheels)
-5
View File
@@ -1,5 +0,0 @@
{
"CIBW_MANYLINUX1_X86_64_IMAGE": "dockcross/manylinux-x64",
"CIBW_MANYLINUX1_I686_IMAGE": "dockcross/manylinux-x86",
"CIBW_SKIP": "cp37-manylinux*"
}
+9
View File
@@ -0,0 +1,9 @@
import os
import utils
def test():
project_dir = os.path.dirname(__file__)
# this test checks that SSL is working in the build environment using
# some checks in setup.py.
utils.cibuildwheel_run(project_dir)
+100
View File
@@ -0,0 +1,100 @@
'''
Utility functions used by the cibuildwheel tests.
This file is added to the PYTHONPATH in the test runner at bin/run_test.py.
'''
import subprocess, sys, os
def cibuildwheel_get_build_identifiers(project_path, env=None):
'''
Returns the list of build identifiers that cibuildwheel will try to build
for the current platform.
'''
cmd_output = subprocess.check_output(
[sys.executable, '-m', 'cibuildwheel', '--print-build-identifiers', project_path],
universal_newlines=True,
env=env,
)
return cmd_output.strip().split('\n')
def cibuildwheel_run(project_path, env=None, add_env=None):
'''
Runs cibuildwheel as a subprocess, building the project at project_path.
Uses the current Python interpreter.
Configure settings using env.
'''
if env is None:
env = os.environ.copy()
if add_env is not None:
env.update(add_env)
subprocess.check_call(
[sys.executable, '-m', 'cibuildwheel', project_path],
env=env,
)
def expected_wheels(package_name, package_version):
'''
Returns a list of expected wheels from a run of cibuildwheel.
'''
if platform == 'linux':
templates = [
'{package_name}-{package_version}-cp27-cp27m-manylinux1_x86_64.whl',
'{package_name}-{package_version}-cp27-cp27mu-manylinux1_x86_64.whl',
'{package_name}-{package_version}-cp34-cp34m-manylinux1_x86_64.whl',
'{package_name}-{package_version}-cp35-cp35m-manylinux1_x86_64.whl',
'{package_name}-{package_version}-cp36-cp36m-manylinux1_x86_64.whl',
'{package_name}-{package_version}-cp37-cp37m-manylinux1_x86_64.whl',
'{package_name}-{package_version}-cp27-cp27m-manylinux1_i686.whl',
'{package_name}-{package_version}-cp27-cp27mu-manylinux1_i686.whl',
'{package_name}-{package_version}-cp34-cp34m-manylinux1_i686.whl',
'{package_name}-{package_version}-cp35-cp35m-manylinux1_i686.whl',
'{package_name}-{package_version}-cp36-cp36m-manylinux1_i686.whl',
'{package_name}-{package_version}-cp37-cp37m-manylinux1_i686.whl',
]
elif platform == 'windows':
templates = [
'{package_name}-{package_version}-cp27-cp27m-win32.whl',
'{package_name}-{package_version}-cp34-cp34m-win32.whl',
'{package_name}-{package_version}-cp35-cp35m-win32.whl',
'{package_name}-{package_version}-cp36-cp36m-win32.whl',
'{package_name}-{package_version}-cp37-cp37m-win32.whl',
'{package_name}-{package_version}-cp27-cp27m-win_amd64.whl',
'{package_name}-{package_version}-cp34-cp34m-win_amd64.whl',
'{package_name}-{package_version}-cp35-cp35m-win_amd64.whl',
'{package_name}-{package_version}-cp36-cp36m-win_amd64.whl',
'{package_name}-{package_version}-cp37-cp37m-win_amd64.whl',
]
elif platform == 'macos':
templates = [
'{package_name}-{package_version}-cp27-cp27m-macosx_10_6_intel.whl',
'{package_name}-{package_version}-cp34-cp34m-macosx_10_6_intel.whl',
'{package_name}-{package_version}-cp35-cp35m-macosx_10_6_intel.whl',
'{package_name}-{package_version}-cp36-cp36m-macosx_10_6_intel.whl',
'{package_name}-{package_version}-cp37-cp37m-macosx_10_6_intel.whl',
]
else:
raise Exception('unsupported platform')
return [filename.format(package_name=package_name, package_version=package_version)
for filename in templates]
platform = None
if 'CIBW_PLATFORM' in os.environ:
platform = os.environ['CIBW_PLATFORM']
elif sys.platform.startswith('linux'):
platform = 'linux'
elif sys.platform.startswith('darwin'):
platform = 'macos'
elif sys.platform in ['win32', 'cygwin']:
platform = 'windows'
else:
raise Exception('Unsupported platform')