From f0f992e37f1e1bac21b70648f9e2000fbafda724 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 19 Mar 2017 20:57:51 +0000 Subject: [PATCH] Initial commit --- .gitignore | 94 +++++++++++++ README.md | 218 +++++++++++++++++++++++++++++ cibuildwheel.py | 292 +++++++++++++++++++++++++++++++++++++++ cibuildwheel/__init__.py | 0 cibuildwheel/__main__.py | 292 +++++++++++++++++++++++++++++++++++++++ setup.py | 36 +++++ 6 files changed, 932 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 cibuildwheel.py create mode 100644 cibuildwheel/__init__.py create mode 100644 cibuildwheel/__main__.py create mode 100644 setup.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..62c1e736 --- /dev/null +++ b/.gitignore @@ -0,0 +1,94 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +env/ +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*,cover +.hypothesis/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# dotenv +.env + +# virtualenv +.venv +venv/ +ENV/ + +# Spyder project settings +.spyderproject + +# Rope project settings +.ropeproject diff --git a/README.md b/README.md new file mode 100644 index 00000000..245d4812 --- /dev/null +++ b/README.md @@ -0,0 +1,218 @@ +cibuildwheel +============ + +Python wheels are great. Building them across **Mac, Linux, Windows**, on **multiple versions of Python**, is not. + +`cibuildwheel` is here to help. `cibuildwheel` runs on your CI server - currently it supports Travis CI and Appveyor - and it builds and tests your wheels across all of your platforms. + +**`cibuildwheel` is in beta**. It's brand new - I'd love for you to try it and help make it better! + +What does it do? +---------------- + +| | macOS 10.6+ | manylinux i686 | manylinux x86_64 | Windows 32bit | Windows 64bit | +|---|---|---|---|---|---| +| Python 2.7 | ✅ | ✅ | ✅ | ✅ | ✅ | +| Python 3.3 | | ✅ | ✅ | ✅ | ✅ | +| Python 3.4 | ✅ | ✅ | ✅ | ✅ | ✅ | +| Python 3.5 | ✅ | ✅ | ✅ | ✅ | ✅ | +| Python 3.6 | ✅ | ✅ | ✅ | ✅ | ✅ | + +- Builds manylinux, macOS and Windows (32 and 64bit) wheels +- Bundles shared library dependencies on Linux and macOS through [auditwheel](https://github.com/pypa/auditwheel) and [delocate](https://github.com/matthew-brett/delocate) +- Runs the library test suite against the wheel-installed version of your library + +Usage +----- + +`cibuildwheel` currently works on **Travis CI** to build Linux and Mac wheels, and **Appveyor** to build Windows wheels. + +`cibuildwheel` is not intended to run on your development machine. It will try to install packages globally; this is no good. Travis CI and Appveyor run their builds in isolated environments, so are ideal for this kind of script. + +### Minimal setup + +- Create a `.travis.yml` file in your repo. + + ``` + matrix: + include: + - sudo: required + services: + - docker + - os: osx + + script: + - pip install cibuildwheel + - cibuildwheel --output-dir wheelhouse + ``` + + Then setup a deployment method by following the [Travis CI deployment docs](https://docs.travis-ci.com/user/deployment/). + +- Create an `appveyor.yml` file in your repo. + + ``` + build_script: + - pip install cibuildwheel + - cibuildwheel --output-dir wheelhouse + artifacts: + - path: "wheelhouse\\*.whl" + name: Wheels + ``` + + Appveyor will store the built wheels for you - you can access them from the project console. Alternatively, you may want to store them in the same place as the Travis CI build. See [Appveyor deployment docs](https://www.appveyor.com/docs/deployment/) for more info. + +- Commit those files, enable building of your repo on Travis CI and Appveyor, and push. + +All being well, you should get wheels delivered to you in a few minutes. + +> Got an error? Check the [checklist](#it-didnt-work) below. + + +Options +------- + +``` +usage: cibuildwheel [-h] + [--output-dir OUTPUT_DIR] + [--platform PLATFORM] + [project_dir] + +Build wheels for all the platforms. + +positional arguments: + project_dir Path to the project that you want wheels for. + Default: the current directory. + +optional arguments: + -h, --help show this help message and exit + --platform {auto,linux,macos,windows} + Platform to build for. For "linux" you need docker + running, on Mac or Linux. For "macos", you need a Mac + machine, and note that this script is going to + automatically install MacPython on your system, so + don't run on your development machine. For "windows", + you need to run in Windows, and it will build and test + for all versions of Python at C:\PythonXX[-x64]. + --output-dir OUTPUT_DIR + Destination folder for the wheels. + +``` + +Most of the config is via environment variables. These go into `.travis.yml` and `appveyor.yml` nicely. + +| Environment variable: `CIBW_PLATFORM` | Command line argument: `--platform` +| --- | --- + +Options: `auto` `linux` `macos` `windows` + +Default: `auto` + +`auto` will auto-detect platform using environment variables, such as `TRAVIS_OS_NAME`/`APPVEYOR`. + +For `linux` you need Docker running, on Mac or Linux. For `macos`, you need a Mac machine, and note that this script is going to automatically install MacPython on your system, so don't run on your development machine. For `windows`, you need to run in Windows, and it will build and test for all versions of Python at `C:\PythonXX[-x64]`. + +| Environment variable: `CIBW_TEST_COMMAND` +| --- + +Optional. + +Shell command to run the tests. The project root should be included in the command as "{project}". The wheel will be installed automatically and available for import from the tests. + +Example: `nosetests {project}/tests` + +| Environment variable: `CIBW_TEST_REQUIRES` +| --- + +Optional. + +Space-separated list of dependencies required for running the tests. + +Example: `pytest` +Example: `nose==1.3.7 moto==0.4.31` + +| Environment variable: `CIBW_SKIP` | 🔶 coming soon 🔶 +| --- | --- + +Optional. + +Space-separated list of builds to skip. Each build has an identifier like `cp27-linux_x86_64` or `cp34-macosx_10_6_intel` - you can list ones to skip here and `cibuildwheel` won't try to build them. + +The format is `python_tag-platform_tag`. The tags are as defined in [PEP 0425](https://www.python.org/dev/peps/pep-0425/#details). + +Python tags look like `cp27` `cp34` `cp35` `cp36` + +Platform tags look like `macosx_10_6_intel` `linux_x86_64` `linux_i386` `win32` `win_amd64` + +You can also use shell-style globbing syntax (as per `fnmatch`) + +Example: `cp27-macosx_10_6_intel ` (don't build on Python 2 on Mac) +Example: `cp27-win*` (don't build on Python 2.7 on Windows) +Example: `cp34-*` (don't build on Python 3.4) + +-- + +#### Example YML syntax + + + + +
example .travis.yml environment variables
env:
+  global:
+    - CIBW_TEST_REQUIRES=nose
+    - CIBW_TEST_COMMAND="nosetests {project}/tests"
+
example appveyor.yml environment variables
environment:
+  global:
+    CIBW_TEST_REQUIRES: nose
+    CIBW_TEST_COMMAND: "nosetests {project}\\tests"
+
+ +It didn't work! +--------------- + +If your wheel didn't compile, check the list below for some debugging tips. + +- A mistake in your config. To quickly test your config without doing a git push and waiting for your code, you can run the Linux build in a Docker container. Try `cibuildwheel --platform linux`. +- Missing dependency. You might need to install something inside on the build machine. You can do this in `.travis.yml` or `appveyor.yml`, with apt-get, brew or whatever Windows uses :P . Given how the Linux build works, we'll probably have to build something into `cibuildwheel`. Let's chat about that over in the issues! +- Windows: missing C feature. The Windows C compiler doesn't support C language features invented after 1990, so you'll have to backport your C code to C90. For me, this mostly involved putting my variable declarations at the top of the function like an animal. + +Legal note +---------- + +Since `cibuildwheel` runs the wheel through delocate or auditwheel, it will automatically bundle library dependencies. This is similar to static linking - it might have some licence implications. Check the license for any code you're pulling in to make sure that's allowed. + + +Contributing +============ + +Wheel-building is pretty complex. I expect users to find many edge-cases - please help the rest of the community out by documenting these, adding features to support them, and reporting bugs. + +I plan to be pretty liberal in accepting pull requests, as long as they align with the design goals below. + +`cibuildwheel` is indie open source. I'm not paid to work on this. + +Design Goals +------------ + +- `cibuildwheel` should wrap the complexity of wheel building. +- The user interface to `cibuildwheel` is the build script (e.g. `.travis.yml`). Feature additions should not increase the complexity of this script. +- Options should be environment variables (these lend themselves better to YML config files). They should be prefixed with `CIBW_`. +- Options should be generalise to all platforms. If platform-specific options are required, they should be namespaced e.g. `CIBW_TEST_COMMAND_MACOS` + +Other notes: + +- The platforms are very similar, until they're not. I'd rather have straight-forward code than totally DRY code, so let's keep airy platfrom abstractions to a minimum. +- I might want to break the options into a shared config file one day, so that config is more easily shared. That has motivated some of the design decisions. + +Credits +------- + +`cibuildwheel` stands on the shoulders of giants. Massive props to- + +- ⭐️ @matthew-brett for matthew-brett/multibuild and matthew-brett/delocate +- @PyPA for the manylinux Docker images pypa/manylinux +- @ogrisel for wheelhouse-uploader and `run_with_env.cmd` + +See also +-------- + +If `cibuildwheel` is too limited for your needs, consider matthew-brett/multibuild. `multibuild` is a toolbox for building a wheel on various platforms. It can do a lot more than this project - it's used to build SciPy! diff --git a/cibuildwheel.py b/cibuildwheel.py new file mode 100644 index 00000000..c638e233 --- /dev/null +++ b/cibuildwheel.py @@ -0,0 +1,292 @@ +from __future__ import print_function +import argparse, os, subprocess, sys, tempfile, shlex +from collections import namedtuple +from glob import glob + +try: + # Python 3 + import urllib.request as urllib2 +except ImportError: + # Python 2 + import urllib2 + +try: + from shlex import quote as shlex_quote +except ImportError: + from pipes import quote as shlex_quote + + +def linux_build(project_dir, package_name, output_dir, test_command, test_requires): + for docker_image in ['quay.io/pypa/manylinux1_x86_64', 'quay.io/pypa/manylinux1_i686']: + bash_script = ''' + set -o errexit + set -o xtrace + cd /project + + for PYBIN in /opt/python/*/bin; do + "$PYBIN/pip" wheel . -w /tmp/linux_wheels + done + + for whl in /tmp/linux_wheels/*.whl; do + auditwheel repair "$whl" -w /output + done + + # Install packages and test + for PYBIN in /opt/python/*/bin/; do + # Install the wheel we just built + "$PYBIN/pip" install {package_name} --no-index -f /output + + # Install any requirements to run the tests + if [ ! -z "{test_requires}" ]; then + "$PYBIN/pip" install {test_requires} + fi + + # Run the tests from a different directory + if [ ! -z {test_command} ]; then + (cd "$HOME" && export PATH=$PYBIN:$PATH && sh -c {test_command}) + fi + done + '''.format( + package_name=package_name, + test_requires=' '.join(test_requires), + test_command=shlex_quote(test_command.format(project='/project') if test_command else ''), + ) + + docker_process = subprocess.Popen([ + 'docker', + 'run', + '--rm', + '-i', + '-v', '%s:/project' % os.path.abspath(project_dir), + '-v', '%s:/output' % os.path.abspath(output_dir), + docker_image, + '/bin/bash'], + stdin=subprocess.PIPE) + + docker_process.communicate(bash_script) + + if docker_process.returncode != 0: + exit(1) + + +def windows_build(project_dir, package_name, output_dir, test_command, test_requires): + # run_with_env is a cmd file that sets the right environment variables to + run_with_env = os.path.join(tempfile.gettempdir(), 'appveyor_run_with_env.cmd') + if not os.path.exists(run_with_env): + with open(run_with_env, 'wb') as f: + request = urllib2.urlopen('https://github.com/ogrisel/python-appveyor-demo/raw/09a1c8672e5015a74d8f69d07add6ee803c176ec/appveyor/run_with_env.cmd') + f.write(request.read()) + + def shell(args, env=None, cwd=None): + # print the command executing for the logs + print('+ ' + ' '.join(args)) + args = ['cmd', '/E:ON', '/V:ON', '/C', run_with_env] + args + return subprocess.check_call(' '.join(args), env=env, cwd=cwd) + + PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'path']) + python_configurations = [ + PythonConfiguration(version='2.7.x', arch="32", path='C:\Python27'), + PythonConfiguration(version='2.7.x', arch="64", path='C:\Python27-x64'), + PythonConfiguration(version='3.3.x', arch="32", path='C:\Python33'), + PythonConfiguration(version='3.3.x', arch="64", path='C:\Python33-x64'), + PythonConfiguration(version='3.4.x', arch="32", path='C:\Python34'), + PythonConfiguration(version='3.4.x', arch="64", path='C:\Python34-x64'), + PythonConfiguration(version='3.5.x', arch="32", path='C:\Python35'), + PythonConfiguration(version='3.5.x', arch="64", path='C:\Python35-x64'), + PythonConfiguration(version='3.6.x', arch="32", path='C:\Python36'), + PythonConfiguration(version='3.6.x', arch="64", path='C:\Python36-x64'), + ] + + for config in python_configurations: + env = os.environ.copy() + # set up environment variables for run_with_env + env['PYTHON_VERSION'] = config.version + env['PYTHON_ARCH'] = config.arch + env['PATH'] = os.pathsep.join([ + config.path, + os.path.join(config.path, 'Scripts'), + env['PATH'] + ]) + + # for the logs - check we're running the right version of python + shell(['python', '--version'], env=env) + shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)\"'], env=env) + + # prepare the Python environment + shell(['pip', 'install', '--disable-pip-version-check', '--user', '--upgrade', 'pip'], env=env) + shell(['pip', 'install', 'wheel'], env=env) + + # build the wheel + shell(['pip', 'wheel', project_dir, '-w', output_dir], env=env) + + # install the wheel + shell(['pip', 'install', package_name, '--no-index', '-f', output_dir], env=env) + + # test the wheel + if test_requires: + shell(['pip', 'install'] + test_requires, env=env) + if test_command: + # run the tests from c:\, with an absolute path in the command + # (this ensures that Python runs the tests against the installed wheel + # and not the repo code) + abs_project_dir = os.path.abspath(project_dir) + test_command_absolute = test_command.format(project=abs_project_dir) + shell([test_command_absolute], cwd='c:\\', env=env) + + +def macos_build(project_dir, package_name, output_dir, test_command, test_requires): + PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'url']) + python_configurations = [ + PythonConfiguration(version='2.7', url='https://www.python.org/ftp/python/2.7.13/python-2.7.13-macosx10.6.pkg'), + PythonConfiguration(version='3.4', url='https://www.python.org/ftp/python/3.4.4/python-3.4.4-macosx10.6.pkg'), + PythonConfiguration(version='3.5', url='https://www.python.org/ftp/python/3.5.3/python-3.5.3-macosx10.6.pkg'), + PythonConfiguration(version='3.6', url='https://www.python.org/ftp/python/3.6.0/python-3.6.0-macosx10.6.pkg'), + ] + + def shell(args, env=None, cwd=None): + # print the command executing for the logs + print('+ ' + ' '.join(shlex_quote(a) for a in args)) + return subprocess.check_call(args, env=env, cwd=cwd) + + for config in python_configurations: + # download the pkg + shell(['curl', '-L', '-o', '/tmp/Python.pkg', config.url]) + # install + shell(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/']) + + env = os.environ.copy() + env['PATH'] = os.pathsep.join([ + '/Library/Frameworks/Python.framework/Versions/%s/bin' % config.version, + env['PATH'], + ]) + + python = 'python3' if config.version[0] == '3' else 'python2' + pip = 'pip3' if config.version[0] == '3' else 'pip2' + + # check what version we're on + shell(['which', python], env=env) + shell([python, '--version'], env=env) + + # install pip & wheel + shell([python, '-m', 'ensurepip', '--upgrade'], env=env) + shell(['which', pip], env=env) # todo: remove + shell([pip, '--version']) + shell([pip, 'install', 'wheel'], env=env) + shell([pip, 'install', 'delocate'], env=env) + + # build the wheel to temp dir + temp_wheel_dir = '/tmp/tmpwheel%s' % config.version + shell([pip, 'wheel', project_dir, '-w', temp_wheel_dir], env=env) + temp_wheel = glob(temp_wheel_dir+'/*.whl')[0] + + # list the dependencies + shell(['delocate-listdeps', temp_wheel], env=env) + # rebuild the wheel with shared libraries included and place in output dir + shell(['delocate-wheel', '-w', output_dir, temp_wheel], env=env) + + # install the wheel + shell([pip, 'install', package_name, '--no-index', '--find-links', output_dir], env=env) + + # test the wheel + if test_requires: + shell([pip, '-v', 'install'] + test_requires, env=env) + if test_command: + # run the tests from $HOME, with an absolute path in the command + # (this ensures that Python runs the tests against the installed wheel + # and not the repo code) + abs_project_dir = os.path.abspath(project_dir) + test_command_absolute = test_command.format(project=abs_project_dir) + shell(['which', 'nosetests'], env=env) # todo: remove + shell(['which', 'nosetests'], cwd=os.environ['HOME'], env=env) # todo: remove + shell(['nosetests', '-v'], env=env) # todo: remove + shell(['nosetests', '-v'], cwd=os.environ['HOME'], env=env) # todo: remove + shell(shlex.split(test_command_absolute), cwd=os.environ['HOME'], env=env) + + +def main(): + parser = argparse.ArgumentParser( + description='Build wheels for all the platforms.', + epilog=('Most options are supplied via environment variables. ' + 'See https://github.com/joerick/cibuildwheel#options for info.')) + + parser.add_argument('--platform', + choices=['auto', 'linux', 'macos', 'windows'], + default='auto', + help=('Platform to build for. For "linux" you need docker running, on Mac ' + 'or Linux. For "macos", you need a Mac machine, and note that this ' + 'script is going to automatically install MacPython on your system, ' + 'so don\'t run on your development machine. For "windows", you need to ' + 'run in Windows, and it will build and test for all versions of ' + 'Python at C:\\PythonXX[-x64]. Default: auto.')) + parser.add_argument('--output-dir', + default=os.environ.get('CIBW_OUTPUT_DIR', 'wheelhouse'), + help='Destination folder for the wheels.') + parser.add_argument('project_dir', + default='.', + nargs='?', + help=('Path to the project that you want wheels for. Default: the current ' + 'directory.')) + + args = parser.parse_args() + + output_dir = args.output_dir + test_command = os.environ.get('CIBW_TEST_COMMAND', None) + test_requires = os.environ.get('CIBW_TEST_REQUIRES', '').split() + project_dir = args.project_dir + + try: + project_setup_py = os.path.join(project_dir, 'setup.py') + package_name = subprocess.check_output([sys.executable, project_setup_py, '--name']) + except subprocess.CalledProcessError as err: + if not os.path.exists(project_setup_py): + print('cibuildwheel: Could not find setup.py at root of project', file=sys.stderr) + exit(2) + else: + print('cibuildwheel: Failed to get name of the package. Command was %s' % err.cmd, + file=sys.stderr) + exit(2) + + package_name = package_name.strip() + + if package_name == '' or package_name == 'UNKNOWN': + print('cibuildwheel: Invalid package name "%s". Check your setup.py' % package_name, + file=sys.stderr) + exit(2) + + build_args = dict( + project_dir=project_dir, + package_name=package_name, + output_dir=output_dir, + test_command=test_command, + test_requires=test_requires + ) + + if args.platform != 'auto': + platform = args.platform + else: + if os.environ.get('TRAVIS_OS_NAME') == 'linux': + platform = 'linux' + elif os.environ.get('TRAVIS_OS_NAME') == 'osx': + platform = 'macos' + elif 'APPVEYOR' in os.environ: + platform = 'windows' + else: + print('Unable to detect platform. cibuildwheel should run on your CI server, ' + 'Travis CI and Appveyor are supported. You can run on your development ' + 'machine using the --platform argument. Check --help output for more ' + 'information.', + file=sys.stderr) + exit(2) + + if platform == 'linux': + linux_build(**build_args) + elif platform == 'windows': + windows_build(**build_args) + elif platform == 'macos': + macos_build(**build_args) + else: + print('cibuildwheel: Unsupported platform %s' % platform, file=sys.stderr) + exit(1) + +if __name__ == '__main__': + main() diff --git a/cibuildwheel/__init__.py b/cibuildwheel/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py new file mode 100644 index 00000000..c638e233 --- /dev/null +++ b/cibuildwheel/__main__.py @@ -0,0 +1,292 @@ +from __future__ import print_function +import argparse, os, subprocess, sys, tempfile, shlex +from collections import namedtuple +from glob import glob + +try: + # Python 3 + import urllib.request as urllib2 +except ImportError: + # Python 2 + import urllib2 + +try: + from shlex import quote as shlex_quote +except ImportError: + from pipes import quote as shlex_quote + + +def linux_build(project_dir, package_name, output_dir, test_command, test_requires): + for docker_image in ['quay.io/pypa/manylinux1_x86_64', 'quay.io/pypa/manylinux1_i686']: + bash_script = ''' + set -o errexit + set -o xtrace + cd /project + + for PYBIN in /opt/python/*/bin; do + "$PYBIN/pip" wheel . -w /tmp/linux_wheels + done + + for whl in /tmp/linux_wheels/*.whl; do + auditwheel repair "$whl" -w /output + done + + # Install packages and test + for PYBIN in /opt/python/*/bin/; do + # Install the wheel we just built + "$PYBIN/pip" install {package_name} --no-index -f /output + + # Install any requirements to run the tests + if [ ! -z "{test_requires}" ]; then + "$PYBIN/pip" install {test_requires} + fi + + # Run the tests from a different directory + if [ ! -z {test_command} ]; then + (cd "$HOME" && export PATH=$PYBIN:$PATH && sh -c {test_command}) + fi + done + '''.format( + package_name=package_name, + test_requires=' '.join(test_requires), + test_command=shlex_quote(test_command.format(project='/project') if test_command else ''), + ) + + docker_process = subprocess.Popen([ + 'docker', + 'run', + '--rm', + '-i', + '-v', '%s:/project' % os.path.abspath(project_dir), + '-v', '%s:/output' % os.path.abspath(output_dir), + docker_image, + '/bin/bash'], + stdin=subprocess.PIPE) + + docker_process.communicate(bash_script) + + if docker_process.returncode != 0: + exit(1) + + +def windows_build(project_dir, package_name, output_dir, test_command, test_requires): + # run_with_env is a cmd file that sets the right environment variables to + run_with_env = os.path.join(tempfile.gettempdir(), 'appveyor_run_with_env.cmd') + if not os.path.exists(run_with_env): + with open(run_with_env, 'wb') as f: + request = urllib2.urlopen('https://github.com/ogrisel/python-appveyor-demo/raw/09a1c8672e5015a74d8f69d07add6ee803c176ec/appveyor/run_with_env.cmd') + f.write(request.read()) + + def shell(args, env=None, cwd=None): + # print the command executing for the logs + print('+ ' + ' '.join(args)) + args = ['cmd', '/E:ON', '/V:ON', '/C', run_with_env] + args + return subprocess.check_call(' '.join(args), env=env, cwd=cwd) + + PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'path']) + python_configurations = [ + PythonConfiguration(version='2.7.x', arch="32", path='C:\Python27'), + PythonConfiguration(version='2.7.x', arch="64", path='C:\Python27-x64'), + PythonConfiguration(version='3.3.x', arch="32", path='C:\Python33'), + PythonConfiguration(version='3.3.x', arch="64", path='C:\Python33-x64'), + PythonConfiguration(version='3.4.x', arch="32", path='C:\Python34'), + PythonConfiguration(version='3.4.x', arch="64", path='C:\Python34-x64'), + PythonConfiguration(version='3.5.x', arch="32", path='C:\Python35'), + PythonConfiguration(version='3.5.x', arch="64", path='C:\Python35-x64'), + PythonConfiguration(version='3.6.x', arch="32", path='C:\Python36'), + PythonConfiguration(version='3.6.x', arch="64", path='C:\Python36-x64'), + ] + + for config in python_configurations: + env = os.environ.copy() + # set up environment variables for run_with_env + env['PYTHON_VERSION'] = config.version + env['PYTHON_ARCH'] = config.arch + env['PATH'] = os.pathsep.join([ + config.path, + os.path.join(config.path, 'Scripts'), + env['PATH'] + ]) + + # for the logs - check we're running the right version of python + shell(['python', '--version'], env=env) + shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)\"'], env=env) + + # prepare the Python environment + shell(['pip', 'install', '--disable-pip-version-check', '--user', '--upgrade', 'pip'], env=env) + shell(['pip', 'install', 'wheel'], env=env) + + # build the wheel + shell(['pip', 'wheel', project_dir, '-w', output_dir], env=env) + + # install the wheel + shell(['pip', 'install', package_name, '--no-index', '-f', output_dir], env=env) + + # test the wheel + if test_requires: + shell(['pip', 'install'] + test_requires, env=env) + if test_command: + # run the tests from c:\, with an absolute path in the command + # (this ensures that Python runs the tests against the installed wheel + # and not the repo code) + abs_project_dir = os.path.abspath(project_dir) + test_command_absolute = test_command.format(project=abs_project_dir) + shell([test_command_absolute], cwd='c:\\', env=env) + + +def macos_build(project_dir, package_name, output_dir, test_command, test_requires): + PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'url']) + python_configurations = [ + PythonConfiguration(version='2.7', url='https://www.python.org/ftp/python/2.7.13/python-2.7.13-macosx10.6.pkg'), + PythonConfiguration(version='3.4', url='https://www.python.org/ftp/python/3.4.4/python-3.4.4-macosx10.6.pkg'), + PythonConfiguration(version='3.5', url='https://www.python.org/ftp/python/3.5.3/python-3.5.3-macosx10.6.pkg'), + PythonConfiguration(version='3.6', url='https://www.python.org/ftp/python/3.6.0/python-3.6.0-macosx10.6.pkg'), + ] + + def shell(args, env=None, cwd=None): + # print the command executing for the logs + print('+ ' + ' '.join(shlex_quote(a) for a in args)) + return subprocess.check_call(args, env=env, cwd=cwd) + + for config in python_configurations: + # download the pkg + shell(['curl', '-L', '-o', '/tmp/Python.pkg', config.url]) + # install + shell(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/']) + + env = os.environ.copy() + env['PATH'] = os.pathsep.join([ + '/Library/Frameworks/Python.framework/Versions/%s/bin' % config.version, + env['PATH'], + ]) + + python = 'python3' if config.version[0] == '3' else 'python2' + pip = 'pip3' if config.version[0] == '3' else 'pip2' + + # check what version we're on + shell(['which', python], env=env) + shell([python, '--version'], env=env) + + # install pip & wheel + shell([python, '-m', 'ensurepip', '--upgrade'], env=env) + shell(['which', pip], env=env) # todo: remove + shell([pip, '--version']) + shell([pip, 'install', 'wheel'], env=env) + shell([pip, 'install', 'delocate'], env=env) + + # build the wheel to temp dir + temp_wheel_dir = '/tmp/tmpwheel%s' % config.version + shell([pip, 'wheel', project_dir, '-w', temp_wheel_dir], env=env) + temp_wheel = glob(temp_wheel_dir+'/*.whl')[0] + + # list the dependencies + shell(['delocate-listdeps', temp_wheel], env=env) + # rebuild the wheel with shared libraries included and place in output dir + shell(['delocate-wheel', '-w', output_dir, temp_wheel], env=env) + + # install the wheel + shell([pip, 'install', package_name, '--no-index', '--find-links', output_dir], env=env) + + # test the wheel + if test_requires: + shell([pip, '-v', 'install'] + test_requires, env=env) + if test_command: + # run the tests from $HOME, with an absolute path in the command + # (this ensures that Python runs the tests against the installed wheel + # and not the repo code) + abs_project_dir = os.path.abspath(project_dir) + test_command_absolute = test_command.format(project=abs_project_dir) + shell(['which', 'nosetests'], env=env) # todo: remove + shell(['which', 'nosetests'], cwd=os.environ['HOME'], env=env) # todo: remove + shell(['nosetests', '-v'], env=env) # todo: remove + shell(['nosetests', '-v'], cwd=os.environ['HOME'], env=env) # todo: remove + shell(shlex.split(test_command_absolute), cwd=os.environ['HOME'], env=env) + + +def main(): + parser = argparse.ArgumentParser( + description='Build wheels for all the platforms.', + epilog=('Most options are supplied via environment variables. ' + 'See https://github.com/joerick/cibuildwheel#options for info.')) + + parser.add_argument('--platform', + choices=['auto', 'linux', 'macos', 'windows'], + default='auto', + help=('Platform to build for. For "linux" you need docker running, on Mac ' + 'or Linux. For "macos", you need a Mac machine, and note that this ' + 'script is going to automatically install MacPython on your system, ' + 'so don\'t run on your development machine. For "windows", you need to ' + 'run in Windows, and it will build and test for all versions of ' + 'Python at C:\\PythonXX[-x64]. Default: auto.')) + parser.add_argument('--output-dir', + default=os.environ.get('CIBW_OUTPUT_DIR', 'wheelhouse'), + help='Destination folder for the wheels.') + parser.add_argument('project_dir', + default='.', + nargs='?', + help=('Path to the project that you want wheels for. Default: the current ' + 'directory.')) + + args = parser.parse_args() + + output_dir = args.output_dir + test_command = os.environ.get('CIBW_TEST_COMMAND', None) + test_requires = os.environ.get('CIBW_TEST_REQUIRES', '').split() + project_dir = args.project_dir + + try: + project_setup_py = os.path.join(project_dir, 'setup.py') + package_name = subprocess.check_output([sys.executable, project_setup_py, '--name']) + except subprocess.CalledProcessError as err: + if not os.path.exists(project_setup_py): + print('cibuildwheel: Could not find setup.py at root of project', file=sys.stderr) + exit(2) + else: + print('cibuildwheel: Failed to get name of the package. Command was %s' % err.cmd, + file=sys.stderr) + exit(2) + + package_name = package_name.strip() + + if package_name == '' or package_name == 'UNKNOWN': + print('cibuildwheel: Invalid package name "%s". Check your setup.py' % package_name, + file=sys.stderr) + exit(2) + + build_args = dict( + project_dir=project_dir, + package_name=package_name, + output_dir=output_dir, + test_command=test_command, + test_requires=test_requires + ) + + if args.platform != 'auto': + platform = args.platform + else: + if os.environ.get('TRAVIS_OS_NAME') == 'linux': + platform = 'linux' + elif os.environ.get('TRAVIS_OS_NAME') == 'osx': + platform = 'macos' + elif 'APPVEYOR' in os.environ: + platform = 'windows' + else: + print('Unable to detect platform. cibuildwheel should run on your CI server, ' + 'Travis CI and Appveyor are supported. You can run on your development ' + 'machine using the --platform argument. Check --help output for more ' + 'information.', + file=sys.stderr) + exit(2) + + if platform == 'linux': + linux_build(**build_args) + elif platform == 'windows': + windows_build(**build_args) + elif platform == 'macos': + macos_build(**build_args) + else: + print('cibuildwheel: Unsupported platform %s' % platform, file=sys.stderr) + exit(1) + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..03f77a8e --- /dev/null +++ b/setup.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +try: + from setuptools import setup +except ImportError: + from distutils.core import setup + +setup( + name='cibuildwheel', + version='0.1.0', + description="Build Python wheels on CI with minimal configuration.", + long_description='For readme please see http://github.com/joerick/cibuildwheel', + author="Joe Rickerby", + author_email='joerick@mac.com', + url='https://github.com/joerick/cibuildwheel', + packages=['cibuildwheel',], + license="BSD", + zip_safe=False, + keywords='ci wheel packaging pypi travis appveyor macos linux windows', + classifiers=[ + 'Intended Audience :: Developers', + 'Natural Language :: English', + 'Programming Language :: Python :: 2', + 'Programming Language :: Python :: 3', + 'Development Status :: 4 - Beta', + 'License :: OSI Approved :: BSD License', + 'Programming Language :: Python :: Implementation :: CPython', + 'Topic :: Software Development :: Build Tools', + ], + entry_points={ + 'console_scripts': [ + 'cibuildwheel = cibuildwheel.__main__:main', + ], + }, +)