From 9f7fdbf456a78adc413013d6d8b36f5eb2119c75 Mon Sep 17 00:00:00 2001 From: Kyle Altendorf Date: Wed, 5 Sep 2018 04:15:32 -0400 Subject: [PATCH] Add CircleCI support (#91) * Initial guess at supporting CircleCI joerick/cibuildwheel#90 * Add note about setup_remote_docker on Circle * Add diagnostic print of entire docker command * Add more diagnostic prints * Retrigger for Circle build * Preliminary .circleci/config.yml * Add workflows to Circle * drop osx on circle * call python3 directly on circle * drop repeat pip install * use a virtualenv on circle * use a virtualenv on circle... again * use a virtualenv on circle... again... again * setup_remote_docker on circle * try to manually copy into docker on circle * correct Host * correct docker calls * refactor docker calls * take 2 on the docker cp destination * more docker cp debugging * more docker cp debugging... again * docker cp explicitly to /project * wait until docker container is up * wait until docker container is up... again * and copy results back * and copy results back... again * create /output in container * create /output in container... again * copy contents of container's /output * Retrigger ci * fix merge typo * fix monotonic * always install monotonic * always install monotonic... again * correct to use project_dir * wait a bit longer and a bit smarter * wait a bit longer and a bit smarter... again * add back circle osx * add back circle osx... again * py2/3 for circle * py2/3 for circle... again * use python2.7 image on circle linux * (mostly) remove unused ci host identification code * skip inspecting docker for the container, just retry copy on failure * cleanup docker container and images * fully remove ci host detection * docs and tidy * add circle badge --- .circleci/config.yml | 81 ++++++++++++++++++++++++++++++++++++ .circleci/prepare.sh | 7 ++++ README.md | 67 ++++++++++++++++++++++++++---- cibuildwheel/__main__.py | 12 +++++- cibuildwheel/linux.py | 89 +++++++++++++++++++++++++++++++++------- setup.py | 5 ++- 6 files changed, 235 insertions(+), 26 deletions(-) create mode 100644 .circleci/config.yml create mode 100644 .circleci/prepare.sh diff --git a/.circleci/config.yml b/.circleci/config.yml new file mode 100644 index 00000000..a0398cab --- /dev/null +++ b/.circleci/config.yml @@ -0,0 +1,81 @@ +version: 2 + +jobs: + osx-python2: + macos: + xcode: "10.0.0" + environment: + PYTHON: python2 + steps: + - checkout + + - run: + name: Prepare the environment. + command: | + bash .circleci/prepare.sh + - run: + name: Test. + command: | + venv/bin/python ./bin/run_tests.py + + osx-python3: + macos: + xcode: "10.0.0" + environment: + PYTHON: python3 + steps: + - checkout + + - run: + name: Prepare the environment. + command: | + bash .circleci/prepare.sh + - run: + name: Test. + command: | + venv/bin/python ./bin/run_tests.py + + linux-python2: + docker: + - image: circleci/python:2.7 + environment: + PYTHON: python2 + steps: + - checkout + - setup_remote_docker + + - run: + name: Prepare the environment. + command: | + bash .circleci/prepare.sh + - run: + name: Test. + command: | + venv/bin/python ./bin/run_tests.py + + linux-python3: + docker: + - image: circleci/python:3.6 + environment: + PYTHON: python3 + steps: + - checkout + - setup_remote_docker + + - run: + name: Prepare the environment. + command: | + bash .circleci/prepare.sh + - run: + name: Test. + command: | + venv/bin/python ./bin/run_tests.py + +workflows: + version: 2 + all-tests: + jobs: + - osx-python2 + - osx-python3 + - linux-python2 + - linux-python3 diff --git a/.circleci/prepare.sh b/.circleci/prepare.sh new file mode 100644 index 00000000..dfd3aeef --- /dev/null +++ b/.circleci/prepare.sh @@ -0,0 +1,7 @@ +$PYTHON --version +$PYTHON -m pip --version +$PYTHON -m pip install -q --user --ignore-installed --upgrade virtualenv +$PYTHON -m virtualenv -p $PYTHON venv +venv/bin/python -m pip install -r requirements-dev.txt +venv/bin/python -m pip freeze +venv/bin/python --version diff --git a/README.md b/README.md index 113b9384..88c17125 100644 --- a/README.md +++ b/README.md @@ -1,11 +1,11 @@ cibuildwheel ============ -[![PyPI](https://img.shields.io/pypi/v/cibuildwheel.svg)](https://pypi.python.org/pypi/cibuildwheel) [![Build Status](https://travis-ci.org/joerick/cibuildwheel.svg?branch=master)](https://travis-ci.org/joerick/cibuildwheel) [![Build status](https://ci.appveyor.com/api/projects/status/wbsgxshp05tt1tif/branch/master?svg=true)](https://ci.appveyor.com/project/joerick/cibuildwheel/branch/master) +[![PyPI](https://img.shields.io/pypi/v/cibuildwheel.svg)](https://pypi.python.org/pypi/cibuildwheel) [![Build Status](https://travis-ci.org/joerick/cibuildwheel.svg?branch=master)](https://travis-ci.org/joerick/cibuildwheel) [![Build status](https://ci.appveyor.com/api/projects/status/wbsgxshp05tt1tif/branch/master?svg=true)](https://ci.appveyor.com/project/joerick/cibuildwheel/branch/master) [![CircleCI](https://circleci.com/gh/joerick/cibuildwheel.svg?style=svg)](https://circleci.com/gh/joerick/cibuildwheel) Python wheels are great. Building them across **Mac, Linux, Windows**, on **multiple versions of Python**, is not. -`cibuildwheel` is here to help. `cibuildwheel` runs on your CI server - currently it supports Travis CI and Appveyor - and it builds and tests your wheels across all of your platforms. +`cibuildwheel` is here to help. `cibuildwheel` runs on your CI server - currently it supports Travis CI, Appveyor, and Circle CI - and it builds and tests your wheels across all of your platforms. **`cibuildwheel` is in beta**. It's brand new - I'd love for you to try it and help make it better! @@ -23,16 +23,16 @@ What does it do? [⚠️=deprecated] -- Builds manylinux, macOS and Windows (32 and 64bit) wheels using Travis CI and Appveyor +- Builds manylinux, macOS and Windows (32 and 64bit) wheels using Travis CI, Appveyor, and CircleCI - Bundles shared library dependencies on Linux and macOS through [auditwheel](https://github.com/pypa/auditwheel) and [delocate](https://github.com/matthew-brett/delocate) - Runs the library test suite against the wheel-installed version of your library Usage ----- -`cibuildwheel` currently works on **Travis CI** to build Linux and Mac wheels, and **Appveyor** to build Windows wheels. +`cibuildwheel` currently works on **Travis CI** and **Circle CI** to build Linux and Mac wheels, and **Appveyor** to build Windows wheels. -`cibuildwheel` is not intended to run on your development machine. It will try to install packages globally; this is no good. Travis CI and Appveyor run their builds in isolated environments, so are ideal for this kind of script. +`cibuildwheel` is not intended to run on your development machine. It will try to install packages globally; this is no good. Travis CI, Circle CI, and Appveyor run their builds in isolated environments, so are ideal for this kind of script. ### Minimal setup @@ -58,6 +58,55 @@ Usage Then setup a deployment method by following the [Travis CI deployment docs](https://docs.travis-ci.com/user/deployment/), or see [Delivering to PyPI](#delivering-to-pypi) below. +- Create a `.circleci/config.yml` file in your repo, + + ``` + version: 2 + + jobs: + linux-wheels: + working_directory: ~/linux-wheels + docker: + - image: circleci/python:3.6 + steps: + - checkout + - setup_remote_docker + + - run: + name: Build the Linux wheels. + command: | + pip install --user cibuildwheel + cibuildwheel --output-dir wheelhouse + + - store_artifacts: + path: wheelhouse/ + + osx-wheels: + working_directory: ~/osx-wheels + macos: + xcode: "10.0.0" + steps: + - checkout + + - run: + name: Build the OS X wheels. + command: | + pip install --user cibuildwheel + cibuildwheel --output-dir wheelhouse + + - store_artifacts: + path: wheelhouse/ + + workflows: + version: 2 + all-tests: + jobs: + - linux-wheels + - osx-wheels + ``` + + Circle CI will store the built wheels for you - you can access them from the project console. + - Create an `appveyor.yml` file in your repo. ``` @@ -99,7 +148,7 @@ A more detailed description of the options, the allowed values, and some example Linux wheels are built in the [`manylinux1` docker images](https://github.com/pypa/manylinux) to provide binary compatible wheels on Linux, according to [PEP 513](https://www.python.org/dev/peps/pep-0513/). Because of this, when building with `cibuildwheel` on Linux, a few things should be taken into account: - Programs and libraries cannot be installed on the Travis CI Ubuntu host with `apt-get`, but can be installed inside of the Docker image using `yum` or manually. The same goes for environment variables that are potentially needed to customize the wheel building. `cibuildwheel` supports this by providing the `CIBW_ENVIRONMENT` and `CIBW_BEFORE_BUILD` options to setup the build environment inside the running Docker image. See [below](#options) for details on these options. -- The project directory is mounted in the running Docker instance as `/project`, the output directory for the wheels as `/output`. In general, this is handled transparently by `cibuildwheel`. For a more finegrained level of control however, the root of the host file system is mounted as `/host`, allowing for example to access shared files, caches, etc. on the host file system. +- The project directory is mounted in the running Docker instance as `/project`, the output directory for the wheels as `/output`. In general, this is handled transparently by `cibuildwheel`. For a more finegrained level of control however, the root of the host file system is mounted as `/host`, allowing for example to access shared files, caches, etc. on the host file system. Note that this is not available on CircleCI due to their Docker policies. - Alternative dockers images can be specified with the `CIBW_MANYLINUX1_X86_64_IMAGE` and `CIBW_MANYLINUX1_I686_IMAGE` options to allow for a custom, preconfigured build environment for the Linux builds. See [below](#options) for more details. @@ -133,7 +182,7 @@ optional arguments: ``` -Most of the config is via environment variables. These go into `.travis.yml` and `appveyor.yml` nicely. +Most of the config is via environment variables. These go into `.travis.yml`, `appveyor.yml`, and `.circleci/config.yml` nicely. *** @@ -144,7 +193,7 @@ Options: `auto` `linux` `macos` `windows` Default: `auto` -`auto` will auto-detect platform using environment variables, such as `TRAVIS_OS_NAME`/`APPVEYOR`. +`auto` will auto-detect platform using environment variables, such as `TRAVIS_OS_NAME`/`APPVEYOR`/`CIRCLECI`. For `linux` you need Docker running, on Mac or Linux. For `macos`, you need a Mac machine, and note that this script is going to automatically install MacPython on your system, so don't run on your development machine. For `windows`, you need to run in Windows, and it will build and test for all versions of Python at `C:\PythonXX[-x64]`. @@ -331,7 +380,7 @@ It didn't work! If your wheel didn't compile, check the list below for some debugging tips. - A mistake in your config. To quickly test your config without doing a git push and waiting for your code to build on CI, you can run the Linux build in a Docker container. On Mac or Linux, with Docker running, try `cibuildwheel --platform linux`. You'll have to bring your config into the current environment first. -- Missing dependency. You might need to install something on the build machine. You can do this in `.travis.yml` or `appveyor.yml`, with apt-get, brew or whatever Windows uses :P . Given how the Linux build works, we'll probably have to build something into `cibuildwheel`. Let's chat about that over in the issues! +- Missing dependency. You might need to install something on the build machine. You can do this in `.travis.yml`, `appveyor.yml`, or `.circleci/config.yml`, with apt-get, brew or whatever Windows uses :P . Given how the Linux build works, we'll probably have to build something into `cibuildwheel`. Let's chat about that over in the issues! - Windows: missing C feature. The Windows C compiler doesn't support C language features invented after 1990, so you'll have to backport your C code to C90. For me, this mostly involved putting my variable declarations at the top of the function like an animal. Working examples diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index e1132100..dc00ce54 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -53,6 +53,8 @@ def main(): if args.platform != 'auto': platform = args.platform else: + platform = None + if os.environ.get('TRAVIS_OS_NAME') == 'linux': platform = 'linux' elif os.environ.get('TRAVIS_OS_NAME') == 'osx': @@ -61,9 +63,15 @@ def main(): platform = 'windows' elif 'BITRISE_BUILD_NUMBER' in os.environ: platform = 'macos' - else: + elif os.environ.get('CIRCLECI'): + if sys.platform.startswith('linux'): + platform = 'linux' + elif sys.platform.startswith('darwin'): + platform = 'macos' + + if platform is None: print('cibuildwheel: Unable to detect platform. cibuildwheel should run on your CI server, ' - 'Travis CI and Appveyor are supported. You can run on your development ' + 'Travis CI, Appveyor, and CircleCI are supported. You can run on your development ' 'machine using the --platform argument. Check --help output for more ' 'information.', file=sys.stderr) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index b2b0f44d..662d552d 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -1,20 +1,27 @@ from __future__ import print_function -import os, subprocess, sys +import os, subprocess, sys, time, uuid from collections import namedtuple from .util import prepare_command, get_build_verbosity_extra_flags +import monotonic + try: from shlex import quote as shlex_quote except ImportError: from pipes import quote as shlex_quote +class DockerRunTimeoutError(Exception): + pass + + def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, build_verbosity, skip, environment, manylinux1_images): try: subprocess.check_call(['docker', '--version']) except: print('cibuildwheel: Docker not found. Docker is required to run Linux builds. ' - 'If you\'re building on Travis CI, add `services: [docker]` to your .travis.yml.', + 'If you\'re building on Travis CI, add `services: [docker]` to your .travis.yml.' + 'If you\'re building on Circle CI in Linux, add a `setup_remote_docker` step to your .circleci/config.yml', file=sys.stderr) exit(2) @@ -50,6 +57,7 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be bash_script = ''' set -o errexit set -o xtrace + mkdir /output cd /project {environment_exports} @@ -65,6 +73,12 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be PATH="$PYBIN:$PATH" sh -c {before_build} fi + pwd + ls -l . + ls -l /project + ls -l /output + ls -l /host + # Build that wheel PATH="$PYBIN:$PATH" "$PYBIN/pip" wheel . -w /tmp/built_wheel --no-deps {build_verbosity_flag} built_wheel=(/tmp/built_wheel/*.whl) @@ -115,19 +129,49 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be gid=os.getgid(), ) - docker_process = subprocess.Popen([ + container_name = 'cibuildwheel-{}'.format(uuid.uuid4()) + + command = [ + 'docker', + 'run', + '--env', + 'CIBUILDWHEEL', + '--name', container_name, + '-i', + '-v', '/:/host', # ignored on Circle + docker_image, + '/bin/bash', + ] + print('docker command: {}'.format(command)) + docker_process = subprocess.Popen( + command, + stdin=subprocess.PIPE, + universal_newlines=True, + ) + + # It will take a bit for docker to get the container up and + # running. Keep trying to copy in the project directory until + # it succeeds. There is a timeout to avoid spinning forever. + + timeout = 180 + start = monotonic.monotonic() + while True: + time.sleep(1) + command = [ 'docker', - 'run', - '--env', - 'CIBUILDWHEEL', - '--rm', - '-i', - '-v', '%s:/project' % os.path.abspath(project_dir), - '-v', '%s:/output' % os.path.abspath(output_dir), - '-v', '/:/host', - docker_image, - '/bin/bash'], - stdin=subprocess.PIPE, universal_newlines=True) + 'cp', + '{}/.'.format(os.path.abspath(project_dir)), + '{}:/project'.format(container_name), + ] + print('docker command: {}'.format(command)) + if subprocess.call(command) == 0: + break + + if monotonic.monotonic() - start > timeout: + raise DockerRunTimeoutError( + 'Unable to successfully copy project directory' + ' within {} seconds'.format(timeout) + ) try: docker_process.communicate(bash_script) @@ -135,5 +179,22 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be docker_process.kill() docker_process.wait() + command = [ + 'docker', + 'cp', + '{}:/output/.'.format(container_name), + os.path.abspath(output_dir), + ] + print('docker command: {}'.format(command)) + subprocess.check_call(command) + + command = [ + 'docker', + 'rm', + '-v', container_name, + ] + print('docker command: {}'.format(command)) + subprocess.check_call(command) + if docker_process.returncode != 0: exit(1) diff --git a/setup.py b/setup.py index 11002a93..d906564e 100644 --- a/setup.py +++ b/setup.py @@ -9,7 +9,10 @@ except ImportError: setup( name='cibuildwheel', version='0.9.4', - install_requires=['bashlex'], + install_requires=[ + 'bashlex', + 'monotonic', + ], description="Build Python wheels on CI with minimal configuration.", long_description='For readme please see http://github.com/joerick/cibuildwheel', author="Joe Rickerby",