Add CircleCI support (#91)

* Initial guess at supporting CircleCI joerick/cibuildwheel#90 * Add note about setup_remote_docker on Circle * Add diagnostic print of entire docker command * Add more diagnostic prints * Retrigger for Circle build * Preliminary .circleci/config.yml * Add workflows to Circle * drop osx on circle * call python3 directly on circle * drop repeat pip install * use a virtualenv on circle * use a virtualenv on circle... again * use a virtualenv on circle... again... again * setup_remote_docker on circle * try to manually copy into docker on circle * correct Host * correct docker calls * refactor docker calls * take 2 on the docker cp destination * more docker cp debugging * more docker cp debugging... again * docker cp explicitly to /project * wait until docker container is up * wait until docker container is up... again * and copy results back * and copy results back... again * create /output in container * create /output in container... again * copy contents of container's /output * Retrigger ci * fix merge typo * fix monotonic * always install monotonic * always install monotonic... again * correct to use project_dir * wait a bit longer and a bit smarter * wait a bit longer and a bit smarter... again * add back circle osx * add back circle osx... again * py2/3 for circle * py2/3 for circle... again * use python2.7 image on circle linux * (mostly) remove unused ci host identification code * skip inspecting docker for the container, just retry copy on failure * cleanup docker container and images * fully remove ci host detection * docs and tidy * add circle badge
2018-09-05 09:15:32 +01:00
parent 7c6b331e15
commit 9f7fdbf456
6 changed files with 235 additions and 26 deletions
@@ -53,6 +53,8 @@ def main():
    if args.platform != 'auto':
        platform = args.platform
    else:
+        platform = None
+
        if os.environ.get('TRAVIS_OS_NAME') == 'linux':
            platform = 'linux'
        elif os.environ.get('TRAVIS_OS_NAME') == 'osx':
@@ -61,9 +63,15 @@ def main():
            platform = 'windows'
        elif 'BITRISE_BUILD_NUMBER' in os.environ:
            platform = 'macos'
-        else:
+        elif os.environ.get('CIRCLECI'):
+            if sys.platform.startswith('linux'):
+                platform = 'linux'
+            elif sys.platform.startswith('darwin'):
+                platform = 'macos'
+
+        if platform is None:
            print('cibuildwheel: Unable to detect platform. cibuildwheel should run on your CI server, '
-                  'Travis CI and Appveyor are supported. You can run on your development '
+                  'Travis CI, Appveyor, and CircleCI are supported. You can run on your development '
                  'machine using the --platform argument. Check --help output for more '
                  'information.',
                  file=sys.stderr)
@@ -1,20 +1,27 @@
 from __future__ import print_function
-import os, subprocess, sys
+import os, subprocess, sys, time, uuid
 from collections import namedtuple
 from .util import prepare_command, get_build_verbosity_extra_flags

+import monotonic
+
 try:
    from shlex import quote as shlex_quote
 except ImportError:
    from pipes import quote as shlex_quote


+class DockerRunTimeoutError(Exception):
+    pass
+
+
 def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, build_verbosity, skip, environment, manylinux1_images):
    try:
        subprocess.check_call(['docker', '--version'])
    except:
        print('cibuildwheel: Docker not found. Docker is required to run Linux builds. '
-              'If you\'re building on Travis CI, add `services: [docker]` to your .travis.yml.',
+              'If you\'re building on Travis CI, add `services: [docker]` to your .travis.yml.'
+              'If you\'re building on Circle CI in Linux, add a `setup_remote_docker` step to your .circleci/config.yml',
              file=sys.stderr)
        exit(2)

@@ -50,6 +57,7 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be
        bash_script = '''
            set -o errexit
            set -o xtrace
+            mkdir /output
            cd /project

            {environment_exports}
@@ -65,6 +73,12 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be
                    PATH="$PYBIN:$PATH" sh -c {before_build}
                fi

+                pwd
+                ls -l .
+                ls -l /project
+                ls -l /output
+                ls -l /host
+
                # Build that wheel
                PATH="$PYBIN:$PATH" "$PYBIN/pip" wheel . -w /tmp/built_wheel --no-deps {build_verbosity_flag}
                built_wheel=(/tmp/built_wheel/*.whl)
@@ -115,19 +129,49 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be
            gid=os.getgid(),
        )

-        docker_process = subprocess.Popen([
+        container_name = 'cibuildwheel-{}'.format(uuid.uuid4())
+
+        command = [
+            'docker',
+            'run',
+            '--env',
+            'CIBUILDWHEEL',
+            '--name', container_name,
+            '-i',
+            '-v', '/:/host', # ignored on Circle
+            docker_image,
+            '/bin/bash',
+        ]
+        print('docker command: {}'.format(command))
+        docker_process = subprocess.Popen(
+            command,
+            stdin=subprocess.PIPE,
+            universal_newlines=True,
+        )
+
+        # It will take a bit for docker to get the container up and
+        # running.  Keep trying to copy in the project directory until
+        # it succeeds.  There is a timeout to avoid spinning forever.
+
+        timeout = 180
+        start = monotonic.monotonic()
+        while True:
+            time.sleep(1)
+            command = [
                'docker',
-                'run',
-                '--env',
-                'CIBUILDWHEEL',
-                '--rm',
-                '-i',
-                '-v', '%s:/project' % os.path.abspath(project_dir),
-                '-v', '%s:/output' % os.path.abspath(output_dir),
-                '-v', '/:/host',
-                docker_image,
-                '/bin/bash'],
-            stdin=subprocess.PIPE, universal_newlines=True)
+                'cp',
+                '{}/.'.format(os.path.abspath(project_dir)),
+                '{}:/project'.format(container_name),
+            ]
+            print('docker command: {}'.format(command))
+            if subprocess.call(command) == 0:
+                break
+
+            if monotonic.monotonic() - start > timeout:
+                raise DockerRunTimeoutError(
+                    'Unable to successfully copy project directory'
+                    ' within {} seconds'.format(timeout)
+                )

        try:
            docker_process.communicate(bash_script)
@@ -135,5 +179,22 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be
            docker_process.kill()
            docker_process.wait()

+        command = [
+            'docker',
+            'cp',
+            '{}:/output/.'.format(container_name),
+            os.path.abspath(output_dir),
+        ]
+        print('docker command: {}'.format(command))
+        subprocess.check_call(command)
+
+        command = [
+            'docker',
+            'rm',
+            '-v', container_name,
+        ]
+        print('docker command: {}'.format(command))
+        subprocess.check_call(command)
+
        if docker_process.returncode != 0:
            exit(1)