From 037cda5ef3114dba7066871b253b36aebb9636d3 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 14 Aug 2017 23:19:39 +0100 Subject: [PATCH 01/14] Add Python 3 CI runners --- .travis.yml | 22 ++++++++++++++++++---- appveyor.yml | 9 +++++++-- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 9ac8583b..e4db986f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,17 +1,31 @@ matrix: include: + # Linux - sudo: required services: - docker + env: + - "PYTHON=python2" + - sudo: required + services: + - docker + env: + - "PYTHON=python3" + # macOS - os: osx + env: + - "PYTHON=python2" + - os: osx + env: + - "PYTHON=python3" script: - | if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then - pip install . - python ./run_tests.py + $PYTHON -m pip install . + $PYTHON ./run_tests.py else # linux test requires root to clean up the wheelhouse (docker runs as root) - sudo pip install . - sudo python ./run_tests.py + sudo $PYTHON -m pip install . + sudo $PYTHON ./run_tests.py fi diff --git a/appveyor.yml b/appveyor.yml index 1f8d71f2..9fd5db4c 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,5 +1,10 @@ +environment: + matrix: + - PYTHON: "C:\\Python27\\python.exe" + - PYTHON: "C:\\Python35\\python.exe" + build_script: - - pip install . + - "%PYTHON% -m pip install ." # the '-u' flag is required so the output is in the correct order. # See https://github.com/joerick/cibuildwheel/pull/24 for more info. - - python -u ./run_tests.py + - "%PYTHON% -u ./run_tests.py' From e66ab50909abcc0faa85649704024bfdeeee1867 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 14 Aug 2017 23:20:41 +0100 Subject: [PATCH 02/14] Fix syntax error --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 9fd5db4c..5c3ba990 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -7,4 +7,4 @@ build_script: - "%PYTHON% -m pip install ." # the '-u' flag is required so the output is in the correct order. # See https://github.com/joerick/cibuildwheel/pull/24 for more info. - - "%PYTHON% -u ./run_tests.py' + - "%PYTHON% -u ./run_tests.py'" From f07979ec2398cfa56b85592f95a9954c4275be71 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 14 Aug 2017 23:20:41 +0100 Subject: [PATCH 03/14] Actually fix syntax error --- appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/appveyor.yml b/appveyor.yml index 5c3ba990..9cb28331 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -7,4 +7,4 @@ build_script: - "%PYTHON% -m pip install ." # the '-u' flag is required so the output is in the correct order. # See https://github.com/joerick/cibuildwheel/pull/24 for more info. - - "%PYTHON% -u ./run_tests.py'" + - "%PYTHON% -u ./run_tests.py" From 6abacb46e49bfb5459c086ec5b4e0091ffbe128e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 14 Aug 2017 23:37:06 +0100 Subject: [PATCH 04/14] Run the tests with the same interpreter that runs `run_tests.py` --- run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/run_tests.py b/run_tests.py index cc86cf48..3eb33b5b 100644 --- a/run_tests.py +++ b/run_tests.py @@ -26,7 +26,7 @@ for project_path in test_projects: project_env = {str(k): str(v) for k, v in project_env.items()} # unicode not allowed in env env.update(project_env) print('Building %s with environment %s' % (project_path, project_env)) - subprocess.check_call(['cibuildwheel', project_path], env=env) + subprocess.check_call([sys.executable, '-m', 'cibuildwheel', project_path], env=env) wheels = glob('wheelhouse/*.whl') print('%s built successfully. %i wheels built.' % (project_path, len(wheels))) From 92304597bd06531c0139db9dd8766562f38cf19e Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Thu, 17 Aug 2017 21:54:08 +0200 Subject: [PATCH 05/14] Replacing unbuffered 'os.fdopen' by proxy object to have a unbuffered stdout solution for Windows and AppVeyor that works on Python 3 --- cibuildwheel/util.py | 17 +++++++++++++++++ cibuildwheel/windows.py | 4 ++-- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index d20f986a..d860beeb 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -22,3 +22,20 @@ class BuildSkipper(object): def __repr__(self): return 'BuildSkipper(%r)' % ' '.join(self.patterns) + + +# Taken from https://stackoverflow.com/a/107717 +class Unbuffered(object): + def __init__(self, stream): + self.stream = stream + + def write(self, data): + self.stream.write(data) + self.stream.flush() + + def writelines(self, datas): + self.stream.writelines(datas) + self.stream.flush() + + def __getattr__(self, attr): + return getattr(self.stream, attr) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index feaf12be..f0a1eaba 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -7,13 +7,13 @@ except ImportError: from collections import namedtuple from glob import glob -from .util import prepare_command +from .util import prepare_command, Unbuffered def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, skip): # Python under AppVeyor/Windows seems to be buffering by default, giving problems interleaving subprocess call output with unflushed calls to 'print' sys.stdout.flush() - sys.stdout = os.fdopen(sys.stdout.fileno(), 'w', 0) + sys.stdout = Unbuffered(sys.stdout) # run_with_env is a cmd file that sets the right environment variables to run_with_env = os.path.join(tempfile.gettempdir(), 'appveyor_run_with_env.cmd') From 267e566c43a1291fdf78bc85fcdcd68227fc100e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 1 Sep 2017 13:49:39 +0100 Subject: [PATCH 06/14] Add `language: python` to the travis build --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index e4db986f..bf4bb19a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,5 @@ +language: python + matrix: include: # Linux From 1cd65ccbb6c9063085196b3ade35a464653577b6 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 7 Sep 2017 18:58:08 +0100 Subject: [PATCH 07/14] Add ensurepip for Linux build to ensure setuptools/pip are installed --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 34a945d2..d2ddc48a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,6 +28,7 @@ script: $PYTHON ./bin/run_tests.py else # linux test requires root to clean up the wheelhouse (docker runs as root) + sudo $PYTHON -m ensurepip sudo $PYTHON -m pip install -r requirements-dev.txt sudo $PYTHON ./bin/run_tests.py fi From 7420e2a9241cfe466be207424cf97f45f496fc16 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 7 Sep 2017 21:45:43 +0100 Subject: [PATCH 08/14] Use apt to install pip --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d2ddc48a..953e4bfd 100644 --- a/.travis.yml +++ b/.travis.yml @@ -28,7 +28,8 @@ script: $PYTHON ./bin/run_tests.py else # linux test requires root to clean up the wheelhouse (docker runs as root) - sudo $PYTHON -m ensurepip + sudo apt-get -qq update + sudo apt-get install -y python3-pip sudo $PYTHON -m pip install -r requirements-dev.txt sudo $PYTHON ./bin/run_tests.py fi From 510a6ce84a39739d8b8dd66e1c86707582166089 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 7 Sep 2017 21:50:38 +0100 Subject: [PATCH 09/14] Only install python3 pip when doing that build --- .travis.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 953e4bfd..60258498 100644 --- a/.travis.yml +++ b/.travis.yml @@ -27,9 +27,11 @@ script: $PYTHON -m pip install -r requirements-dev.txt $PYTHON ./bin/run_tests.py else + if [[ "$PYTHON" == python3 ]]; then + sudo apt-get -qq update + sudo apt-get install -y python3-pip + fi # linux test requires root to clean up the wheelhouse (docker runs as root) - sudo apt-get -qq update - sudo apt-get install -y python3-pip sudo $PYTHON -m pip install -r requirements-dev.txt sudo $PYTHON ./bin/run_tests.py fi From 0cd8228db1f27bb1e9ea2b8140d663c600d06719 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 7 Sep 2017 21:59:04 +0100 Subject: [PATCH 10/14] Use same python interpreter for running pytest --- bin/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/run_tests.py b/bin/run_tests.py index 9264e2be..bcae61c5 100755 --- a/bin/run_tests.py +++ b/bin/run_tests.py @@ -10,7 +10,7 @@ if __name__ == '__main__': ### run the unit tests - subprocess.check_call(['python', '-m', 'pytest', 'unit_test']) + subprocess.check_call([sys.executable, '-m', 'pytest', 'unit_test']) ### run the integration tests From f4eafa901b67cd89328c750a1f7437f65a5112ac Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 7 Sep 2017 22:38:32 +0100 Subject: [PATCH 11/14] Ignore some venv locations --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 5721e556..e78281bc 100644 --- a/.gitignore +++ b/.gitignore @@ -85,7 +85,10 @@ celerybeat-schedule # virtualenv .venv venv/ +venv3/ ENV/ +env/ +env3/ # Spyder project settings .spyderproject From e4138635316c0b42ba20c8a28e77de8616fa989e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 7 Sep 2017 22:38:52 +0100 Subject: [PATCH 12/14] Fix running command nodes on python 3 --- cibuildwheel/bashlex_eval.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/bashlex_eval.py b/cibuildwheel/bashlex_eval.py index 90a399fe..af1ef5ba 100644 --- a/cibuildwheel/bashlex_eval.py +++ b/cibuildwheel/bashlex_eval.py @@ -1,4 +1,4 @@ -import subprocess, shlex +import subprocess, shlex, sys from collections import namedtuple import bashlex @@ -60,8 +60,12 @@ def evaluate_word_node(node, context): def evaluate_command_node(node, context): words = [evaluate_node(part, context=context) for part in node.parts] command = ' '.join(words) - return subprocess.check_output(shlex.split(command), env=context.environment) + output = subprocess.check_output(shlex.split(command), env=context.environment) + if sys.version_info[0] >= 3: + return output.decode('utf8', 'replace') + else: + return output def evaluate_parameter_node(node, context): return context.environment.get(node.value, '') From a7f81b3c586a495b5f17d27400023caca7824f94 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 10 Sep 2017 21:29:01 +0100 Subject: [PATCH 13/14] Use language: generic for Mac builds on Travis https://github.com/travis-ci/travis-ci/issues/2312 --- .travis.yml | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 60258498..5d1465fb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,25 +1,38 @@ -language: python +language: generic matrix: include: - # Linux + # Linux Python 2 - sudo: required + language: python services: - docker env: - "PYTHON=python2" + + # Linux Python 3 - sudo: required + language: python services: - docker env: - "PYTHON=python3" - # macOS + before_install: + - sudo apt-get -qq update + - sudo apt-get install -y python3-pip + + # macOS Python 2 - os: osx env: - "PYTHON=python2" + + # macOS Python 3 - os: osx env: - "PYTHON=python3" + before_install: + - brew update + - brew install python3 script: - | @@ -27,10 +40,6 @@ script: $PYTHON -m pip install -r requirements-dev.txt $PYTHON ./bin/run_tests.py else - if [[ "$PYTHON" == python3 ]]; then - sudo apt-get -qq update - sudo apt-get install -y python3-pip - fi # linux test requires root to clean up the wheelhouse (docker runs as root) sudo $PYTHON -m pip install -r requirements-dev.txt sudo $PYTHON ./bin/run_tests.py From 53b4ea5125fd23a01484e31c9202a40f5d9de6e2 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 10 Sep 2017 22:02:36 +0100 Subject: [PATCH 14/14] Maybe bitrise will run the macos builds! --- cibuildwheel/__main__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 46b87cf8..1a157495 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -59,6 +59,8 @@ def main(): platform = 'macos' elif 'APPVEYOR' in os.environ: platform = 'windows' + elif 'BITRISE_BUILD_NUMBER' in os.environ: + platform = 'macos' else: print('cibuildwheel: Unable to detect platform. cibuildwheel should run on your CI server, ' 'Travis CI and Appveyor are supported. You can run on your development '