From 0124d174bee508fa1982d5e42ec331e4288acc22 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 2 Feb 2020 14:33:36 +0000 Subject: [PATCH 01/13] Raise a warning if .so files were found in a project that fails to build Fix #139 --- cibuildwheel/linux.py | 32 ++++++++++++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index eb420516..30841794 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -1,5 +1,5 @@ from __future__ import print_function -import os, subprocess, sys, uuid +import os, subprocess, sys, uuid, textwrap from collections import namedtuple from .util import prepare_command, get_build_verbosity_extra_flags @@ -173,8 +173,36 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef run_docker(['cp', os.path.abspath(project_dir) + '/.', container_name + ':/project']) run_docker(['start', '-i', '-a', container_name], stdin_str=bash_script) run_docker(['cp', container_name + ':/output/.', os.path.abspath(output_dir)]) - except subprocess.CalledProcessError: + except subprocess.CalledProcessError as error: + troubleshoot(project_dir, error) exit(1) finally: # Still gets executed, even when 'exit(1)' gets called run_docker(['rm', '--force', '-v', container_name]) + + +def troubleshoot(project_dir, error): + if (isinstance(error, subprocess.CalledProcessError) and 'start' in error.cmd): + # the bash script failed + print('Checking for common errors...') + so_files = [] + for root, dirs, files in os.walk(project_dir): + for name in files: + _, ext = os.path.splitext(name) + if ext == '.so': + so_files.append(os.path.join(root, name)) + + if so_files: + print(textwrap.dedent(''' + NOTE: Shared object (.so) files found in this project. + + These files might be built against the wrong OS, causing problems with + auditwheel. + + If you're using Cython and have previously done an in-place build, + remove those build files (*.so and *.c) before starting cibuildwheel. + ''')) + + print(' Files detected:') + print('\n'.join([' '+f for f in so_files])) + print('') From ffec9aecf872c04d1247fc696843f76bf30fb676 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Sun, 2 Feb 2020 20:07:22 +0100 Subject: [PATCH 02/13] Update line in docs/cpp_standards.md on setting MACOSX_DEPLOYMENT_TARGET >= 10.9 (see #220) --- docs/cpp_standards.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/cpp_standards.md b/docs/cpp_standards.md index d330da48..39666c66 100644 --- a/docs/cpp_standards.md +++ b/docs/cpp_standards.md @@ -20,7 +20,7 @@ OS X/macOS allows you to specify a so-called "deployment target" version that wi However, to enable modern C++ standards, the deploment target needs to be set high enough (since older OS X/macOS versions did not have the necessary modern C++ standard library). -To get C++11 and C++14 support, set `MACOSX_DEPLOYMENT_TARGET` to (at least) `"10.9"`. +To get C++11 and C++14 support, `MACOSX_DEPLOYMENT_TARGET` needs to be set to (at least) `"10.9"`. By default, `cibuildwheel` already does this, building 64-bit-only wheels for macOS 10.9 and later. To get C++17 support, set `MACOSX_DEPLOYMENT_TARGET` to (at least) `"10.13"` or `"10.14"` (macOS 10.13 offers partial C++17 support; e.g., the filesystem header is in experimental, offering `#include ` instead of `#include `; macOS 10.14 has full C++17 support). From 692c64ffd942fbaa0ea8197e0d1c257156536793 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Mon, 3 Feb 2020 15:31:51 +0100 Subject: [PATCH 03/13] Removing Python 2 configurations for CI runs --- .circleci/config.yml | 33 ++++++++------------------------- .travis.yml | 7 ------- CI.md | 9 +++++++++ appveyor.yml | 4 ++-- azure-pipelines.yml | 28 +++++++++++++++++++++++++--- 5 files changed, 44 insertions(+), 37 deletions(-) create mode 100644 CI.md diff --git a/.circleci/config.yml b/.circleci/config.yml index b881c8a8..3d2d09a1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,11 +1,11 @@ version: 2 jobs: - osx-python2: + osx-python3.6: macos: - xcode: "10.0.0" + xcode: "9.4.1" environment: - PYTHON: python2 + PYTHON: python3 steps: - checkout @@ -16,7 +16,7 @@ jobs: name: Test. command: venv/bin/python ./bin/run_tests.py - osx-python3: + osx-python3.7: macos: xcode: "10.0.0" environment: @@ -31,23 +31,7 @@ jobs: name: Test. command: venv/bin/python ./bin/run_tests.py - linux-python2: - docker: - - image: circleci/python:2.7 - environment: - PYTHON: python2 - steps: - - checkout - - setup_remote_docker - - - run: - name: Prepare the environment. - command: bash .circleci/prepare.sh - - run: - name: Test. - command: venv/bin/python ./bin/run_tests.py - - linux-python3: + linux-python3.6: docker: - image: circleci/python:3.6 environment: @@ -67,7 +51,6 @@ workflows: version: 2 all-tests: jobs: - - osx-python2 - - osx-python3 - - linux-python2 - - linux-python3 + - osx-python3.6 + - osx-python3.7 + - linux-python3.6 diff --git a/.travis.yml b/.travis.yml index ea03f8e0..d5215b04 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,13 +2,6 @@ language: generic matrix: include: - # Linux Python 2 - - sudo: required - language: python - python: 2.7 - services: docker - env: PYTHON=python - # Linux Python 3 - sudo: required language: python diff --git a/CI.md b/CI.md new file mode 100644 index 00000000..3b43a2e6 --- /dev/null +++ b/CI.md @@ -0,0 +1,9 @@ +This is a summary of the Python versions and platforms covered by the different CI platforms: + +| | 3.5 | 3.6 | 3.7 | 3.8 | +|----------|------------------|------------------|-----------------------|------------------| +| Linux | Travis CI | CircleCI | AppVeyor | Azure Pipelines | +| macOS | Azure Pipelines | CircleCI | Travis CI¹ / CircleCI | Azure Pipelines | +| Windows | TravisCI | Azure Pipelines | AppVeyor | Azure Pipelines | + +> ¹ Python version not really pinned, but dependent on the (default) version of image used. diff --git a/appveyor.yml b/appveyor.yml index 6f611d94..d3f321d9 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -3,9 +3,9 @@ image: - Visual Studio 2015 build_script: - - cmd: "C:\\Python27\\python.exe -m pip install -r requirements-dev.txt" + - cmd: "C:\\Python37\\python.exe -m pip install -r requirements-dev.txt" - sh: "${HOME}/.localpython3.7.4/bin/python3 -m pip install -r requirements-dev.txt" # the '-u' flag is required so the output is in the correct order. # See https://github.com/joerick/cibuildwheel/pull/24 for more info. - - cmd: "C:\\Python27\\python.exe -u ./bin/run_tests.py" + - cmd: "C:\\Python37\\python.exe -u ./bin/run_tests.py" - sh: "${HOME}/.localpython3.7.4/bin/python3 ./bin/run_tests.py" diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 72ff2b85..74d78395 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -1,5 +1,5 @@ jobs: -- job: linux +- job: linux_38 pool: {vmImage: 'Ubuntu-18.04'} steps: - task: UsePythonVersion@0 @@ -9,7 +9,17 @@ jobs: python -m pip install -r requirements-dev.txt python ./bin/run_tests.py -- job: macos +- job: macos_35 + pool: {vmImage: 'macOS-10.13'} + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.5' + - bash: | + python -m pip install -r requirements-dev.txt + python ./bin/run_tests.py + +- job: macos_38 pool: {vmImage: 'macOS-10.13'} steps: - task: UsePythonVersion@0 @@ -19,7 +29,19 @@ jobs: python -m pip install -r requirements-dev.txt python ./bin/run_tests.py -- job: windows +- job: windows_36 + pool: {vmImage: 'vs2017-win2016'} + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.6' + - script: choco install vcpython27 -f -y + displayName: Install Visual C++ for Python 2.7 + - bash: | + python -m pip install -r requirements-dev.txt + python ./bin/run_tests.py + +- job: windows_38 pool: {vmImage: 'vs2017-win2016'} steps: - task: UsePythonVersion@0 From 9fe8d29545948eda5402b07a399b9afb70659d3e Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Mon, 3 Feb 2020 17:37:35 +0100 Subject: [PATCH 04/13] Dropping Python 2 from setup.py --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index b521a6f5..89537bdf 100644 --- a/setup.py +++ b/setup.py @@ -28,12 +28,11 @@ setup( 'cibuildwheel': ['resources/*'], }, # Supported python versions - python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*', + python_requires='>=3.5', keywords='ci wheel packaging pypi travis appveyor macos linux windows', classifiers=[ 'Intended Audience :: Developers', 'Natural Language :: English', - 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 3', 'Development Status :: 5 - Production/Stable', 'License :: OSI Approved :: BSD License', From b900b2d51eec1164eb892f00cc483c013a160e93 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Mon, 3 Feb 2020 17:26:41 +0100 Subject: [PATCH 05/13] Ditching run_docker for subprocess.run --- cibuildwheel/linux.py | 36 +++++++++++------------------------- 1 file changed, 11 insertions(+), 25 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 30841794..043325c7 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -147,38 +147,24 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef gid=os.getgid(), ) - def run_docker(command, stdin_str=None): - print('docker command: docker {}'.format(' '.join(map(shlex_quote, command)))) - if stdin_str is None: - subprocess.check_call(['docker'] + command) - else: - args = ['docker'] + command - process = subprocess.Popen(args, stdin=subprocess.PIPE, universal_newlines=True) - try: - process.communicate(stdin_str) - except KeyboardInterrupt: - process.kill() - process.wait() - if process.returncode != 0: - raise subprocess.CalledProcessError(process.returncode, args) - container_name = 'cibuildwheel-{}'.format(uuid.uuid4()) try: - run_docker(['create', - '--env', 'CIBUILDWHEEL', - '--name', container_name, - '-i', - '-v', '/:/host', # ignored on Circle - docker_image, '/bin/bash']) - run_docker(['cp', os.path.abspath(project_dir) + '/.', container_name + ':/project']) - run_docker(['start', '-i', '-a', container_name], stdin_str=bash_script) - run_docker(['cp', container_name + ':/output/.', os.path.abspath(output_dir)]) + subprocess.run(['docker', 'create', + '--env', 'CIBUILDWHEEL', + '--name', container_name, + '-i', + '-v', '/:/host', # ignored on CircleCI + docker_image, '/bin/bash'], + check=True) + subprocess.run(['docker', 'cp', os.path.abspath(project_dir) + '/.', container_name + ':/project'], check=True) + subprocess.run(['docker', 'start', '-i', '-a', container_name], input=bash_script, universal_newlines=True, check=True) + subprocess.run(['docker', 'cp', container_name + ':/output/.', os.path.abspath(output_dir)], check=True) except subprocess.CalledProcessError as error: troubleshoot(project_dir, error) exit(1) finally: # Still gets executed, even when 'exit(1)' gets called - run_docker(['rm', '--force', '-v', container_name]) + subprocess.run(['docker', 'rm', '--force', '-v', container_name], check=True) def troubleshoot(project_dir, error): From 67ec8fe99f2b0ca791bfd5da73f33eb97b43b287 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Mon, 3 Feb 2020 17:34:55 +0100 Subject: [PATCH 06/13] Removing Python 2-compatible imports of urllib.request.open and shlex.quote --- cibuildwheel/linux.py | 13 ++++--------- cibuildwheel/macos.py | 6 +----- cibuildwheel/util.py | 9 ++------- cibuildwheel/windows.py | 5 ----- 4 files changed, 7 insertions(+), 26 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 043325c7..c2b37840 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -1,13 +1,8 @@ from __future__ import print_function -import os, subprocess, sys, uuid, textwrap +import os, shlex, subprocess, sys, textwrap, uuid from collections import namedtuple from .util import prepare_command, get_build_verbosity_extra_flags -try: - from shlex import quote as shlex_quote -except ImportError: - from pipes import quote as shlex_quote - def get_python_configurations(build_selector): PythonConfiguration = namedtuple('PythonConfiguration', ['identifier', 'path']) @@ -132,14 +127,14 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef pybin_paths=' '.join(c.path+'/bin' for c in platform_configs), test_requires=' '.join(test_requires), test_extras=test_extras, - test_command=shlex_quote( + test_command=shlex.quote( prepare_command(test_command, project='/project') if test_command else '' ), - before_build=shlex_quote( + before_build=shlex.quote( prepare_command(before_build, project='/project') if before_build else '' ), build_verbosity_flag=' '.join(get_build_verbosity_extra_flags(build_verbosity)), - repair_command=shlex_quote( + repair_command=shlex.quote( prepare_command(repair_command, wheel='"$1"', dest_dir='/tmp/repaired_wheels') if repair_command else '' ), environment_exports='\n'.join(environment.as_shell_commands()), diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 7480990f..6e71d673 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -3,10 +3,6 @@ import tempfile import os, subprocess, shlex, sys, shutil from collections import namedtuple from glob import glob -try: - from shlex import quote as shlex_quote -except ImportError: - from pipes import quote as shlex_quote from .util import prepare_command, get_build_verbosity_extra_flags, download @@ -46,7 +42,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef if shell: print('+ %s' % args) else: - print('+ ' + ' '.join(shlex_quote(a) for a in args)) + print('+ ' + ' '.join(shlex.quote(a) for a in args)) return subprocess.check_call(args, env=env, cwd=cwd, shell=shell) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 0aedbcac..14839ba3 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -1,13 +1,8 @@ from fnmatch import fnmatch import warnings -import os +import os, urllib.request from time import sleep -try: - from urllib.request import urlopen -except ImportError: - from urllib2 import urlopen - def prepare_command(command, **kwargs): ''' @@ -68,7 +63,7 @@ def download(url, dest): repeat_num = 3 for i in range(repeat_num): try: - response = urlopen(url) + response = urllib.request.urlopen(url) except: if i == repeat_num - 1: raise diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index c50c9810..26b93b7f 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -3,11 +3,6 @@ import os, tempfile, subprocess, shutil, sys from collections import namedtuple from glob import glob -try: - from shlex import quote as shlex_quote -except ImportError: - from pipes import quote as shlex_quote - from .util import prepare_command, get_build_verbosity_extra_flags, download From e8d121552c9a23e87f35369aba79563f72b3bba1 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Mon, 3 Feb 2020 20:11:00 +0100 Subject: [PATCH 07/13] Remove a few more Python 2-compatibility workarounds --- cibuildwheel/bashlex_eval.py | 6 +----- cibuildwheel/macos.py | 4 +--- setup.py | 1 - 3 files changed, 2 insertions(+), 9 deletions(-) diff --git a/cibuildwheel/bashlex_eval.py b/cibuildwheel/bashlex_eval.py index af1ef5ba..1caddd8f 100644 --- a/cibuildwheel/bashlex_eval.py +++ b/cibuildwheel/bashlex_eval.py @@ -60,12 +60,8 @@ def evaluate_word_node(node, context): def evaluate_command_node(node, context): words = [evaluate_node(part, context=context) for part in node.parts] command = ' '.join(words) - output = subprocess.check_output(shlex.split(command), env=context.environment) + return subprocess.check_output(shlex.split(command), env=context.environment, universal_newlines=True) - if sys.version_info[0] >= 3: - return output.decode('utf8', 'replace') - else: - return output def evaluate_parameter_node(node, context): return context.environment.get(node.value, '') diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 6e71d673..a649ac91 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -32,9 +32,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef get_pip_url = 'https://bootstrap.pypa.io/get-pip.py' get_pip_script = '/tmp/get-pip.py' - pkgs_output = subprocess.check_output(['pkgutil', '--pkgs']) - if sys.version_info[0] >= 3: - pkgs_output = pkgs_output.decode('utf8') + pkgs_output = subprocess.check_output(['pkgutil', '--pkgs'], universal_newlines=True) installed_system_packages = pkgs_output.splitlines() def call(args, env=None, cwd=None, shell=False): diff --git a/setup.py b/setup.py index 89537bdf..bb6db2ca 100644 --- a/setup.py +++ b/setup.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # -*- coding: utf-8 -*- import os, io From 48055273f1701afd4fd01bef3fba993dd18a9f43 Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Mon, 3 Feb 2020 20:42:55 +0100 Subject: [PATCH 08/13] Remove inheritance from object base --- cibuildwheel/environment.py | 4 ++-- cibuildwheel/util.py | 4 ++-- unit_test/main_util_fixtures.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cibuildwheel/environment.py b/cibuildwheel/environment.py index 1b2adec4..5d548f57 100644 --- a/cibuildwheel/environment.py +++ b/cibuildwheel/environment.py @@ -41,7 +41,7 @@ def split_env_items(env_string): return result -class EnvironmentAssignment(object): +class EnvironmentAssignment: def __init__(self, assignment): name, equals, value = assignment.partition('=') if not equals: @@ -60,7 +60,7 @@ class EnvironmentAssignment(object): return '%s=%s' % (self.name, self.value) -class ParsedEnvironment(object): +class ParsedEnvironment: def __init__(self, assignments): self.assignments = assignments diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 14839ba3..fbee1728 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -23,7 +23,7 @@ def get_build_verbosity_extra_flags(level): return [] -class BuildSelector(object): +class BuildSelector: def __init__(self, build_config, skip_config): self.build_patterns = build_config.split() self.skip_patterns = skip_config.split() @@ -38,7 +38,7 @@ class BuildSelector(object): # Taken from https://stackoverflow.com/a/107717 -class Unbuffered(object): +class Unbuffered: def __init__(self, stream): self.stream = stream diff --git a/unit_test/main_util_fixtures.py b/unit_test/main_util_fixtures.py index 3c993b35..21a7f36a 100644 --- a/unit_test/main_util_fixtures.py +++ b/unit_test/main_util_fixtures.py @@ -7,7 +7,7 @@ import subprocess from cibuildwheel import linux, macos, windows, util -class ArgsInterceptor(object): +class ArgsInterceptor: def __call__(self, *args, **kwargs): self.args = args self.kwargs = kwargs From 165e51aec59d1b844170fdb1a6ad8a59b57e9645 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Mon, 3 Feb 2020 23:05:45 +0100 Subject: [PATCH 09/13] Remove 'from __future__ import print_function' lines --- bin/run_test.py | 3 +-- bin/run_tests.py | 3 +-- cibuildwheel/__main__.py | 1 - cibuildwheel/linux.py | 1 - cibuildwheel/macos.py | 1 - cibuildwheel/windows.py | 1 - 6 files changed, 2 insertions(+), 8 deletions(-) diff --git a/bin/run_test.py b/bin/run_test.py index 70a7825c..075c4d08 100755 --- a/bin/run_test.py +++ b/bin/run_test.py @@ -1,6 +1,5 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -from __future__ import print_function import os, sys, subprocess, shutil diff --git a/bin/run_tests.py b/bin/run_tests.py index 279a992c..a0413dcf 100755 --- a/bin/run_tests.py +++ b/bin/run_tests.py @@ -1,6 +1,5 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 -from __future__ import print_function import os, sys, subprocess, shutil, json from glob import glob diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index bdc34aaf..314512a9 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -1,4 +1,3 @@ -from __future__ import print_function import argparse, os, subprocess, sys, textwrap import cibuildwheel diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index c2b37840..11a15413 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -1,4 +1,3 @@ -from __future__ import print_function import os, shlex, subprocess, sys, textwrap, uuid from collections import namedtuple from .util import prepare_command, get_build_verbosity_extra_flags diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index a649ac91..6185070f 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -1,4 +1,3 @@ -from __future__ import print_function import tempfile import os, subprocess, shlex, sys, shutil from collections import namedtuple diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 26b93b7f..175edda3 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -1,4 +1,3 @@ -from __future__ import print_function import os, tempfile, subprocess, shutil, sys from collections import namedtuple from glob import glob From 691174cf1104d761baf2dcd05aedb1bd710e8de3 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Fri, 14 Feb 2020 13:32:30 +0100 Subject: [PATCH 10/13] Keeping virtualenv<20 on macOS, until problems get fixed by new release --- .circleci/prepare.sh | 2 +- cibuildwheel/macos.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/prepare.sh b/.circleci/prepare.sh index dfd3aeef..083c6d19 100644 --- a/.circleci/prepare.sh +++ b/.circleci/prepare.sh @@ -1,6 +1,6 @@ $PYTHON --version $PYTHON -m pip --version -$PYTHON -m pip install -q --user --ignore-installed --upgrade virtualenv +$PYTHON -m pip install -q --user --ignore-installed --upgrade "virtualenv<20" $PYTHON -m virtualenv -p $PYTHON venv venv/bin/python -m pip install -r requirements-dev.txt venv/bin/python -m pip freeze diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 6185070f..304e594a 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -130,7 +130,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef if test_command: # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. - call(['pip', 'install', 'virtualenv'], env=env) + call(['pip', 'install', 'virtualenv<20'], env=env) venv_dir = tempfile.mkdtemp() call(['python', '-m', 'virtualenv', venv_dir], env=env) From cab6ecef24cae19ed615c4996d251571a0cc73c7 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 12 Nov 2019 23:51:27 +0000 Subject: [PATCH 11/13] Fix all PEP-008 issues --- bin/run_test.py | 7 +++--- bin/run_tests.py | 8 ++++--- cibuildwheel/__main__.py | 22 +++++++++++------ cibuildwheel/bashlex_eval.py | 11 ++++++--- cibuildwheel/environment.py | 1 + cibuildwheel/linux.py | 24 ++++++++++++------- cibuildwheel/macos.py | 13 +++++++--- cibuildwheel/util.py | 6 ++--- cibuildwheel/windows.py | 14 ++++++++--- .../mkdocs_include_markdown_plugin/plugin.py | 13 ++++++---- setup.cfg | 3 +++ setup.py | 5 ++-- test/01_basic/cibuildwheel_test.py | 5 ++-- test/01_basic/setup.py | 5 +++- test/02_test/cibuildwheel_test.py | 6 ++++- test/02_test/setup.py | 5 +++- test/02_test/test/spam_test.py | 1 + test/03_before_build/cibuildwheel_test.py | 2 ++ test/03_before_build/setup.py | 9 +++++-- test/04_build_skip/cibuildwheel_test.py | 2 ++ test/04_build_skip/setup.py | 6 ++++- test/05_environment/cibuildwheel_test.py | 2 ++ test/05_environment/setup.py | 9 ++++--- test/06_docker_images/cibuildwheel_test.py | 6 ++++- test/06_docker_images/setup.py | 8 +++++-- test/07_ssl/cibuildwheel_test.py | 2 ++ test/07_ssl/setup.py | 6 ++++- .../cibuildwheel_test.py | 5 +++- test/08_manylinuxXXXX_only/setup.py | 5 +++- test/09_setup_cfg/cibuildwheel_test.py | 3 ++- test/09_setup_cfg/setup.py | 8 +++---- test/09_setup_cfg/spam/__init__.py | 2 +- test/shared/utils.py | 8 ++++--- unit_test/environment_test.py | 8 +++++++ 34 files changed, 175 insertions(+), 65 deletions(-) diff --git a/bin/run_test.py b/bin/run_test.py index 075c4d08..bee65de4 100755 --- a/bin/run_test.py +++ b/bin/run_test.py @@ -1,6 +1,9 @@ #!/usr/bin/env python3 -import os, sys, subprocess, shutil +import argparse +import os +import subprocess +import sys def single_run(test_project): @@ -11,8 +14,6 @@ def single_run(test_project): if __name__ == '__main__': - import argparse - parser = argparse.ArgumentParser() parser.add_argument("test_project_dir") args = parser.parse_args() diff --git a/bin/run_tests.py b/bin/run_tests.py index a0413dcf..4963a883 100755 --- a/bin/run_tests.py +++ b/bin/run_tests.py @@ -1,17 +1,19 @@ #!/usr/bin/env python3 -import os, sys, subprocess, shutil, json +import os +import subprocess +import sys from glob import glob if __name__ == '__main__': # move cwd to the project root os.chdir(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - ### run the unit tests + # run the unit tests subprocess.check_call([sys.executable, '-m', 'pytest', 'unit_test']) - ### run the integration tests + # run the integration tests test_projects = sorted(glob('test/??_*')) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 314512a9..2d309788 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -1,10 +1,20 @@ -import argparse, os, subprocess, sys, textwrap +import argparse +import os +import sys +import textwrap +import traceback import cibuildwheel -import cibuildwheel.linux, cibuildwheel.windows, cibuildwheel.macos -from cibuildwheel.environment import parse_environment, EnvironmentParseError +import cibuildwheel.linux +import cibuildwheel.macos +import cibuildwheel.windows +from cibuildwheel.environment import ( + EnvironmentParseError, + parse_environment, +) from cibuildwheel.util import BuildSelector, Unbuffered + def get_option_from_environment(option_name, platform=None, default=None): ''' Returns an option from the environment, optionally scoped by the platform. @@ -84,7 +94,6 @@ def main(): file=sys.stderr) exit(2) - output_dir = args.output_dir test_command = get_option_from_environment('CIBW_TEST_COMMAND', platform=platform) test_requires = get_option_from_environment('CIBW_TEST_REQUIRES', platform=platform, default='').split() @@ -112,9 +121,8 @@ def main(): try: environment = parse_environment(environment_config) - except (EnvironmentParseError, ValueError) as e: + except (EnvironmentParseError, ValueError): print('cibuildwheel: Malformed environment option "%s"' % environment_config, file=sys.stderr) - import traceback traceback.print_exc(None, sys.stderr) exit(2) @@ -210,6 +218,7 @@ def detect_obsolete_options(): )) os.environ[option] = os.environ[option].replace(deprecated, alternative) + def print_preamble(platform, build_options): print(textwrap.dedent(''' _ _ _ _ _ _ _ @@ -220,7 +229,6 @@ def print_preamble(platform, build_options): print('cibuildwheel version %s\n' % cibuildwheel.__version__) - print('Build options:') print(' platform: %r' % platform) for option, value in sorted(build_options.items()): diff --git a/cibuildwheel/bashlex_eval.py b/cibuildwheel/bashlex_eval.py index 1caddd8f..7e2d7f4a 100644 --- a/cibuildwheel/bashlex_eval.py +++ b/cibuildwheel/bashlex_eval.py @@ -1,9 +1,13 @@ -import subprocess, shlex, sys +import shlex +import subprocess +import sys from collections import namedtuple + import bashlex NodeExecutionContext = namedtuple('NodeExecutionContext', ['environment', 'input']) + def evaluate(value, environment): if not value: # empty string evaluates to empty string @@ -16,9 +20,9 @@ def evaluate(value, environment): raise ValueError('"%s" has too many parts' % value) value_word_node = command_node.parts[0] - + return evaluate_node( - value_word_node, + value_word_node, context=NodeExecutionContext(environment=environment, input=value) ) @@ -63,5 +67,6 @@ def evaluate_command_node(node, context): return subprocess.check_output(shlex.split(command), env=context.environment, universal_newlines=True) + def evaluate_parameter_node(node, context): return context.environment.get(node.value, '') diff --git a/cibuildwheel/environment.py b/cibuildwheel/environment.py index 5d548f57..f016c780 100644 --- a/cibuildwheel/environment.py +++ b/cibuildwheel/environment.py @@ -1,4 +1,5 @@ import bashlex + from . import bashlex_eval diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 11a15413..24a034d6 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -1,6 +1,15 @@ -import os, shlex, subprocess, sys, textwrap, uuid +import os +import shlex +import subprocess +import sys +import textwrap +import uuid from collections import namedtuple -from .util import prepare_command, get_build_verbosity_extra_flags + +from .util import ( + get_build_verbosity_extra_flags, + prepare_command, +) def get_python_configurations(build_selector): @@ -27,7 +36,7 @@ def get_python_configurations(build_selector): def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, manylinux_images): try: subprocess.check_call(['docker', '--version']) - except: + except Exception: print('cibuildwheel: Docker not found. Docker is required to run Linux builds. ' 'If you\'re building on Travis CI, add `services: [docker]` to your .travis.yml.' 'If you\'re building on Circle CI in Linux, add a `setup_remote_docker` step to your .circleci/config.yml', @@ -123,7 +132,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef for repaired_wheel in "${{repaired_wheels[@]}}"; do chown {uid}:{gid} "/output/$(basename "$repaired_wheel")"; done done '''.format( - pybin_paths=' '.join(c.path+'/bin' for c in platform_configs), + pybin_paths=' '.join(c.path + '/bin' for c in platform_configs), test_requires=' '.join(test_requires), test_extras=test_extras, test_command=shlex.quote( @@ -147,9 +156,8 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef '--env', 'CIBUILDWHEEL', '--name', container_name, '-i', - '-v', '/:/host', # ignored on CircleCI - docker_image, '/bin/bash'], - check=True) + '-v', '/:/host', # ignored on CircleCI + docker_image, '/bin/bash'], check=True) subprocess.run(['docker', 'cp', os.path.abspath(project_dir) + '/.', container_name + ':/project'], check=True) subprocess.run(['docker', 'start', '-i', '-a', container_name], input=bash_script, universal_newlines=True, check=True) subprocess.run(['docker', 'cp', container_name + ':/output/.', os.path.abspath(output_dir)], check=True) @@ -184,5 +192,5 @@ def troubleshoot(project_dir, error): ''')) print(' Files detected:') - print('\n'.join([' '+f for f in so_files])) + print('\n'.join([' ' + f for f in so_files])) print('') diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 304e594a..c79c5a95 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -1,9 +1,16 @@ +import os +import shlex +import shutil +import subprocess import tempfile -import os, subprocess, shlex, sys, shutil from collections import namedtuple from glob import glob -from .util import prepare_command, get_build_verbosity_extra_flags, download +from .util import ( + download, + get_build_verbosity_extra_flags, + prepare_command, +) def get_python_configurations(build_selector): @@ -31,7 +38,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef get_pip_url = 'https://bootstrap.pypa.io/get-pip.py' get_pip_script = '/tmp/get-pip.py' - pkgs_output = subprocess.check_output(['pkgutil', '--pkgs'], universal_newlines=True) + pkgs_output = subprocess.check_output(['pkgutil', '--pkgs'], universal_newlines=True) installed_system_packages = pkgs_output.splitlines() def call(args, env=None, cwd=None, shell=False): diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index fbee1728..4b80a399 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -1,6 +1,6 @@ +import os +import urllib.request from fnmatch import fnmatch -import warnings -import os, urllib.request from time import sleep @@ -64,7 +64,7 @@ def download(url, dest): for i in range(repeat_num): try: response = urllib.request.urlopen(url) - except: + except Exception: if i == repeat_num - 1: raise sleep(3) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 175edda3..abb3a0a2 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -1,8 +1,15 @@ -import os, tempfile, subprocess, shutil, sys +import os +import shutil +import subprocess +import tempfile from collections import namedtuple from glob import glob -from .util import prepare_command, get_build_verbosity_extra_flags, download +from .util import ( + download, + get_build_verbosity_extra_flags, + prepare_command, +) IS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache') @@ -20,6 +27,7 @@ def get_nuget_args(configuration): python_name = python_name + "x86" return [python_name, "-Version", configuration.version, "-OutputDirectory", "C:/cibw/python"] + def get_python_configurations(build_selector): PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier']) python_configurations = [ @@ -40,7 +48,7 @@ def get_python_configurations(build_selector): # try with (and similar): msiexec /i VCForPython27.msi ALLUSERS=1 ACCEPT=YES /passive python_configurations = [c for c in python_configurations if not c.version.startswith('2.7.')] - # skip builds as required + # skip builds as required python_configurations = [c for c in python_configurations if build_selector(c.identifier)] return python_configurations diff --git a/docs/mkdocs_include_markdown_plugin/mkdocs_include_markdown_plugin/plugin.py b/docs/mkdocs_include_markdown_plugin/mkdocs_include_markdown_plugin/plugin.py index 55fadf2f..2f43d0f3 100644 --- a/docs/mkdocs_include_markdown_plugin/mkdocs_include_markdown_plugin/plugin.py +++ b/docs/mkdocs_include_markdown_plugin/mkdocs_include_markdown_plugin/plugin.py @@ -1,4 +1,9 @@ -import mkdocs, re, os, io, cgi +import cgi +import io +import os +import re + +import mkdocs INCLUDE_TAG_REGEX = re.compile( r''' @@ -28,6 +33,7 @@ INCLUDEMARKDOWN_TAG_REGEX = re.compile( flags=re.VERBOSE, ) + class ImportMarkdownPlugin(mkdocs.plugins.BasePlugin): def on_page_markdown(self, markdown, page, **kwargs): page_src_path = page.file.abs_src_path @@ -62,13 +68,13 @@ class ImportMarkdownPlugin(mkdocs.plugins.BasePlugin): with io.open(file_path_abs, encoding='utf8') as f: text_to_include = f.read() - + if start: _, _, text_to_include = text_to_include.partition(start) if end: text_to_include, _, _ = text_to_include.partition(end) - + return ( '\n' % ( filename, cgi.escape(start or ''), cgi.escape(end or '') @@ -80,4 +86,3 @@ class ImportMarkdownPlugin(mkdocs.plugins.BasePlugin): markdown = re.sub(INCLUDE_TAG_REGEX, found_include_tag, markdown) markdown = re.sub(INCLUDEMARKDOWN_TAG_REGEX, found_includemarkdown_tag, markdown) return markdown - \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 42b782c4..6c170e29 100644 --- a/setup.cfg +++ b/setup.cfg @@ -19,3 +19,6 @@ replace = cibuildwheel=={new_version} [bdist_wheel] universal = 1 +[flake8] +ignore = E501,W503 +application-import-names = cibuildwheel diff --git a/setup.py b/setup.py index bb6db2ca..010d5621 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- -import os, io +import io +import os try: from setuptools import setup @@ -20,7 +21,7 @@ setup( author="Joe Rickerby", author_email='joerick@mac.com', url='https://github.com/joerick/cibuildwheel', - packages=['cibuildwheel',], + packages=['cibuildwheel', ], license="BSD", zip_safe=False, package_data={ diff --git a/test/01_basic/cibuildwheel_test.py b/test/01_basic/cibuildwheel_test.py index ea6abb39..76dd4857 100644 --- a/test/01_basic/cibuildwheel_test.py +++ b/test/01_basic/cibuildwheel_test.py @@ -1,9 +1,10 @@ import os + import utils - project_dir = os.path.dirname(__file__) + def test(): # build the wheels actual_wheels = utils.cibuildwheel_run(project_dir) @@ -20,6 +21,6 @@ def test_build_identifiers(): # can be multiple wheels for each wheel, though, so we need to limit # the expected wheels expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0') - if not '-manylinux' in w or '-manylinux1' in w] + if '-manylinux' not in w or '-manylinux1' in w] build_identifiers = utils.cibuildwheel_get_build_identifiers(project_dir) assert len(expected_wheels) == len(build_identifiers) diff --git a/test/01_basic/setup.py b/test/01_basic/setup.py index 48d7c00b..83f6d53d 100644 --- a/test/01_basic/setup.py +++ b/test/01_basic/setup.py @@ -1,6 +1,9 @@ import os -from setuptools import setup, Extension +from setuptools import ( + Extension, + setup, +) if os.environ.get('CIBUILDWHEEL', '0') != '1': raise Exception('CIBUILDWHEEL environment variable is not set to 1') diff --git a/test/02_test/cibuildwheel_test.py b/test/02_test/cibuildwheel_test.py index e197a16d..ca084d65 100644 --- a/test/02_test/cibuildwheel_test.py +++ b/test/02_test/cibuildwheel_test.py @@ -1,7 +1,11 @@ -import os, subprocess +import os +import subprocess + import pytest + import utils + def test(): project_dir = os.path.dirname(__file__) diff --git a/test/02_test/setup.py b/test/02_test/setup.py index 1609aad6..3a150b3d 100644 --- a/test/02_test/setup.py +++ b/test/02_test/setup.py @@ -1,4 +1,7 @@ -from setuptools import setup, Extension +from setuptools import ( + Extension, + setup, +) setup( name="spam", diff --git a/test/02_test/test/spam_test.py b/test/02_test/test/spam_test.py index a4c17e93..faf7e6bc 100644 --- a/test/02_test/test/spam_test.py +++ b/test/02_test/test/spam_test.py @@ -1,4 +1,5 @@ from unittest import TestCase + import spam diff --git a/test/03_before_build/cibuildwheel_test.py b/test/03_before_build/cibuildwheel_test.py index e3d8ce55..8b0bfcb6 100644 --- a/test/03_before_build/cibuildwheel_test.py +++ b/test/03_before_build/cibuildwheel_test.py @@ -1,6 +1,8 @@ import os + import utils + def test(): project_dir = os.path.dirname(__file__) diff --git a/test/03_before_build/setup.py b/test/03_before_build/setup.py index b7b802fe..fd78c005 100644 --- a/test/03_before_build/setup.py +++ b/test/03_before_build/setup.py @@ -1,5 +1,10 @@ -from setuptools import setup, Extension -import sys, os +import os +import sys + +from setuptools import ( + Extension, + setup, +) # assert that the Python version as written to pythonversion.txt in the CIBW_BEFORE_BUILD step # is the same one as is currently running. diff --git a/test/04_build_skip/cibuildwheel_test.py b/test/04_build_skip/cibuildwheel_test.py index a2999a73..2c6a622c 100644 --- a/test/04_build_skip/cibuildwheel_test.py +++ b/test/04_build_skip/cibuildwheel_test.py @@ -1,6 +1,8 @@ import os + import utils + def test(): project_dir = os.path.dirname(__file__) diff --git a/test/04_build_skip/setup.py b/test/04_build_skip/setup.py index 94f3eb13..2e3fd5c6 100644 --- a/test/04_build_skip/setup.py +++ b/test/04_build_skip/setup.py @@ -1,6 +1,10 @@ -from setuptools import setup, Extension import sys +from setuptools import ( + Extension, + setup, +) + # explode if run on Python 2.7 or Python 3.4 (these should be skipped) if sys.version_info[0:2] == (2, 7): raise Exception('Python 2.7 should not be built') diff --git a/test/05_environment/cibuildwheel_test.py b/test/05_environment/cibuildwheel_test.py index 9efeafc1..4f0b922d 100644 --- a/test/05_environment/cibuildwheel_test.py +++ b/test/05_environment/cibuildwheel_test.py @@ -1,6 +1,8 @@ import os + import utils + def test(): project_dir = os.path.dirname(__file__) diff --git a/test/05_environment/setup.py b/test/05_environment/setup.py index 61abd814..da5e8d59 100644 --- a/test/05_environment/setup.py +++ b/test/05_environment/setup.py @@ -1,5 +1,9 @@ -from setuptools import setup, Extension -import sys, os +import os + +from setuptools import ( + Extension, + setup, +) # explode if environment isn't correct, as set in CIBW_ENVIRONMENT CIBW_TEST_VAR = os.environ.get('CIBW_TEST_VAR') @@ -15,7 +19,6 @@ if '/opt/cibw_test_path' not in PATH: if '$PATH' in PATH: raise Exception('$PATH should be expanded in PATH. It was "%s"' % PATH) - setup( name="spam", ext_modules=[Extension('spam', sources=['spam.c'])], diff --git a/test/06_docker_images/cibuildwheel_test.py b/test/06_docker_images/cibuildwheel_test.py index 1bfc50a7..9f243664 100644 --- a/test/06_docker_images/cibuildwheel_test.py +++ b/test/06_docker_images/cibuildwheel_test.py @@ -1,6 +1,10 @@ -import os, pytest +import os + +import pytest + import utils + def test(): project_dir = os.path.dirname(__file__) diff --git a/test/06_docker_images/setup.py b/test/06_docker_images/setup.py index 8084d9fe..5680349d 100644 --- a/test/06_docker_images/setup.py +++ b/test/06_docker_images/setup.py @@ -1,6 +1,10 @@ -import os, sys +import os +import sys -from setuptools import setup, Extension +from setuptools import ( + Extension, + setup, +) # check that we're running in the correct docker image as specified in the # environment options CIBW_MANYLINUX1_*_IMAGE diff --git a/test/07_ssl/cibuildwheel_test.py b/test/07_ssl/cibuildwheel_test.py index fd3fff87..787ba866 100644 --- a/test/07_ssl/cibuildwheel_test.py +++ b/test/07_ssl/cibuildwheel_test.py @@ -1,6 +1,8 @@ import os + import utils + def test(): project_dir = os.path.dirname(__file__) # this test checks that SSL is working in the build environment using diff --git a/test/07_ssl/setup.py b/test/07_ssl/setup.py index aa60f382..4b6a4f24 100644 --- a/test/07_ssl/setup.py +++ b/test/07_ssl/setup.py @@ -1,6 +1,10 @@ import ssl import sys -from setuptools import setup, Extension + +from setuptools import ( + Extension, + setup, +) if sys.version_info[0] == 2: from urllib2 import urlopen diff --git a/test/08_manylinuxXXXX_only/cibuildwheel_test.py b/test/08_manylinuxXXXX_only/cibuildwheel_test.py index 69c28fcd..439f428f 100644 --- a/test/08_manylinuxXXXX_only/cibuildwheel_test.py +++ b/test/08_manylinuxXXXX_only/cibuildwheel_test.py @@ -1,4 +1,7 @@ -import os, pytest +import os + +import pytest + import utils diff --git a/test/08_manylinuxXXXX_only/setup.py b/test/08_manylinuxXXXX_only/setup.py index 866fa22c..209f45b0 100644 --- a/test/08_manylinuxXXXX_only/setup.py +++ b/test/08_manylinuxXXXX_only/setup.py @@ -1,4 +1,7 @@ -from setuptools import setup, Extension +from setuptools import ( + Extension, + setup, +) setup( name="spam", diff --git a/test/09_setup_cfg/cibuildwheel_test.py b/test/09_setup_cfg/cibuildwheel_test.py index 9e9ffce6..b6c3de01 100644 --- a/test/09_setup_cfg/cibuildwheel_test.py +++ b/test/09_setup_cfg/cibuildwheel_test.py @@ -1,9 +1,10 @@ import os + import utils - project_dir = os.path.dirname(__file__) + def test(): # build the wheels actual_wheels = utils.cibuildwheel_run(project_dir) diff --git a/test/09_setup_cfg/setup.py b/test/09_setup_cfg/setup.py index 0fbbe75e..a4c19252 100644 --- a/test/09_setup_cfg/setup.py +++ b/test/09_setup_cfg/setup.py @@ -1,7 +1,7 @@ -import os - -from setuptools import setup, Extension - +from setuptools import ( + Extension, + setup, +) setup( ext_modules=[Extension('spam.spam', sources=['spam/spam.c'])], diff --git a/test/09_setup_cfg/spam/__init__.py b/test/09_setup_cfg/spam/__init__.py index a68927d6..3dc1f76b 100644 --- a/test/09_setup_cfg/spam/__init__.py +++ b/test/09_setup_cfg/spam/__init__.py @@ -1 +1 @@ -__version__ = "0.1.0" \ No newline at end of file +__version__ = "0.1.0" diff --git a/test/shared/utils.py b/test/shared/utils.py index 09923326..4d6b96d0 100644 --- a/test/shared/utils.py +++ b/test/shared/utils.py @@ -4,10 +4,12 @@ Utility functions used by the cibuildwheel tests. This file is added to the PYTHONPATH in the test runner at bin/run_test.py. ''' -import subprocess, sys, os, shutil -from tempfile import mkdtemp +import os +import shutil +import subprocess +import sys from contextlib import contextmanager - +from tempfile import mkdtemp IS_WINDOWS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache') IS_WINDOWS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows' diff --git a/unit_test/environment_test.py b/unit_test/environment_test.py index 1bc95374..b8b961a8 100644 --- a/unit_test/environment_test.py +++ b/unit_test/environment_test.py @@ -1,4 +1,5 @@ import os + from cibuildwheel.environment import parse_environment @@ -13,6 +14,7 @@ def test_basic_parsing(): assert environment_dict == {'VAR': '1', 'VBR': '2'} assert environment_cmds == ['export VAR=1', 'export VBR=2'] + def test_quotes(): environment_recipe = parse_environment('A=1 VAR="1 NOT_A_VAR=2" VBR=\'vbr\'') @@ -24,6 +26,7 @@ def test_quotes(): assert environment_dict == {'A': '1', 'VAR': '1 NOT_A_VAR=2', 'VBR': 'vbr'} assert environment_cmds == ['export A=1', 'export VAR="1 NOT_A_VAR=2"', 'export VBR=\'vbr\''] + def test_inheritance(): environment_recipe = parse_environment('PATH=$PATH:/usr/local/bin') @@ -35,6 +38,7 @@ def test_inheritance(): assert environment_dict == {'PATH': '/usr/bin:/usr/local/bin'} assert environment_cmds == ['export PATH=$PATH:/usr/local/bin'] + def test_shell_eval(): environment_recipe = parse_environment('VAR="$(echo "a test" string)"') @@ -49,6 +53,7 @@ def test_shell_eval(): assert environment_dict['VAR'] == 'a test string' assert environment_cmds == ['export VAR="$(echo "a test" string)"'] + def test_shell_eval_and_env(): environment_recipe = parse_environment('VAR="$(echo "$PREV_VAR" string)"') @@ -60,6 +65,7 @@ def test_shell_eval_and_env(): assert environment_dict == {'PREV_VAR': '1 2 3', 'VAR': '1 2 3 string'} assert environment_cmds == ['export VAR="$(echo "$PREV_VAR" string)"'] + def test_empty_var(): environment_recipe = parse_environment('CFLAGS=') @@ -71,6 +77,7 @@ def test_empty_var(): assert environment_dict == {'CFLAGS': ''} assert environment_cmds == ['export CFLAGS='] + def test_no_vars(): environment_recipe = parse_environment('') @@ -80,6 +87,7 @@ def test_no_vars(): assert environment_dict == {} assert environment_cmds == [] + def test_no_vars_pass_through(): environment_recipe = parse_environment('') From c15d6567267da1c5eaa9e3114fe1b6e0993bb110 Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Sun, 17 Nov 2019 19:45:59 +0000 Subject: [PATCH 12/13] Add a CircleCI job for flake8 --- .circleci/config.yml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 3d2d09a1..4914ed1a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -1,6 +1,19 @@ version: 2 jobs: + flake8: + docker: + - image: circleci/python:3.6 + steps: + - checkout + + - run: + name: Install flake8 + command: sudo python -m pip install flake8 + - run: + name: Test. + command: flake8 --exclude=.git,.venv,site . + osx-python3.6: macos: xcode: "9.4.1" @@ -51,6 +64,7 @@ workflows: version: 2 all-tests: jobs: + - flake8 - osx-python3.6 - osx-python3.7 - linux-python3.6 From c738481e820f95f7cca60e0168eda7e950231fbe Mon Sep 17 00:00:00 2001 From: Nicola Soranzo Date: Tue, 7 Jan 2020 02:42:38 +0000 Subject: [PATCH 13/13] Rename main_util_fixtures.py to conftest.py to allow fixture auto discovery Also fix new PEP-008 issues. --- cibuildwheel/__main__.py | 5 ++++- cibuildwheel/bashlex_eval.py | 2 -- .../mkdocs_include_markdown_plugin/plugin.py | 1 - test/10_cpp_standards/cibuildwheel_test.py | 17 ++++++++--------- test/10_cpp_standards/setup.py | 8 ++++++-- .../conftest.py} | 17 ++++++++++++----- unit_test/{ => main_tests}/main_options_test.py | 10 +++------- .../{ => main_tests}/main_platform_test.py | 6 +++--- 8 files changed, 36 insertions(+), 30 deletions(-) rename unit_test/{main_util_fixtures.py => main_tests/conftest.py} (95%) rename unit_test/{ => main_tests}/main_options_test.py (98%) rename unit_test/{ => main_tests}/main_platform_test.py (93%) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 2d309788..b3aaacf2 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -12,7 +12,10 @@ from cibuildwheel.environment import ( EnvironmentParseError, parse_environment, ) -from cibuildwheel.util import BuildSelector, Unbuffered +from cibuildwheel.util import ( + BuildSelector, + Unbuffered +) def get_option_from_environment(option_name, platform=None, default=None): diff --git a/cibuildwheel/bashlex_eval.py b/cibuildwheel/bashlex_eval.py index 7e2d7f4a..8e2c0831 100644 --- a/cibuildwheel/bashlex_eval.py +++ b/cibuildwheel/bashlex_eval.py @@ -1,6 +1,5 @@ import shlex import subprocess -import sys from collections import namedtuple import bashlex @@ -67,6 +66,5 @@ def evaluate_command_node(node, context): return subprocess.check_output(shlex.split(command), env=context.environment, universal_newlines=True) - def evaluate_parameter_node(node, context): return context.environment.get(node.value, '') diff --git a/docs/mkdocs_include_markdown_plugin/mkdocs_include_markdown_plugin/plugin.py b/docs/mkdocs_include_markdown_plugin/mkdocs_include_markdown_plugin/plugin.py index 2f43d0f3..b6b7a723 100644 --- a/docs/mkdocs_include_markdown_plugin/mkdocs_include_markdown_plugin/plugin.py +++ b/docs/mkdocs_include_markdown_plugin/mkdocs_include_markdown_plugin/plugin.py @@ -55,7 +55,6 @@ class ImportMarkdownPlugin(mkdocs.plugins.BasePlugin): return text_to_include - def found_includemarkdown_tag(match): filename = match.group('filename') start = match.group('start') diff --git a/test/10_cpp_standards/cibuildwheel_test.py b/test/10_cpp_standards/cibuildwheel_test.py index 2fc172e0..9b2cb724 100644 --- a/test/10_cpp_standards/cibuildwheel_test.py +++ b/test/10_cpp_standards/cibuildwheel_test.py @@ -1,13 +1,12 @@ import os -import sys import pytest import utils - project_dir = os.path.dirname(__file__) + def test_cpp11(tmp_path): # This test checks that the C++11 standard is supported @@ -18,8 +17,8 @@ def test_cpp11(tmp_path): actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env) expected_wheels = [x for x in utils.expected_wheels( - 'spam', '0.1.0', macosx_deployment_target='10.9') - if 'cp27-cp27m-win' not in x] + 'spam', '0.1.0', macosx_deployment_target='10.9') + if 'cp27-cp27m-win' not in x] assert set(actual_wheels) == set(expected_wheels) @@ -35,15 +34,15 @@ def test_cpp14(): actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env) expected_wheels = [x for x in utils.expected_wheels( - 'spam', '0.1.0', macosx_deployment_target='10.9') - if 'cp27-cp27m-win' not in x and 'cp35-cp35m-win' not in x] + 'spam', '0.1.0', macosx_deployment_target='10.9') + if 'cp27-cp27m-win' not in x and 'cp35-cp35m-win' not in x] assert set(actual_wheels) == set(expected_wheels) def test_cpp17(): # This test checks that the C++17 standard is supported - # Python 2.7 uses the `register` keyword which is forbidden in the C++17 standard + # Python 2.7 uses the `register` keyword which is forbidden in the C++17 standard # The manylinux1 docker image does not have a compiler which supports C++11 # Python 3.4 and 3.5 are compiled with MSVC 10, which does not support C++17 if os.environ.get('APPVEYOR_BUILD_WORKER_IMAGE', '') == 'Visual Studio 2015': @@ -55,6 +54,6 @@ def test_cpp17(): actual_wheels = utils.cibuildwheel_run(project_dir, add_env=add_env) expected_wheels = [x for x in utils.expected_wheels( - 'spam', '0.1.0', macosx_deployment_target='10.13') - if 'cp27-cp27m-win' not in x and 'cp35-cp35m-win' not in x] + 'spam', '0.1.0', macosx_deployment_target='10.13') + if 'cp27-cp27m-win' not in x and 'cp35-cp35m-win' not in x] assert set(actual_wheels) == set(expected_wheels) diff --git a/test/10_cpp_standards/setup.py b/test/10_cpp_standards/setup.py index 969ca39b..f848d0b1 100644 --- a/test/10_cpp_standards/setup.py +++ b/test/10_cpp_standards/setup.py @@ -1,7 +1,11 @@ -import os, sys -from setuptools import setup, Extension +import os import platform +from setuptools import ( + Extension, + setup, +) + standard = os.environ["STANDARD"] language_standard = "/std:c++" + standard if platform.system() == "Windows" else "-std=c++" + standard diff --git a/unit_test/main_util_fixtures.py b/unit_test/main_tests/conftest.py similarity index 95% rename from unit_test/main_util_fixtures.py rename to unit_test/main_tests/conftest.py index 21a7f36a..227b8df7 100644 --- a/unit_test/main_util_fixtures.py +++ b/unit_test/main_tests/conftest.py @@ -1,10 +1,15 @@ -import pytest - -import sys import os import subprocess +import sys -from cibuildwheel import linux, macos, windows, util +import pytest + +from cibuildwheel import ( + linux, + macos, + util, + windows, +) class ArgsInterceptor: @@ -15,6 +20,7 @@ class ArgsInterceptor: MOCK_PROJECT_DIR = 'some_project_dir' + @pytest.fixture(autouse=True) def mock_protection(monkeypatch): ''' @@ -31,13 +37,14 @@ def mock_protection(monkeypatch): monkeypatch.setattr(linux, 'build', fail_on_call) monkeypatch.setattr(macos, 'build', fail_on_call) + @pytest.fixture(autouse=True) def fake_project_dir(monkeypatch): ''' Monkey-patch enough for the main() function to run ''' - real_os_path_exists = os.path.exists + def mock_os_path_exists(path): if path == os.path.join(MOCK_PROJECT_DIR, 'setup.py'): return True diff --git a/unit_test/main_options_test.py b/unit_test/main_tests/main_options_test.py similarity index 98% rename from unit_test/main_options_test.py rename to unit_test/main_tests/main_options_test.py index 44d92da8..05a20afc 100644 --- a/unit_test/main_options_test.py +++ b/unit_test/main_tests/main_options_test.py @@ -1,14 +1,11 @@ -import pytest - import sys +import pytest + from cibuildwheel.__main__ import main from cibuildwheel.environment import ParsedEnvironment from cibuildwheel.util import BuildSelector -from main_util_fixtures import mock_protection, fake_project_dir, platform, intercepted_build_args - - # CIBW_PLATFORM is tested in main_platform_test.py @@ -93,6 +90,7 @@ def get_default_repair_command(platform): else: raise ValueError('Unknown platform', platform) + @pytest.mark.parametrize('repair_command', [None, 'repair', 'repair -w {dest_dir} {wheel}']) @pytest.mark.parametrize('platform_specific', [False, True]) def test_repair_command(repair_command, platform_specific, platform, intercepted_build_args, monkeypatch): @@ -225,5 +223,3 @@ def test_build_selector_migrations(intercepted_build_args, monkeypatch, option_n assert intercepted_build_selector.build_patterns == build_selector_patterns else: assert intercepted_build_selector.skip_patterns == build_selector_patterns - - diff --git a/unit_test/main_platform_test.py b/unit_test/main_tests/main_platform_test.py similarity index 93% rename from unit_test/main_platform_test.py rename to unit_test/main_tests/main_platform_test.py index 3de3be29..dae0c596 100644 --- a/unit_test/main_platform_test.py +++ b/unit_test/main_tests/main_platform_test.py @@ -1,10 +1,10 @@ -import pytest - import sys +import pytest + from cibuildwheel.__main__ import main -from main_util_fixtures import MOCK_PROJECT_DIR, mock_protection, fake_project_dir, platform, intercepted_build_args +from conftest import MOCK_PROJECT_DIR # noqa: I100 def test_unknown_platform_non_ci(monkeypatch, capsys):