From a5f89fa7aad09c4d8b28c95e2b77d351c3c8e41d Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Sun, 16 Sep 2018 15:16:30 +0200 Subject: [PATCH 1/5] Adding CIBW_BUILD option and changing util.BuildSkipper to util.BuildSelector --- cibuildwheel/__main__.py | 8 ++++---- cibuildwheel/linux.py | 4 ++-- cibuildwheel/macos.py | 4 ++-- cibuildwheel/util.py | 13 ++++++++----- cibuildwheel/windows.py | 4 ++-- 5 files changed, 18 insertions(+), 15 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index dc00ce54..2ae108d3 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -4,7 +4,7 @@ import argparse, os, subprocess, sys, textwrap import cibuildwheel import cibuildwheel.linux, cibuildwheel.windows, cibuildwheel.macos from cibuildwheel.environment import parse_environment, EnvironmentParseError -from cibuildwheel.util import BuildSkipper, Unbuffered +from cibuildwheel.util import BuildSelector, Unbuffered def get_option_from_environment(option_name, platform=None, default=None): ''' @@ -83,7 +83,7 @@ def main(): project_dir = args.project_dir before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform) build_verbosity = get_option_from_environment('CIBW_BUILD_VERBOSITY', platform=platform, default='') - skip_config = os.environ.get('CIBW_SKIP', '') + build_config, skip_config = os.environ.get('CIBW_BUILD', '*'), os.environ.get('CIBW_SKIP', '') environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform, default='') try: @@ -99,7 +99,7 @@ def main(): traceback.print_exc(None, sys.stderr) exit(2) - skip = BuildSkipper(skip_config) + selection = BuildSelector(build_config, skip_config) # Add CIBUILDWHEEL environment variable # This needs to be passed on to the docker container in linux.py @@ -134,7 +134,7 @@ def main(): test_requires=test_requires, before_build=before_build, build_verbosity=build_verbosity, - skip=skip, + selection=selection, environment=environment, ) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index a3eb19f9..400295cc 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -9,7 +9,7 @@ except ImportError: from pipes import quote as shlex_quote -def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, build_verbosity, skip, environment, manylinux1_images): +def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, build_verbosity, selection, environment, manylinux1_images): try: subprocess.check_call(['docker', '--version']) except: @@ -36,7 +36,7 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be ] # skip builds as required - python_configurations = [c for c in python_configurations if not skip(c.identifier)] + python_configurations = [c for c in python_configurations if selection(c.identifier)] platforms = [ ('manylinux1_x86_64', manylinux1_images.get('x86_64') or 'quay.io/pypa/manylinux1_x86_64'), diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 6ba8dcdb..f45bb1f6 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -10,7 +10,7 @@ except ImportError: from .util import prepare_command, get_build_verbosity_extra_flags -def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, build_verbosity, skip, environment): +def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, build_verbosity, selection, environment): PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'url']) python_configurations = [ PythonConfiguration(version='2.7', identifier='cp27-macosx_10_6_intel', url='https://www.python.org/ftp/python/2.7.15/python-2.7.15-macosx10.6.pkg'), @@ -42,7 +42,7 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be call(['curl', '-L', '-o', get_pip_script, get_pip_url]) for config in python_configurations: - if skip(config.identifier): + if not selection(config.identifier): print('cibuildwheel: Skipping build %s' % config.identifier, file=sys.stderr) continue diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 7db3edaa..160e8c20 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -21,15 +21,18 @@ def get_build_verbosity_extra_flags(level): return [] -class BuildSkipper(object): - def __init__(self, skip_config): - self.patterns = skip_config.split() +class BuildSelector(object): + def __init__(self, build_config, skip_config): + self.build_patterns = build_config.split() + self.skip_patterns = skip_config.split() def __call__(self, build_id): - return any(fnmatch(build_id, pattern) for pattern in self.patterns) + def match_any(patterns): + return any(fnmatch(build_id, pattern) for pattern in patterns) + return match_any(self.build_patterns) and not match_any(self.skip_patterns) def __repr__(self): - return 'BuildSkipper(%r)' % ' '.join(self.patterns) + return 'BuildSelector({!r} - {!r})'.format(' '.join(self.build_patterns), ' '.join(self.skip_patterns)) # Taken from https://stackoverflow.com/a/107717 diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 4118bd33..34dafb0c 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -10,7 +10,7 @@ from glob import glob from .util import prepare_command, get_build_verbosity_extra_flags -def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, build_verbosity, skip, environment): +def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, build_verbosity, selection, environment): # run_with_env is a cmd file that sets the right environment variables to run_with_env = os.path.join(tempfile.gettempdir(), 'appveyor_run_with_env.cmd') if not os.path.exists(run_with_env): @@ -45,7 +45,7 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be built_wheel_dir = os.path.join(temp_dir, 'built_wheel') for config in python_configurations: - if skip(config.identifier): + if not selection(config.identifier): print('cibuildwheel: Skipping build %s' % config.identifier, file=sys.stderr) continue From 05a054245a7af4c5b0524e53c3ee73850a87b132 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Sun, 16 Sep 2018 15:28:07 +0200 Subject: [PATCH 2/5] Updating README.md to include CIBW_BUILD and interation with CIBW_SKIP --- README.md | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 4e5625a9..11118e34 100644 --- a/README.md +++ b/README.md @@ -131,6 +131,7 @@ All being well, you should get wheels delivered to you in a few minutes. | | Option | | |---|---|---| | **Target wheels** | `CIBW_PLATFORM` | Override the auto-detected target platform | +| | `CIBW_BUILD` | Build only certain Python versions | | | `CIBW_SKIP` | Skip certain Python versions | | **Build parameters** | `CIBW_BUILD_VERBOSITY` | Increase or decrease the output of `pip wheel` | | **Build environment** | `CIBW_ENVIRONMENT` | Set environment variables needed during the build | @@ -197,12 +198,14 @@ For `linux` you need Docker running, on Mac or Linux. For `macos`, you need a Ma *** -| Environment variable: `CIBW_SKIP` +| Environment variables: `CIBW_BUILD` and `CIBW_SKIP` | --- Optional. -Space-separated list of builds to skip. Each build has an identifier like `cp27-manylinux1_x86_64` or `cp34-macosx_10_6_intel` - you can list ones to skip here and `cibuildwheel` won't try to build them. +Space-separated list of builds to build and skip. Each build has an identifier like `cp27-manylinux1_x86_64` or `cp34-macosx_10_6_intel` - you can list specific ones to build and `cibuildwheel` will only build those, and/or list ones to skip and `cibuildwheel` won't try to build them. + +When both options are specified, both conditions are applied and only builds with a tag that matches `CIBW_BUILD` and does not match `CIBW_SKIP` will be built. The format is `python_tag-platform_tag`. The tags are as defined in [PEP 0425](https://www.python.org/dev/peps/pep-0425/#details). @@ -213,15 +216,16 @@ Platform tags look like `macosx_10_6_intel` `manylinux1_x86_64` `manylinux1_i686 You can also use shell-style globbing syntax (as per `fnmatch`) Examples: -- Skip building on Python 2.7 on the Mac: `cp27-macosx_10_6_intel` -- Skip building on Python 2.7 on all platforms: `cp27-*` -- Skip Python 2.7 on Windows: `cp27-win*` -- Skip Python 2.7 on 32bit Windows: `cp27-win32` -- Skip Python 3.4 and Python 3.5: `cp34-* cp35-*` -- Skip Python 3.6 on Linux: `cp36-manylinux*` -- Only build on Python 3.6: `cp27-* cp34-* cp35-*` +- Only build on Python 3.6: `CIBW_BUILD`:`cp36-*` +- Skip building on Python 2.7 on the Mac: `CIBW_SKIP`:`cp27-macosx_10_6_intel` +- Skip building on Python 2.7 on all platforms: `CIBW_SKIP`:`cp27-*` +- Skip Python 2.7 on Windows: `CIBW_SKIP`:`cp27-win*` +- Skip Python 2.7 on 32-bit Windows: `CIBW_SKIP`:`cp27-win32` +- Skip Python 3.4 and Python 3.5: `CIBW_SKIP`:`cp34-* cp35-*` +- Skip Python 3.6 on Linux: `CIBW_SKIP`:`cp36-manylinux*` +- Only build on Python 3 and skip 32-bit builds: `CIBW_BUILD`:`cp3?-*` and `CIBW_SKIP`:`*-win32 *-manylinux1_i686` -*** +** | Environment variable: `CIBW_BUILD_VERBOSITY` | --- From c108807e161ea50cbe542b93eb92a1ee5de577f6 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Sun, 16 Sep 2018 15:31:05 +0200 Subject: [PATCH 3/5] Renaming test 04_skip to 04_build_skip and amending it to test CIBW_BUILD as well --- test/04_build_skip/environment.json | 4 ++++ test/{04_skip => 04_build_skip}/setup.py | 6 +++--- test/{04_skip => 04_build_skip}/spam.c | 0 test/{04_skip => 04_build_skip}/version.txt | 0 test/04_skip/environment.json | 3 --- 5 files changed, 7 insertions(+), 6 deletions(-) create mode 100644 test/04_build_skip/environment.json rename test/{04_skip => 04_build_skip}/setup.py (64%) rename test/{04_skip => 04_build_skip}/spam.c (100%) rename test/{04_skip => 04_build_skip}/version.txt (100%) delete mode 100644 test/04_skip/environment.json diff --git a/test/04_build_skip/environment.json b/test/04_build_skip/environment.json new file mode 100644 index 00000000..06a0eb59 --- /dev/null +++ b/test/04_build_skip/environment.json @@ -0,0 +1,4 @@ +{ + "CIBW_BUILD": "cp3?-*", + "CIBW_SKIP": "cp33-*" +} diff --git a/test/04_skip/setup.py b/test/04_build_skip/setup.py similarity index 64% rename from test/04_skip/setup.py rename to test/04_build_skip/setup.py index 8d25ca60..f4434cf0 100644 --- a/test/04_skip/setup.py +++ b/test/04_build_skip/setup.py @@ -2,9 +2,9 @@ from setuptools import setup, Extension import sys if sys.argv[-1] != '--name': - # explode if run on Python 2.6 or Python 3.3 (these should be skipped) - if sys.version_info[0:2] == (2, 6): - raise Exception('Python 2.6 should be skipped') + # explode if run on Python 2.7 or Python 3.3 (these should be skipped) + if sys.version_info[0:2] == (2, 7): + raise Exception('Python 2.7 should not be built') if sys.version_info[0:2] == (3, 3): raise Exception('Python 3.3 should be skipped') diff --git a/test/04_skip/spam.c b/test/04_build_skip/spam.c similarity index 100% rename from test/04_skip/spam.c rename to test/04_build_skip/spam.c diff --git a/test/04_skip/version.txt b/test/04_build_skip/version.txt similarity index 100% rename from test/04_skip/version.txt rename to test/04_build_skip/version.txt diff --git a/test/04_skip/environment.json b/test/04_skip/environment.json deleted file mode 100644 index a19912ba..00000000 --- a/test/04_skip/environment.json +++ /dev/null @@ -1,3 +0,0 @@ -{ - "CIBW_SKIP": "cp26-* cp33-*" -} From 7362075b61a2bde09780d38ce6e80606e7b87cd1 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Sun, 16 Sep 2018 22:49:38 +0200 Subject: [PATCH 4/5] Changing variable name of BuildSelector instance, updating Python version in test 04_build_skip, and adding unit tests for BuildSelector --- cibuildwheel/__main__.py | 4 +- cibuildwheel/linux.py | 4 +- cibuildwheel/macos.py | 4 +- cibuildwheel/windows.py | 4 +- test/04_build_skip/environment.json | 2 +- test/04_build_skip/setup.py | 4 +- unit_test/build_selector_test.py | 61 +++++++++++++++++++++++++++++ 7 files changed, 72 insertions(+), 11 deletions(-) create mode 100644 unit_test/build_selector_test.py diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 2ae108d3..99fea065 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -99,7 +99,7 @@ def main(): traceback.print_exc(None, sys.stderr) exit(2) - selection = BuildSelector(build_config, skip_config) + build_selector = BuildSelector(build_config, skip_config) # Add CIBUILDWHEEL environment variable # This needs to be passed on to the docker container in linux.py @@ -134,7 +134,7 @@ def main(): test_requires=test_requires, before_build=before_build, build_verbosity=build_verbosity, - selection=selection, + build_selector=build_selector, environment=environment, ) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 400295cc..ee221443 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -9,7 +9,7 @@ except ImportError: from pipes import quote as shlex_quote -def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, build_verbosity, selection, environment, manylinux1_images): +def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, build_verbosity, build_selector, environment, manylinux1_images): try: subprocess.check_call(['docker', '--version']) except: @@ -36,7 +36,7 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be ] # skip builds as required - python_configurations = [c for c in python_configurations if selection(c.identifier)] + python_configurations = [c for c in python_configurations if build_selector(c.identifier)] platforms = [ ('manylinux1_x86_64', manylinux1_images.get('x86_64') or 'quay.io/pypa/manylinux1_x86_64'), diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index f45bb1f6..74e1990c 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -10,7 +10,7 @@ except ImportError: from .util import prepare_command, get_build_verbosity_extra_flags -def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, build_verbosity, selection, environment): +def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, build_verbosity, build_selector, environment): PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'url']) python_configurations = [ PythonConfiguration(version='2.7', identifier='cp27-macosx_10_6_intel', url='https://www.python.org/ftp/python/2.7.15/python-2.7.15-macosx10.6.pkg'), @@ -42,7 +42,7 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be call(['curl', '-L', '-o', get_pip_script, get_pip_url]) for config in python_configurations: - if not selection(config.identifier): + if not build_selector(config.identifier): print('cibuildwheel: Skipping build %s' % config.identifier, file=sys.stderr) continue diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 34dafb0c..fe9d496d 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -10,7 +10,7 @@ from glob import glob from .util import prepare_command, get_build_verbosity_extra_flags -def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, build_verbosity, selection, environment): +def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, build_verbosity, build_selector, environment): # run_with_env is a cmd file that sets the right environment variables to run_with_env = os.path.join(tempfile.gettempdir(), 'appveyor_run_with_env.cmd') if not os.path.exists(run_with_env): @@ -45,7 +45,7 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be built_wheel_dir = os.path.join(temp_dir, 'built_wheel') for config in python_configurations: - if not selection(config.identifier): + if not build_selector(config.identifier): print('cibuildwheel: Skipping build %s' % config.identifier, file=sys.stderr) continue diff --git a/test/04_build_skip/environment.json b/test/04_build_skip/environment.json index 06a0eb59..a47c85d2 100644 --- a/test/04_build_skip/environment.json +++ b/test/04_build_skip/environment.json @@ -1,4 +1,4 @@ { "CIBW_BUILD": "cp3?-*", - "CIBW_SKIP": "cp33-*" + "CIBW_SKIP": "cp34-*" } diff --git a/test/04_build_skip/setup.py b/test/04_build_skip/setup.py index f4434cf0..75b1232b 100644 --- a/test/04_build_skip/setup.py +++ b/test/04_build_skip/setup.py @@ -5,8 +5,8 @@ if sys.argv[-1] != '--name': # explode if run on Python 2.7 or Python 3.3 (these should be skipped) if sys.version_info[0:2] == (2, 7): raise Exception('Python 2.7 should not be built') - if sys.version_info[0:2] == (3, 3): - raise Exception('Python 3.3 should be skipped') + if sys.version_info[0:2] == (3, 4): + raise Exception('Python 3.4 should be skipped') setup( name="spam", diff --git a/unit_test/build_selector_test.py b/unit_test/build_selector_test.py new file mode 100644 index 00000000..8737ee6b --- /dev/null +++ b/unit_test/build_selector_test.py @@ -0,0 +1,61 @@ +from cibuildwheel.util import BuildSelector + + +def test_build(): + build_selector = BuildSelector(build_config="cp3?-* *-manylinux1*", skip_config="") + + assert build_selector('cp27-manylinux1_x86_64') + assert build_selector('cp36-manylinux1_x86_64') + assert build_selector('cp37-manylinux1_x86_64') + assert build_selector('cp27-manylinux1_i686') + assert build_selector('cp36-manylinux1_i686') + assert build_selector('cp37-manylinux1_i686') + assert not build_selector('cp27-macosx_10_6_intel') + assert build_selector('cp36-macosx_10_6_intel') + assert build_selector('cp37-macosx_10_6_intel') + assert not build_selector('cp27-win32') + assert build_selector('cp36-win32') + assert build_selector('cp37-win32') + assert not build_selector('cp27-win_amd64') + assert build_selector('cp36-win_amd64') + assert build_selector('cp37-win_amd64') + + +def test_skip(): + build_selector = BuildSelector(build_config="*", skip_config="cp27-* cp3?-manylinux1_i686 cp36-win* *-win32") + + assert not build_selector('cp27-manylinux1_x86_64') + assert build_selector('cp36-manylinux1_x86_64') + assert build_selector('cp37-manylinux1_x86_64') + assert not build_selector('cp27-manylinux1_i686') + assert not build_selector('cp36-manylinux1_i686') + assert not build_selector('cp37-manylinux1_i686') + assert not build_selector('cp27-macosx_10_6_intel') + assert build_selector('cp36-macosx_10_6_intel') + assert build_selector('cp37-macosx_10_6_intel') + assert not build_selector('cp27-win32') + assert not build_selector('cp36-win32') + assert not build_selector('cp37-win32') + assert not build_selector('cp27-win_amd64') + assert not build_selector('cp36-win_amd64') + assert build_selector('cp37-win_amd64') + + +def test_build_and_skip(): + build_selector = BuildSelector(build_config="cp36-* cp37-macosx* *-manylinux1*", skip_config="cp27-* cp37-manylinux1_i686") + + assert not build_selector('cp27-manylinux1_x86_64') + assert build_selector('cp36-manylinux1_x86_64') + assert build_selector('cp37-manylinux1_x86_64') + assert not build_selector('cp27-manylinux1_i686') + assert build_selector('cp36-manylinux1_i686') + assert not build_selector('cp37-manylinux1_i686') + assert not build_selector('cp27-macosx_10_6_intel') + assert build_selector('cp36-macosx_10_6_intel') + assert build_selector('cp37-macosx_10_6_intel') + assert not build_selector('cp27-win32') + assert build_selector('cp36-win32') + assert not build_selector('cp37-win32') + assert not build_selector('cp27-win_amd64') + assert build_selector('cp36-win_amd64') + assert not build_selector('cp37-win_amd64') From 22a6f0c7a98c978d0d889f41e2d70e0978736904 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Mon, 17 Sep 2018 10:11:27 +0200 Subject: [PATCH 5/5] Fixing tests by reducing minimum number of built wheels --- bin/run_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/run_test.py b/bin/run_test.py index efcf3de5..94db6512 100755 --- a/bin/run_test.py +++ b/bin/run_test.py @@ -22,7 +22,7 @@ def single_run(test_project): print('%s built successfully. %i wheels built.' % (test_project, len(wheels))) # check some wheels were actually built - assert len(wheels) >= 4 + assert len(wheels) >= 3 # clean up shutil.rmtree('wheelhouse')