Merge branch 'master' of github.com:joerick/cibuildwheel into min-macosx-deployment-target

appveyor.yml

@@ -14,7 +14,7 @@ build: off
 init:
 - cmd: set PATH=C:\Python37;C:\Python37\Scripts;%PATH%
 
-install: python -m pip install -r requirements-dev.txt
+install: python -m pip install --retries 3 -r requirements-dev.txt
 
 # the '-u' flag is required so the output is in the correct order.
 # See https://github.com/joerick/cibuildwheel/pull/24 for more info.

bin/make_dependency_update_pr.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+
+import os
+import time
+from pathlib import Path
+from subprocess import run
+
+import click
+import textwrap
+
+
+def shell(cmd, **kwargs):
+    return run([cmd], shell=True, **kwargs)
+
+
+def git_repo_has_changes():
+    unstaged_changes = shell('git diff-index --quiet HEAD --').returncode != 0
+    staged_changes = shell('git diff-index --quiet --cached HEAD --').returncode != 0
+    return unstaged_changes or staged_changes
+
+
+@click.command()
+def main():
+    project_root = Path(__file__).parent / '..'
+    os.chdir(project_root)
+
+    if git_repo_has_changes():
+        print('Your git repo has uncommitted changes. Commit or stash before continuing.')
+        exit(1)
+
+    previous_branch = shell('git rev-parse --abbrev-ref HEAD',
+                            check=True,
+                            capture_output=True,
+                            encoding='utf8').stdout.strip()
+
+    shell('git fetch origin', check=True)
+
+    timestamp = time.strftime('%Y-%m-%dT%H-%M-%S', time.gmtime())
+    branch_name = f'update-constraints-{timestamp}'
+
+    shell(f'git checkout -b {branch_name} origin/master', check=True)
+
+    try:
+        shell('bin/update_dependencies.py', check=True)
+
+        if not git_repo_has_changes():
+            print('Done: no constraint updates required.')
+            return
+
+        shell('git commit -a -m "Update dependencies"', check=True)
+        run(
+            [
+                'gh', 'pr', 'create',
+                '--repo', 'joerick/cibuildwheel',
+                '--base', 'master',
+                '--title', 'Update dependencies',
+                '--body', textwrap.dedent(f'''
+                    Update the versions of our dependencies.
+
+                    PR generated by `{os.path.basename(__file__)}`.
+                ''')
+            ],
+            check=True
+        )
+
+        print('Done.')
+    finally:
+        # remove any local changes
+        shell('git checkout -- .')
+        shell(f'git checkout {previous_branch}', check=True)
+        shell(f'git branch -D --force {branch_name}', check=True)
+
+
+if __name__ == '__main__':
+    main.main(standalone_mode=True)

bin/update_dependencies.py

@@ -84,7 +84,9 @@ for image in images:
         digest = latest_tag['images'][0]['digest']
 
         pinned_tag = next(
-            tag for tag in tags if tag['images'][0]['digest'] == digest
+            tag
+            for tag in tags
+            if tag != latest_tag and tag['images'][0]['digest'] == digest
         )
         tag_name = pinned_tag['name']
 

cibuildwheel/macos.py

@@ -108,19 +108,11 @@ def install_pypy(version, url):
     return installation_bin_path
 
 
-def build(project_dir, output_dir, test_command, before_test, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, dependency_constraints):
+def setup_python(python_configuration, dependency_constraint_flags, environment):
-    abs_project_dir = os.path.abspath(project_dir)
+    if python_configuration.identifier.startswith('cp'):
-    temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
+        installation_bin_path = install_cpython(python_configuration.version, python_configuration.url)
-    built_wheel_dir = os.path.join(temp_dir, 'built_wheel')
+    elif python_configuration.identifier.startswith('pp'):
-    repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel')
+        installation_bin_path = install_pypy(python_configuration.version, python_configuration.url)
-
-    python_configurations = get_python_configurations(build_selector)
-
-    for config in python_configurations:
-        if config.identifier.startswith('cp'):
-            installation_bin_path = install_cpython(config.version, config.url)
-        elif config.identifier.startswith('pp'):
-            installation_bin_path = install_pypy(config.version, config.url)
     else:
         raise ValueError("Unknown Python implementation")
     
@@ -148,12 +140,6 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes
         print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr)
         exit(1)
         
-        dependency_constraint_flags = []
-        if dependency_constraints:
-            dependency_constraint_flags = [
-                '-c', dependency_constraints.get_for_python_version(config.version)
-            ]
-
         # install pip & wheel
         call(['python', get_pip_script] + dependency_constraint_flags, env=env, cwd="/tmp")
         assert os.path.exists(os.path.join(installation_bin_path, 'pip'))
@@ -177,6 +163,28 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes
             # https://github.com/python/cpython/blob/a5ed2fe0eedefa1649aa93ee74a0bafc8e628a10/Lib/_osx_support.py#L260
             env.setdefault('ARCHFLAGS', '-arch x86_64')
 
+    return env
+
+
+def build(project_dir, output_dir, test_command, before_test, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, dependency_constraints):
+    abs_project_dir = os.path.abspath(project_dir)
+    temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
+    built_wheel_dir = os.path.join(temp_dir, 'built_wheel')
+    repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel')
+
+    python_configurations = get_python_configurations(build_selector)
+
+    for config in python_configurations:
+
+        dependency_constraint_flags = []
+        if dependency_constraints:
+            dependency_constraint_flags = [
+                '-c', dependency_constraints.get_for_python_version(config.version)
+            ]
+        
+        env = setup_python(config, dependency_constraint_flags, environment)
+
+
         # run the before_build command
         if before_build:
             before_build_prepared = prepare_command(before_build, project=abs_project_dir)

cibuildwheel/windows.py

@@ -89,23 +89,15 @@ def install_pypy(version, arch, url):
     return installation_path
 
 
-def build(project_dir, output_dir, test_command, before_test, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, dependency_constraints):
+def setup_python(python_configuration, dependency_constraint_flags, environment):
-    abs_project_dir = os.path.abspath(project_dir)
-    temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
-    built_wheel_dir = os.path.join(temp_dir, 'built_wheel')
-    repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel')
-
-    # install nuget as best way to provide python
     nuget = 'C:\\cibw\\nuget.exe'
+    if not os.path.exists(nuget):
         download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget)
 
-    python_configurations = get_python_configurations(build_selector)
+    if python_configuration.identifier.startswith('cp'):
-    for config in python_configurations:
+        installation_path = install_cpython(python_configuration.version, python_configuration.arch, nuget)
-        # install Python
+    elif python_configuration.identifier.startswith('pp'):
-        if config.identifier.startswith('cp'):
+        installation_path = install_pypy(python_configuration.version, python_configuration.arch, python_configuration.url)
-            installation_path = install_cpython(config.version, config.arch, nuget)
-        elif config.identifier.startswith('pp'):
-            installation_path = install_pypy(config.version, config.arch, config.url)
     else:
         raise ValueError("Unknown Python implementation")
 
@@ -113,8 +105,8 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes
 
     # set up PATH and environment variables for run_with_env
     env = os.environ.copy()
-        env['PYTHON_VERSION'] = config.version
+    env['PYTHON_VERSION'] = python_configuration.version
-        env['PYTHON_ARCH'] = config.arch
+    env['PYTHON_ARCH'] = python_configuration.arch
     env['PATH'] = os.pathsep.join([
         installation_path,
         os.path.join(installation_path, 'Scripts'),
@@ -132,12 +124,6 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes
         print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr)
         exit(1)
 
-        dependency_constraint_flags = []
-        if dependency_constraints:
-            dependency_constraint_flags = [
-                '-c', dependency_constraints.get_for_python_version(config.version)
-            ]
-
     # make sure pip is installed
     if not os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe')):
         shell(['python', get_pip_script] + dependency_constraint_flags, env=env, cwd="C:\\cibw")
@@ -152,6 +138,30 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes
     shell(['pip', '--version'], env=env)
     shell(['pip', 'install', '--upgrade', 'setuptools', 'wheel'] + dependency_constraint_flags, env=env)
 
+    return env
+
+
+def build(project_dir, output_dir, test_command, before_test, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, dependency_constraints):
+    abs_project_dir = os.path.abspath(project_dir)
+    temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
+    built_wheel_dir = os.path.join(temp_dir, 'built_wheel')
+    repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel')
+
+    # install nuget as best way to provide python
+    nuget = 'C:\\cibw\\nuget.exe'
+    download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget)
+
+    python_configurations = get_python_configurations(build_selector)
+    for config in python_configurations:
+        dependency_constraint_flags = []
+        if dependency_constraints:
+            dependency_constraint_flags = [
+                '-c', dependency_constraints.get_for_python_version(config.version)
+            ]
+
+        # install Python
+        env = setup_python(config, dependency_constraint_flags, environment)
+
         # run the before_build command
         if before_build:
             before_build_prepared = prepare_command(before_build, project=abs_project_dir)

requirements-dev.txt

@@ -4,3 +4,4 @@ pytest
 mkdocs==1.0.4
 pip-tools
 requests
+click

setup.cfg

@@ -8,5 +8,6 @@ exclude =
   .git/,
   env/,
   env2/,
+  env??/,
   .venv/,
   site/

test/shared/utils.py

@@ -81,59 +81,60 @@ def expected_wheels(package_name, package_version, manylinux_versions=None,
     # {python tag} and {abi tag} are closely related to the python interpreter used to build the wheel
     # so we'll merge them below as python_abi_tag
 
+    if manylinux_versions is None:
+        if pm.machine() == 'x86_64':
+            manylinux_versions = ['manylinux1', 'manylinux2010']
+        else:
+            manylinux_versions = ['manylinux2014']
+
     python_abi_tags = ['cp35-cp35m', 'cp36-cp36m', 'cp37-cp37m', 'cp38-cp38']
-    extra_x86_python_abi_tags = ['cp27-cp27m', 'pp27-pypy_73', 'pp36-pypy36_pp73']
+
+    if pm.machine() in ['x86_64', 'AMD64', 'x86']:
+        python_abi_tags += ['cp27-cp27m', 'pp27-pypy_73', 'pp36-pypy36_pp73']
 
         if platform == 'linux':
-        if pm.machine() not in ['x86_64', 'i686']:
-            if manylinux_versions is None:
-                manylinux_versions = ['manylinux2014']
-            architectures = {'cp': [pm.machine()]}
-        else:
-            if manylinux_versions is None:
-                manylinux_versions = ['manylinux1', 'manylinux2010']
-            python_abi_tags += extra_x86_python_abi_tags
             python_abi_tags.append('cp27-cp27mu')  # python 2.7 has 2 different ABI on manylinux
-            architectures = {'cp': ['x86_64', 'i686'], 'pp': ['x86_64']}
+
-        platform_tags = {}
+    wheels = []
-        for python_implemention in architectures:
+
-            platform_tags[python_implemention] = [
+    for python_abi_tag in python_abi_tags:
-                '{manylinux_version}_{architecture}'.format(
+        platform_tags = []
-                    manylinux_version=manylinux_version, architecture=architecture)
+
-                for architecture in architectures[python_implemention]
+        if platform == 'linux':
+            architectures = [pm.machine()]
+
+            if pm.machine() == 'x86_64' and python_abi_tag.startswith('cp'):
+                architectures.append('i686')
+
+            platform_tags = [
+                '{}_{}'.format(manylinux_version, architecture)
+                for architecture in architectures
                 for manylinux_version in manylinux_versions
             ]
 
-        def get_platform_tags(python_abi_tag):
-            return platform_tags[python_abi_tag[:2]]
         elif platform == 'windows':
-        python_abi_tags += extra_x86_python_abi_tags
+            if python_abi_tag.startswith('cp'):
-        platform_tags = {'cp': ['win32', 'win_amd64'], 'pp': ['win32']}
+                platform_tags = ['win32', 'win_amd64']
-
+            else:
-        def get_platform_tags(python_abi_tag):
+                platform_tags = ['win32']
-            return platform_tags[python_abi_tag[:2]]
 
         elif platform == 'macos':
-        python_abi_tags += extra_x86_python_abi_tags
+            platform_tags = ['macosx_{}_x86_64'.format(macosx_deployment_target.replace('.', '_'))]
 
-        def get_platform_tags(python_abi_tag):
-            return ['macosx_{}_x86_64'.format(macosx_deployment_target.replace('.', '_'))]
         else:
             raise Exception('unsupported platform')
 
-    templates = []
+        for platform_tag in platform_tags:
-    for python_abi_tag in python_abi_tags:
+            wheels.append('{package_name}-{package_version}-{python_abi_tag}-{platform_tag}.whl'.format(
-        for platform_tag in get_platform_tags(python_abi_tag):
-            templates.append('{package_name}-{package_version}-{python_abi_tag}-{platform_tag}.whl'.format(
                 package_name=package_name, package_version=package_version,
                 python_abi_tag=python_abi_tag, platform_tag=platform_tag
             ))
 
     if IS_WINDOWS_RUNNING_ON_TRAVIS:
         # Python 2.7 isn't supported on Travis.
-        templates = [t for t in templates if '-cp27-' not in t and '-pp2' not in t]
+        wheels = [w for w in wheels if '-cp27-' not in w and '-pp2' not in w]
 
-    return templates
+    return wheels
 
 
 platform = None