Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/master' into deterministic-builds

Browse Source
This commit is contained in:
Joe Rickerby
2020-02-15 11:45:54 +00:00
parent ada46aaf43 d9948da29a
commit 103381774c
46 changed files with 314 additions and 184 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cibuildwheel/__main__.py
+21 -13
View File
@@ -1,14 +1,24 @@
from __future__ import print_function
import argparse, os, subprocess, sys, textwrap
try:
from configparser import ConfigParser
except ImportError:
from ConfigParser import ConfigParser
import argparse
import os
import sys
import textwrap
import traceback
from configparser import ConfigParser
import cibuildwheel
import cibuildwheel.linux, cibuildwheel.windows, cibuildwheel.macos
from cibuildwheel.environment import parse_environment, EnvironmentParseError
from cibuildwheel.util import BuildSelector, DependencyConstraints, Unbuffered
import cibuildwheel.linux
import cibuildwheel.macos
import cibuildwheel.windows
from cibuildwheel.environment import (
EnvironmentParseError,
parse_environment,
)
from cibuildwheel.util import (
BuildSelector,
DependencyConstraints,
Unbuffered
)
def get_option_from_environment(option_name, platform=None, default=None):
'''
@@ -89,7 +99,6 @@ def main():
file=sys.stderr)
exit(2)
output_dir = args.output_dir
test_command = get_option_from_environment('CIBW_TEST_COMMAND', platform=platform)
test_requires = get_option_from_environment('CIBW_TEST_REQUIRES', platform=platform, default='').split()
@@ -125,9 +134,8 @@ def main():
try:
environment = parse_environment(environment_config)
except (EnvironmentParseError, ValueError) as e:
except (EnvironmentParseError, ValueError):
print('cibuildwheel: Malformed environment option "%s"' % environment_config, file=sys.stderr)
import traceback
traceback.print_exc(None, sys.stderr)
exit(2)
@@ -239,6 +247,7 @@ def detect_obsolete_options():
))
os.environ[option] = os.environ[option].replace(deprecated, alternative)
def print_preamble(platform, build_options):
print(textwrap.dedent('''
_ _ _ _ _ _ _
@@ -249,7 +258,6 @@ def print_preamble(platform, build_options):
print('cibuildwheel version %s\n' % cibuildwheel.__version__)
print('Build options:')
print(' platform: %r' % platform)
for option, value in sorted(build_options.items()):
cibuildwheel/bashlex_eval.py
+7 -8
View File
@@ -1,9 +1,12 @@
import subprocess, shlex, sys
import shlex
import subprocess
from collections import namedtuple
import bashlex
NodeExecutionContext = namedtuple('NodeExecutionContext', ['environment', 'input'])
def evaluate(value, environment):
if not value:
# empty string evaluates to empty string
@@ -16,9 +19,9 @@ def evaluate(value, environment):
raise ValueError('"%s" has too many parts' % value)
value_word_node = command_node.parts[0]
return evaluate_node(
value_word_node,
value_word_node,
context=NodeExecutionContext(environment=environment, input=value)
)
@@ -60,12 +63,8 @@ def evaluate_word_node(node, context):
def evaluate_command_node(node, context):
words = [evaluate_node(part, context=context) for part in node.parts]
command = ' '.join(words)
output = subprocess.check_output(shlex.split(command), env=context.environment)
return subprocess.check_output(shlex.split(command), env=context.environment, universal_newlines=True)
if sys.version_info[0] >= 3:
return output.decode('utf8', 'replace')
else:
return output
def evaluate_parameter_node(node, context):
return context.environment.get(node.value, '')
cibuildwheel/environment.py
+3 -2
View File
@@ -1,4 +1,5 @@
import bashlex
from . import bashlex_eval
@@ -41,7 +42,7 @@ def split_env_items(env_string):
return result
class EnvironmentAssignment(object):
class EnvironmentAssignment:
def __init__(self, assignment):
name, equals, value = assignment.partition('=')
if not equals:
@@ -60,7 +61,7 @@ class EnvironmentAssignment(object):
return '%s=%s' % (self.name, self.value)
class ParsedEnvironment(object):
class ParsedEnvironment:
def __init__(self, assignments):
self.assignments = assignments
cibuildwheel/linux.py
+46 -18
View File
@@ -1,12 +1,15 @@
from __future__ import print_function
import os, subprocess, sys, uuid
import os
import shlex
import subprocess
import sys
import textwrap
import uuid
from collections import namedtuple
from .util import prepare_command, get_build_verbosity_extra_flags
try:
from shlex import quote as shlex_quote
except ImportError:
from pipes import quote as shlex_quote
from .util import (
get_build_verbosity_extra_flags,
prepare_command,
)
def get_python_configurations(build_selector):
@@ -49,7 +52,7 @@ def run_docker(command, stdin_str=None):
def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, manylinux_images, dependency_constraints):
try:
subprocess.check_call(['docker', '--version'])
except:
except Exception:
print('cibuildwheel: Docker not found. Docker is required to run Linux builds. '
'If you\'re building on Travis CI, add `services: [docker]` to your .travis.yml.'
'If you\'re building on Circle CI in Linux, add a `setup_remote_docker` step to your .circleci/config.yml',
@@ -69,20 +72,20 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
container_name = 'cibuildwheel-{}'.format(uuid.uuid4())
try:
run_docker(['create',
'--env', 'CIBUILDWHEEL',
'--name', container_name,
'-i',
'-v', '/:/host', # ignored on CircleCI
docker_image, '/bin/bash'])
run_docker(['cp', os.path.abspath(project_dir) + '/.', container_name + ':/project'])
subprocess.run(['docker', 'create',
'--env', 'CIBUILDWHEEL',
'--name', container_name,
'-i',
'-v', '/:/host', # ignored on CircleCI
docker_image, '/bin/bash'])
subprocess.run(['docker', 'cp', os.path.abspath(project_dir) + '/.', container_name + ':/project'])
for config in platform_configs:
if dependency_constraints:
constraints_file = dependency_constraints.get_for_python_version(config.version)
run_docker(['cp', os.path.abspath(constraints_file), container_name + ':/constraints.txt'])
run_docker(['start', '-i', '-a', container_name], stdin_str='''
subprocess.run(['docker', 'start', '-i', '-a', container_name], stdin_str='''
set -o errexit
set -o xtrace
mkdir -p /output
@@ -188,11 +191,36 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
))
# copy the output back into the host
run_docker(['cp', container_name + ':/output/.', os.path.abspath(output_dir)])
subprocess.run(['docker', 'cp', container_name + ':/output/.', os.path.abspath(output_dir)])
except subprocess.CalledProcessError:
exit(1)
finally:
# Still gets executed, even when 'exit(1)' gets called
run_docker(['rm', '--force', '-v', container_name])
subprocess.run(['docker', 'rm', '--force', '-v', container_name], check=True)
def troubleshoot(project_dir, error):
if (isinstance(error, subprocess.CalledProcessError) and 'start' in error.cmd):
# the bash script failed
print('Checking for common errors...')
so_files = []
for root, dirs, files in os.walk(project_dir):
for name in files:
_, ext = os.path.splitext(name)
if ext == '.so':
so_files.append(os.path.join(root, name))
if so_files:
print(textwrap.dedent('''
NOTE: Shared object (.so) files found in this project.
These files might be built against the wrong OS, causing problems with
auditwheel.
If you're using Cython and have previously done an in-place build,
remove those build files (*.so and *.c) before starting cibuildwheel.
'''))
print(' Files detected:')
print('\n'.join([' ' + f for f in so_files]))
print('')
cibuildwheel/macos.py
+12 -11
View File
@@ -1,14 +1,17 @@
from __future__ import print_function
import os
import shlex
import shutil
import subprocess
import tempfile
import os, subprocess, shlex, sys, shutil
from collections import namedtuple
from glob import glob
try:
from shlex import quote as shlex_quote
except ImportError:
from pipes import quote as shlex_quote
from .util import prepare_command, get_build_verbosity_extra_flags, download, get_pip_script
from .util import (
download,
get_build_verbosity_extra_flags,
prepare_command,
get_pip_script
)
def get_python_configurations(build_selector):
@@ -33,9 +36,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
python_configurations = get_python_configurations(build_selector)
pkgs_output = subprocess.check_output(['pkgutil', '--pkgs'])
if sys.version_info[0] >= 3:
pkgs_output = pkgs_output.decode('utf8')
pkgs_output = subprocess.check_output(['pkgutil', '--pkgs'], universal_newlines=True)
installed_system_packages = pkgs_output.splitlines()
def call(args, env=None, cwd=None, shell=False):
@@ -43,7 +44,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
if shell:
print('+ %s' % args)
else:
print('+ ' + ' '.join(shlex_quote(a) for a in args))
print('+ ' + ' '.join(shlex.quote(a) for a in args))
return subprocess.check_call(args, env=env, cwd=cwd, shell=shell)
cibuildwheel/util.py
+6 -11
View File
@@ -1,13 +1,8 @@
from fnmatch import fnmatch
import warnings
import os
import urllib.request
from fnmatch import fnmatch
from time import sleep
try:
from urllib.request import urlopen
except ImportError:
from urllib2 import urlopen
def prepare_command(command, **kwargs):
'''
@@ -28,7 +23,7 @@ def get_build_verbosity_extra_flags(level):
return []
class BuildSelector(object):
class BuildSelector:
def __init__(self, build_config, skip_config):
self.build_patterns = build_config.split()
self.skip_patterns = skip_config.split()
@@ -43,7 +38,7 @@ class BuildSelector(object):
# Taken from https://stackoverflow.com/a/107717
class Unbuffered(object):
class Unbuffered:
def __init__(self, stream):
self.stream = stream
@@ -68,8 +63,8 @@ def download(url, dest):
repeat_num = 3
for i in range(repeat_num):
try:
response = urlopen(url)
except:
response = urllib.request.urlopen(url)
except Exception:
if i == repeat_num - 1:
raise
sleep(3)
cibuildwheel/windows.py
+12 -9
View File
@@ -1,14 +1,16 @@
from __future__ import print_function
import os, tempfile, subprocess, shutil, sys
import os
import shutil
import subprocess
import tempfile
from collections import namedtuple
from glob import glob
try:
from shlex import quote as shlex_quote
except ImportError:
from pipes import quote as shlex_quote
from .util import prepare_command, get_build_verbosity_extra_flags, download, get_pip_script
from .util import (
download,
get_build_verbosity_extra_flags,
prepare_command,
get_pip_script,
)
IS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache')
@@ -26,6 +28,7 @@ def get_nuget_args(configuration):
python_name = python_name + "x86"
return [python_name, "-Version", configuration.version, "-OutputDirectory", "C:/cibw/python"]
def get_python_configurations(build_selector):
PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier'])
python_configurations = [
@@ -46,7 +49,7 @@ def get_python_configurations(build_selector):
# try with (and similar): msiexec /i VCForPython27.msi ALLUSERS=1 ACCEPT=YES /passive
python_configurations = [c for c in python_configurations if not c.version.startswith('2.7.')]
# skip builds as required
# skip builds as required
python_configurations = [c for c in python_configurations if build_selector(c.identifier)]
return python_configurations