Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #311 from Czaki/pack_arguments

Browse Source 
Pack build arguments
This commit is contained in:
Joe Rickerby
2020-04-09 10:46:42 +01:00
committed by GitHub
parent b8dbedcdbe ef935187d0
commit eff997d12e
7 changed files with 130 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cibuildwheel/__main__.py
+26 -24
View File
@@ -16,7 +16,8 @@ from cibuildwheel.environment import (
from cibuildwheel.util import (
BuildSelector,
DependencyConstraints,
Unbuffered
Unbuffered,
BuildOptions
)
@@ -154,21 +155,6 @@ def main():
print_build_identifiers(platform, build_selector)
exit(0)
build_options = dict(
project_dir=project_dir,
output_dir=output_dir,
test_command=test_command,
test_requires=test_requires,
test_extras=test_extras,
before_build=before_build,
build_verbosity=build_verbosity,
build_selector=build_selector,
repair_command=repair_command,
environment=environment,
before_test=before_test,
dependency_constraints=dependency_constraints,
)
if platform == 'linux':
pinned_docker_images_file = os.path.join(
os.path.dirname(__file__), 'resources', 'pinned_docker_images.cfg'
@@ -199,9 +185,25 @@ def main():
manylinux_images[build_platform] = image
build_options.update(
manylinux_images=manylinux_images
)
else:
manylinux_images = None
build_options = BuildOptions(
project_dir=project_dir,
output_dir=output_dir,
test_command=test_command,
test_requires=test_requires,
test_extras=test_extras,
before_build=before_build,
build_verbosity=build_verbosity,
build_selector=build_selector,
repair_command=repair_command,
environment=environment,
before_test=before_test,
dependency_constraints=dependency_constraints,
manylinux_images=manylinux_images
)
# Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print'
sys.stdout = Unbuffered(sys.stdout)
@@ -212,11 +214,11 @@ def main():
os.makedirs(output_dir)
if platform == 'linux':
cibuildwheel.linux.build(**build_options)
cibuildwheel.linux.build(build_options)
elif platform == 'windows':
cibuildwheel.windows.build(**build_options)
cibuildwheel.windows.build(build_options)
elif platform == 'macos':
cibuildwheel.macos.build(**build_options)
cibuildwheel.macos.build(build_options)
else:
print('cibuildwheel: Unsupported platform: {}'.format(platform), file=sys.stderr)
exit(2)
@@ -261,7 +263,7 @@ def print_preamble(platform, build_options):
print('Build options:')
print(' platform: %r' % platform)
for option, value in sorted(build_options.items()):
for option, value in sorted(build_options._asdict().items()):
print(' %s: %r' % (option, value))
warnings = detect_warnings(platform, build_options)
@@ -292,7 +294,7 @@ def detect_warnings(platform, build_options):
# warn about deprecated {python} and {pip}
for option_name in ['test_command', 'before_build']:
option_value = build_options.get(option_name)
option_value = getattr(build_options, option_name)
if option_value:
if '{python}' in option_value or '{pip}' in option_value:
cibuildwheel/linux.py
+22 -21
View File
@@ -10,6 +10,7 @@ from collections import namedtuple
from .util import (
get_build_verbosity_extra_flags,
prepare_command,
BuildOptions
)
@@ -75,7 +76,7 @@ def get_python_configurations(build_selector):
return [c for c in python_configurations if matches_platform(c.identifier) and build_selector(c.identifier)]
def build(project_dir, output_dir, test_command, before_test, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, manylinux_images, dependency_constraints):
def build(options: BuildOptions):
try:
subprocess.check_call(['docker', '--version'])
except Exception:
@@ -85,14 +86,14 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes
file=sys.stderr)
exit(2)
python_configurations = get_python_configurations(build_selector)
python_configurations = get_python_configurations(options.build_selector)
platforms = [
('cp', 'manylinux_x86_64', manylinux_images['x86_64']),
('cp', 'manylinux_i686', manylinux_images['i686']),
('cp', 'manylinux_aarch64', manylinux_images['aarch64']),
('cp', 'manylinux_ppc64le', manylinux_images['ppc64le']),
('cp', 'manylinux_s390x', manylinux_images['s390x']),
('pp', 'manylinux_x86_64', manylinux_images['pypy_x86_64']),
('cp', 'manylinux_x86_64', options.manylinux_images['x86_64']),
('cp', 'manylinux_i686', options.manylinux_images['i686']),
('cp', 'manylinux_aarch64', options.manylinux_images['aarch64']),
('cp', 'manylinux_ppc64le', options.manylinux_images['ppc64le']),
('cp', 'manylinux_s390x', options.manylinux_images['s390x']),
('pp', 'manylinux_x86_64', options.manylinux_images['pypy_x86_64']),
]
for implementation, platform_tag, docker_image in platforms:
@@ -111,14 +112,14 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes
'/bin/bash'])
call(['docker', 'cp',
os.path.abspath(project_dir) + '/.',
os.path.abspath(options.project_dir) + '/.',
container_name + ':/project'])
call(['docker', 'start', container_name])
for config in platform_configs:
if dependency_constraints:
constraints_file = dependency_constraints.get_for_python_version(config.version)
if options.dependency_constraints:
constraints_file = options.dependency_constraints.get_for_python_version(config.version)
# `docker cp` causes 'no space left on device' error when
# a container is running and the host filesystem is
@@ -233,32 +234,32 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes
done
'''.format(
config_python_bin=config.path + '/bin',
test_requires=' '.join(test_requires),
test_extras=test_extras,
test_requires=' '.join(options.test_requires),
test_extras=options.test_extras,
test_command=shlex.quote(
prepare_command(test_command, project='/project') if test_command else ''
prepare_command(options.test_command, project='/project') if options.test_command else ''
),
before_build=shlex.quote(
prepare_command(before_build, project='/project') if before_build else ''
prepare_command(options.before_build, project='/project') if options.before_build else ''
),
build_verbosity_flag=' '.join(get_build_verbosity_extra_flags(build_verbosity)),
build_verbosity_flag=' '.join(get_build_verbosity_extra_flags(options.build_verbosity)),
repair_command=shlex.quote(
prepare_command(repair_command, wheel='"$1"', dest_dir='/tmp/repaired_wheels') if repair_command else ''
prepare_command(options.repair_command, wheel='"$1"', dest_dir='/tmp/repaired_wheels') if options.repair_command else ''
),
environment_exports='\n'.join(environment.as_shell_commands()),
environment_exports='\n'.join(options.environment.as_shell_commands()),
uid=os.getuid(),
gid=os.getgid(),
before_test=shlex.quote(
prepare_command(before_test, project='/project') if before_test else ''
prepare_command(options.before_test, project='/project') if options.before_test else ''
),
dependency_install_flags='-c /constraints.txt' if dependency_constraints else '',
dependency_install_flags='-c /constraints.txt' if options.dependency_constraints else '',
)
)
# copy the output back into the host
call(['docker', 'cp',
container_name + ':/output/.',
os.path.abspath(output_dir)])
os.path.abspath(options.output_dir)])
except subprocess.CalledProcessError:
exit(1)
finally:
cibuildwheel/macos.py
+21 -20
View File
@@ -12,6 +12,7 @@ from .util import (
get_build_verbosity_extra_flags,
prepare_command,
get_pip_script,
BuildOptions
)
@@ -166,48 +167,48 @@ def setup_python(python_configuration, dependency_constraint_flags, environment)
return env
def build(project_dir, output_dir, test_command, before_test, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, dependency_constraints):
abs_project_dir = os.path.abspath(project_dir)
def build(options: BuildOptions):
abs_project_dir = os.path.abspath(options.project_dir)
temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
built_wheel_dir = os.path.join(temp_dir, 'built_wheel')
repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel')
python_configurations = get_python_configurations(build_selector)
python_configurations = get_python_configurations(options.build_selector)
for config in python_configurations:
dependency_constraint_flags = []
if dependency_constraints:
if options.dependency_constraints:
dependency_constraint_flags = [
'-c', dependency_constraints.get_for_python_version(config.version)
'-c', options.dependency_constraints.get_for_python_version(config.version)
]
env = setup_python(config, dependency_constraint_flags, environment)
env = setup_python(config, dependency_constraint_flags, options.environment)
# run the before_build command
if before_build:
before_build_prepared = prepare_command(before_build, project=abs_project_dir)
if options.before_build:
before_build_prepared = prepare_command(options.before_build, project=abs_project_dir)
call(before_build_prepared, env=env, shell=True)
# build the wheel
if os.path.exists(built_wheel_dir):
shutil.rmtree(built_wheel_dir)
os.makedirs(built_wheel_dir)
call(['pip', 'wheel', abs_project_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(build_verbosity), env=env)
call(['pip', 'wheel', abs_project_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env)
built_wheel = glob(os.path.join(built_wheel_dir, '*.whl'))[0]
# repair the wheel
if os.path.exists(repaired_wheel_dir):
shutil.rmtree(repaired_wheel_dir)
os.makedirs(repaired_wheel_dir)
if built_wheel.endswith('none-any.whl') or not repair_command:
if built_wheel.endswith('none-any.whl') or not options.repair_command:
# pure Python wheel or empty repair command
shutil.move(built_wheel, repaired_wheel_dir)
else:
repair_command_prepared = prepare_command(repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir)
repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir)
call(repair_command_prepared, env=env, shell=True)
repaired_wheel = glob(os.path.join(repaired_wheel_dir, '*.whl'))[0]
if test_command:
if options.test_command:
# set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
call(['pip', 'install', 'virtualenv'] + dependency_constraint_flags, env=env)
@@ -227,26 +228,26 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes
# check that we are using the Python from the virtual environment
call(['which', 'python'], env=virtualenv_env)
if before_test:
before_test_prepared = prepare_command(before_test, project=abs_project_dir)
if options.before_test:
before_test_prepared = prepare_command(options.before_test, project=abs_project_dir)
call(before_test_prepared, env=virtualenv_env, shell=True)
# install the wheel
call(['pip', 'install', repaired_wheel + test_extras], env=virtualenv_env)
call(['pip', 'install', repaired_wheel + options.test_extras], env=virtualenv_env)
# test the wheel
if test_requires:
call(['pip', 'install'] + test_requires, env=virtualenv_env)
if options.test_requires:
call(['pip', 'install'] + options.test_requires, env=virtualenv_env)
# run the tests from $HOME, with an absolute path in the command
# (this ensures that Python runs the tests against the installed wheel
# and not the repo code)
test_command_prepared = prepare_command(test_command, project=abs_project_dir)
test_command_prepared = prepare_command(options.test_command, project=abs_project_dir)
call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True)
# clean up
shutil.rmtree(venv_dir)
# we're all done here; move it to output (overwrite existing)
dst = os.path.join(output_dir, os.path.basename(repaired_wheel))
dst = os.path.join(options.output_dir, os.path.basename(repaired_wheel))
shutil.move(repaired_wheel, dst)
cibuildwheel/util.py
+25
View File
@@ -1,7 +1,11 @@
import os
import sys
import urllib.request
from fnmatch import fnmatch
from time import sleep
from typing import NamedTuple, List, Optional, Dict
from .environment import ParsedEnvironment
def prepare_command(command, **kwargs):
@@ -103,5 +107,26 @@ class DependencyConstraints:
return self.base_file_path
BuildOptions = NamedTuple("BuildOptions", [
("project_dir", str),
("output_dir", str),
("test_command", Optional[str]),
("test_requires", List[str]),
("test_extras", str),
("before_build", Optional[str]),
("build_verbosity", int),
("build_selector", BuildSelector),
("repair_command", str),
("environment", ParsedEnvironment),
("before_test", str),
("dependency_constraints", Optional[DependencyConstraints]),
("manylinux_images", Optional[Dict[str, str]]),
])
"""
Replace this definition with a class-style NamedTuple in the
PEP526 style when Python 3.5 host support is dropped
"""
resources_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'resources'))
get_pip_script = os.path.join(resources_dir, 'get-pip.py')
cibuildwheel/windows.py
+20 -19
View File
@@ -12,6 +12,7 @@ from .util import (
get_build_verbosity_extra_flags,
prepare_command,
get_pip_script,
BuildOptions
)
@@ -141,8 +142,8 @@ def setup_python(python_configuration, dependency_constraint_flags, environment)
return env
def build(project_dir, output_dir, test_command, before_test, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, dependency_constraints):
abs_project_dir = os.path.abspath(project_dir)
def build(options: BuildOptions):
abs_project_dir = os.path.abspath(options.project_dir)
temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
built_wheel_dir = os.path.join(temp_dir, 'built_wheel')
repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel')
@@ -151,42 +152,42 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes
nuget = 'C:\\cibw\\nuget.exe'
download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget)
python_configurations = get_python_configurations(build_selector)
python_configurations = get_python_configurations(options.build_selector)
for config in python_configurations:
dependency_constraint_flags = []
if dependency_constraints:
if options.dependency_constraints:
dependency_constraint_flags = [
'-c', dependency_constraints.get_for_python_version(config.version)
'-c', options.dependency_constraints.get_for_python_version(config.version)
]
# install Python
env = setup_python(config, dependency_constraint_flags, environment)
env = setup_python(config, dependency_constraint_flags, options.environment)
# run the before_build command
if before_build:
before_build_prepared = prepare_command(before_build, project=abs_project_dir)
if options.before_build:
before_build_prepared = prepare_command(options.before_build, project=abs_project_dir)
shell([before_build_prepared], env=env)
# build the wheel
if os.path.exists(built_wheel_dir):
shutil.rmtree(built_wheel_dir)
os.makedirs(built_wheel_dir)
shell(['pip', 'wheel', abs_project_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(build_verbosity), env=env)
shell(['pip', 'wheel', abs_project_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env)
built_wheel = glob(os.path.join(built_wheel_dir, '*.whl'))[0]
# repair the wheel
if os.path.exists(repaired_wheel_dir):
shutil.rmtree(repaired_wheel_dir)
os.makedirs(repaired_wheel_dir)
if built_wheel.endswith('none-any.whl') or not repair_command:
if built_wheel.endswith('none-any.whl') or not options.repair_command:
# pure Python wheel or empty repair command
shutil.move(built_wheel, repaired_wheel_dir)
else:
repair_command_prepared = prepare_command(repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir)
repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir)
shell([repair_command_prepared], env=env)
repaired_wheel = glob(os.path.join(repaired_wheel_dir, '*.whl'))[0]
if test_command:
if options.test_command:
# set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
shell(['pip', 'install', 'virtualenv'] + dependency_constraint_flags, env=env)
@@ -211,28 +212,28 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes
# check that we are using the Python from the virtual environment
shell(['which', 'python'], env=virtualenv_env)
if before_test:
before_test_prepared = prepare_command(before_test, project=abs_project_dir)
if options.before_test:
before_test_prepared = prepare_command(options.before_test, project=abs_project_dir)
shell([before_test_prepared], env=virtualenv_env)
# install the wheel
shell(['pip', 'install', repaired_wheel + test_extras], env=virtualenv_env)
shell(['pip', 'install', repaired_wheel + options.test_extras], env=virtualenv_env)
# test the wheel
if test_requires:
shell(['pip', 'install'] + test_requires, env=virtualenv_env)
if options.test_requires:
shell(['pip', 'install'] + options.test_requires, env=virtualenv_env)
# run the tests from c:\, with an absolute path in the command
# (this ensures that Python runs the tests against the installed wheel
# and not the repo code)
test_command_prepared = prepare_command(test_command, project=abs_project_dir)
test_command_prepared = prepare_command(options.test_command, project=abs_project_dir)
shell([test_command_prepared], cwd='c:\\', env=virtualenv_env)
# clean up
shutil.rmtree(venv_dir)
# we're all done here; move it to output (remove if already exists)
dst = os.path.join(output_dir, os.path.basename(repaired_wheel))
dst = os.path.join(options.output_dir, os.path.basename(repaired_wheel))
if os.path.isfile(dst):
os.remove(dst)
shutil.move(repaired_wheel, dst)
unit_test/main_tests/main_options_test.py
+14 -14
View File
@@ -18,13 +18,13 @@ def test_output_dir(platform, intercepted_build_args, monkeypatch):
main()
assert intercepted_build_args.kwargs['output_dir'] == OUTPUT_DIR
assert intercepted_build_args.args[0].output_dir == OUTPUT_DIR
def test_output_dir_default(platform, intercepted_build_args, monkeypatch):
main()
assert intercepted_build_args.kwargs['output_dir'] == 'wheelhouse'
assert intercepted_build_args.args[0].output_dir == 'wheelhouse'
@pytest.mark.parametrize('also_set_environment', [False, True])
@@ -37,7 +37,7 @@ def test_output_dir_argument(also_set_environment, platform, intercepted_build_a
main()
assert intercepted_build_args.kwargs['output_dir'] == OUTPUT_DIR
assert intercepted_build_args.args[0].output_dir == OUTPUT_DIR
def test_build_selector(platform, intercepted_build_args, monkeypatch):
@@ -49,7 +49,7 @@ def test_build_selector(platform, intercepted_build_args, monkeypatch):
main()
intercepted_build_selector = intercepted_build_args.kwargs['build_selector']
intercepted_build_selector = intercepted_build_args.args[0].build_selector
assert isinstance(intercepted_build_selector, BuildSelector)
assert intercepted_build_selector('build-this')
assert not intercepted_build_selector('skip-that')
@@ -82,11 +82,11 @@ def test_manylinux_images(architecture, image, full_image, platform, intercepted
if platform == 'linux':
assert fnmatch(
intercepted_build_args.kwargs['manylinux_images'][architecture],
intercepted_build_args.args[0].manylinux_images[architecture],
full_image
)
else:
assert 'manylinux_images' not in intercepted_build_args.kwargs
assert intercepted_build_args.args[0].manylinux_images is None
def get_default_repair_command(platform):
@@ -113,7 +113,7 @@ def test_repair_command(repair_command, platform_specific, platform, intercepted
main()
expected_repair = repair_command or get_default_repair_command(platform)
assert intercepted_build_args.kwargs['repair_command'] == expected_repair
assert intercepted_build_args.args[0].repair_command == expected_repair
@pytest.mark.parametrize('environment', [
@@ -132,7 +132,7 @@ def test_environment(environment, platform_specific, platform, intercepted_build
main()
intercepted_environment = intercepted_build_args.kwargs['environment']
intercepted_environment = intercepted_build_args.args[0].environment
assert isinstance(intercepted_environment, ParsedEnvironment)
assert intercepted_environment.as_dictionary(prev_environment={}) == environment
@@ -149,7 +149,7 @@ def test_test_requires(test_requires, platform_specific, platform, intercepted_b
main()
assert intercepted_build_args.kwargs['test_requires'] == (test_requires or '').split()
assert intercepted_build_args.args[0].test_requires == (test_requires or '').split()
@pytest.mark.parametrize('test_extras', [None, 'extras'])
@@ -164,7 +164,7 @@ def test_test_extras(test_extras, platform_specific, platform, intercepted_build
main()
assert intercepted_build_args.kwargs['test_extras'] == ('[' + test_extras + ']' if test_extras else '')
assert intercepted_build_args.args[0],test_extras == ('[' + test_extras + ']' if test_extras else '')
@pytest.mark.parametrize('test_command', [None, 'test --command'])
@@ -179,7 +179,7 @@ def test_test_command(test_command, platform_specific, platform, intercepted_bui
main()
assert intercepted_build_args.kwargs['test_command'] == test_command
assert intercepted_build_args.args[0].test_command == test_command
@pytest.mark.parametrize('before_build', [None, 'before --build'])
@@ -194,7 +194,7 @@ def test_before_build(before_build, platform_specific, platform, intercepted_bui
main()
assert intercepted_build_args.kwargs['before_build'] == before_build
assert intercepted_build_args.args[0].before_build == before_build
@pytest.mark.parametrize('build_verbosity', [None, 0, 2, -2, 4, -4])
@@ -210,7 +210,7 @@ def test_build_verbosity(build_verbosity, platform_specific, platform, intercept
main()
expected_verbosity = max(-3, min(3, int(build_verbosity or 0)))
assert intercepted_build_args.kwargs['build_verbosity'] == expected_verbosity
assert intercepted_build_args.args[0].build_verbosity == expected_verbosity
@pytest.mark.parametrize('option_name', ['CIBW_BUILD', 'CIBW_SKIP'])
@@ -225,7 +225,7 @@ def test_build_selector_migrations(intercepted_build_args, monkeypatch, option_n
main()
intercepted_build_selector = intercepted_build_args.kwargs['build_selector']
intercepted_build_selector = intercepted_build_args.args[0].build_selector
assert isinstance(intercepted_build_selector, BuildSelector)
if option_name == 'CIBW_BUILD':
unit_test/main_tests/main_platform_test.py
+2 -2
View File
@@ -51,10 +51,10 @@ def test_platform_argument(platform, intercepted_build_args, monkeypatch):
main()
assert intercepted_build_args.kwargs['project_dir'] == MOCK_PROJECT_DIR
assert intercepted_build_args.args[0].project_dir == MOCK_PROJECT_DIR
def test_platform_environment(platform, intercepted_build_args, monkeypatch):
main()
assert intercepted_build_args.kwargs['project_dir'] == MOCK_PROJECT_DIR
assert intercepted_build_args.args[0].project_dir == MOCK_PROJECT_DIR