diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 3de75657..ffb0b684 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -16,7 +16,8 @@ from cibuildwheel.environment import ( from cibuildwheel.util import ( BuildSelector, DependencyConstraints, - Unbuffered + Unbuffered, + BuildOptions ) @@ -154,21 +155,6 @@ def main(): print_build_identifiers(platform, build_selector) exit(0) - build_options = dict( - project_dir=project_dir, - output_dir=output_dir, - test_command=test_command, - test_requires=test_requires, - test_extras=test_extras, - before_build=before_build, - build_verbosity=build_verbosity, - build_selector=build_selector, - repair_command=repair_command, - environment=environment, - before_test=before_test, - dependency_constraints=dependency_constraints, - ) - if platform == 'linux': pinned_docker_images_file = os.path.join( os.path.dirname(__file__), 'resources', 'pinned_docker_images.cfg' @@ -199,9 +185,25 @@ def main(): manylinux_images[build_platform] = image - build_options.update( - manylinux_images=manylinux_images - ) + else: + manylinux_images = None + + build_options = BuildOptions( + project_dir=project_dir, + output_dir=output_dir, + test_command=test_command, + test_requires=test_requires, + test_extras=test_extras, + before_build=before_build, + build_verbosity=build_verbosity, + build_selector=build_selector, + repair_command=repair_command, + environment=environment, + before_test=before_test, + dependency_constraints=dependency_constraints, + manylinux_images=manylinux_images + + ) # Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print' sys.stdout = Unbuffered(sys.stdout) @@ -212,11 +214,11 @@ def main(): os.makedirs(output_dir) if platform == 'linux': - cibuildwheel.linux.build(**build_options) + cibuildwheel.linux.build(build_options) elif platform == 'windows': - cibuildwheel.windows.build(**build_options) + cibuildwheel.windows.build(build_options) elif platform == 'macos': - cibuildwheel.macos.build(**build_options) + cibuildwheel.macos.build(build_options) else: print('cibuildwheel: Unsupported platform: {}'.format(platform), file=sys.stderr) exit(2) @@ -261,7 +263,7 @@ def print_preamble(platform, build_options): print('Build options:') print(' platform: %r' % platform) - for option, value in sorted(build_options.items()): + for option, value in sorted(build_options._asdict().items()): print(' %s: %r' % (option, value)) warnings = detect_warnings(platform, build_options) @@ -292,7 +294,7 @@ def detect_warnings(platform, build_options): # warn about deprecated {python} and {pip} for option_name in ['test_command', 'before_build']: - option_value = build_options.get(option_name) + option_value = getattr(build_options, option_name) if option_value: if '{python}' in option_value or '{pip}' in option_value: diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 089aecbf..31e794f7 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -10,6 +10,7 @@ from collections import namedtuple from .util import ( get_build_verbosity_extra_flags, prepare_command, + BuildOptions ) @@ -75,7 +76,7 @@ def get_python_configurations(build_selector): return [c for c in python_configurations if matches_platform(c.identifier) and build_selector(c.identifier)] -def build(project_dir, output_dir, test_command, before_test, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, manylinux_images, dependency_constraints): +def build(opt: BuildOptions): try: subprocess.check_call(['docker', '--version']) except Exception: @@ -85,14 +86,14 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes file=sys.stderr) exit(2) - python_configurations = get_python_configurations(build_selector) + python_configurations = get_python_configurations(opt.build_selector) platforms = [ - ('cp', 'manylinux_x86_64', manylinux_images['x86_64']), - ('cp', 'manylinux_i686', manylinux_images['i686']), - ('cp', 'manylinux_aarch64', manylinux_images['aarch64']), - ('cp', 'manylinux_ppc64le', manylinux_images['ppc64le']), - ('cp', 'manylinux_s390x', manylinux_images['s390x']), - ('pp', 'manylinux_x86_64', manylinux_images['pypy_x86_64']), + ('cp', 'manylinux_x86_64', opt.manylinux_images['x86_64']), + ('cp', 'manylinux_i686', opt.manylinux_images['i686']), + ('cp', 'manylinux_aarch64', opt.manylinux_images['aarch64']), + ('cp', 'manylinux_ppc64le', opt.manylinux_images['ppc64le']), + ('cp', 'manylinux_s390x', opt.manylinux_images['s390x']), + ('pp', 'manylinux_x86_64', opt.manylinux_images['pypy_x86_64']), ] for implementation, platform_tag, docker_image in platforms: @@ -111,14 +112,14 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes '/bin/bash']) call(['docker', 'cp', - os.path.abspath(project_dir) + '/.', + os.path.abspath(opt.project_dir) + '/.', container_name + ':/project']) call(['docker', 'start', container_name]) for config in platform_configs: - if dependency_constraints: - constraints_file = dependency_constraints.get_for_python_version(config.version) + if opt.dependency_constraints: + constraints_file = opt.dependency_constraints.get_for_python_version(config.version) # `docker cp` causes 'no space left on device' error when # a container is running and the host filesystem is @@ -233,32 +234,32 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes done '''.format( config_python_bin=config.path + '/bin', - test_requires=' '.join(test_requires), - test_extras=test_extras, + test_requires=' '.join(opt.test_requires), + test_extras=opt.test_extras, test_command=shlex.quote( - prepare_command(test_command, project='/project') if test_command else '' + prepare_command(opt.test_command, project='/project') if opt.test_command else '' ), before_build=shlex.quote( - prepare_command(before_build, project='/project') if before_build else '' + prepare_command(opt.before_build, project='/project') if opt.before_build else '' ), - build_verbosity_flag=' '.join(get_build_verbosity_extra_flags(build_verbosity)), + build_verbosity_flag=' '.join(get_build_verbosity_extra_flags(opt.build_verbosity)), repair_command=shlex.quote( - prepare_command(repair_command, wheel='"$1"', dest_dir='/tmp/repaired_wheels') if repair_command else '' + prepare_command(opt.repair_command, wheel='"$1"', dest_dir='/tmp/repaired_wheels') if opt.repair_command else '' ), - environment_exports='\n'.join(environment.as_shell_commands()), + environment_exports='\n'.join(opt.environment.as_shell_commands()), uid=os.getuid(), gid=os.getgid(), before_test=shlex.quote( - prepare_command(before_test, project='/project') if before_test else '' + prepare_command(opt.before_test, project='/project') if opt.before_test else '' ), - dependency_install_flags='-c /constraints.txt' if dependency_constraints else '', + dependency_install_flags='-c /constraints.txt' if opt.dependency_constraints else '', ) ) # copy the output back into the host call(['docker', 'cp', container_name + ':/output/.', - os.path.abspath(output_dir)]) + os.path.abspath(opt.output_dir)]) except subprocess.CalledProcessError: exit(1) finally: diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index c897e67a..30d02558 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -12,6 +12,7 @@ from .util import ( get_build_verbosity_extra_flags, prepare_command, get_pip_script, + BuildOptions ) @@ -166,48 +167,48 @@ def setup_python(python_configuration, dependency_constraint_flags, environment) return env -def build(project_dir, output_dir, test_command, before_test, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, dependency_constraints): - abs_project_dir = os.path.abspath(project_dir) +def build(opt: BuildOptions): + abs_project_dir = os.path.abspath(opt.project_dir) temp_dir = tempfile.mkdtemp(prefix='cibuildwheel') built_wheel_dir = os.path.join(temp_dir, 'built_wheel') repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel') - python_configurations = get_python_configurations(build_selector) + python_configurations = get_python_configurations(opt.build_selector) for config in python_configurations: dependency_constraint_flags = [] - if dependency_constraints: + if opt.dependency_constraints: dependency_constraint_flags = [ - '-c', dependency_constraints.get_for_python_version(config.version) + '-c', opt.dependency_constraints.get_for_python_version(config.version) ] - - env = setup_python(config, dependency_constraint_flags, environment) + + env = setup_python(config, dependency_constraint_flags, opt.environment) # run the before_build command - if before_build: - before_build_prepared = prepare_command(before_build, project=abs_project_dir) + if opt.before_build: + before_build_prepared = prepare_command(opt.before_build, project=abs_project_dir) call(before_build_prepared, env=env, shell=True) # build the wheel if os.path.exists(built_wheel_dir): shutil.rmtree(built_wheel_dir) os.makedirs(built_wheel_dir) - call(['pip', 'wheel', abs_project_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(build_verbosity), env=env) + call(['pip', 'wheel', abs_project_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(opt.build_verbosity), env=env) built_wheel = glob(os.path.join(built_wheel_dir, '*.whl'))[0] # repair the wheel if os.path.exists(repaired_wheel_dir): shutil.rmtree(repaired_wheel_dir) os.makedirs(repaired_wheel_dir) - if built_wheel.endswith('none-any.whl') or not repair_command: + if built_wheel.endswith('none-any.whl') or not opt.repair_command: # pure Python wheel or empty repair command shutil.move(built_wheel, repaired_wheel_dir) else: - repair_command_prepared = prepare_command(repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) + repair_command_prepared = prepare_command(opt.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) call(repair_command_prepared, env=env, shell=True) repaired_wheel = glob(os.path.join(repaired_wheel_dir, '*.whl'))[0] - if test_command: + if opt.test_command: # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. call(['pip', 'install', 'virtualenv'] + dependency_constraint_flags, env=env) @@ -227,26 +228,26 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes # check that we are using the Python from the virtual environment call(['which', 'python'], env=virtualenv_env) - if before_test: - before_test_prepared = prepare_command(before_test, project=abs_project_dir) + if opt.before_test: + before_test_prepared = prepare_command(opt.before_test, project=abs_project_dir) call(before_test_prepared, env=virtualenv_env, shell=True) # install the wheel - call(['pip', 'install', repaired_wheel + test_extras], env=virtualenv_env) + call(['pip', 'install', repaired_wheel + opt.test_extras], env=virtualenv_env) # test the wheel - if test_requires: - call(['pip', 'install'] + test_requires, env=virtualenv_env) + if opt.test_requires: + call(['pip', 'install'] + opt.test_requires, env=virtualenv_env) # run the tests from $HOME, with an absolute path in the command # (this ensures that Python runs the tests against the installed wheel # and not the repo code) - test_command_prepared = prepare_command(test_command, project=abs_project_dir) + test_command_prepared = prepare_command(opt.test_command, project=abs_project_dir) call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True) # clean up shutil.rmtree(venv_dir) # we're all done here; move it to output (overwrite existing) - dst = os.path.join(output_dir, os.path.basename(repaired_wheel)) + dst = os.path.join(opt.output_dir, os.path.basename(repaired_wheel)) shutil.move(repaired_wheel, dst) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 16bd832a..953d7943 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -2,6 +2,9 @@ import os import urllib.request from fnmatch import fnmatch from time import sleep +from typing import NamedTuple, List, Optional, Dict + +from .environment import ParsedEnvironment def prepare_command(command, **kwargs): @@ -103,5 +106,21 @@ class DependencyConstraints(object): return self.base_file_path +class BuildOptions(NamedTuple): + project_dir: str + output_dir: str + test_command: str + test_requires: List[str] + test_extras: str + before_build: str + build_verbosity: int + build_selector: BuildSelector + repair_command: str + environment: ParsedEnvironment + before_test: str + dependency_constraints: Optional[DependencyConstraints] + manylinux_images: Optional[Dict[str, str]] + + resources_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'resources')) get_pip_script = os.path.join(resources_dir, 'get-pip.py') diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 11cc33f1..f720786c 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -12,6 +12,7 @@ from .util import ( get_build_verbosity_extra_flags, prepare_command, get_pip_script, + BuildOptions ) @@ -141,8 +142,8 @@ def setup_python(python_configuration, dependency_constraint_flags, environment) return env -def build(project_dir, output_dir, test_command, before_test, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, dependency_constraints): - abs_project_dir = os.path.abspath(project_dir) +def build(opt: BuildOptions): + abs_project_dir = os.path.abspath(opt.project_dir) temp_dir = tempfile.mkdtemp(prefix='cibuildwheel') built_wheel_dir = os.path.join(temp_dir, 'built_wheel') repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel') @@ -151,42 +152,42 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes nuget = 'C:\\cibw\\nuget.exe' download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget) - python_configurations = get_python_configurations(build_selector) + python_configurations = get_python_configurations(opt.build_selector) for config in python_configurations: dependency_constraint_flags = [] - if dependency_constraints: + if opt.dependency_constraints: dependency_constraint_flags = [ - '-c', dependency_constraints.get_for_python_version(config.version) + '-c', opt.dependency_constraints.get_for_python_version(config.version) ] # install Python - env = setup_python(config, dependency_constraint_flags, environment) + env = setup_python(config, dependency_constraint_flags, opt.environment) # run the before_build command - if before_build: - before_build_prepared = prepare_command(before_build, project=abs_project_dir) + if opt.before_build: + before_build_prepared = prepare_command(opt.before_build, project=abs_project_dir) shell([before_build_prepared], env=env) # build the wheel if os.path.exists(built_wheel_dir): shutil.rmtree(built_wheel_dir) os.makedirs(built_wheel_dir) - shell(['pip', 'wheel', abs_project_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(build_verbosity), env=env) + shell(['pip', 'wheel', abs_project_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(opt.build_verbosity), env=env) built_wheel = glob(os.path.join(built_wheel_dir, '*.whl'))[0] # repair the wheel if os.path.exists(repaired_wheel_dir): shutil.rmtree(repaired_wheel_dir) os.makedirs(repaired_wheel_dir) - if built_wheel.endswith('none-any.whl') or not repair_command: + if built_wheel.endswith('none-any.whl') or not opt.repair_command: # pure Python wheel or empty repair command shutil.move(built_wheel, repaired_wheel_dir) else: - repair_command_prepared = prepare_command(repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) + repair_command_prepared = prepare_command(opt.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) shell([repair_command_prepared], env=env) repaired_wheel = glob(os.path.join(repaired_wheel_dir, '*.whl'))[0] - if test_command: + if opt.test_command: # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. shell(['pip', 'install', 'virtualenv'] + dependency_constraint_flags, env=env) @@ -211,28 +212,28 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes # check that we are using the Python from the virtual environment shell(['which', 'python'], env=virtualenv_env) - if before_test: - before_test_prepared = prepare_command(before_test, project=abs_project_dir) + if opt.before_test: + before_test_prepared = prepare_command(opt.before_test, project=abs_project_dir) shell([before_test_prepared], env=virtualenv_env) # install the wheel - shell(['pip', 'install', repaired_wheel + test_extras], env=virtualenv_env) + shell(['pip', 'install', repaired_wheel + opt.test_extras], env=virtualenv_env) # test the wheel - if test_requires: - shell(['pip', 'install'] + test_requires, env=virtualenv_env) + if opt.test_requires: + shell(['pip', 'install'] + opt.test_requires, env=virtualenv_env) # run the tests from c:\, with an absolute path in the command # (this ensures that Python runs the tests against the installed wheel # and not the repo code) - test_command_prepared = prepare_command(test_command, project=abs_project_dir) + test_command_prepared = prepare_command(opt.test_command, project=abs_project_dir) shell([test_command_prepared], cwd='c:\\', env=virtualenv_env) # clean up shutil.rmtree(venv_dir) # we're all done here; move it to output (remove if already exists) - dst = os.path.join(output_dir, os.path.basename(repaired_wheel)) + dst = os.path.join(opt.output_dir, os.path.basename(repaired_wheel)) if os.path.isfile(dst): os.remove(dst) shutil.move(repaired_wheel, dst) diff --git a/unit_test/main_tests/main_options_test.py b/unit_test/main_tests/main_options_test.py index 88243175..9439daf6 100644 --- a/unit_test/main_tests/main_options_test.py +++ b/unit_test/main_tests/main_options_test.py @@ -18,13 +18,13 @@ def test_output_dir(platform, intercepted_build_args, monkeypatch): main() - assert intercepted_build_args.kwargs['output_dir'] == OUTPUT_DIR + assert intercepted_build_args.args[0].output_dir == OUTPUT_DIR def test_output_dir_default(platform, intercepted_build_args, monkeypatch): main() - assert intercepted_build_args.kwargs['output_dir'] == 'wheelhouse' + assert intercepted_build_args.args[0].output_dir == 'wheelhouse' @pytest.mark.parametrize('also_set_environment', [False, True]) @@ -37,7 +37,7 @@ def test_output_dir_argument(also_set_environment, platform, intercepted_build_a main() - assert intercepted_build_args.kwargs['output_dir'] == OUTPUT_DIR + assert intercepted_build_args.args[0].output_dir == OUTPUT_DIR def test_build_selector(platform, intercepted_build_args, monkeypatch): @@ -49,7 +49,7 @@ def test_build_selector(platform, intercepted_build_args, monkeypatch): main() - intercepted_build_selector = intercepted_build_args.kwargs['build_selector'] + intercepted_build_selector = intercepted_build_args.args[0].build_selector assert isinstance(intercepted_build_selector, BuildSelector) assert intercepted_build_selector('build-this') assert not intercepted_build_selector('skip-that') @@ -82,11 +82,11 @@ def test_manylinux_images(architecture, image, full_image, platform, intercepted if platform == 'linux': assert fnmatch( - intercepted_build_args.kwargs['manylinux_images'][architecture], + intercepted_build_args.args[0].manylinux_images[architecture], full_image ) else: - assert 'manylinux_images' not in intercepted_build_args.kwargs + assert intercepted_build_args.args[0].manylinux_images is None def get_default_repair_command(platform): @@ -113,7 +113,7 @@ def test_repair_command(repair_command, platform_specific, platform, intercepted main() expected_repair = repair_command or get_default_repair_command(platform) - assert intercepted_build_args.kwargs['repair_command'] == expected_repair + assert intercepted_build_args.args[0].repair_command == expected_repair @pytest.mark.parametrize('environment', [ @@ -132,7 +132,7 @@ def test_environment(environment, platform_specific, platform, intercepted_build main() - intercepted_environment = intercepted_build_args.kwargs['environment'] + intercepted_environment = intercepted_build_args.args[0].environment assert isinstance(intercepted_environment, ParsedEnvironment) assert intercepted_environment.as_dictionary(prev_environment={}) == environment @@ -149,7 +149,7 @@ def test_test_requires(test_requires, platform_specific, platform, intercepted_b main() - assert intercepted_build_args.kwargs['test_requires'] == (test_requires or '').split() + assert intercepted_build_args.args[0].test_requires == (test_requires or '').split() @pytest.mark.parametrize('test_extras', [None, 'extras']) @@ -164,7 +164,7 @@ def test_test_extras(test_extras, platform_specific, platform, intercepted_build main() - assert intercepted_build_args.kwargs['test_extras'] == ('[' + test_extras + ']' if test_extras else '') + assert intercepted_build_args.args[0],test_extras == ('[' + test_extras + ']' if test_extras else '') @pytest.mark.parametrize('test_command', [None, 'test --command']) @@ -179,7 +179,7 @@ def test_test_command(test_command, platform_specific, platform, intercepted_bui main() - assert intercepted_build_args.kwargs['test_command'] == test_command + assert intercepted_build_args.args[0].test_command == test_command @pytest.mark.parametrize('before_build', [None, 'before --build']) @@ -194,7 +194,7 @@ def test_before_build(before_build, platform_specific, platform, intercepted_bui main() - assert intercepted_build_args.kwargs['before_build'] == before_build + assert intercepted_build_args.args[0].before_build == before_build @pytest.mark.parametrize('build_verbosity', [None, 0, 2, -2, 4, -4]) @@ -210,7 +210,7 @@ def test_build_verbosity(build_verbosity, platform_specific, platform, intercept main() expected_verbosity = max(-3, min(3, int(build_verbosity or 0))) - assert intercepted_build_args.kwargs['build_verbosity'] == expected_verbosity + assert intercepted_build_args.args[0].build_verbosity == expected_verbosity @pytest.mark.parametrize('option_name', ['CIBW_BUILD', 'CIBW_SKIP']) @@ -225,7 +225,7 @@ def test_build_selector_migrations(intercepted_build_args, monkeypatch, option_n main() - intercepted_build_selector = intercepted_build_args.kwargs['build_selector'] + intercepted_build_selector = intercepted_build_args.args[0].build_selector assert isinstance(intercepted_build_selector, BuildSelector) if option_name == 'CIBW_BUILD': diff --git a/unit_test/main_tests/main_platform_test.py b/unit_test/main_tests/main_platform_test.py index 5b33a4d5..9159586b 100644 --- a/unit_test/main_tests/main_platform_test.py +++ b/unit_test/main_tests/main_platform_test.py @@ -51,10 +51,10 @@ def test_platform_argument(platform, intercepted_build_args, monkeypatch): main() - assert intercepted_build_args.kwargs['project_dir'] == MOCK_PROJECT_DIR + assert intercepted_build_args.args[0].project_dir == MOCK_PROJECT_DIR def test_platform_environment(platform, intercepted_build_args, monkeypatch): main() - assert intercepted_build_args.kwargs['project_dir'] == MOCK_PROJECT_DIR + assert intercepted_build_args.args[0].project_dir == MOCK_PROJECT_DIR