From 1e08d29cfc11314669fd8e1305ec2230bd8bc439 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20=C5=A0?= Date: Fri, 13 Mar 2020 13:45:14 +0100 Subject: [PATCH 01/13] Use package_dir to specify path to Python package --- cibuildwheel/__main__.py | 13 +++++++------ cibuildwheel/linux.py | 15 +++++++++++---- cibuildwheel/macos.py | 11 ++++++----- cibuildwheel/windows.py | 11 ++++++----- 4 files changed, 30 insertions(+), 20 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index f8b7631e..db5c7c23 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -60,10 +60,10 @@ def main(): parser.add_argument('--output-dir', default=os.environ.get('CIBW_OUTPUT_DIR', 'wheelhouse'), help='Destination folder for the wheels.') - parser.add_argument('project_dir', + parser.add_argument('package_dir', default='.', nargs='?', - help=('Path to the project that you want wheels for. Default: the current ' + help=('Path to the package that you want wheels for. Default: the current ' 'directory.')) parser.add_argument('--print-build-identifiers', @@ -101,7 +101,7 @@ def main(): test_command = get_option_from_environment('CIBW_TEST_COMMAND', platform=platform) test_requires = get_option_from_environment('CIBW_TEST_REQUIRES', platform=platform, default='').split() test_extras = get_option_from_environment('CIBW_TEST_EXTRAS', platform=platform, default='') - project_dir = args.project_dir + package_dir = args.package_dir before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform) build_verbosity = get_option_from_environment('CIBW_BUILD_VERBOSITY', platform=platform, default='') build_config, skip_config = os.environ.get('CIBW_BUILD', '*'), os.environ.get('CIBW_SKIP', '') @@ -136,8 +136,8 @@ def main(): # This needs to be passed on to the docker container in linux.py os.environ['CIBUILDWHEEL'] = '1' - if not os.path.exists(os.path.join(project_dir, 'setup.py')): - print('cibuildwheel: Could not find setup.py at root of project', file=sys.stderr) + if not os.path.exists(os.path.join(package_dir, 'setup.py')): + print('cibuildwheel: Could not find setup.py at root of package', file=sys.stderr) exit(2) if args.print_build_identifiers: @@ -145,7 +145,8 @@ def main(): exit(0) build_options = dict( - project_dir=project_dir, + project_dir='.', + package_dir=package_dir, output_dir=output_dir, test_command=test_command, test_requires=test_requires, diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 3113f309..4d2d4de6 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -88,6 +88,12 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes ('pp', 'manylinux_x86_64', manylinux_images['pypy_x86_64']), ] + abs_project_dir = os.path.abspath(project_dir) + abs_package_dir = os.path.abspath(package_dir) + + container_project_dir = '/project' + container_package_dir = os.path.join(container_project_dir, os.path.relpath(abs_package_dir, os.path.commonprefix([abs_project_dir, abs_package_dir]))), + for implementation, platform_tag, docker_image in platforms: platform_configs = [c for c in python_configurations if c.identifier.startswith(implementation) and c.identifier.endswith(platform_tag)] if not platform_configs: @@ -122,7 +128,7 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes # Build the wheel rm -rf /tmp/built_wheel mkdir /tmp/built_wheel - pip wheel . -w /tmp/built_wheel --no-deps {build_verbosity_flag} + pip wheel {package_dir} -w /tmp/built_wheel --no-deps {build_verbosity_flag} built_wheel=(/tmp/built_wheel/*.whl) # repair the wheel @@ -191,13 +197,14 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes ) done '''.format( pybin_paths=' '.join(c.path + '/bin' for c in platform_configs), + package_dir=container_package_dir, test_requires=' '.join(test_requires), test_extras=test_extras, test_command=shlex.quote( - prepare_command(test_command, project='/project') if test_command else '' + prepare_command(test_command, project=container_project_dir, package=container_package_dir) if test_command else '' ), before_build=shlex.quote( - prepare_command(before_build, project='/project') if before_build else '' + prepare_command(before_build, project=container_project_dir, package=container_package_dir) if before_build else '' ), build_verbosity_flag=' '.join(get_build_verbosity_extra_flags(build_verbosity)), repair_command=shlex.quote( @@ -207,7 +214,7 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes uid=os.getuid(), gid=os.getgid(), before_test=shlex.quote( - prepare_command(before_test, project='/project') if before_test else '' + prepare_command(before_test, project=container_project_dir, package=container_package_dir) if before_test else '' ), ) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 59106262..329102b7 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -105,8 +105,9 @@ def install_pypy(version, url): return installation_bin_path -def build(project_dir, output_dir, test_command, before_test, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment): +def build(project_dir, package_dir, output_dir, test_command, before_test, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment): abs_project_dir = os.path.abspath(project_dir) + abs_package_dir = os.path.abspath(package_dir) temp_dir = tempfile.mkdtemp(prefix='cibuildwheel') built_wheel_dir = os.path.join(temp_dir, 'built_wheel') repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel') @@ -175,14 +176,14 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes # run the before_build command if before_build: - before_build_prepared = prepare_command(before_build, project=abs_project_dir) + before_build_prepared = prepare_command(before_build, project=abs_project_dir, package=abs_package_dir) call(before_build_prepared, env=env, shell=True) # build the wheel if os.path.exists(built_wheel_dir): shutil.rmtree(built_wheel_dir) os.makedirs(built_wheel_dir) - call(['pip', 'wheel', abs_project_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(build_verbosity), env=env) + call(['pip', 'wheel', abs_package_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(build_verbosity), env=env) built_wheel = glob(os.path.join(built_wheel_dir, '*.whl'))[0] # repair the wheel @@ -215,7 +216,7 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes call(['which', 'python'], env=virtualenv_env) if before_test: - before_test_prepared = prepare_command(before_test, project=abs_project_dir) + before_test_prepared = prepare_command(before_test, project=abs_project_dir, package=abs_package_dir) call(before_test_prepared, env=virtualenv_env, shell=True) # install the wheel @@ -228,7 +229,7 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes # run the tests from $HOME, with an absolute path in the command # (this ensures that Python runs the tests against the installed wheel # and not the repo code) - test_command_prepared = prepare_command(test_command, project=abs_project_dir) + test_command_prepared = prepare_command(test_command, project=abs_project_dir, package=abs_package_dir) call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True) # clean up diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 8030cf31..9ff642e3 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -86,8 +86,9 @@ def install_pypy(version, arch, url): return installation_path -def build(project_dir, output_dir, test_command, before_test, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment): +def build(project_dir, package_dir, output_dir, test_command, before_test, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment): abs_project_dir = os.path.abspath(project_dir) + abs_package_dir = os.path.abspath(package_dir) temp_dir = tempfile.mkdtemp(prefix='cibuildwheel') built_wheel_dir = os.path.join(temp_dir, 'built_wheel') repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel') @@ -148,14 +149,14 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes # run the before_build command if before_build: - before_build_prepared = prepare_command(before_build, project=abs_project_dir) + before_build_prepared = prepare_command(before_build, project=abs_project_dir, package=abs_package_dir) shell([before_build_prepared], env=env) # build the wheel if os.path.exists(built_wheel_dir): shutil.rmtree(built_wheel_dir) os.makedirs(built_wheel_dir) - shell(['pip', 'wheel', abs_project_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(build_verbosity), env=env) + shell(['pip', 'wheel', abs_package_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(build_verbosity), env=env) built_wheel = glob(os.path.join(built_wheel_dir, '*.whl'))[0] # repair the wheel @@ -193,7 +194,7 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes shell(['which', 'python'], env=virtualenv_env) if before_test: - before_test_prepared = prepare_command(before_test, project=abs_project_dir) + before_test_prepared = prepare_command(before_test, project=abs_project_dir, package=abs_package_dir) shell([before_test_prepared], env=virtualenv_env) # install the wheel @@ -206,7 +207,7 @@ def build(project_dir, output_dir, test_command, before_test, test_requires, tes # run the tests from c:\, with an absolute path in the command # (this ensures that Python runs the tests against the installed wheel # and not the repo code) - test_command_prepared = prepare_command(test_command, project=abs_project_dir) + test_command_prepared = prepare_command(test_command, project=abs_project_dir, package=abs_package_dir) shell([test_command_prepared], cwd='c:\\', env=virtualenv_env) # clean up From bbb77aa488b4c910709fb376c514b6eb0d9e8051 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20=C5=A0?= Date: Fri, 13 Mar 2020 13:46:51 +0100 Subject: [PATCH 02/13] Update docs --- docs/options.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/options.md b/docs/options.md index d2feaa0f..27c50123 100644 --- a/docs/options.md +++ b/docs/options.md @@ -358,13 +358,13 @@ Platform-specific variants also available:
#### Examples ```yaml -# install test dependencies with overwritten environment variables. +# install test dependencies with overwritten environment variables. CIBW_BEFORE_TEST: CC=gcc CXX=g++ pip install -r requirements.txt # chain commands using && CIBW_BEFORE_TEST: rm -rf ./data/cache && mkdir -p ./data/cache -# install non pip python package +# install non pip python package CIBW_BEFORE_TEST: cd some_dir; ./configure; make; make install # install python packages that are required to install test dependencies @@ -399,8 +399,8 @@ usage: cibuildwheel [-h] [--platform {auto,linux,macos,windows}] Build wheels for all the platforms. positional arguments: - project_dir Path to the project that you want wheels for. - Default: the current directory. + package_dir Path to the package that you want wheels for. Default: + the current directory. optional arguments: -h, --help show this help message and exit From ed50cfd20a64f55302745ad725ad4e7be551b8d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Filip=20=C5=A0?= Date: Sat, 28 Mar 2020 16:25:57 +0100 Subject: [PATCH 03/13] Fix tests --- unit_test/main_tests/conftest.py | 8 ++++---- unit_test/main_tests/main_platform_test.py | 6 +++--- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/unit_test/main_tests/conftest.py b/unit_test/main_tests/conftest.py index 227b8df7..cd896915 100644 --- a/unit_test/main_tests/conftest.py +++ b/unit_test/main_tests/conftest.py @@ -18,7 +18,7 @@ class ArgsInterceptor: self.kwargs = kwargs -MOCK_PROJECT_DIR = 'some_project_dir' +MOCK_PACKAGE_DIR = 'some_package_dir' @pytest.fixture(autouse=True) @@ -39,20 +39,20 @@ def mock_protection(monkeypatch): @pytest.fixture(autouse=True) -def fake_project_dir(monkeypatch): +def fake_package_dir(monkeypatch): ''' Monkey-patch enough for the main() function to run ''' real_os_path_exists = os.path.exists def mock_os_path_exists(path): - if path == os.path.join(MOCK_PROJECT_DIR, 'setup.py'): + if path == os.path.join(MOCK_PACKAGE_DIR, 'setup.py'): return True else: return real_os_path_exists(path) monkeypatch.setattr(os.path, 'exists', mock_os_path_exists) - monkeypatch.setattr(sys, 'argv', ['cibuildwheel', MOCK_PROJECT_DIR]) + monkeypatch.setattr(sys, 'argv', ['cibuildwheel', MOCK_PACKAGE_DIR]) @pytest.fixture(params=['linux', 'macos', 'windows']) diff --git a/unit_test/main_tests/main_platform_test.py b/unit_test/main_tests/main_platform_test.py index 5b33a4d5..18d10e13 100644 --- a/unit_test/main_tests/main_platform_test.py +++ b/unit_test/main_tests/main_platform_test.py @@ -4,7 +4,7 @@ import pytest from cibuildwheel.__main__ import main -from conftest import MOCK_PROJECT_DIR # noqa: I100 +from conftest import MOCK_PACKAGE_DIR # noqa: I100 def test_unknown_platform_non_ci(monkeypatch, capsys): @@ -51,10 +51,10 @@ def test_platform_argument(platform, intercepted_build_args, monkeypatch): main() - assert intercepted_build_args.kwargs['project_dir'] == MOCK_PROJECT_DIR + assert intercepted_build_args.kwargs['package_dir'] == MOCK_PACKAGE_DIR def test_platform_environment(platform, intercepted_build_args, monkeypatch): main() - assert intercepted_build_args.kwargs['project_dir'] == MOCK_PROJECT_DIR + assert intercepted_build_args.kwargs['package_dir'] == MOCK_PACKAGE_DIR From 95bef468329a898ab9c039182994ef897d191e08 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 9 Apr 2020 15:51:00 +0100 Subject: [PATCH 04/13] Update docs for package_dir --- cibuildwheel/__main__.py | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 9d7331da..916b117e 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -48,26 +48,34 @@ def strtobool(val): def main(): parser = argparse.ArgumentParser( description='Build wheels for all the platforms.', - epilog=('Most options are supplied via environment variables. ' - 'See https://github.com/joerick/cibuildwheel#options for info.')) + epilog=''' + Most options are supplied via environment variables. + See https://github.com/joerick/cibuildwheel#options for info. + ''') parser.add_argument('--platform', choices=['auto', 'linux', 'macos', 'windows'], default=os.environ.get('CIBW_PLATFORM', 'auto'), - help=('Platform to build for. For "linux" you need docker running, on Mac ' - 'or Linux. For "macos", you need a Mac machine, and note that this ' - 'script is going to automatically install MacPython on your system, ' - 'so don\'t run on your development machine. For "windows", you need to ' - 'run in Windows, and it will build and test for all versions of ' - 'Python. Default: auto.')) + help=''' + Platform to build for. For "linux" you need docker running, on Mac + or Linux. For "macos", you need a Mac machine, and note that this + script is going to automatically install MacPython on your system, + so don't run on your development machine. For "windows", you need to + run in Windows, and it will build and test for all versions of + Python. Default: auto. + ''') parser.add_argument('--output-dir', default=os.environ.get('CIBW_OUTPUT_DIR', 'wheelhouse'), help='Destination folder for the wheels.') parser.add_argument('package_dir', default='.', nargs='?', - help=('Path to the package that you want wheels for. Default: the current ' - 'directory.')) + help=''' + Path to the package that you want wheels for. Must be a subdirectory of + the working directory. When set, the working directory is still + considered the 'project' and is copied into the Docker container on + Linux. Default: the working directory. + ''') parser.add_argument('--print-build-identifiers', action='store_true', From 7e59d8c61cf84db4e583630358975d1c768032cf Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 9 Apr 2020 16:19:36 +0100 Subject: [PATCH 05/13] Change how paths are handled, using 'project is working dir' mentality --- cibuildwheel/__main__.py | 1 - cibuildwheel/linux.py | 14 +++++++------- cibuildwheel/macos.py | 14 ++++++++------ cibuildwheel/util.py | 1 - cibuildwheel/windows.py | 18 ++++++++++++------ 5 files changed, 27 insertions(+), 21 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 916b117e..7099bc8f 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -197,7 +197,6 @@ def main(): manylinux_images = None build_options = BuildOptions( - project_dir='.', package_dir=package_dir, output_dir=output_dir, test_command=test_command, diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 2942c9af..f7e34586 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -96,11 +96,10 @@ def build(options: BuildOptions): ('pp', 'manylinux_x86_64', options.manylinux_images['pypy_x86_64']), ] - abs_project_dir = os.path.abspath(options.project_dir) - abs_package_dir = os.path.abspath(options.package_dir) + if not os.path.realpath(options.package_dir).startswith(os.path.realpath('.')): + raise Exception('package_dir must be inside the working directory') - container_project_dir = '/project' - container_package_dir = os.path.join(container_project_dir, os.path.relpath(abs_package_dir, os.path.commonprefix([abs_project_dir, abs_package_dir]))), + container_package_dir = os.path.join('/project', os.path.relpath(options.package_dir, '.')) for implementation, platform_tag, docker_image in platforms: platform_configs = [c for c in python_configurations if c.identifier.startswith(implementation) and c.identifier.endswith(platform_tag)] @@ -108,6 +107,7 @@ def build(options: BuildOptions): continue container_name = 'cibuildwheel-{}'.format(uuid.uuid4()) + try: call(['docker', 'create', '--env', 'CIBUILDWHEEL', @@ -244,10 +244,10 @@ def build(options: BuildOptions): test_requires=' '.join(options.test_requires), test_extras=options.test_extras, test_command=shlex.quote( - prepare_command(options.test_command, project='/project') if options.test_command else '' + prepare_command(options.test_command, project='/project', package=container_package_dir) if options.test_command else '' ), before_build=shlex.quote( - prepare_command(options.before_build, project='/project') if options.before_build else '' + prepare_command(options.before_build, project='/project', package=container_package_dir) if options.before_build else '' ), build_verbosity_flag=' '.join(get_build_verbosity_extra_flags(options.build_verbosity)), repair_command=shlex.quote( @@ -257,7 +257,7 @@ def build(options: BuildOptions): uid=os.getuid(), gid=os.getgid(), before_test=shlex.quote( - prepare_command(options.before_test, project='/project') if options.before_test else '' + prepare_command(options.before_test, project='/project', package=container_package_dir) if options.before_test else '' ), dependency_install_flags='-c /constraints.txt' if options.dependency_constraints else '', ) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index b903dcb9..599e9b23 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -168,8 +168,6 @@ def setup_python(python_configuration, dependency_constraint_flags, environment) def build(options: BuildOptions): - abs_project_dir = os.path.abspath(options.project_dir) - abs_package_dir = os.path.abspath(options.package_dir) temp_dir = tempfile.mkdtemp(prefix='cibuildwheel') built_wheel_dir = os.path.join(temp_dir, 'built_wheel') repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel') @@ -187,14 +185,14 @@ def build(options: BuildOptions): # run the before_build command if options.before_build: - before_build_prepared = prepare_command(options.before_build, project=abs_project_dir, package=abs_package_dir) + before_build_prepared = prepare_command(options.before_build, project='.', package=options.package_dir) call(before_build_prepared, env=env, shell=True) # build the wheel if os.path.exists(built_wheel_dir): shutil.rmtree(built_wheel_dir) os.makedirs(built_wheel_dir) - call(['pip', 'wheel', abs_package_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env) + call(['pip', 'wheel', options.package_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env) built_wheel = glob(os.path.join(built_wheel_dir, '*.whl'))[0] # repair the wheel @@ -230,7 +228,7 @@ def build(options: BuildOptions): call(['which', 'python'], env=virtualenv_env) if options.before_test: - before_test_prepared = prepare_command(options.before_test, project=abs_project_dir, package=abs_package_dir) + before_test_prepared = prepare_command(options.before_test, project='.', package=options.package_dir) call(before_test_prepared, env=virtualenv_env, shell=True) # install the wheel @@ -243,7 +241,11 @@ def build(options: BuildOptions): # run the tests from $HOME, with an absolute path in the command # (this ensures that Python runs the tests against the installed wheel # and not the repo code) - test_command_prepared = prepare_command(options.test_command, project=abs_project_dir, package=abs_package_dir) + test_command_prepared = prepare_command( + options.test_command, + project=os.path.abspath('.'), + package=os.path.abspath(options.package_dir) + ) call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True) # clean up diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index d42dcc38..64f561a1 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -107,7 +107,6 @@ class DependencyConstraints: BuildOptions = NamedTuple("BuildOptions", [ - ("project_dir", str), ("package_dir", str), ("output_dir", str), ("test_command", Optional[str]), diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 2a5df54d..e6f14ed9 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -143,8 +143,6 @@ def setup_python(python_configuration, dependency_constraint_flags, environment) def build(options: BuildOptions): - abs_project_dir = os.path.abspath(options.project_dir) - abs_package_dir = os.path.abspath(options.package_dir) temp_dir = tempfile.mkdtemp(prefix='cibuildwheel') built_wheel_dir = os.path.join(temp_dir, 'built_wheel') repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel') @@ -166,14 +164,14 @@ def build(options: BuildOptions): # run the before_build command if options.before_build: - before_build_prepared = prepare_command(options.before_build, project=abs_project_dir, package=abs_package_dir) + before_build_prepared = prepare_command(options.before_build, project='.', package=options.package_dir) shell([before_build_prepared], env=env) # build the wheel if os.path.exists(built_wheel_dir): shutil.rmtree(built_wheel_dir) os.makedirs(built_wheel_dir) - shell(['pip', 'wheel', abs_package_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env) + shell(['pip', 'wheel', options.package_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env) built_wheel = glob(os.path.join(built_wheel_dir, '*.whl'))[0] # repair the wheel @@ -214,7 +212,11 @@ def build(options: BuildOptions): shell(['which', 'python'], env=virtualenv_env) if options.before_test: - before_test_prepared = prepare_command(options.before_test, project=abs_project_dir, package=abs_package_dir) + before_test_prepared = prepare_command( + options.before_test, + project='.', + package=options.package_dir + ) shell([before_test_prepared], env=virtualenv_env) # install the wheel @@ -227,7 +229,11 @@ def build(options: BuildOptions): # run the tests from c:\, with an absolute path in the command # (this ensures that Python runs the tests against the installed wheel # and not the repo code) - test_command_prepared = prepare_command(options.test_command, project=abs_project_dir, package=abs_package_dir) + test_command_prepared = prepare_command( + options.test_command, + project=os.path.abspath('.'), + package=os.path.abspath(options.package_dir) + ) shell([test_command_prepared], cwd='c:\\', env=virtualenv_env) # clean up From f84388fe2601facce5f2a74727857620d5070e94 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 10 Apr 2020 12:18:21 +0100 Subject: [PATCH 06/13] Changes that let me run the entire the entire linux test suite locally --- unit_test/main_tests/main_platform_test.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/unit_test/main_tests/main_platform_test.py b/unit_test/main_tests/main_platform_test.py index 41ca1373..009fb1b4 100644 --- a/unit_test/main_tests/main_platform_test.py +++ b/unit_test/main_tests/main_platform_test.py @@ -12,6 +12,7 @@ def test_unknown_platform_non_ci(monkeypatch, capsys): monkeypatch.delenv('BITRISE_BUILD_NUMBER', raising=False) monkeypatch.delenv('AZURE_HTTP_USER_AGENT', raising=False) monkeypatch.delenv('GITHUB_WORKFLOW', raising=False) + monkeypatch.delenv('CIBW_PLATFORM', raising=False) with pytest.raises(SystemExit) as exit: main() @@ -25,6 +26,7 @@ def test_unknown_platform_non_ci(monkeypatch, capsys): def test_unknown_platform_on_ci(monkeypatch, capsys): monkeypatch.setenv('CI', 'true') monkeypatch.setattr(sys, 'platform', 'nonexistent') + monkeypatch.delenv('CIBW_PLATFORM', raising=False) with pytest.raises(SystemExit) as exit: main() From e07d8038c672b867d3f3c3005302faaeceea1fd7 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 10 Apr 2020 12:27:26 +0100 Subject: [PATCH 07/13] Remove lingering project_dir references --- cibuildwheel/linux.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index f7e34586..664a8fb2 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -117,9 +117,7 @@ def build(options: BuildOptions): docker_image, '/bin/bash']) - call(['docker', 'cp', - os.path.abspath(options.project_dir) + '/.', - container_name + ':/project']) + call(['docker', 'cp', '.', container_name + ':/project']) call(['docker', 'start', container_name]) @@ -274,12 +272,12 @@ def build(options: BuildOptions): call(['docker', 'rm', '--force', '-v', container_name]) -def troubleshoot(project_dir, error): +def troubleshoot(package_dir, error): if (isinstance(error, subprocess.CalledProcessError) and 'exec' in error.cmd): # the bash script failed print('Checking for common errors...') so_files = [] - for root, dirs, files in os.walk(project_dir): + for root, dirs, files in os.walk(package_dir): for name in files: _, ext = os.path.splitext(name) if ext == '.so': From 397393ba9180d5317f9076c31dcd88df1c9ea1ef Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 10 Apr 2020 12:27:53 +0100 Subject: [PATCH 08/13] Use cwd to run test projects, not package_dir --- test/shared/utils.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/shared/utils.py b/test/shared/utils.py index 0b463334..d1e1f0a1 100644 --- a/test/shared/utils.py +++ b/test/shared/utils.py @@ -64,8 +64,9 @@ def cibuildwheel_run(project_path, env=None, add_env=None, output_dir=None): with TemporaryDirectoryIfNone(output_dir) as _output_dir: subprocess.check_call( - [sys.executable, '-m', 'cibuildwheel', '--output-dir', str(_output_dir), project_path], + [sys.executable, '-m', 'cibuildwheel', '--output-dir', str(_output_dir)], env=env, + cwd=project_path, ) wheels = os.listdir(_output_dir) return wheels From 83b39bcda2c53b94eadb58b5359b4242c106a4cf Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 10 Apr 2020 12:59:22 +0100 Subject: [PATCH 09/13] Update docs on the package/project split --- docs/options.md | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/docs/options.md b/docs/options.md index 08e1955a..8ecaaf23 100644 --- a/docs/options.md +++ b/docs/options.md @@ -199,7 +199,7 @@ A shell command to run before building the wheel. This option allows you to run If dependencies are required to build your wheel (for example if you include a header from a Python module), set this to `pip install .`, and the dependencies will be installed automatically by pip. However, this means your package will be built twice - if your package takes a long time to build, you might wish to manually list the dependencies here instead. -The active Python binary can be accessed using `python`, and pip with `pip`; `cibuildwheel` makes sure the right version of Python and pip will be executed. `{project}` can be used as a placeholder for the absolute path to the project's root and will be replaced by `cibuildwheel`. +The active Python binary can be accessed using `python`, and pip with `pip`; `cibuildwheel` makes sure the right version of Python and pip will be executed. The placeholder `{package}` can be used here; it will be replaced by the path to the package being built by `cibuildwheel`. The command is run in a shell, so you can write things like `cmd1 && cmd2`. @@ -217,8 +217,11 @@ CIBW_BEFORE_BUILD: pip install pybind11 # chain commands using && CIBW_BEFORE_BUILD: yum install -y libffi-dev && pip install . -# run a script that's inside your repo +# run a script that's inside your project CIBW_BEFORE_BUILD: bash scripts/prepare_for_build.sh + +# if cibuildwheel is called with a package_dir argument, it's available as {package} +CIBW_BEFORE_BUILD: "{package}/bin/prepare_for_build.sh" ``` @@ -348,7 +351,10 @@ CIBW_DEPENDENCY_VERSIONS: ./constraints.txt ### `CIBW_TEST_COMMAND` {: #test-command} > Execute a shell command to test each built wheel -Shell command to run tests after the build. The wheel will be installed automatically and available for import from the tests. `{project}` can be used as a placeholder for the absolute path to the project's root and will be replaced by `cibuildwheel`. +Shell command to run tests after the build. The wheel will be installed automatically and available for import from the tests. To ensure the wheel is imported by your tests (instead of your source copy), tests are run from a different directory. Use the placeholders `{project}` and `{package}` when specifying paths in your project. + +- `{project}` is an absolute path to the project root - the working directory where cibuildwheel was called. +- `{package}` is the path to the package being built - the `package_dir` argument supplied to cibuildwheel on the command line. The command is run in a shell, so you can write things like `cmd1 && cmd2`. @@ -361,8 +367,8 @@ Platform-specific variants also available:
# run the project tests against the installed wheel using `nose` CIBW_TEST_COMMAND: nosetests {project}/tests -# run the project tests using `pytest` -CIBW_TEST_COMMAND: pytest {project}/tests +# run the package tests using `pytest` +CIBW_TEST_COMMAND: pytest {package}/tests ``` @@ -411,7 +417,7 @@ CIBW_TEST_EXTRAS: test,qt A shell command to run in **each** test virtual environment, before your wheel is installed and tested. This is useful if you need to install a non pip package, change values of environment variables or perform multi step pip installation (e.g. installing `scikit-build` or `cython` before install test package) -The active Python binary can be accessed using `python`, and pip with `pip`; `cibuildwheel` makes sure the right version of Python and pip will be executed. `{project}` can be used as a placeholder for the absolute path to the project's root and will be replaced by `cibuildwheel`. +The active Python binary can be accessed using `python`, and pip with `pip`; `cibuildwheel` makes sure the right version of Python and pip will be executed. The placeholder `{package}` can be used here; it will be replaced by the path to the package being built by `cibuildwheel`. The command is run in a shell, so you can write things like `cmd1 && cmd2`. @@ -456,13 +462,16 @@ CIBW_BUILD_VERBOSITY: 1 ```text usage: cibuildwheel [-h] [--platform {auto,linux,macos,windows}] [--output-dir OUTPUT_DIR] [--print-build-identifiers] - [project_dir] + [package_dir] Build wheels for all the platforms. positional arguments: - package_dir Path to the package that you want wheels for. Default: - the current directory. + package_dir Path to the package that you want wheels for. Must be + a subdirectory of the working directory. When set, the + working directory is still considered the 'project' + and is copied into the Docker container on Linux. + Default: the working directory. optional arguments: -h, --help show this help message and exit @@ -473,13 +482,12 @@ optional arguments: automatically install MacPython on your system, so don't run on your development machine. For "windows", you need to run in Windows, and it will build and test - for all versions of Python at C:\PythonXX[-x64]. + for all versions of Python. Default: auto. --output-dir OUTPUT_DIR Destination folder for the wheels. --print-build-identifiers Print the build identifiers matched by the current invocation and exit. - ```