Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #319 from joerick/support-subdirectory-joerick

Browse Source 
Provide `package_dir` for setting subdirectory of project [alternative]
This commit is contained in:
Joe Rickerby
2020-04-14 19:12:50 +01:00
committed by GitHub
parent f7cda33d96 603f77a0c1
commit 95678f3d5d
14 changed files with 175 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cibuildwheel/__main__.py
+24 -16
View File
@@ -48,26 +48,34 @@ def strtobool(val):
def main():
parser = argparse.ArgumentParser(
description='Build wheels for all the platforms.',
epilog=('Most options are supplied via environment variables. '
'See https://github.com/joerick/cibuildwheel#options for info.'))
epilog='''
Most options are supplied via environment variables.
See https://github.com/joerick/cibuildwheel#options for info.
''')
parser.add_argument('--platform',
choices=['auto', 'linux', 'macos', 'windows'],
default=os.environ.get('CIBW_PLATFORM', 'auto'),
help=('Platform to build for. For "linux" you need docker running, on Mac '
'or Linux. For "macos", you need a Mac machine, and note that this '
'script is going to automatically install MacPython on your system, '
'so don\'t run on your development machine. For "windows", you need to '
'run in Windows, and it will build and test for all versions of '
'Python. Default: auto.'))
help='''
Platform to build for. For "linux" you need docker running, on Mac
or Linux. For "macos", you need a Mac machine, and note that this
script is going to automatically install MacPython on your system,
so don't run on your development machine. For "windows", you need to
run in Windows, and it will build and test for all versions of
Python. Default: auto.
''')
parser.add_argument('--output-dir',
default=os.environ.get('CIBW_OUTPUT_DIR', 'wheelhouse'),
help='Destination folder for the wheels.')
parser.add_argument('project_dir',
parser.add_argument('package_dir',
default='.',
nargs='?',
help=('Path to the project that you want wheels for. Default: the current '
'directory.'))
help='''
Path to the package that you want wheels for. Must be a subdirectory of
the working directory. When set, the working directory is still
considered the 'project' and is copied into the Docker container on
Linux. Default: the working directory.
''')
parser.add_argument('--print-build-identifiers',
action='store_true',
@@ -104,7 +112,7 @@ def main():
test_command = get_option_from_environment('CIBW_TEST_COMMAND', platform=platform)
test_requires = get_option_from_environment('CIBW_TEST_REQUIRES', platform=platform, default='').split()
test_extras = get_option_from_environment('CIBW_TEST_EXTRAS', platform=platform, default='')
project_dir = args.project_dir
package_dir = args.package_dir
before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform)
build_verbosity = get_option_from_environment('CIBW_BUILD_VERBOSITY', platform=platform, default='')
build_config, skip_config = os.environ.get('CIBW_BUILD', '*'), os.environ.get('CIBW_SKIP', '')
@@ -147,8 +155,8 @@ def main():
# This needs to be passed on to the docker container in linux.py
os.environ['CIBUILDWHEEL'] = '1'
if not os.path.exists(os.path.join(project_dir, 'setup.py')):
print('cibuildwheel: Could not find setup.py at root of project', file=sys.stderr)
if not os.path.exists(os.path.join(package_dir, 'setup.py')):
print('cibuildwheel: Could not find setup.py at root of package', file=sys.stderr)
exit(2)
if args.print_build_identifiers:
@@ -189,7 +197,7 @@ def main():
manylinux_images = None
build_options = BuildOptions(
project_dir=project_dir,
package_dir=package_dir,
output_dir=output_dir,
test_command=test_command,
test_requires=test_requires,
@@ -201,7 +209,7 @@ def main():
environment=environment,
before_test=before_test,
dependency_constraints=dependency_constraints,
manylinux_images=manylinux_images
manylinux_images=manylinux_images,
)
# Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print'
cibuildwheel/linux.py
+14 -9
View File
@@ -96,12 +96,18 @@ def build(options: BuildOptions):
('pp', 'manylinux_x86_64', options.manylinux_images['pypy_x86_64']),
]
if not os.path.realpath(options.package_dir).startswith(os.path.realpath('.')):
raise Exception('package_dir must be inside the working directory')
container_package_dir = os.path.join('/project', os.path.relpath(options.package_dir, '.'))
for implementation, platform_tag, docker_image in platforms:
platform_configs = [c for c in python_configurations if c.identifier.startswith(implementation) and c.identifier.endswith(platform_tag)]
if not platform_configs:
continue
container_name = 'cibuildwheel-{}'.format(uuid.uuid4())
try:
call(['docker', 'create',
'--env', 'CIBUILDWHEEL',
@@ -111,9 +117,7 @@ def build(options: BuildOptions):
docker_image,
'/bin/bash'])
call(['docker', 'cp',
os.path.abspath(options.project_dir) + '/.',
container_name + ':/project'])
call(['docker', 'cp', '.', container_name + ':/project'])
call(['docker', 'start', container_name])
@@ -165,7 +169,7 @@ def build(options: BuildOptions):
# Build the wheel
rm -rf /tmp/built_wheel
mkdir /tmp/built_wheel
pip wheel . -w /tmp/built_wheel --no-deps {build_verbosity_flag}
pip wheel {package_dir} -w /tmp/built_wheel --no-deps {build_verbosity_flag}
built_wheel=(/tmp/built_wheel/*.whl)
# repair the wheel
@@ -234,13 +238,14 @@ def build(options: BuildOptions):
done
'''.format(
config_python_bin=config.path + '/bin',
package_dir=container_package_dir,
test_requires=' '.join(options.test_requires),
test_extras=options.test_extras,
test_command=shlex.quote(
prepare_command(options.test_command, project='/project') if options.test_command else ''
prepare_command(options.test_command, project='/project', package=container_package_dir) if options.test_command else ''
),
before_build=shlex.quote(
prepare_command(options.before_build, project='/project') if options.before_build else ''
prepare_command(options.before_build, project='/project', package=container_package_dir) if options.before_build else ''
),
build_verbosity_flag=' '.join(get_build_verbosity_extra_flags(options.build_verbosity)),
repair_command=shlex.quote(
@@ -250,7 +255,7 @@ def build(options: BuildOptions):
uid=os.getuid(),
gid=os.getgid(),
before_test=shlex.quote(
prepare_command(options.before_test, project='/project') if options.before_test else ''
prepare_command(options.before_test, project='/project', package=container_package_dir) if options.before_test else ''
),
dependency_install_flags='-c /constraints.txt' if options.dependency_constraints else '',
)
@@ -268,12 +273,12 @@ def build(options: BuildOptions):
call(['docker', 'rm', '--force', '-v', container_name])
def troubleshoot(project_dir, error):
def troubleshoot(package_dir, error):
if (isinstance(error, subprocess.CalledProcessError) and 'exec' in error.cmd):
# the bash script failed
print('Checking for common errors...')
so_files = []
for root, dirs, files in os.walk(project_dir):
for root, dirs, files in os.walk(package_dir):
for name in files:
_, ext = os.path.splitext(name)
if ext == '.so':
cibuildwheel/macos.py
+8 -5
View File
@@ -168,7 +168,6 @@ def setup_python(python_configuration, dependency_constraint_flags, environment)
def build(options: BuildOptions):
abs_project_dir = os.path.abspath(options.project_dir)
temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
built_wheel_dir = os.path.join(temp_dir, 'built_wheel')
repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel')
@@ -186,14 +185,14 @@ def build(options: BuildOptions):
# run the before_build command
if options.before_build:
before_build_prepared = prepare_command(options.before_build, project=abs_project_dir)
before_build_prepared = prepare_command(options.before_build, project='.', package=options.package_dir)
call(before_build_prepared, env=env, shell=True)
# build the wheel
if os.path.exists(built_wheel_dir):
shutil.rmtree(built_wheel_dir)
os.makedirs(built_wheel_dir)
call(['pip', 'wheel', abs_project_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env)
call(['pip', 'wheel', options.package_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env)
built_wheel = glob(os.path.join(built_wheel_dir, '*.whl'))[0]
# repair the wheel
@@ -229,7 +228,7 @@ def build(options: BuildOptions):
call(['which', 'python'], env=virtualenv_env)
if options.before_test:
before_test_prepared = prepare_command(options.before_test, project=abs_project_dir)
before_test_prepared = prepare_command(options.before_test, project='.', package=options.package_dir)
call(before_test_prepared, env=virtualenv_env, shell=True)
# install the wheel
@@ -242,7 +241,11 @@ def build(options: BuildOptions):
# run the tests from $HOME, with an absolute path in the command
# (this ensures that Python runs the tests against the installed wheel
# and not the repo code)
test_command_prepared = prepare_command(options.test_command, project=abs_project_dir)
test_command_prepared = prepare_command(
options.test_command,
project=os.path.abspath('.'),
package=os.path.abspath(options.package_dir)
)
call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True)
# clean up
cibuildwheel/util.py
+1 -1
View File
@@ -107,7 +107,7 @@ class DependencyConstraints:
BuildOptions = NamedTuple("BuildOptions", [
("project_dir", str),
("package_dir", str),
("output_dir", str),
("test_command", Optional[str]),
("test_requires", List[str]),
cibuildwheel/windows.py
+12 -5
View File
@@ -143,7 +143,6 @@ def setup_python(python_configuration, dependency_constraint_flags, environment)
def build(options: BuildOptions):
abs_project_dir = os.path.abspath(options.project_dir)
temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
built_wheel_dir = os.path.join(temp_dir, 'built_wheel')
repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel')
@@ -165,14 +164,14 @@ def build(options: BuildOptions):
# run the before_build command
if options.before_build:
before_build_prepared = prepare_command(options.before_build, project=abs_project_dir)
before_build_prepared = prepare_command(options.before_build, project='.', package=options.package_dir)
shell([before_build_prepared], env=env)
# build the wheel
if os.path.exists(built_wheel_dir):
shutil.rmtree(built_wheel_dir)
os.makedirs(built_wheel_dir)
shell(['pip', 'wheel', abs_project_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env)
shell(['pip', 'wheel', options.package_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env)
built_wheel = glob(os.path.join(built_wheel_dir, '*.whl'))[0]
# repair the wheel
@@ -213,7 +212,11 @@ def build(options: BuildOptions):
shell(['which', 'python'], env=virtualenv_env)
if options.before_test:
before_test_prepared = prepare_command(options.before_test, project=abs_project_dir)
before_test_prepared = prepare_command(
options.before_test,
project='.',
package=options.package_dir
)
shell([before_test_prepared], env=virtualenv_env)
# install the wheel
@@ -226,7 +229,11 @@ def build(options: BuildOptions):
# run the tests from c:\, with an absolute path in the command
# (this ensures that Python runs the tests against the installed wheel
# and not the repo code)
test_command_prepared = prepare_command(options.test_command, project=abs_project_dir)
test_command_prepared = prepare_command(
options.test_command,
project=os.path.abspath('.'),
package=os.path.abspath(options.package_dir)
)
shell([test_command_prepared], cwd='c:\\', env=virtualenv_env)
# clean up
docs/options.md
+19 -11
View File
@@ -199,7 +199,7 @@ A shell command to run before building the wheel. This option allows you to run
If dependencies are required to build your wheel (for example if you include a header from a Python module), set this to `pip install .`, and the dependencies will be installed automatically by pip. However, this means your package will be built twice - if your package takes a long time to build, you might wish to manually list the dependencies here instead.
The active Python binary can be accessed using `python`, and pip with `pip`; `cibuildwheel` makes sure the right version of Python and pip will be executed. `{project}` can be used as a placeholder for the absolute path to the project's root and will be replaced by `cibuildwheel`.
The active Python binary can be accessed using `python`, and pip with `pip`; `cibuildwheel` makes sure the right version of Python and pip will be executed. The placeholder `{package}` can be used here; it will be replaced by the path to the package being built by `cibuildwheel`.
The command is run in a shell, so you can write things like `cmd1 && cmd2`.
@@ -217,8 +217,11 @@ CIBW_BEFORE_BUILD: pip install pybind11
# chain commands using &&
CIBW_BEFORE_BUILD: yum install -y libffi-dev && pip install .
# run a script that's inside your repo
# run a script that's inside your project
CIBW_BEFORE_BUILD: bash scripts/prepare_for_build.sh
# if cibuildwheel is called with a package_dir argument, it's available as {package}
CIBW_BEFORE_BUILD: "{package}/bin/prepare_for_build.sh"
```
@@ -348,7 +351,10 @@ CIBW_DEPENDENCY_VERSIONS: ./constraints.txt
### `CIBW_TEST_COMMAND` {: #test-command}
> Execute a shell command to test each built wheel
Shell command to run tests after the build. The wheel will be installed automatically and available for import from the tests. `{project}` can be used as a placeholder for the absolute path to the project's root and will be replaced by `cibuildwheel`.
Shell command to run tests after the build. The wheel will be installed automatically and available for import from the tests. To ensure the wheel is imported by your tests (instead of your source copy), tests are run from a different directory. Use the placeholders `{project}` and `{package}` when specifying paths in your project.
- `{project}` is an absolute path to the project root - the working directory where cibuildwheel was called.
- `{package}` is the path to the package being built - the `package_dir` argument supplied to cibuildwheel on the command line.
The command is run in a shell, so you can write things like `cmd1 && cmd2`.
@@ -361,8 +367,8 @@ Platform-specific variants also available:<br/>
# run the project tests against the installed wheel using `nose`
CIBW_TEST_COMMAND: nosetests {project}/tests
# run the project tests using `pytest`
CIBW_TEST_COMMAND: pytest {project}/tests
# run the package tests using `pytest`
CIBW_TEST_COMMAND: pytest {package}/tests
```
@@ -411,7 +417,7 @@ CIBW_TEST_EXTRAS: test,qt
A shell command to run in **each** test virtual environment, before your wheel is installed and tested. This is useful if you need to install a non pip package, change values of environment variables
or perform multi step pip installation (e.g. installing `scikit-build` or `cython` before install test package)
The active Python binary can be accessed using `python`, and pip with `pip`; `cibuildwheel` makes sure the right version of Python and pip will be executed. `{project}` can be used as a placeholder for the absolute path to the project's root and will be replaced by `cibuildwheel`.
The active Python binary can be accessed using `python`, and pip with `pip`; `cibuildwheel` makes sure the right version of Python and pip will be executed. The placeholder `{package}` can be used here; it will be replaced by the path to the package being built by `cibuildwheel`.
The command is run in a shell, so you can write things like `cmd1 && cmd2`.
@@ -456,13 +462,16 @@ CIBW_BUILD_VERBOSITY: 1
```text
usage: cibuildwheel [-h] [--platform {auto,linux,macos,windows}]
[--output-dir OUTPUT_DIR] [--print-build-identifiers]
[project_dir]
[package_dir]
Build wheels for all the platforms.
positional arguments:
project_dir Path to the project that you want wheels for.
Default: the current directory.
package_dir Path to the package that you want wheels for. Must be
a subdirectory of the working directory. When set, the
working directory is still considered the 'project'
and is copied into the Docker container on Linux.
Default: the working directory.
optional arguments:
-h, --help show this help message and exit
@@ -473,13 +482,12 @@ optional arguments:
automatically install MacPython on your system, so
don't run on your development machine. For "windows",
you need to run in Windows, and it will build and test
for all versions of Python at C:\PythonXX[-x64].
for all versions of Python. Default: auto.
--output-dir OUTPUT_DIR
Destination folder for the wheels.
--print-build-identifiers
Print the build identifiers matched by the current
invocation and exit.
```
<style>
test/13_subdir_package/bin/before_build.py
Executable
+1
View File
@@ -0,0 +1 @@
print('before_build.py executed!')
test/13_subdir_package/cibuildwheel_test.py
+25
View File
@@ -0,0 +1,25 @@
import os
import utils
project_dir = os.path.dirname(__file__)
def test(capfd):
package_dir = os.path.join(project_dir, 'src', 'spam')
# build the wheels
actual_wheels = utils.cibuildwheel_run(project_dir, package_dir=package_dir, add_env={
'CIBW_BEFORE_BUILD': 'python {project}/bin/before_build.py',
'CIBW_TEST_COMMAND': 'python {package}/test/run_tests.py',
# this shouldn't depend on the version of python, so build only
# CPython 3.6
'CIBW_BUILD': 'cp36-*',
})
# check that the expected wheels are produced
expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0')
if 'cp36' in w]
assert set(actual_wheels) == set(expected_wheels)
captured = capfd.readouterr()
assert "before_build.py executed!" in captured.out
assert "run_tests.py executed!" in captured.out
test/13_subdir_package/src/spam/setup.py
+8
View File
@@ -0,0 +1,8 @@
from setuptools import Extension, setup
setup(
name="spam",
ext_modules=[Extension('spam', sources=['spam.c'])],
version="0.1.0",
)
test/13_subdir_package/src/spam/spam.c
+48
View File
@@ -0,0 +1,48 @@
#include <Python.h>
static PyObject *
spam_system(PyObject *self, PyObject *args)
{
const char *command;
int sts;
if (!PyArg_ParseTuple(args, "s", &command))
return NULL;
sts = system(command);
return PyLong_FromLong(sts);
}
/* Module initialization */
#if PY_MAJOR_VERSION >= 3
#define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void)
#define MOD_DEF(m, name, doc, methods, module_state_size) \
static struct PyModuleDef moduledef = { \
PyModuleDef_HEAD_INIT, name, doc, module_state_size, methods, }; \
m = PyModule_Create(&moduledef);
#define MOD_RETURN(m) return m;
#else
#define MOD_INIT(name) PyMODINIT_FUNC init##name(void)
#define MOD_DEF(m, name, doc, methods, module_state_size) \
m = Py_InitModule3(name, methods, doc);
#define MOD_RETURN(m) return;
#endif
static PyMethodDef module_methods[] = {
{"system", (PyCFunction)spam_system, METH_VARARGS,
"Execute a shell command."},
{NULL} /* Sentinel */
};
MOD_INIT(spam)
{
PyObject* m;
MOD_DEF(m,
"spam",
"Example module",
module_methods,
-1)
MOD_RETURN(m)
}
test/13_subdir_package/src/spam/test/run_tests.py
+1
View File
@@ -0,0 +1 @@
print('run_tests.py executed!')
test/shared/utils.py
+5 -2
View File
@@ -41,13 +41,15 @@ def cibuildwheel_get_build_identifiers(project_path, env=None):
return cmd_output.strip().split('\n')
def cibuildwheel_run(project_path, env=None, add_env=None, output_dir=None):
def cibuildwheel_run(project_path, package_dir='.', env=None, add_env=None, output_dir=None):
'''
Runs cibuildwheel as a subprocess, building the project at project_path.
Uses the current Python interpreter.
:param project_path: path of the project to be built.
:param package_dir: path of the package to be built. Can be absolute, or
relative to project_path.
:param env: full environment to be used, os.environ if None
:param add_env: environment used to update env
:param output_dir: directory where wheels are saved. If None, a temporary
@@ -64,8 +66,9 @@ def cibuildwheel_run(project_path, env=None, add_env=None, output_dir=None):
with TemporaryDirectoryIfNone(output_dir) as _output_dir:
subprocess.check_call(
[sys.executable, '-m', 'cibuildwheel', '--output-dir', str(_output_dir), project_path],
[sys.executable, '-m', 'cibuildwheel', '--output-dir', str(_output_dir), package_dir],
env=env,
cwd=project_path,
)
wheels = os.listdir(_output_dir)
return wheels
unit_test/main_tests/conftest.py
+4 -4
View File
@@ -18,7 +18,7 @@ class ArgsInterceptor:
self.kwargs = kwargs
MOCK_PROJECT_DIR = 'some_project_dir'
MOCK_PACKAGE_DIR = 'some_package_dir'
@pytest.fixture(autouse=True)
@@ -39,20 +39,20 @@ def mock_protection(monkeypatch):
@pytest.fixture(autouse=True)
def fake_project_dir(monkeypatch):
def fake_package_dir(monkeypatch):
'''
Monkey-patch enough for the main() function to run
'''
real_os_path_exists = os.path.exists
def mock_os_path_exists(path):
if path == os.path.join(MOCK_PROJECT_DIR, 'setup.py'):
if path == os.path.join(MOCK_PACKAGE_DIR, 'setup.py'):
return True
else:
return real_os_path_exists(path)
monkeypatch.setattr(os.path, 'exists', mock_os_path_exists)
monkeypatch.setattr(sys, 'argv', ['cibuildwheel', MOCK_PROJECT_DIR])
monkeypatch.setattr(sys, 'argv', ['cibuildwheel', MOCK_PACKAGE_DIR])
@pytest.fixture(params=['linux', 'macos', 'windows'])
unit_test/main_tests/main_platform_test.py
+5 -3
View File
@@ -4,7 +4,7 @@ import pytest
from cibuildwheel.__main__ import main
from conftest import MOCK_PROJECT_DIR # noqa: I100
from conftest import MOCK_PACKAGE_DIR # noqa: I100
def test_unknown_platform_non_ci(monkeypatch, capsys):
@@ -12,6 +12,7 @@ def test_unknown_platform_non_ci(monkeypatch, capsys):
monkeypatch.delenv('BITRISE_BUILD_NUMBER', raising=False)
monkeypatch.delenv('AZURE_HTTP_USER_AGENT', raising=False)
monkeypatch.delenv('GITHUB_WORKFLOW', raising=False)
monkeypatch.delenv('CIBW_PLATFORM', raising=False)
with pytest.raises(SystemExit) as exit:
main()
@@ -25,6 +26,7 @@ def test_unknown_platform_non_ci(monkeypatch, capsys):
def test_unknown_platform_on_ci(monkeypatch, capsys):
monkeypatch.setenv('CI', 'true')
monkeypatch.setattr(sys, 'platform', 'nonexistent')
monkeypatch.delenv('CIBW_PLATFORM', raising=False)
with pytest.raises(SystemExit) as exit:
main()
@@ -51,10 +53,10 @@ def test_platform_argument(platform, intercepted_build_args, monkeypatch):
main()
assert intercepted_build_args.args[0].project_dir == MOCK_PROJECT_DIR
assert intercepted_build_args.args[0].package_dir == MOCK_PACKAGE_DIR
def test_platform_environment(platform, intercepted_build_args, monkeypatch):
main()
assert intercepted_build_args.args[0].project_dir == MOCK_PROJECT_DIR
assert intercepted_build_args.args[0].package_dir == MOCK_PACKAGE_DIR