diff --git a/README.md b/README.md index c75881ea..d8e85c55 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,20 @@ Space-separated list of dependencies required for running the tests. Example: `pytest` Example: `nose==1.3.7 moto==0.4.31` +| Environment variable: `CIBW_BEFORE_BUILD` +| --- + +Optional. + +Shell command to run before building the wheel. This option allows you to run a command in **each** Python environment before the `pip wheel` command. This is useful if you need to set up some dependency so it's available during the build. + +The active Python binary can be accessed using `{python}`, and pip with `{pip}`. These are useful when you need to write `python3` or `pip3` on a Python 3.x build. + +Example: `{pip} install pybind11` + +Platform-specific variants also available: + `CIBW_BEFORE_BUILD_MACOS` | `CIBW_BEFORE_BUILD_WINDOWS` | `CIBW_BEFORE_BUILD_LINUX` + | Environment variable: `CIBW_SKIP` | 🔶 [coming soon](https://github.com/joerick/cibuildwheel/issues/3) 🔶 | --- | --- diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 8f2b640f..348d486e 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -3,6 +3,24 @@ import argparse, os, subprocess, sys from cibuildwheel import linux, windows, macos +def get_option_from_environment(option_name, platform=None): + ''' + Returns an option from the environment, optionally scoped by the platform. + + Example: + get_option_from_environment('CIBW_COLOR', platform='macos') + + This will return the value of CIBW_COLOR_MACOS if it exists, otherwise the value of + CIBW_COLOR. + ''' + if platform: + option = os.environ.get('%s_%s' % (option_name, platform.upper())) + if option is not None: + return option + + return os.environ.get(option_name) + + def main(): parser = argparse.ArgumentParser( description='Build wheels for all the platforms.', @@ -29,10 +47,28 @@ def main(): args = parser.parse_args() + if args.platform != 'auto': + platform = args.platform + else: + if os.environ.get('TRAVIS_OS_NAME') == 'linux': + platform = 'linux' + elif os.environ.get('TRAVIS_OS_NAME') == 'osx': + platform = 'macos' + elif 'APPVEYOR' in os.environ: + platform = 'windows' + else: + print('Unable to detect platform. cibuildwheel should run on your CI server, ' + 'Travis CI and Appveyor are supported. You can run on your development ' + 'machine using the --platform argument. Check --help output for more ' + 'information.', + file=sys.stderr) + exit(2) + output_dir = args.output_dir test_command = os.environ.get('CIBW_TEST_COMMAND', None) test_requires = os.environ.get('CIBW_TEST_REQUIRES', '').split() project_dir = args.project_dir + before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform) try: project_setup_py = os.path.join(project_dir, 'setup.py') @@ -58,26 +94,10 @@ def main(): package_name=package_name, output_dir=output_dir, test_command=test_command, - test_requires=test_requires + test_requires=test_requires, + before_build=before_build, ) - if args.platform != 'auto': - platform = args.platform - else: - if os.environ.get('TRAVIS_OS_NAME') == 'linux': - platform = 'linux' - elif os.environ.get('TRAVIS_OS_NAME') == 'osx': - platform = 'macos' - elif 'APPVEYOR' in os.environ: - platform = 'windows' - else: - print('Unable to detect platform. cibuildwheel should run on your CI server, ' - 'Travis CI and Appveyor are supported. You can run on your development ' - 'machine using the --platform argument. Check --help output for more ' - 'information.', - file=sys.stderr) - exit(2) - if platform == 'linux': linux.build(**build_args) elif platform == 'windows': diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 6ca27361..057cd510 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -1,5 +1,6 @@ from __future__ import print_function import os, subprocess +from .util import prepare_command try: from shlex import quote as shlex_quote @@ -7,7 +8,7 @@ except ImportError: from pipes import quote as shlex_quote -def build(project_dir, package_name, output_dir, test_command, test_requires): +def build(project_dir, package_name, output_dir, test_command, test_requires, before_build): for docker_image in ['quay.io/pypa/manylinux1_x86_64', 'quay.io/pypa/manylinux1_i686']: bash_script = ''' set -o errexit @@ -15,6 +16,10 @@ def build(project_dir, package_name, output_dir, test_command, test_requires): cd /project for PYBIN in /opt/python/*/bin; do + if [ ! -z {before_build} ]; then + PATH=$PYBIN:$PATH sh -c {before_build} + fi + "$PYBIN/pip" wheel . -w /tmp/linux_wheels done @@ -34,13 +39,20 @@ def build(project_dir, package_name, output_dir, test_command, test_requires): # Run the tests from a different directory if [ ! -z {test_command} ]; then - (cd "$HOME" && export PATH=$PYBIN:$PATH && sh -c {test_command}) + pushd $HOME + PATH=$PYBIN:$PATH sh -c {test_command} + popd fi done '''.format( package_name=package_name, test_requires=' '.join(test_requires), - test_command=shlex_quote(test_command.format(project='/project') if test_command else ''), + test_command=shlex_quote( + test_command.format(project='/project') if test_command else '' + ), + before_build=shlex_quote( + prepare_command(before_build, python='python', pip='pip') if before_build else '' + ), ) docker_process = subprocess.Popen([ diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 3c68d73f..3ea2271a 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -7,8 +7,10 @@ try: except ImportError: from pipes import quote as shlex_quote +from .util import prepare_command -def build(project_dir, package_name, output_dir, test_command, test_requires): + +def build(project_dir, package_name, output_dir, test_command, test_requires, before_build): PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'url']) python_configurations = [ PythonConfiguration(version='2.7', url='https://www.python.org/ftp/python/2.7.13/python-2.7.13-macosx10.6.pkg'), @@ -43,10 +45,15 @@ def build(project_dir, package_name, output_dir, test_command, test_requires): # install pip & wheel shell([python, '-m', 'ensurepip', '--upgrade'], env=env) - shell([pip, '--version']) + shell([pip, '--version'], env=env) shell([pip, 'install', 'wheel'], env=env) shell([pip, 'install', 'delocate'], env=env) + # run the before_build command + if before_build: + before_build_prepared = prepare_command(before_build, python=python, pip=pip) + shell(shlex.split(before_build_prepared), env=env) + # build the wheel to temp dir temp_wheel_dir = '/tmp/tmpwheel%s' % config.version shell([pip, 'wheel', project_dir, '-w', temp_wheel_dir], env=env) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py new file mode 100644 index 00000000..c93a17c4 --- /dev/null +++ b/cibuildwheel/util.py @@ -0,0 +1,11 @@ + +def prepare_command(command, python, pip): + ''' + Preprocesses a command by expanding variables like {python} or {pip}. + + For example, used for the before_build option, where the user would + like to run a command like `python setup.py test`. If the command should run on + Python 3, the user could write `{python} setup.py test`. This command would expand + it out to python2 or python3 as appropriate. + ''' + return command.format(python=python, pip=pip) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 1ac226a6..af28bdc2 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -2,8 +2,10 @@ from __future__ import print_function import os, tempfile, subprocess, urllib2 from collections import namedtuple +from .util import prepare_command -def build(project_dir, package_name, output_dir, test_command, test_requires): + +def build(project_dir, package_name, output_dir, test_command, test_requires, before_build): # run_with_env is a cmd file that sets the right environment variables to run_with_env = os.path.join(tempfile.gettempdir(), 'appveyor_run_with_env.cmd') if not os.path.exists(run_with_env): @@ -51,6 +53,11 @@ def build(project_dir, package_name, output_dir, test_command, test_requires): env=env) shell(['pip', 'install', 'wheel'], env=env) + # run the before_build command + if before_build: + before_build_prepared = prepare_command(before_build, python='python', pip='pip') + shell([before_build_prepared], env=env) + # build the wheel shell(['pip', 'wheel', project_dir, '-w', output_dir], env=env) diff --git a/test/03_before_build/environment.json b/test/03_before_build/environment.json new file mode 100644 index 00000000..04db7726 --- /dev/null +++ b/test/03_before_build/environment.json @@ -0,0 +1,4 @@ +{ + "CIBW_BEFORE_BUILD": "{python} -c \"import sys; open('/tmp/pythonversion.txt', 'w').write(sys.version)\"", + "CIBW_BEFORE_BUILD_WINDOWS": "{python} -c \"import sys; open('c:\\pythonversion.txt', 'w').write(sys.version)\"" +} diff --git a/test/03_before_build/setup.py b/test/03_before_build/setup.py new file mode 100644 index 00000000..b3400640 --- /dev/null +++ b/test/03_before_build/setup.py @@ -0,0 +1,18 @@ +from setuptools import setup, Extension +import sys + +# here we assert that the Python version as written to version.txt in the CIBW_BEFORE_BUILD step +# is the same one as is currently running. +if sys.argv[-1] != '--name': + version_file = 'c:\\pythonversion.txt' if sys.platform == 'win32' else '/tmp/pythonversion.txt' + with open(version_file) as f: + stored_version = f.read() + print('stored_version', stored_version) + print('sys.version', sys.version) + assert stored_version == sys.version + +setup( + name="spam", + ext_modules=[Extension('spam', sources=['spam.c'])], + version="0.1.0", +) diff --git a/test/03_before_build/spam.c b/test/03_before_build/spam.c new file mode 100644 index 00000000..d1ab0f22 --- /dev/null +++ b/test/03_before_build/spam.c @@ -0,0 +1,48 @@ +#include + +static PyObject * +spam_system(PyObject *self, PyObject *args) +{ + const char *command; + int sts; + + if (!PyArg_ParseTuple(args, "s", &command)) + return NULL; + sts = system(command); + return PyLong_FromLong(sts); +} + +/* Module initialization */ + +#if PY_MAJOR_VERSION >= 3 + #define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void) + #define MOD_DEF(m, name, doc, methods, module_state_size) \ + static struct PyModuleDef moduledef = { \ + PyModuleDef_HEAD_INIT, name, doc, module_state_size, methods, }; \ + m = PyModule_Create(&moduledef); + #define MOD_RETURN(m) return m; +#else + #define MOD_INIT(name) PyMODINIT_FUNC init##name(void) + #define MOD_DEF(m, name, doc, methods, module_state_size) \ + m = Py_InitModule3(name, methods, doc); + #define MOD_RETURN(m) return; +#endif + +static PyMethodDef module_methods[] = { + {"system", (PyCFunction)spam_system, METH_VARARGS, + "Execute a shell command."}, + {NULL} /* Sentinel */ +}; + +MOD_INIT(spam) +{ + PyObject* m; + + MOD_DEF(m, + "spam", + "Example module", + module_methods, + -1) + + MOD_RETURN(m) +} diff --git a/test/03_before_build/version.txt b/test/03_before_build/version.txt new file mode 100644 index 00000000..3e4354cb --- /dev/null +++ b/test/03_before_build/version.txt @@ -0,0 +1,2 @@ +3.6.0 (default, Feb 7 2017, 23:55:32) +[GCC 4.8.2 20140120 (Red Hat 4.8.2-15)] \ No newline at end of file