From 5c91340ad07b1d44f24ed8715b3d706d2999bcc3 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 10 Apr 2017 21:40:03 +0100 Subject: [PATCH 01/10] Run the pip version check in the right env --- cibuildwheel/macos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 3c68d73f..cd3962b1 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -43,7 +43,7 @@ def build(project_dir, package_name, output_dir, test_command, test_requires): # install pip & wheel shell([python, '-m', 'ensurepip', '--upgrade'], env=env) - shell([pip, '--version']) + shell([pip, '--version'], env=env) shell([pip, 'install', 'wheel'], env=env) shell([pip, 'install', 'delocate'], env=env) From 56f9a3716550174bd41534febc008e7b819b6f60 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 10 Apr 2017 21:41:05 +0100 Subject: [PATCH 02/10] Don't run tests in a subshell so the errexit option applies --- cibuildwheel/linux.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 6ca27361..a8d6c128 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -34,7 +34,9 @@ def build(project_dir, package_name, output_dir, test_command, test_requires): # Run the tests from a different directory if [ ! -z {test_command} ]; then - (cd "$HOME" && export PATH=$PYBIN:$PATH && sh -c {test_command}) + pushd $HOME + PATH=$PYBIN:$PATH sh -c {test_command} + popd fi done '''.format( From 2679cb7f46a31aa11b1af567af498b92166ac05e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 10 Apr 2017 21:42:12 +0100 Subject: [PATCH 03/10] Add before_build option, that runs a shell command before 'pip wheel' --- cibuildwheel/__main__.py | 56 ++++++++++++++++++--------- cibuildwheel/linux.py | 14 ++++++- cibuildwheel/macos.py | 9 ++++- cibuildwheel/util.py | 11 ++++++ cibuildwheel/windows.py | 9 ++++- test/03_before_build/environment.json | 3 ++ test/03_before_build/setup.py | 17 ++++++++ test/03_before_build/spam.c | 48 +++++++++++++++++++++++ test/03_before_build/version.txt | 2 + 9 files changed, 147 insertions(+), 22 deletions(-) create mode 100644 cibuildwheel/util.py create mode 100644 test/03_before_build/environment.json create mode 100644 test/03_before_build/setup.py create mode 100644 test/03_before_build/spam.c create mode 100644 test/03_before_build/version.txt diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 8f2b640f..348d486e 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -3,6 +3,24 @@ import argparse, os, subprocess, sys from cibuildwheel import linux, windows, macos +def get_option_from_environment(option_name, platform=None): + ''' + Returns an option from the environment, optionally scoped by the platform. + + Example: + get_option_from_environment('CIBW_COLOR', platform='macos') + + This will return the value of CIBW_COLOR_MACOS if it exists, otherwise the value of + CIBW_COLOR. + ''' + if platform: + option = os.environ.get('%s_%s' % (option_name, platform.upper())) + if option is not None: + return option + + return os.environ.get(option_name) + + def main(): parser = argparse.ArgumentParser( description='Build wheels for all the platforms.', @@ -29,10 +47,28 @@ def main(): args = parser.parse_args() + if args.platform != 'auto': + platform = args.platform + else: + if os.environ.get('TRAVIS_OS_NAME') == 'linux': + platform = 'linux' + elif os.environ.get('TRAVIS_OS_NAME') == 'osx': + platform = 'macos' + elif 'APPVEYOR' in os.environ: + platform = 'windows' + else: + print('Unable to detect platform. cibuildwheel should run on your CI server, ' + 'Travis CI and Appveyor are supported. You can run on your development ' + 'machine using the --platform argument. Check --help output for more ' + 'information.', + file=sys.stderr) + exit(2) + output_dir = args.output_dir test_command = os.environ.get('CIBW_TEST_COMMAND', None) test_requires = os.environ.get('CIBW_TEST_REQUIRES', '').split() project_dir = args.project_dir + before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform) try: project_setup_py = os.path.join(project_dir, 'setup.py') @@ -58,26 +94,10 @@ def main(): package_name=package_name, output_dir=output_dir, test_command=test_command, - test_requires=test_requires + test_requires=test_requires, + before_build=before_build, ) - if args.platform != 'auto': - platform = args.platform - else: - if os.environ.get('TRAVIS_OS_NAME') == 'linux': - platform = 'linux' - elif os.environ.get('TRAVIS_OS_NAME') == 'osx': - platform = 'macos' - elif 'APPVEYOR' in os.environ: - platform = 'windows' - else: - print('Unable to detect platform. cibuildwheel should run on your CI server, ' - 'Travis CI and Appveyor are supported. You can run on your development ' - 'machine using the --platform argument. Check --help output for more ' - 'information.', - file=sys.stderr) - exit(2) - if platform == 'linux': linux.build(**build_args) elif platform == 'windows': diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index a8d6c128..057cd510 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -1,5 +1,6 @@ from __future__ import print_function import os, subprocess +from .util import prepare_command try: from shlex import quote as shlex_quote @@ -7,7 +8,7 @@ except ImportError: from pipes import quote as shlex_quote -def build(project_dir, package_name, output_dir, test_command, test_requires): +def build(project_dir, package_name, output_dir, test_command, test_requires, before_build): for docker_image in ['quay.io/pypa/manylinux1_x86_64', 'quay.io/pypa/manylinux1_i686']: bash_script = ''' set -o errexit @@ -15,6 +16,10 @@ def build(project_dir, package_name, output_dir, test_command, test_requires): cd /project for PYBIN in /opt/python/*/bin; do + if [ ! -z {before_build} ]; then + PATH=$PYBIN:$PATH sh -c {before_build} + fi + "$PYBIN/pip" wheel . -w /tmp/linux_wheels done @@ -42,7 +47,12 @@ def build(project_dir, package_name, output_dir, test_command, test_requires): '''.format( package_name=package_name, test_requires=' '.join(test_requires), - test_command=shlex_quote(test_command.format(project='/project') if test_command else ''), + test_command=shlex_quote( + test_command.format(project='/project') if test_command else '' + ), + before_build=shlex_quote( + prepare_command(before_build, python='python', pip='pip') if before_build else '' + ), ) docker_process = subprocess.Popen([ diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index cd3962b1..3ea2271a 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -7,8 +7,10 @@ try: except ImportError: from pipes import quote as shlex_quote +from .util import prepare_command -def build(project_dir, package_name, output_dir, test_command, test_requires): + +def build(project_dir, package_name, output_dir, test_command, test_requires, before_build): PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'url']) python_configurations = [ PythonConfiguration(version='2.7', url='https://www.python.org/ftp/python/2.7.13/python-2.7.13-macosx10.6.pkg'), @@ -47,6 +49,11 @@ def build(project_dir, package_name, output_dir, test_command, test_requires): shell([pip, 'install', 'wheel'], env=env) shell([pip, 'install', 'delocate'], env=env) + # run the before_build command + if before_build: + before_build_prepared = prepare_command(before_build, python=python, pip=pip) + shell(shlex.split(before_build_prepared), env=env) + # build the wheel to temp dir temp_wheel_dir = '/tmp/tmpwheel%s' % config.version shell([pip, 'wheel', project_dir, '-w', temp_wheel_dir], env=env) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py new file mode 100644 index 00000000..c93a17c4 --- /dev/null +++ b/cibuildwheel/util.py @@ -0,0 +1,11 @@ + +def prepare_command(command, python, pip): + ''' + Preprocesses a command by expanding variables like {python} or {pip}. + + For example, used for the before_build option, where the user would + like to run a command like `python setup.py test`. If the command should run on + Python 3, the user could write `{python} setup.py test`. This command would expand + it out to python2 or python3 as appropriate. + ''' + return command.format(python=python, pip=pip) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 1ac226a6..af28bdc2 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -2,8 +2,10 @@ from __future__ import print_function import os, tempfile, subprocess, urllib2 from collections import namedtuple +from .util import prepare_command -def build(project_dir, package_name, output_dir, test_command, test_requires): + +def build(project_dir, package_name, output_dir, test_command, test_requires, before_build): # run_with_env is a cmd file that sets the right environment variables to run_with_env = os.path.join(tempfile.gettempdir(), 'appveyor_run_with_env.cmd') if not os.path.exists(run_with_env): @@ -51,6 +53,11 @@ def build(project_dir, package_name, output_dir, test_command, test_requires): env=env) shell(['pip', 'install', 'wheel'], env=env) + # run the before_build command + if before_build: + before_build_prepared = prepare_command(before_build, python='python', pip='pip') + shell([before_build_prepared], env=env) + # build the wheel shell(['pip', 'wheel', project_dir, '-w', output_dir], env=env) diff --git a/test/03_before_build/environment.json b/test/03_before_build/environment.json new file mode 100644 index 00000000..77cdbb10 --- /dev/null +++ b/test/03_before_build/environment.json @@ -0,0 +1,3 @@ +{ + "CIBW_BEFORE_BUILD": "{python} -c \"import sys; open('version.txt', 'w').write(sys.version)\"" +} diff --git a/test/03_before_build/setup.py b/test/03_before_build/setup.py new file mode 100644 index 00000000..3c2724a4 --- /dev/null +++ b/test/03_before_build/setup.py @@ -0,0 +1,17 @@ +from setuptools import setup, Extension +import sys + +# here we assert that the Python version as written to version.txt in the CIBW_BEFORE_BUILD step +# is the same one as is currently running. +sys.stderr.write('sys.argv \n' + str(sys.argv)) +if sys.argv[-1] != '--name': + with open('version.txt') as f: + stored_version = f.read() + print('version is', stored_version) + assert stored_version == sys.version + +setup( + name="spam", + ext_modules=[Extension('spam', sources=['spam.c'])], + version="0.1.0", +) diff --git a/test/03_before_build/spam.c b/test/03_before_build/spam.c new file mode 100644 index 00000000..d1ab0f22 --- /dev/null +++ b/test/03_before_build/spam.c @@ -0,0 +1,48 @@ +#include + +static PyObject * +spam_system(PyObject *self, PyObject *args) +{ + const char *command; + int sts; + + if (!PyArg_ParseTuple(args, "s", &command)) + return NULL; + sts = system(command); + return PyLong_FromLong(sts); +} + +/* Module initialization */ + +#if PY_MAJOR_VERSION >= 3 + #define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void) + #define MOD_DEF(m, name, doc, methods, module_state_size) \ + static struct PyModuleDef moduledef = { \ + PyModuleDef_HEAD_INIT, name, doc, module_state_size, methods, }; \ + m = PyModule_Create(&moduledef); + #define MOD_RETURN(m) return m; +#else + #define MOD_INIT(name) PyMODINIT_FUNC init##name(void) + #define MOD_DEF(m, name, doc, methods, module_state_size) \ + m = Py_InitModule3(name, methods, doc); + #define MOD_RETURN(m) return; +#endif + +static PyMethodDef module_methods[] = { + {"system", (PyCFunction)spam_system, METH_VARARGS, + "Execute a shell command."}, + {NULL} /* Sentinel */ +}; + +MOD_INIT(spam) +{ + PyObject* m; + + MOD_DEF(m, + "spam", + "Example module", + module_methods, + -1) + + MOD_RETURN(m) +} diff --git a/test/03_before_build/version.txt b/test/03_before_build/version.txt new file mode 100644 index 00000000..3e4354cb --- /dev/null +++ b/test/03_before_build/version.txt @@ -0,0 +1,2 @@ +3.6.0 (default, Feb 7 2017, 23:55:32) +[GCC 4.8.2 20140120 (Red Hat 4.8.2-15)] \ No newline at end of file From 520daa4c230d3c66628190353fe54b42fb7fb6f7 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 10 Apr 2017 21:56:26 +0100 Subject: [PATCH 04/10] A little more debugging info for Windows --- test/03_before_build/setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/03_before_build/setup.py b/test/03_before_build/setup.py index 3c2724a4..72431cf2 100644 --- a/test/03_before_build/setup.py +++ b/test/03_before_build/setup.py @@ -7,7 +7,8 @@ sys.stderr.write('sys.argv \n' + str(sys.argv)) if sys.argv[-1] != '--name': with open('version.txt') as f: stored_version = f.read() - print('version is', stored_version) + print('stored_version', stored_version) + print('sys.version', sys.version) assert stored_version == sys.version setup( From 278661df77b1cf23938b2495287c78800cf36237 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 10 Apr 2017 22:11:10 +0100 Subject: [PATCH 05/10] remove redundant line --- test/03_before_build/setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/03_before_build/setup.py b/test/03_before_build/setup.py index 72431cf2..afc30692 100644 --- a/test/03_before_build/setup.py +++ b/test/03_before_build/setup.py @@ -3,7 +3,6 @@ import sys # here we assert that the Python version as written to version.txt in the CIBW_BEFORE_BUILD step # is the same one as is currently running. -sys.stderr.write('sys.argv \n' + str(sys.argv)) if sys.argv[-1] != '--name': with open('version.txt') as f: stored_version = f.read() From 881cd66fae6b153155259b76266da6de42cb510c Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 10 Apr 2017 22:12:26 +0100 Subject: [PATCH 06/10] Check the path is okay on windows --- test/03_before_build/environment.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/03_before_build/environment.json b/test/03_before_build/environment.json index 77cdbb10..ef0a0fab 100644 --- a/test/03_before_build/environment.json +++ b/test/03_before_build/environment.json @@ -1,3 +1,3 @@ { - "CIBW_BEFORE_BUILD": "{python} -c \"import sys; open('version.txt', 'w').write(sys.version)\"" + "CIBW_BEFORE_BUILD": "echo $PATH && {python} -c \"import sys; open('version.txt', 'w').write(sys.version)\"" } From ff942ad76682f9042d29c167ef91cd589fdbd97f Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 10 Apr 2017 22:25:34 +0100 Subject: [PATCH 07/10] Perhaps a filename collision --- test/03_before_build/environment.json | 2 +- test/03_before_build/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/03_before_build/environment.json b/test/03_before_build/environment.json index ef0a0fab..d541431a 100644 --- a/test/03_before_build/environment.json +++ b/test/03_before_build/environment.json @@ -1,3 +1,3 @@ { - "CIBW_BEFORE_BUILD": "echo $PATH && {python} -c \"import sys; open('version.txt', 'w').write(sys.version)\"" + "CIBW_BEFORE_BUILD": "{python} -c \"import sys; open('pythonversion.txt', 'w').write(sys.version)\"" } diff --git a/test/03_before_build/setup.py b/test/03_before_build/setup.py index afc30692..a05deba2 100644 --- a/test/03_before_build/setup.py +++ b/test/03_before_build/setup.py @@ -4,7 +4,7 @@ import sys # here we assert that the Python version as written to version.txt in the CIBW_BEFORE_BUILD step # is the same one as is currently running. if sys.argv[-1] != '--name': - with open('version.txt') as f: + with open('pythonversion.txt') as f: stored_version = f.read() print('stored_version', stored_version) print('sys.version', sys.version) From 7a2e951f488c6ca15a1e54f577843c1211c5b59e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 10 Apr 2017 22:36:06 +0100 Subject: [PATCH 08/10] Change version file path to absolute on Windows --- test/03_before_build/environment.json | 3 ++- test/03_before_build/setup.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/test/03_before_build/environment.json b/test/03_before_build/environment.json index d541431a..70fda075 100644 --- a/test/03_before_build/environment.json +++ b/test/03_before_build/environment.json @@ -1,3 +1,4 @@ { - "CIBW_BEFORE_BUILD": "{python} -c \"import sys; open('pythonversion.txt', 'w').write(sys.version)\"" + "CIBW_BEFORE_BUILD": "{python} -c \"import sys; open('pythonversion.txt', 'w').write(sys.version)\"", + "CIBW_BEFORE_BUILD_WINDOWS": "{python} -c \"import sys; open('c:\\pythonversion.txt', 'w').write(sys.version)\"" } diff --git a/test/03_before_build/setup.py b/test/03_before_build/setup.py index a05deba2..bfad2c15 100644 --- a/test/03_before_build/setup.py +++ b/test/03_before_build/setup.py @@ -4,7 +4,8 @@ import sys # here we assert that the Python version as written to version.txt in the CIBW_BEFORE_BUILD step # is the same one as is currently running. if sys.argv[-1] != '--name': - with open('pythonversion.txt') as f: + version_file = 'c:\\pythonversion.txt' if os.platform == 'win32' else 'pythonversion.txt' + with open(version_file) as f: stored_version = f.read() print('stored_version', stored_version) print('sys.version', sys.version) From 89c87357193a829354cade9fef09292186a643e7 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 10 Apr 2017 22:41:08 +0100 Subject: [PATCH 09/10] Fix os/sys typo and use an absolute path on *nix too --- test/03_before_build/environment.json | 2 +- test/03_before_build/setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/test/03_before_build/environment.json b/test/03_before_build/environment.json index 70fda075..04db7726 100644 --- a/test/03_before_build/environment.json +++ b/test/03_before_build/environment.json @@ -1,4 +1,4 @@ { - "CIBW_BEFORE_BUILD": "{python} -c \"import sys; open('pythonversion.txt', 'w').write(sys.version)\"", + "CIBW_BEFORE_BUILD": "{python} -c \"import sys; open('/tmp/pythonversion.txt', 'w').write(sys.version)\"", "CIBW_BEFORE_BUILD_WINDOWS": "{python} -c \"import sys; open('c:\\pythonversion.txt', 'w').write(sys.version)\"" } diff --git a/test/03_before_build/setup.py b/test/03_before_build/setup.py index bfad2c15..b3400640 100644 --- a/test/03_before_build/setup.py +++ b/test/03_before_build/setup.py @@ -4,7 +4,7 @@ import sys # here we assert that the Python version as written to version.txt in the CIBW_BEFORE_BUILD step # is the same one as is currently running. if sys.argv[-1] != '--name': - version_file = 'c:\\pythonversion.txt' if os.platform == 'win32' else 'pythonversion.txt' + version_file = 'c:\\pythonversion.txt' if sys.platform == 'win32' else '/tmp/pythonversion.txt' with open(version_file) as f: stored_version = f.read() print('stored_version', stored_version) From 2c0d6e3db1590a9f6902c6d46a8ff27aeb4f1add Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 10 Apr 2017 22:48:04 +0100 Subject: [PATCH 10/10] Add documentation for CIBW_BEFORE_BUILD --- README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/README.md b/README.md index 2685b518..eb28f1a8 100644 --- a/README.md +++ b/README.md @@ -132,6 +132,20 @@ Space-separated list of dependencies required for running the tests. Example: `pytest` Example: `nose==1.3.7 moto==0.4.31` +| Environment variable: `CIBW_BEFORE_BUILD` +| --- + +Optional. + +Shell command to run before building the wheel. This option allows you to run a command in **each** Python environment before the `pip wheel` command. This is useful if you need to set up some dependency so it's available during the build. + +The active Python binary can be accessed using `{python}`, and pip with `{pip}`. These are useful when you need to write `python3` or `pip3` on a Python 3.x build. + +Example: `{pip} install pybind11` + +Platform-specific variants also available: + `CIBW_BEFORE_BUILD_MACOS` | `CIBW_BEFORE_BUILD_WINDOWS` | `CIBW_BEFORE_BUILD_LINUX` + | Environment variable: `CIBW_SKIP` | 🔶 coming soon 🔶 | --- | ---