From 4e0b3c0f314fe51745808ccfa4781498977b8f91 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Sun, 29 Oct 2017 11:30:29 +0100 Subject: [PATCH 1/6] First try at implementing optionsfor allowing different manylinux1 docker images --- cibuildwheel/__main__.py | 4 ++- cibuildwheel/linux.py | 6 ++-- test/06_docker_images/environment.json | 5 +++ test/06_docker_images/setup.py | 12 +++++++ test/06_docker_images/spam.c | 48 ++++++++++++++++++++++++++ 5 files changed, 71 insertions(+), 4 deletions(-) create mode 100644 test/06_docker_images/environment.json create mode 100644 test/06_docker_images/setup.py create mode 100644 test/06_docker_images/spam.c diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 1a157495..2a3d6610 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -76,6 +76,8 @@ def main(): before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform) skip_config = os.environ.get('CIBW_SKIP', '') environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform) or '' + manylinux1_x86_64_image = os.environ.get('CIBW_MANYLINUX1_X86_64_IMAGE', None) + manylinux1_i686_image = os.environ.get('CIBW_MANYLINUX1_I686_IMAGE', None) try: environment = parse_environment(environment_config) @@ -128,7 +130,7 @@ def main(): os.makedirs(output_dir) if platform == 'linux': - cibuildwheel.linux.build(**build_options) + cibuildwheel.linux.build(manylinux1_x86_64_image=manylinux1_x86_64_image, manylinux1_i686_image=manylinux1_i686_image, **build_options) elif platform == 'windows': cibuildwheel.windows.build(**build_options) elif platform == 'macos': diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 68b0349e..5525f0d8 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -9,7 +9,7 @@ except ImportError: from pipes import quote as shlex_quote -def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, skip, environment): +def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, skip, environment, manylinux1_x86_64_image, manylinux1_i686_image): try: subprocess.check_call(['docker', '--version']) except: @@ -38,8 +38,8 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be python_configurations = [c for c in python_configurations if not skip(c.identifier)] platforms = [ - ('manylinux1_x86_64', 'quay.io/pypa/manylinux1_x86_64'), - ('manylinux1_i686', 'quay.io/pypa/manylinux1_i686'), + ('manylinux1_x86_64', manylinux1_x86_64_image or 'quay.io/pypa/manylinux1_x86_64'), + ('manylinux1_i686', manylinux1_i686_image or 'quay.io/pypa/manylinux1_i686'), ] for platform_tag, docker_image in platforms: diff --git a/test/06_docker_images/environment.json b/test/06_docker_images/environment.json new file mode 100644 index 00000000..e5e73cc8 --- /dev/null +++ b/test/06_docker_images/environment.json @@ -0,0 +1,5 @@ +{ + "CIBW_MANYLINUX1_X86_64_IMAGE": "dockcross/manylinux-x64", + "CIBW_MANYLINUX1_I686_IMAGE": "dockcross/manylinux-x86", + "CIBW_SKIP": "*macos* *win*" +} diff --git a/test/06_docker_images/setup.py b/test/06_docker_images/setup.py new file mode 100644 index 00000000..48d7c00b --- /dev/null +++ b/test/06_docker_images/setup.py @@ -0,0 +1,12 @@ +import os + +from setuptools import setup, Extension + +if os.environ.get('CIBUILDWHEEL', '0') != '1': + raise Exception('CIBUILDWHEEL environment variable is not set to 1') + +setup( + name="spam", + ext_modules=[Extension('spam', sources=['spam.c'])], + version="0.1.0", +) diff --git a/test/06_docker_images/spam.c b/test/06_docker_images/spam.c new file mode 100644 index 00000000..d1ab0f22 --- /dev/null +++ b/test/06_docker_images/spam.c @@ -0,0 +1,48 @@ +#include + +static PyObject * +spam_system(PyObject *self, PyObject *args) +{ + const char *command; + int sts; + + if (!PyArg_ParseTuple(args, "s", &command)) + return NULL; + sts = system(command); + return PyLong_FromLong(sts); +} + +/* Module initialization */ + +#if PY_MAJOR_VERSION >= 3 + #define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void) + #define MOD_DEF(m, name, doc, methods, module_state_size) \ + static struct PyModuleDef moduledef = { \ + PyModuleDef_HEAD_INIT, name, doc, module_state_size, methods, }; \ + m = PyModule_Create(&moduledef); + #define MOD_RETURN(m) return m; +#else + #define MOD_INIT(name) PyMODINIT_FUNC init##name(void) + #define MOD_DEF(m, name, doc, methods, module_state_size) \ + m = Py_InitModule3(name, methods, doc); + #define MOD_RETURN(m) return; +#endif + +static PyMethodDef module_methods[] = { + {"system", (PyCFunction)spam_system, METH_VARARGS, + "Execute a shell command."}, + {NULL} /* Sentinel */ +}; + +MOD_INIT(spam) +{ + PyObject* m; + + MOD_DEF(m, + "spam", + "Example module", + module_methods, + -1) + + MOD_RETURN(m) +} From d24774371320eaf27ecf2e24e245ac9d2bcab4e4 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Sun, 19 Nov 2017 23:18:10 +0100 Subject: [PATCH 2/6] Adding platform-specific arguments (for manylinux1 images) --- cibuildwheel/__main__.py | 16 +++++++++++++--- cibuildwheel/linux.py | 6 +++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 2a3d6610..b8f1ac07 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -76,8 +76,6 @@ def main(): before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform) skip_config = os.environ.get('CIBW_SKIP', '') environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform) or '' - manylinux1_x86_64_image = os.environ.get('CIBW_MANYLINUX1_X86_64_IMAGE', None) - manylinux1_i686_image = os.environ.get('CIBW_MANYLINUX1_I686_IMAGE', None) try: environment = parse_environment(environment_config) @@ -124,13 +122,25 @@ def main(): environment=environment, ) + if platform == 'linux': + manylinux1_x86_64_image = os.environ.get('CIBW_MANYLINUX1_X86_64_IMAGE', None) + manylinux1_i686_image = os.environ.get('CIBW_MANYLINUX1_I686_IMAGE', None) + + build_options.update( + manylinux1_images={'x86_64': manylinux1_x86_64_image, 'i686': manylinux1_i686_image}, + ) + elif platform == 'macos': + pass + elif platform == 'windows': + pass + print_preamble(platform, build_options) if not os.path.exists(output_dir): os.makedirs(output_dir) if platform == 'linux': - cibuildwheel.linux.build(manylinux1_x86_64_image=manylinux1_x86_64_image, manylinux1_i686_image=manylinux1_i686_image, **build_options) + cibuildwheel.linux.build(**build_options) elif platform == 'windows': cibuildwheel.windows.build(**build_options) elif platform == 'macos': diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 5525f0d8..cd7a7136 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -9,7 +9,7 @@ except ImportError: from pipes import quote as shlex_quote -def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, skip, environment, manylinux1_x86_64_image, manylinux1_i686_image): +def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, skip, environment, manylinux1_images): try: subprocess.check_call(['docker', '--version']) except: @@ -38,8 +38,8 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be python_configurations = [c for c in python_configurations if not skip(c.identifier)] platforms = [ - ('manylinux1_x86_64', manylinux1_x86_64_image or 'quay.io/pypa/manylinux1_x86_64'), - ('manylinux1_i686', manylinux1_i686_image or 'quay.io/pypa/manylinux1_i686'), + ('manylinux1_x86_64', manylinux1_images.get('x86_64') or 'quay.io/pypa/manylinux1_x86_64'), + ('manylinux1_i686', manylinux1_images.get('i686') or 'quay.io/pypa/manylinux1_i686'), ] for platform_tag, docker_image in platforms: From 65dccc46b21801d04e0e63cf7219bdc5ee7c675f Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Sun, 19 Nov 2017 23:18:55 +0100 Subject: [PATCH 3/6] Disabling skipping of all macos/windows tests for 06_docker_images --- test/06_docker_images/environment.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/test/06_docker_images/environment.json b/test/06_docker_images/environment.json index e5e73cc8..d91a26ef 100644 --- a/test/06_docker_images/environment.json +++ b/test/06_docker_images/environment.json @@ -1,5 +1,4 @@ { "CIBW_MANYLINUX1_X86_64_IMAGE": "dockcross/manylinux-x64", - "CIBW_MANYLINUX1_I686_IMAGE": "dockcross/manylinux-x86", - "CIBW_SKIP": "*macos* *win*" + "CIBW_MANYLINUX1_I686_IMAGE": "dockcross/manylinux-x86" } From e40f9b2345fe351bd7554e9170a2c62ded1573ff Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Sun, 19 Nov 2017 23:59:24 +0100 Subject: [PATCH 4/6] Added documentation on CIBW_MANYLINUX1_..._IMAGE options to the README --- README.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/README.md b/README.md index 3f48891a..49d0b267 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,8 @@ All being well, you should get wheels delivered to you in a few minutes. | | `CIBW_SKIP` | Skip certain Python versions | | **Build environment** | `CIBW_ENVIRONMENT` | Set environment variables needed during the build | | | `CIBW_BEFORE_BUILD` | Execute a shell command preparing each wheel's build | +| | `CIBW_MANYLINUX1_X86_64_IMAGE` | Specify an alternative manylinx1 x86_64 docker image | +| | `CIBW_MANYLINUX1_I686_IMAGE` | Specify an alternative manylinux1 i686 docker image | | **Tests** | `CIBW_TEST_COMMAND` | Execute a shell command to test all built wheels | | | `CIBW_TEST_REQUIRES` | Install Python dependencies before running the tests | @@ -89,6 +91,7 @@ A more detailed description of the options, the allowed values, and some example Linux wheels are built in the [`manylinux1` docker images](https://github.com/pypa/manylinux) to provide binary compatible wheels on Linux, according to [PEP 513](https://www.python.org/dev/peps/pep-0513/). Because of this, when building with `cibuildwheel` on Linux, a few things should be taken into account: - Progams and libraries cannot be installed on the Travis CI Ubuntu host with `apt-get`, but can be installed inside of the Docker image using `yum` or manually. The same goes for environment variables that are potentially needed to customize the wheel building. `cibuildwheel` supports this by providing the `CIBW_ENVIRONMENT` and `CIBW_BEFORE_BUILD` options to setup the build environment inside the running Docker image. See [below](#options) for details on these options. - The project directory is mounted in the running Docker instance as `/project`, the output directory for the wheels as `/output`. In general, this is handled transparently by `cibuildwheel`. For a more finegrained level of control however, the root of the host file system is mounted as `/host`, allowing for example to access shared files, caches, etc. on the host file system. +- Alternative dockers images can be specified with the `CIBW_MANYLINUX1_X86_64_IMAGE` and `CIBW_MANYLINUX1_I686_IMAGE` options to allow for a custom, preconfigured build environment for the Linux builds. See [below](#options) for more details. Options @@ -192,6 +195,15 @@ Example: `yum install -y libffi-dev && {pip} install .` Platform-specific variants also available: `CIBW_BEFORE_BUILD_MACOS` | `CIBW_BEFORE_BUILD_WINDOWS` | `CIBW_BEFORE_BUILD_LINUX` +| Environment variables: `CIBW_MANYLINUX1_X86_64_IMAGE` and `CIBW_MANYLINUX1_I686_IMAGE` +| --- + +Optional. + +An alternative docker image to be used for building [`manylinux1`](https://github.com/pypa/manylinux) wheels. `cibuildwheel` will then pull these instead of the official images, [`quay.io/pypa/manylinux1_x86_64`](https://quay.io/pypa/manylinux1_i686) and [`quay.io/pypa/manylinux1_i686`](https://quay.io/pypa/manylinux1_i686). + +Beware to specify a valid docker image that can be used the same as the official, default docker images: all necessary Python and pip versions need to be present in `/opt/python/`, and the `auditwheel` tool needs to be present for `cibuildwheel` to work. Apart from that, the architecture and relevant shared system libraries need to be manylinux1-compatible in order to produce valid `manylinux1` wheels (see https://github.com/pypa/manylinux and [PEP 513](https://www.python.org/dev/peps/pep-0513/) for more details). + | Environment variable: `CIBW_TEST_COMMAND` | --- From 08194a1db90cac9bf3c44a172c834298ae43aaf6 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 24 Dec 2017 12:53:10 +0000 Subject: [PATCH 5/6] Add check for running inside the dockcross image --- test/06_docker_images/setup.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/test/06_docker_images/setup.py b/test/06_docker_images/setup.py index 48d7c00b..75b922a6 100644 --- a/test/06_docker_images/setup.py +++ b/test/06_docker_images/setup.py @@ -1,9 +1,12 @@ -import os +import os, sys from setuptools import setup, Extension -if os.environ.get('CIBUILDWHEEL', '0') != '1': - raise Exception('CIBUILDWHEEL environment variable is not set to 1') +if sys.argv[-1] != '--name': + # check that we're running in the correct docker image as specified in the + # environment options CIBW_MANYLINUX1_*_IMAGE + if not os.path.exists('/dockcross'): + raise Exception('/dockcross directory not found. Is this test running in the correct docker image?') setup( name="spam", From 9013a7783f397f4254f534ff16b78daa2d6307b0 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Fri, 29 Dec 2017 13:33:11 +0100 Subject: [PATCH 6/6] Adding a check for Linux to test 06_docker_images's check for the '/dockcross' directory --- test/06_docker_images/setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/06_docker_images/setup.py b/test/06_docker_images/setup.py index 75b922a6..d341d5ff 100644 --- a/test/06_docker_images/setup.py +++ b/test/06_docker_images/setup.py @@ -5,7 +5,7 @@ from setuptools import setup, Extension if sys.argv[-1] != '--name': # check that we're running in the correct docker image as specified in the # environment options CIBW_MANYLINUX1_*_IMAGE - if not os.path.exists('/dockcross'): + if 'linux' in sys.platform and not os.path.exists('/dockcross'): raise Exception('/dockcross directory not found. Is this test running in the correct docker image?') setup(