From e194dad0da3962c0271de80c7cdc0a2dac751ea2 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Wed, 14 Aug 2019 18:29:15 +0200 Subject: [PATCH 01/12] Alternative proposal to generalize manylinux builds and support manylinux2010 --- cibuildwheel/__main__.py | 14 +++++++++++--- cibuildwheel/linux.py | 32 +++++++++++++++++--------------- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 516f0d69..68c3ff65 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -141,11 +141,19 @@ def main(): ) if platform == 'linux': - manylinux1_x86_64_image = os.environ.get('CIBW_MANYLINUX1_X86_64_IMAGE', None) - manylinux1_i686_image = os.environ.get('CIBW_MANYLINUX1_I686_IMAGE', None) + # TODO Check CIBW_MANYLINUX1_X86_64_IMAGE and CIBW_MANYLINUX1_I686_IMAGE environment variables and error if they exist + # TODO Check CIBW_BUILD and CIBW_SKIP for "manylinux1" and error if present + # TODO Add documentation on CIBW_ENVIRONMENT and possibility of AUDITWHEEL_PLAT if not defined by custom manylinux image + manylinux_x86_64_image = os.environ.get('CIBW_MANYLINUX_X86_64_IMAGE', '2010') + manylinux_i686_image = os.environ.get('CIBW_MANYLINUX_I686_IMAGE', '') + + default_manylinux_images_x86_64 = {'1': 'quay.io/pypa/manylinux1_x86_64', + '2010': 'quay.io/pypa/manylinux2010_x86_64'} + default_manylinux_images_i686 = {'1': 'quay.io/pypa/manylinux1_i686'} build_options.update( - manylinux1_images={'x86_64': manylinux1_x86_64_image, 'i686': manylinux1_i686_image}, + manylinux_images={'x86_64': default_manylinux_images_x86_64.get(manylinux_x86_64_image) or manylinux_x86_64_image, + 'i686': default_manylinux_images_i686.get(manylinux_i686_image) or manylinux_i686_image}, ) elif platform == 'macos': pass diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index ddcec627..c56a4472 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -12,25 +12,25 @@ except ImportError: def get_python_configurations(build_selector): PythonConfiguration = namedtuple('PythonConfiguration', ['identifier', 'path']) python_configurations = [ - PythonConfiguration(identifier='cp27-manylinux1_x86_64', path='/opt/python/cp27-cp27m'), - PythonConfiguration(identifier='cp27-manylinux1_x86_64', path='/opt/python/cp27-cp27mu'), - PythonConfiguration(identifier='cp34-manylinux1_x86_64', path='/opt/python/cp34-cp34m'), - PythonConfiguration(identifier='cp35-manylinux1_x86_64', path='/opt/python/cp35-cp35m'), - PythonConfiguration(identifier='cp36-manylinux1_x86_64', path='/opt/python/cp36-cp36m'), - PythonConfiguration(identifier='cp37-manylinux1_x86_64', path='/opt/python/cp37-cp37m'), - PythonConfiguration(identifier='cp27-manylinux1_i686', path='/opt/python/cp27-cp27m'), - PythonConfiguration(identifier='cp27-manylinux1_i686', path='/opt/python/cp27-cp27mu'), - PythonConfiguration(identifier='cp34-manylinux1_i686', path='/opt/python/cp34-cp34m'), - PythonConfiguration(identifier='cp35-manylinux1_i686', path='/opt/python/cp35-cp35m'), - PythonConfiguration(identifier='cp36-manylinux1_i686', path='/opt/python/cp36-cp36m'), - PythonConfiguration(identifier='cp37-manylinux1_i686', path='/opt/python/cp37-cp37m'), + PythonConfiguration(identifier='cp27-manylinux_x86_64', path='/opt/python/cp27-cp27m'), + PythonConfiguration(identifier='cp27-manylinux_x86_64', path='/opt/python/cp27-cp27mu'), + PythonConfiguration(identifier='cp34-manylinux_x86_64', path='/opt/python/cp34-cp34m'), + PythonConfiguration(identifier='cp35-manylinux_x86_64', path='/opt/python/cp35-cp35m'), + PythonConfiguration(identifier='cp36-manylinux_x86_64', path='/opt/python/cp36-cp36m'), + PythonConfiguration(identifier='cp37-manylinux_x86_64', path='/opt/python/cp37-cp37m'), + PythonConfiguration(identifier='cp27-manylinux_i686', path='/opt/python/cp27-cp27m'), + PythonConfiguration(identifier='cp27-manylinux_i686', path='/opt/python/cp27-cp27mu'), + PythonConfiguration(identifier='cp34-manylinux_i686', path='/opt/python/cp34-cp34m'), + PythonConfiguration(identifier='cp35-manylinux_i686', path='/opt/python/cp35-cp35m'), + PythonConfiguration(identifier='cp36-manylinux_i686', path='/opt/python/cp36-cp36m'), + PythonConfiguration(identifier='cp37-manylinux_i686', path='/opt/python/cp37-cp37m'), ] # skip builds as required return [c for c in python_configurations if build_selector(c.identifier)] -def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, environment, manylinux1_images): +def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, environment, manylinux_images): try: subprocess.check_call(['docker', '--version']) except: @@ -42,11 +42,13 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef python_configurations = get_python_configurations(build_selector) platforms = [ - ('manylinux1_x86_64', manylinux1_images.get('x86_64') or 'quay.io/pypa/manylinux1_x86_64'), - ('manylinux1_i686', manylinux1_images.get('i686') or 'quay.io/pypa/manylinux1_i686'), + ('manylinux_x86_64', manylinux_images['x86_64']), + ('manylinux_i686', manylinux_images['i686']), ] for platform_tag, docker_image in platforms: + if not docker_image: + continue platform_configs = [c for c in python_configurations if c.identifier.endswith(platform_tag)] if not platform_configs: continue From 7cadc6374a4d2bfe9f5bace433dd933735d0d223 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Wed, 14 Aug 2019 19:19:53 +0200 Subject: [PATCH 02/12] Fixing tests failing because of new manylinux2010 default --- test/shared/utils.py | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/test/shared/utils.py b/test/shared/utils.py index 0284dd4a..2fa3db1f 100644 --- a/test/shared/utils.py +++ b/test/shared/utils.py @@ -42,25 +42,35 @@ def cibuildwheel_run(project_path, env=None, add_env=None): ) -def expected_wheels(package_name, package_version): +def expected_wheels(package_name, package_version, manylinux_version='2010'): ''' Returns a list of expected wheels from a run of cibuildwheel. ''' if platform == 'linux': - templates = [ - '{package_name}-{package_version}-cp27-cp27m-manylinux1_x86_64.whl', - '{package_name}-{package_version}-cp27-cp27mu-manylinux1_x86_64.whl', - '{package_name}-{package_version}-cp34-cp34m-manylinux1_x86_64.whl', - '{package_name}-{package_version}-cp35-cp35m-manylinux1_x86_64.whl', - '{package_name}-{package_version}-cp36-cp36m-manylinux1_x86_64.whl', - '{package_name}-{package_version}-cp37-cp37m-manylinux1_x86_64.whl', - '{package_name}-{package_version}-cp27-cp27m-manylinux1_i686.whl', - '{package_name}-{package_version}-cp27-cp27mu-manylinux1_i686.whl', - '{package_name}-{package_version}-cp34-cp34m-manylinux1_i686.whl', - '{package_name}-{package_version}-cp35-cp35m-manylinux1_i686.whl', - '{package_name}-{package_version}-cp36-cp36m-manylinux1_i686.whl', - '{package_name}-{package_version}-cp37-cp37m-manylinux1_i686.whl', - ] + if manylinux_version == '2010': + templates = [ + '{package_name}-{package_version}-cp27-cp27m-manylinux2010_x86_64.whl', + '{package_name}-{package_version}-cp27-cp27mu-manylinux2010_x86_64.whl', + '{package_name}-{package_version}-cp34-cp34m-manylinux2010_x86_64.whl', + '{package_name}-{package_version}-cp35-cp35m-manylinux2010_x86_64.whl', + '{package_name}-{package_version}-cp36-cp36m-manylinux2010_x86_64.whl', + '{package_name}-{package_version}-cp37-cp37m-manylinux2010_x86_64.whl', + ] + elif manylinux_version == '1': + templates = [ + '{package_name}-{package_version}-cp27-cp27m-manylinux1_x86_64.whl', + '{package_name}-{package_version}-cp27-cp27mu-manylinux1_x86_64.whl', + '{package_name}-{package_version}-cp34-cp34m-manylinux1_x86_64.whl', + '{package_name}-{package_version}-cp35-cp35m-manylinux1_x86_64.whl', + '{package_name}-{package_version}-cp36-cp36m-manylinux1_x86_64.whl', + '{package_name}-{package_version}-cp37-cp37m-manylinux1_x86_64.whl', + '{package_name}-{package_version}-cp27-cp27m-manylinux1_i686.whl', + '{package_name}-{package_version}-cp27-cp27mu-manylinux1_i686.whl', + '{package_name}-{package_version}-cp34-cp34m-manylinux1_i686.whl', + '{package_name}-{package_version}-cp35-cp35m-manylinux1_i686.whl', + '{package_name}-{package_version}-cp36-cp36m-manylinux1_i686.whl', + '{package_name}-{package_version}-cp37-cp37m-manylinux1_i686.whl', + ] elif platform == 'windows': templates = [ '{package_name}-{package_version}-cp27-cp27m-win32.whl', From 7978247fa722def160641fb82aec745a1e769e69 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Sat, 17 Aug 2019 23:43:30 +0200 Subject: [PATCH 03/12] Fixing tests still failing as manylinux1 wheels are also produced by auditwheel --- test/shared/utils.py | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/test/shared/utils.py b/test/shared/utils.py index 2fa3db1f..63f4c557 100644 --- a/test/shared/utils.py +++ b/test/shared/utils.py @@ -42,28 +42,23 @@ def cibuildwheel_run(project_path, env=None, add_env=None): ) -def expected_wheels(package_name, package_version, manylinux_version='2010'): +def expected_wheels(package_name, package_version, manylinux_versions={'1_x86_64', '2010_x86_64'}): ''' Returns a list of expected wheels from a run of cibuildwheel. ''' if platform == 'linux': - if manylinux_version == '2010': - templates = [ - '{package_name}-{package_version}-cp27-cp27m-manylinux2010_x86_64.whl', - '{package_name}-{package_version}-cp27-cp27mu-manylinux2010_x86_64.whl', - '{package_name}-{package_version}-cp34-cp34m-manylinux2010_x86_64.whl', - '{package_name}-{package_version}-cp35-cp35m-manylinux2010_x86_64.whl', - '{package_name}-{package_version}-cp36-cp36m-manylinux2010_x86_64.whl', - '{package_name}-{package_version}-cp37-cp37m-manylinux2010_x86_64.whl', - ] - elif manylinux_version == '1': - templates = [ + templates = [] + if '1_x86_64' in manylinux_versions: + templates += [ '{package_name}-{package_version}-cp27-cp27m-manylinux1_x86_64.whl', '{package_name}-{package_version}-cp27-cp27mu-manylinux1_x86_64.whl', '{package_name}-{package_version}-cp34-cp34m-manylinux1_x86_64.whl', '{package_name}-{package_version}-cp35-cp35m-manylinux1_x86_64.whl', '{package_name}-{package_version}-cp36-cp36m-manylinux1_x86_64.whl', '{package_name}-{package_version}-cp37-cp37m-manylinux1_x86_64.whl', + ] + if '1_i686' in manylinux_versions: + templates += [ '{package_name}-{package_version}-cp27-cp27m-manylinux1_i686.whl', '{package_name}-{package_version}-cp27-cp27mu-manylinux1_i686.whl', '{package_name}-{package_version}-cp34-cp34m-manylinux1_i686.whl', @@ -71,6 +66,15 @@ def expected_wheels(package_name, package_version, manylinux_version='2010'): '{package_name}-{package_version}-cp36-cp36m-manylinux1_i686.whl', '{package_name}-{package_version}-cp37-cp37m-manylinux1_i686.whl', ] + if '2010_x86_64' in manylinux_versions: + templates += [ + '{package_name}-{package_version}-cp27-cp27m-manylinux2010_x86_64.whl', + '{package_name}-{package_version}-cp27-cp27mu-manylinux2010_x86_64.whl', + '{package_name}-{package_version}-cp34-cp34m-manylinux2010_x86_64.whl', + '{package_name}-{package_version}-cp35-cp35m-manylinux2010_x86_64.whl', + '{package_name}-{package_version}-cp36-cp36m-manylinux2010_x86_64.whl', + '{package_name}-{package_version}-cp37-cp37m-manylinux2010_x86_64.whl', + ] elif platform == 'windows': templates = [ '{package_name}-{package_version}-cp27-cp27m-win32.whl', From 54dd72de6647ebeb62c88c2ad8aeab4dc2893430 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Sat, 17 Aug 2019 23:49:35 +0200 Subject: [PATCH 04/12] Copying all created manylinux wheels to the output directory --- cibuildwheel/linux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index c56a4472..d125ca69 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -120,7 +120,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef fi # we're all done here; move it to output - mv "$delocated_wheel" /output + mv "${{delocated_wheel[@]}}" /output chown {uid}:{gid} "/output/$(basename "$delocated_wheel")" done '''.format( From 27517cd8a085371c210e6b88141aae289603fd91 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Sun, 18 Aug 2019 00:42:35 +0200 Subject: [PATCH 05/12] Updating test 06_docker_images to use new CIBW_MANYLINUX_*_IMAGE options --- test/06_docker_images/cibuildwheel_test.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/test/06_docker_images/cibuildwheel_test.py b/test/06_docker_images/cibuildwheel_test.py index a5448485..fda6ed49 100644 --- a/test/06_docker_images/cibuildwheel_test.py +++ b/test/06_docker_images/cibuildwheel_test.py @@ -8,11 +8,13 @@ def test(): pytest.skip('the docker test is only relevant to the linux build') utils.cibuildwheel_run(project_dir, add_env={ - 'CIBW_MANYLINUX1_X86_64_IMAGE': 'dockcross/manylinux-x64', - 'CIBW_MANYLINUX1_I686_IMAGE': 'dockcross/manylinux-x86', + 'CIBW_MANYLINUX_X86_64_IMAGE': 'dockcross/manylinux2010-x64', + 'CIBW_MANYLINUX_I686_IMAGE': 'dockcross/manylinux1-x86', + 'CIBW_BEFORE_BUILD': '/opt/python/cp36-cp36m/bin/pip install -U auditwheel', # Currently necessary on dockcross images to get auditwheel 2.1 supporting AUDITWHEEL_PLAT + 'CIBW_ENVIRONMENT': 'AUDITWHEEL_PLAT=`if [ $(uname -i) == "x86_64" ]; then echo "manylinux2010_x86_64"; else echo "manylinux1_i686"; fi`', }) # also check that we got the right wheels built - expected_wheels = utils.expected_wheels('spam', '0.1.0') + expected_wheels = utils.expected_wheels('spam', '0.1.0', manylinux_versions={'1_x86_64', '2010_x86_64', '1_i686'}) actual_wheels = os.listdir('wheelhouse') assert set(actual_wheels) == set(expected_wheels) From 4571262803aea6744e69d30c020087003fd3d690 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Tue, 20 Aug 2019 17:43:45 +0200 Subject: [PATCH 06/12] Adapting Linux bash script for multiple output wheels from auditwheel --- cibuildwheel/linux.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index d125ca69..573a150c 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -64,9 +64,9 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef for PYBIN in {pybin_paths}; do # Setup rm -rf /tmp/built_wheel - rm -rf /tmp/delocated_wheel + rm -rf /tmp/delocated_wheels mkdir /tmp/built_wheel - mkdir /tmp/delocated_wheel + mkdir /tmp/delocated_wheels if [ ! -z {before_build} ]; then PATH="$PYBIN:$PATH" sh -c {before_build} @@ -81,11 +81,11 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef # the first element if [[ "$built_wheel" == *none-any.whl ]]; then # pure python wheel - just copy - mv "$built_wheel" /tmp/delocated_wheel + mv "$built_wheel" /tmp/delocated_wheels else - auditwheel repair "$built_wheel" -w /tmp/delocated_wheel + auditwheel repair "$built_wheel" -w /tmp/delocated_wheels fi - delocated_wheel=(/tmp/delocated_wheel/*.whl) + delocated_wheels=(/tmp/delocated_wheels/*.whl) if [ ! -z {test_command} ]; then # Set up a virtual environment to install and test from, to make sure @@ -102,7 +102,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef echo "Running tests using `which python`" # Install the wheel we just built - pip install "$delocated_wheel"{test_extras} + pip install "${{delocated_wheels[0]}}"{test_extras} # Install any requirements to run the tests if [ ! -z "{test_requires}" ]; then @@ -120,8 +120,8 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef fi # we're all done here; move it to output - mv "${{delocated_wheel[@]}}" /output - chown {uid}:{gid} "/output/$(basename "$delocated_wheel")" + mv "${{delocated_wheels[@]}}" /output + for delocated_wheel in "${{delocated_wheels[@]}}"; do chown {uid}:{gid} "/output/$(basename "$delocated_wheel")"; done done '''.format( pybin_paths=' '.join(c.path+'/bin' for c in platform_configs), From 4536b23a1d737d4e0c1ad4ab69b79a21c539381b Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Thu, 22 Aug 2019 15:06:03 +0200 Subject: [PATCH 07/12] Adding test accessing manylinux2010-only ABI --- .../cibuildwheel_test.py | 20 +++++++ test/08_manylinux2010_only/setup.py | 7 +++ test/08_manylinux2010_only/spam.c | 57 +++++++++++++++++++ 3 files changed, 84 insertions(+) create mode 100644 test/08_manylinux2010_only/cibuildwheel_test.py create mode 100644 test/08_manylinux2010_only/setup.py create mode 100644 test/08_manylinux2010_only/spam.c diff --git a/test/08_manylinux2010_only/cibuildwheel_test.py b/test/08_manylinux2010_only/cibuildwheel_test.py new file mode 100644 index 00000000..392ec459 --- /dev/null +++ b/test/08_manylinux2010_only/cibuildwheel_test.py @@ -0,0 +1,20 @@ +import os, pytest +import utils + +def test(): + project_dir = os.path.dirname(__file__) + + if utils.platform != 'linux': + pytest.skip('the docker test is only relevant to the linux build') + + # build the wheels + # CFLAGS environment veriable is ecessary to fail on 'malloc_info' (on manylinux1) during compilation/linking, + # rather than when dynamically loading the Python + utils.cibuildwheel_run(project_dir, add_env={ + 'CIBW_ENVIRONMENT': 'CFLAGS="$CFLAGS -Werror=implicit-function-declaration"', + }) + + # also check that we got the right wheels + expected_wheels = utils.expected_wheels('spam', '0.1.0', manylinux_versions={'2010_x86_64'}) + actual_wheels = os.listdir('wheelhouse') + assert set(actual_wheels) == set(expected_wheels) diff --git a/test/08_manylinux2010_only/setup.py b/test/08_manylinux2010_only/setup.py new file mode 100644 index 00000000..866fa22c --- /dev/null +++ b/test/08_manylinux2010_only/setup.py @@ -0,0 +1,7 @@ +from setuptools import setup, Extension + +setup( + name="spam", + ext_modules=[Extension('spam', sources=['spam.c'])], + version="0.1.0", +) diff --git a/test/08_manylinux2010_only/spam.c b/test/08_manylinux2010_only/spam.c new file mode 100644 index 00000000..0e50eba9 --- /dev/null +++ b/test/08_manylinux2010_only/spam.c @@ -0,0 +1,57 @@ +#include +#if defined(__linux__) +#include +#endif + +static PyObject * +spam_system(PyObject *self, PyObject *args) +{ + const char *command; + int sts = 0; + + if (!PyArg_ParseTuple(args, "s", &command)) + return NULL; + +#if defined(__linux__) + sts = malloc_info(0, stdout); +#endif + if (sts == 0) { + sts = system(command); + } + return PyLong_FromLong(sts); +} + +/* Module initialization */ + +#if PY_MAJOR_VERSION >= 3 + #define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void) + #define MOD_DEF(m, name, doc, methods, module_state_size) \ + static struct PyModuleDef moduledef = { \ + PyModuleDef_HEAD_INIT, name, doc, module_state_size, methods, }; \ + m = PyModule_Create(&moduledef); + #define MOD_RETURN(m) return m; +#else + #define MOD_INIT(name) PyMODINIT_FUNC init##name(void) + #define MOD_DEF(m, name, doc, methods, module_state_size) \ + m = Py_InitModule3(name, methods, doc); + #define MOD_RETURN(m) return; +#endif + +static PyMethodDef module_methods[] = { + {"system", (PyCFunction)spam_system, METH_VARARGS, + "Execute a shell command."}, + {NULL} /* Sentinel */ +}; + +MOD_INIT(spam) +{ + PyObject* m; + + MOD_DEF(m, + "spam", + "Example module", + module_methods, + -1) + + MOD_RETURN(m) +} From 179148b0e528f56dce6b499a4432e26084f8c903 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Tue, 27 Aug 2019 16:03:09 +0200 Subject: [PATCH 08/12] Updated preset values for CIBW_MANYLINUX_*_IMAGE, fixed skip message in 08_manylinux2010_only test, and added clarifying comment on multiple wheels in Linux bash script --- cibuildwheel/__main__.py | 9 ++++++--- cibuildwheel/linux.py | 5 +++++ test/06_docker_images/cibuildwheel_test.py | 4 ++-- test/08_manylinux2010_only/cibuildwheel_test.py | 2 +- test/shared/utils.py | 8 ++++---- 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 68c3ff65..0ad6b425 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -147,9 +147,12 @@ def main(): manylinux_x86_64_image = os.environ.get('CIBW_MANYLINUX_X86_64_IMAGE', '2010') manylinux_i686_image = os.environ.get('CIBW_MANYLINUX_I686_IMAGE', '') - default_manylinux_images_x86_64 = {'1': 'quay.io/pypa/manylinux1_x86_64', - '2010': 'quay.io/pypa/manylinux2010_x86_64'} - default_manylinux_images_i686 = {'1': 'quay.io/pypa/manylinux1_i686'} + manylinux_x86_64_image = os.environ.get('CIBW_MANYLINUX_X86_64_IMAGE', 'manylinux2010') + manylinux_i686_image = os.environ.get('CIBW_MANYLINUX_I686_IMAGE', '') + + default_manylinux_images_x86_64 = {'manylinux1': 'quay.io/pypa/manylinux1_x86_64', + 'manylinux2010': 'quay.io/pypa/manylinux2010_x86_64'} + default_manylinux_images_i686 = {'manylinux1': 'quay.io/pypa/manylinux1_i686'} build_options.update( manylinux_images={'x86_64': default_manylinux_images_x86_64.get(manylinux_x86_64_image) or manylinux_x86_64_image, diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 573a150c..84b16e48 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -102,6 +102,11 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef echo "Running tests using `which python`" # Install the wheel we just built + # Note: If auditwheel produced two wheels, it's because the earlier produced wheel + # conforms to multiple manylinux standards. These multiple versions of the wheel are + # functionally the same, differing only in name, wheel metadata, and possibly include + # different external shared libraries. so it doesn't matter which one we run the tests on. + # Let's just pick the first one. pip install "${{delocated_wheels[0]}}"{test_extras} # Install any requirements to run the tests diff --git a/test/06_docker_images/cibuildwheel_test.py b/test/06_docker_images/cibuildwheel_test.py index fda6ed49..c586151b 100644 --- a/test/06_docker_images/cibuildwheel_test.py +++ b/test/06_docker_images/cibuildwheel_test.py @@ -5,7 +5,7 @@ def test(): project_dir = os.path.dirname(__file__) if utils.platform != 'linux': - pytest.skip('the docker test is only relevant to the linux build') + pytest.skip('the test is only relevant to the linux build') utils.cibuildwheel_run(project_dir, add_env={ 'CIBW_MANYLINUX_X86_64_IMAGE': 'dockcross/manylinux2010-x64', @@ -15,6 +15,6 @@ def test(): }) # also check that we got the right wheels built - expected_wheels = utils.expected_wheels('spam', '0.1.0', manylinux_versions={'1_x86_64', '2010_x86_64', '1_i686'}) + expected_wheels = utils.expected_wheels('spam', '0.1.0', manylinux_versions={'manylinux1_x86_64', 'manylinux2010_x86_64', 'manylinux1_i686'}) actual_wheels = os.listdir('wheelhouse') assert set(actual_wheels) == set(expected_wheels) diff --git a/test/08_manylinux2010_only/cibuildwheel_test.py b/test/08_manylinux2010_only/cibuildwheel_test.py index 392ec459..814a364b 100644 --- a/test/08_manylinux2010_only/cibuildwheel_test.py +++ b/test/08_manylinux2010_only/cibuildwheel_test.py @@ -15,6 +15,6 @@ def test(): }) # also check that we got the right wheels - expected_wheels = utils.expected_wheels('spam', '0.1.0', manylinux_versions={'2010_x86_64'}) + expected_wheels = utils.expected_wheels('spam', '0.1.0', manylinux_versions={'manylinux2010_x86_64'}) actual_wheels = os.listdir('wheelhouse') assert set(actual_wheels) == set(expected_wheels) diff --git a/test/shared/utils.py b/test/shared/utils.py index 63f4c557..d4ea60a5 100644 --- a/test/shared/utils.py +++ b/test/shared/utils.py @@ -42,13 +42,13 @@ def cibuildwheel_run(project_path, env=None, add_env=None): ) -def expected_wheels(package_name, package_version, manylinux_versions={'1_x86_64', '2010_x86_64'}): +def expected_wheels(package_name, package_version, manylinux_versions={'manylinux1_x86_64', 'manylinux2010_x86_64'}): ''' Returns a list of expected wheels from a run of cibuildwheel. ''' if platform == 'linux': templates = [] - if '1_x86_64' in manylinux_versions: + if 'manylinux1_x86_64' in manylinux_versions: templates += [ '{package_name}-{package_version}-cp27-cp27m-manylinux1_x86_64.whl', '{package_name}-{package_version}-cp27-cp27mu-manylinux1_x86_64.whl', @@ -57,7 +57,7 @@ def expected_wheels(package_name, package_version, manylinux_versions={'1_x86_64 '{package_name}-{package_version}-cp36-cp36m-manylinux1_x86_64.whl', '{package_name}-{package_version}-cp37-cp37m-manylinux1_x86_64.whl', ] - if '1_i686' in manylinux_versions: + if 'manylinux1_i686' in manylinux_versions: templates += [ '{package_name}-{package_version}-cp27-cp27m-manylinux1_i686.whl', '{package_name}-{package_version}-cp27-cp27mu-manylinux1_i686.whl', @@ -66,7 +66,7 @@ def expected_wheels(package_name, package_version, manylinux_versions={'1_x86_64 '{package_name}-{package_version}-cp36-cp36m-manylinux1_i686.whl', '{package_name}-{package_version}-cp37-cp37m-manylinux1_i686.whl', ] - if '2010_x86_64' in manylinux_versions: + if 'manylinux2010_x86_64' in manylinux_versions: templates += [ '{package_name}-{package_version}-cp27-cp27m-manylinux2010_x86_64.whl', '{package_name}-{package_version}-cp27-cp27mu-manylinux2010_x86_64.whl', From 1220fbbd4bb97faacdb123068ca69f33c3c317a5 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Fri, 30 Aug 2019 18:31:02 +0200 Subject: [PATCH 09/12] Adding detection of old CIBW_MANYLINUX1_*_IMAGE options and manylinux1-* build identifiers in CIBW_BUILD and CIBW_SKIP --- cibuildwheel/__main__.py | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 0ad6b425..8112629b 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -54,6 +54,8 @@ def main(): args = parser.parse_args() + detect_obsolete_options() + if args.platform != 'auto': platform = args.platform else: @@ -141,11 +143,7 @@ def main(): ) if platform == 'linux': - # TODO Check CIBW_MANYLINUX1_X86_64_IMAGE and CIBW_MANYLINUX1_I686_IMAGE environment variables and error if they exist - # TODO Check CIBW_BUILD and CIBW_SKIP for "manylinux1" and error if present # TODO Add documentation on CIBW_ENVIRONMENT and possibility of AUDITWHEEL_PLAT if not defined by custom manylinux image - manylinux_x86_64_image = os.environ.get('CIBW_MANYLINUX_X86_64_IMAGE', '2010') - manylinux_i686_image = os.environ.get('CIBW_MANYLINUX_I686_IMAGE', '') manylinux_x86_64_image = os.environ.get('CIBW_MANYLINUX_X86_64_IMAGE', 'manylinux2010') manylinux_i686_image = os.environ.get('CIBW_MANYLINUX_I686_IMAGE', '') @@ -181,6 +179,26 @@ def main(): raise Exception('Unsupported platform') +def detect_obsolete_options(): + # Check the old 'MANYLINUX1_*_IMAGE' options + for (deprecated, alternative) in [('CIBW_MANYLINUX1_X86_64_IMAGE', 'CIBW_MANYLINUX_X86_64_IMAGE'), + ('CIBW_MANYLINUX1_I686_IMAGE', 'CIBW_MANYLINUX_I686_IMAGE')]: + if deprecated in os.environ: + print("'{}' has been deprecated, and will be removed in a future release. Use the option '{}' instead.".format(deprecated, alternative)) + if alternative not in os.environ: + print("Using value of option '{}' as replacement for '{}'".format(deprecated, alternative)) + os.environ[alternative] = os.environ[deprecated] + else: + print("Option '{}' is not empty. Please unset '{}'".format(alternative, deprecated)) + exit(2) + + # Check for 'manylinux1' in the 'CIBW_BUILD' and 'CIBW_SKIP' options + for deprecated in ['CIBW_BUILD', 'CIBW_SKIP']: + if deprecated in os.environ and 'manylinux1' in os.environ[deprecated]: + print("Build identifiers with 'manylinux1' been deprecated. Replacing all occurences of 'manylinux1' by 'manylinux' in the option '{}'".format(deprecated)) + os.environ[deprecated] = os.environ[deprecated].replace('manylinux1', 'manylinux') + + def print_preamble(platform, build_options): print(textwrap.dedent(''' _ _ _ _ _ _ _ From c6f13828132fd250715dd6c816cda392ba66b19e Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Mon, 16 Sep 2019 12:48:15 +0200 Subject: [PATCH 10/12] Making manylinux1 still the default for manylinux_i686 builds --- cibuildwheel/__main__.py | 2 +- cibuildwheel/linux.py | 2 -- test/01_basic/cibuildwheel_test.py | 5 +++- test/06_docker_images/cibuildwheel_test.py | 2 +- .../cibuildwheel_test.py | 4 +++- test/shared/utils.py | 24 +++++++++---------- 6 files changed, 21 insertions(+), 18 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 8112629b..fb79c9fb 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -146,7 +146,7 @@ def main(): # TODO Add documentation on CIBW_ENVIRONMENT and possibility of AUDITWHEEL_PLAT if not defined by custom manylinux image manylinux_x86_64_image = os.environ.get('CIBW_MANYLINUX_X86_64_IMAGE', 'manylinux2010') - manylinux_i686_image = os.environ.get('CIBW_MANYLINUX_I686_IMAGE', '') + manylinux_i686_image = os.environ.get('CIBW_MANYLINUX_I686_IMAGE', 'manylinux1') default_manylinux_images_x86_64 = {'manylinux1': 'quay.io/pypa/manylinux1_x86_64', 'manylinux2010': 'quay.io/pypa/manylinux2010_x86_64'} diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 84b16e48..233f1f02 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -47,8 +47,6 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef ] for platform_tag, docker_image in platforms: - if not docker_image: - continue platform_configs = [c for c in python_configurations if c.identifier.endswith(platform_tag)] if not platform_configs: continue diff --git a/test/01_basic/cibuildwheel_test.py b/test/01_basic/cibuildwheel_test.py index 93c67829..32da2c6f 100644 --- a/test/01_basic/cibuildwheel_test.py +++ b/test/01_basic/cibuildwheel_test.py @@ -17,6 +17,9 @@ def test(): def test_build_identifiers(): # check that the number of expected wheels matches the number of build # identifiers - expected_wheels = utils.expected_wheels('spam', '0.1.0') + # after adding CIBW_MANYLINUX_IMAGE to support manylinux2010, there + # can be multiple wheels for each wheel, though, so we need to limit + # the expected wheels + expected_wheels = utils.expected_wheels('spam', '0.1.0', manylinux_x86_64_versions={'manylinux2010'}, manylinux_i686_versions={'manylinux1'}) build_identifiers = utils.cibuildwheel_get_build_identifiers(project_dir) assert len(expected_wheels) == len(build_identifiers) diff --git a/test/06_docker_images/cibuildwheel_test.py b/test/06_docker_images/cibuildwheel_test.py index c586151b..f1ef516c 100644 --- a/test/06_docker_images/cibuildwheel_test.py +++ b/test/06_docker_images/cibuildwheel_test.py @@ -15,6 +15,6 @@ def test(): }) # also check that we got the right wheels built - expected_wheels = utils.expected_wheels('spam', '0.1.0', manylinux_versions={'manylinux1_x86_64', 'manylinux2010_x86_64', 'manylinux1_i686'}) + expected_wheels = utils.expected_wheels('spam', '0.1.0') actual_wheels = os.listdir('wheelhouse') assert set(actual_wheels) == set(expected_wheels) diff --git a/test/08_manylinux2010_only/cibuildwheel_test.py b/test/08_manylinux2010_only/cibuildwheel_test.py index 814a364b..203e1fbe 100644 --- a/test/08_manylinux2010_only/cibuildwheel_test.py +++ b/test/08_manylinux2010_only/cibuildwheel_test.py @@ -12,9 +12,11 @@ def test(): # rather than when dynamically loading the Python utils.cibuildwheel_run(project_dir, add_env={ 'CIBW_ENVIRONMENT': 'CFLAGS="$CFLAGS -Werror=implicit-function-declaration"', + 'CIBW_SKIP': '*-manylinux_i686', }) # also check that we got the right wheels - expected_wheels = utils.expected_wheels('spam', '0.1.0', manylinux_versions={'manylinux2010_x86_64'}) + expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0', manylinux_x86_64_versions={'manylinux2010'}) + if '-manylinux1_i686' not in w] actual_wheels = os.listdir('wheelhouse') assert set(actual_wheels) == set(expected_wheels) diff --git a/test/shared/utils.py b/test/shared/utils.py index d4ea60a5..3edba2ee 100644 --- a/test/shared/utils.py +++ b/test/shared/utils.py @@ -42,13 +42,13 @@ def cibuildwheel_run(project_path, env=None, add_env=None): ) -def expected_wheels(package_name, package_version, manylinux_versions={'manylinux1_x86_64', 'manylinux2010_x86_64'}): +def expected_wheels(package_name, package_version, manylinux_x86_64_versions={'manylinux1', 'manylinux2010'}, manylinux_i686_versions={'manylinux1'}): ''' Returns a list of expected wheels from a run of cibuildwheel. ''' if platform == 'linux': templates = [] - if 'manylinux1_x86_64' in manylinux_versions: + if 'manylinux1' in manylinux_x86_64_versions: templates += [ '{package_name}-{package_version}-cp27-cp27m-manylinux1_x86_64.whl', '{package_name}-{package_version}-cp27-cp27mu-manylinux1_x86_64.whl', @@ -57,16 +57,7 @@ def expected_wheels(package_name, package_version, manylinux_versions={'manylinu '{package_name}-{package_version}-cp36-cp36m-manylinux1_x86_64.whl', '{package_name}-{package_version}-cp37-cp37m-manylinux1_x86_64.whl', ] - if 'manylinux1_i686' in manylinux_versions: - templates += [ - '{package_name}-{package_version}-cp27-cp27m-manylinux1_i686.whl', - '{package_name}-{package_version}-cp27-cp27mu-manylinux1_i686.whl', - '{package_name}-{package_version}-cp34-cp34m-manylinux1_i686.whl', - '{package_name}-{package_version}-cp35-cp35m-manylinux1_i686.whl', - '{package_name}-{package_version}-cp36-cp36m-manylinux1_i686.whl', - '{package_name}-{package_version}-cp37-cp37m-manylinux1_i686.whl', - ] - if 'manylinux2010_x86_64' in manylinux_versions: + if 'manylinux2010' in manylinux_x86_64_versions: templates += [ '{package_name}-{package_version}-cp27-cp27m-manylinux2010_x86_64.whl', '{package_name}-{package_version}-cp27-cp27mu-manylinux2010_x86_64.whl', @@ -75,6 +66,15 @@ def expected_wheels(package_name, package_version, manylinux_versions={'manylinu '{package_name}-{package_version}-cp36-cp36m-manylinux2010_x86_64.whl', '{package_name}-{package_version}-cp37-cp37m-manylinux2010_x86_64.whl', ] + if 'manylinux1' in manylinux_i686_versions: + templates += [ + '{package_name}-{package_version}-cp27-cp27m-manylinux1_i686.whl', + '{package_name}-{package_version}-cp27-cp27mu-manylinux1_i686.whl', + '{package_name}-{package_version}-cp34-cp34m-manylinux1_i686.whl', + '{package_name}-{package_version}-cp35-cp35m-manylinux1_i686.whl', + '{package_name}-{package_version}-cp36-cp36m-manylinux1_i686.whl', + '{package_name}-{package_version}-cp37-cp37m-manylinux1_i686.whl', + ] elif platform == 'windows': templates = [ '{package_name}-{package_version}-cp27-cp27m-win32.whl', From 2e0dc40e3f15dcf18f81df9673c6492826cdc415 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Wed, 16 Oct 2019 00:30:17 +0200 Subject: [PATCH 11/12] Adding manylinux2010_i686 image and simplifying tests' utils.expected_wheels() --- cibuildwheel/__main__.py | 5 +- test/01_basic/cibuildwheel_test.py | 3 +- test/06_docker_images/cibuildwheel_test.py | 3 +- .../cibuildwheel_test.py | 5 +- test/shared/utils.py | 56 +++++++++---------- 5 files changed, 36 insertions(+), 36 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index fb79c9fb..fd68da10 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -146,11 +146,12 @@ def main(): # TODO Add documentation on CIBW_ENVIRONMENT and possibility of AUDITWHEEL_PLAT if not defined by custom manylinux image manylinux_x86_64_image = os.environ.get('CIBW_MANYLINUX_X86_64_IMAGE', 'manylinux2010') - manylinux_i686_image = os.environ.get('CIBW_MANYLINUX_I686_IMAGE', 'manylinux1') + manylinux_i686_image = os.environ.get('CIBW_MANYLINUX_I686_IMAGE', 'manylinux2010') default_manylinux_images_x86_64 = {'manylinux1': 'quay.io/pypa/manylinux1_x86_64', 'manylinux2010': 'quay.io/pypa/manylinux2010_x86_64'} - default_manylinux_images_i686 = {'manylinux1': 'quay.io/pypa/manylinux1_i686'} + default_manylinux_images_i686 = {'manylinux1': 'quay.io/pypa/manylinux1_i686', + 'manylinux2010': 'quay.io/pypa/manylinux2010_i686'} build_options.update( manylinux_images={'x86_64': default_manylinux_images_x86_64.get(manylinux_x86_64_image) or manylinux_x86_64_image, diff --git a/test/01_basic/cibuildwheel_test.py b/test/01_basic/cibuildwheel_test.py index 32da2c6f..998b55de 100644 --- a/test/01_basic/cibuildwheel_test.py +++ b/test/01_basic/cibuildwheel_test.py @@ -20,6 +20,7 @@ def test_build_identifiers(): # after adding CIBW_MANYLINUX_IMAGE to support manylinux2010, there # can be multiple wheels for each wheel, though, so we need to limit # the expected wheels - expected_wheels = utils.expected_wheels('spam', '0.1.0', manylinux_x86_64_versions={'manylinux2010'}, manylinux_i686_versions={'manylinux1'}) + expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0') + if not '-manylinux' in w or '-manylinux1' in w] build_identifiers = utils.cibuildwheel_get_build_identifiers(project_dir) assert len(expected_wheels) == len(build_identifiers) diff --git a/test/06_docker_images/cibuildwheel_test.py b/test/06_docker_images/cibuildwheel_test.py index f1ef516c..9067bc0f 100644 --- a/test/06_docker_images/cibuildwheel_test.py +++ b/test/06_docker_images/cibuildwheel_test.py @@ -15,6 +15,7 @@ def test(): }) # also check that we got the right wheels built - expected_wheels = utils.expected_wheels('spam', '0.1.0') + expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0') + if '-manylinux2010_i686' not in w] actual_wheels = os.listdir('wheelhouse') assert set(actual_wheels) == set(expected_wheels) diff --git a/test/08_manylinux2010_only/cibuildwheel_test.py b/test/08_manylinux2010_only/cibuildwheel_test.py index 203e1fbe..dab763e7 100644 --- a/test/08_manylinux2010_only/cibuildwheel_test.py +++ b/test/08_manylinux2010_only/cibuildwheel_test.py @@ -12,11 +12,10 @@ def test(): # rather than when dynamically loading the Python utils.cibuildwheel_run(project_dir, add_env={ 'CIBW_ENVIRONMENT': 'CFLAGS="$CFLAGS -Werror=implicit-function-declaration"', - 'CIBW_SKIP': '*-manylinux_i686', }) # also check that we got the right wheels - expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0', manylinux_x86_64_versions={'manylinux2010'}) - if '-manylinux1_i686' not in w] + expected_wheels = [w for w in utils.expected_wheels('spam', '0.1.0') + if not '-manylinux' in w or '-manylinux2010' in w] actual_wheels = os.listdir('wheelhouse') assert set(actual_wheels) == set(expected_wheels) diff --git a/test/shared/utils.py b/test/shared/utils.py index 3edba2ee..86cb198a 100644 --- a/test/shared/utils.py +++ b/test/shared/utils.py @@ -42,39 +42,37 @@ def cibuildwheel_run(project_path, env=None, add_env=None): ) -def expected_wheels(package_name, package_version, manylinux_x86_64_versions={'manylinux1', 'manylinux2010'}, manylinux_i686_versions={'manylinux1'}): +def expected_wheels(package_name, package_version): ''' Returns a list of expected wheels from a run of cibuildwheel. ''' if platform == 'linux': - templates = [] - if 'manylinux1' in manylinux_x86_64_versions: - templates += [ - '{package_name}-{package_version}-cp27-cp27m-manylinux1_x86_64.whl', - '{package_name}-{package_version}-cp27-cp27mu-manylinux1_x86_64.whl', - '{package_name}-{package_version}-cp34-cp34m-manylinux1_x86_64.whl', - '{package_name}-{package_version}-cp35-cp35m-manylinux1_x86_64.whl', - '{package_name}-{package_version}-cp36-cp36m-manylinux1_x86_64.whl', - '{package_name}-{package_version}-cp37-cp37m-manylinux1_x86_64.whl', - ] - if 'manylinux2010' in manylinux_x86_64_versions: - templates += [ - '{package_name}-{package_version}-cp27-cp27m-manylinux2010_x86_64.whl', - '{package_name}-{package_version}-cp27-cp27mu-manylinux2010_x86_64.whl', - '{package_name}-{package_version}-cp34-cp34m-manylinux2010_x86_64.whl', - '{package_name}-{package_version}-cp35-cp35m-manylinux2010_x86_64.whl', - '{package_name}-{package_version}-cp36-cp36m-manylinux2010_x86_64.whl', - '{package_name}-{package_version}-cp37-cp37m-manylinux2010_x86_64.whl', - ] - if 'manylinux1' in manylinux_i686_versions: - templates += [ - '{package_name}-{package_version}-cp27-cp27m-manylinux1_i686.whl', - '{package_name}-{package_version}-cp27-cp27mu-manylinux1_i686.whl', - '{package_name}-{package_version}-cp34-cp34m-manylinux1_i686.whl', - '{package_name}-{package_version}-cp35-cp35m-manylinux1_i686.whl', - '{package_name}-{package_version}-cp36-cp36m-manylinux1_i686.whl', - '{package_name}-{package_version}-cp37-cp37m-manylinux1_i686.whl', - ] + templates = [ + '{package_name}-{package_version}-cp27-cp27m-manylinux1_x86_64.whl', + '{package_name}-{package_version}-cp27-cp27mu-manylinux1_x86_64.whl', + '{package_name}-{package_version}-cp34-cp34m-manylinux1_x86_64.whl', + '{package_name}-{package_version}-cp35-cp35m-manylinux1_x86_64.whl', + '{package_name}-{package_version}-cp36-cp36m-manylinux1_x86_64.whl', + '{package_name}-{package_version}-cp37-cp37m-manylinux1_x86_64.whl', + '{package_name}-{package_version}-cp27-cp27m-manylinux2010_x86_64.whl', + '{package_name}-{package_version}-cp27-cp27mu-manylinux2010_x86_64.whl', + '{package_name}-{package_version}-cp34-cp34m-manylinux2010_x86_64.whl', + '{package_name}-{package_version}-cp35-cp35m-manylinux2010_x86_64.whl', + '{package_name}-{package_version}-cp36-cp36m-manylinux2010_x86_64.whl', + '{package_name}-{package_version}-cp37-cp37m-manylinux2010_x86_64.whl', + '{package_name}-{package_version}-cp27-cp27m-manylinux1_i686.whl', + '{package_name}-{package_version}-cp27-cp27mu-manylinux1_i686.whl', + '{package_name}-{package_version}-cp34-cp34m-manylinux1_i686.whl', + '{package_name}-{package_version}-cp35-cp35m-manylinux1_i686.whl', + '{package_name}-{package_version}-cp36-cp36m-manylinux1_i686.whl', + '{package_name}-{package_version}-cp37-cp37m-manylinux1_i686.whl', + '{package_name}-{package_version}-cp27-cp27m-manylinux2010_i686.whl', + '{package_name}-{package_version}-cp27-cp27mu-manylinux2010_i686.whl', + '{package_name}-{package_version}-cp34-cp34m-manylinux2010_i686.whl', + '{package_name}-{package_version}-cp35-cp35m-manylinux2010_i686.whl', + '{package_name}-{package_version}-cp36-cp36m-manylinux2010_i686.whl', + '{package_name}-{package_version}-cp37-cp37m-manylinux2010_i686.whl', + ] elif platform == 'windows': templates = [ '{package_name}-{package_version}-cp27-cp27m-win32.whl', From 31d89377dad77ef07de1a9b8fb8accddc47c35e5 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Wed, 16 Oct 2019 01:26:15 +0200 Subject: [PATCH 12/12] Updated README with new CIBW_MANYLINUX_*_IMAGE options --- README.md | 31 +++++++++++++++++++------------ cibuildwheel/__main__.py | 2 -- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index cec3816e..c14b3ad3 100644 --- a/README.md +++ b/README.md @@ -221,8 +221,8 @@ All being well, you should get wheels delivered to you in a few minutes. | **Build parameters** | `CIBW_BUILD_VERBOSITY` | Increase or decrease the output of `pip wheel` | | **Build environment** | `CIBW_ENVIRONMENT` | Set environment variables needed during the build | | | `CIBW_BEFORE_BUILD` | Execute a shell command preparing each wheel's build | -| | `CIBW_MANYLINUX1_X86_64_IMAGE` | Specify an alternative manylinx1 x86_64 docker image | -| | `CIBW_MANYLINUX1_I686_IMAGE` | Specify an alternative manylinux1 i686 docker image | +| | `CIBW_MANYLINUX_X86_64_IMAGE` | Specify an alternative manylinux x86_64 docker image | +| | `CIBW_MANYLINUX_I686_IMAGE` | Specify an alternative manylinux i686 docker image | | **Tests** | `CIBW_TEST_COMMAND` | Execute a shell command to test all built wheels | | | `CIBW_TEST_REQUIRES` | Install Python dependencies before running the tests | | | `CIBW_TEST_EXTRAS` | Install Python dependencies before running the tests using ``extras_require``| @@ -231,10 +231,10 @@ A more detailed description of the options, the allowed values, and some example ### Linux builds on Docker -Linux wheels are built in the [`manylinux1` docker images](https://github.com/pypa/manylinux) to provide binary compatible wheels on Linux, according to [PEP 513](https://www.python.org/dev/peps/pep-0513/). Because of this, when building with `cibuildwheel` on Linux, a few things should be taken into account: +Linux wheels are built in the [`manylinux` docker images](https://github.com/pypa/manylinux) to provide binary compatible wheels on Linux, according to [PEP 571](https://www.python.org/dev/peps/pep-0571/). Because of this, when building with `cibuildwheel` on Linux, a few things should be taken into account: - Programs and libraries cannot be installed on the Travis CI Ubuntu host with `apt-get`, but can be installed inside of the Docker image using `yum` or manually. The same goes for environment variables that are potentially needed to customize the wheel building. `cibuildwheel` supports this by providing the `CIBW_ENVIRONMENT` and `CIBW_BEFORE_BUILD` options to setup the build environment inside the running Docker image. See [below](#options) for details on these options. - The project directory is mounted in the running Docker instance as `/project`, the output directory for the wheels as `/output`. In general, this is handled transparently by `cibuildwheel`. For a more finegrained level of control however, the root of the host file system is mounted as `/host`, allowing for example to access shared files, caches, etc. on the host file system. Note that this is not available on CircleCI due to their Docker policies. -- Alternative dockers images can be specified with the `CIBW_MANYLINUX1_X86_64_IMAGE` and `CIBW_MANYLINUX1_I686_IMAGE` options to allow for a custom, preconfigured build environment for the Linux builds. See [below](#options) for more details. +- Alternative dockers images can be specified with the `CIBW_MANYLINUX_X86_64_IMAGE` and `CIBW_MANYLINUX_I686_IMAGE` options to allow for a custom, preconfigured build environment for the Linux builds. See [below](#options) for more details. Options @@ -291,17 +291,19 @@ For `linux` you need Docker running, on Mac or Linux. For `macos`, you need a Ma Optional. -Space-separated list of builds to build and skip. Each build has an identifier like `cp27-manylinux1_x86_64` or `cp34-macosx_10_6_intel` - you can list specific ones to build and `cibuildwheel` will only build those, and/or list ones to skip and `cibuildwheel` won't try to build them. +Space-separated list of builds to build and skip. Each build has an identifier like `cp27-manylinux_x86_64` or `cp34-macosx_10_6_intel` - you can list specific ones to build and `cibuildwheel` will only build those, and/or list ones to skip and `cibuildwheel` won't try to build them. When both options are specified, both conditions are applied and only builds with a tag that matches `CIBW_BUILD` and does not match `CIBW_SKIP` will be built. -The format is `python_tag-platform_tag`. The tags are as defined in [PEP 0425](https://www.python.org/dev/peps/pep-0425/#details). +The format is `python_tag-platform_tag`. The tags are similar but not identical to the ones defined in [PEP 425](https://www.python.org/dev/peps/pep-0425/#details). Python tags look like `cp27` `cp34` `cp35` `cp36` `cp37` -Platform tags look like `macosx_10_6_intel` `manylinux1_x86_64` `manylinux1_i686` `win32` `win_amd64` +Platform tags look like `macosx_10_6_intel` `manylinux_x86_64` `manylinux_i686` `win32` `win_amd64` -You can also use shell-style globbing syntax (as per `fnmatch`) +You can also use shell-style globbing syntax (as per `fnmatch`). + +The list of supported and currently selected build identifiers can be retrieved by passing the `--print-build-identifiers` flag to `cibuildwheel`. Examples: - Only build on Python 3.6: `CIBW_BUILD`:`cp36-*` @@ -311,7 +313,7 @@ Examples: - Skip Python 2.7 on 32-bit Windows: `CIBW_SKIP`:`cp27-win32` - Skip Python 3.4 and Python 3.5: `CIBW_SKIP`:`cp34-* cp35-*` - Skip Python 3.6 on Linux: `CIBW_SKIP`:`cp36-manylinux*` -- Only build on Python 3 and skip 32-bit builds: `CIBW_BUILD`:`cp3?-*` and `CIBW_SKIP`:`*-win32 *-manylinux1_i686` +- Only build on Python 3 and skip 32-bit builds: `CIBW_BUILD`:`cp3?-*` and `CIBW_SKIP`:`*-win32 *-manylinux_i686` *** @@ -370,15 +372,20 @@ Platform-specific variants also available: *** -| Environment variables: `CIBW_MANYLINUX1_X86_64_IMAGE` and `CIBW_MANYLINUX1_I686_IMAGE` +| Environment variables: `CIBW_MANYLINUX_X86_64_IMAGE` and `CIBW_MANYLINUX_I686_IMAGE` | --- Optional. -An alternative docker image to be used for building [`manylinux1`](https://github.com/pypa/manylinux) wheels. `cibuildwheel` will then pull these instead of the official images, [`quay.io/pypa/manylinux1_x86_64`](https://quay.io/pypa/manylinux1_i686) and [`quay.io/pypa/manylinux1_i686`](https://quay.io/pypa/manylinux1_i686). +An alternative Docker image to be used for building [`manylinux`](https://github.com/pypa/manylinux) wheels. `cibuildwheel` will then pull these instead of the default images, [`quay.io/pypa/manylinux2010_x86_64`](https://quay.io/pypa/manylinux2010_x86_64) and [`quay.io/pypa/manylinux2010_i686`](https://quay.io/pypa/manylinux2010_i686). -Beware to specify a valid docker image that can be used the same as the official, default docker images: all necessary Python and pip versions need to be present in `/opt/python/`, and the `auditwheel` tool needs to be present for `cibuildwheel` to work. Apart from that, the architecture and relevant shared system libraries need to be manylinux1-compatible in order to produce valid `manylinux1` wheels (see https://github.com/pypa/manylinux and [PEP 513](https://www.python.org/dev/peps/pep-0513/) for more details). +The value of this option can either be set to `manylinux1` or `manylinux2010` to use the [official `manylinux` images](https://github.com/pypa/manylinux), or any other valid Docker image name. +Beware to specify a valid Docker image that can be used in the same way as the official, default Docker images: all necessary Python and pip versions need to be present in `/opt/python/`, and the `auditwheel` tool needs to be present for `cibuildwheel` to work. Apart from that, the architecture and relevant shared system libraries need to be manylinux1- or manylinux2010-compatible in order to produce valid `manylinux1`/`manylinux2010` wheels (see https://github.com/pypa/manylinux, [PEP 513](https://www.python.org/dev/peps/pep-0513/), and [PEP 571](https://www.python.org/dev/peps/pep-0571/) for more details). + +Note that `auditwheel` detects the version of the `manylinux` standard in the Docker image through the `AUDITWHEEL_PLAT` environment variable, as `cibuildwheel` has no way of detecting the correct `--plat` command line argument to pass to `auditwheel` for a custom image. If a Docker image does not correctly set this `AUDITWHEEL_PLAT` environment variable, the `CIBW_ENVIRONMENT` option can be used to do so (e.g., `CIBW_ENVIRONMENT="manylinux2010_$(uname -m)"`). + +Example: `manylinux1` Example: `dockcross/manylinux-x64` Example: `dockcross/manylinux-x86` diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index fd68da10..847b608a 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -143,8 +143,6 @@ def main(): ) if platform == 'linux': - # TODO Add documentation on CIBW_ENVIRONMENT and possibility of AUDITWHEEL_PLAT if not defined by custom manylinux image - manylinux_x86_64_image = os.environ.get('CIBW_MANYLINUX_X86_64_IMAGE', 'manylinux2010') manylinux_i686_image = os.environ.get('CIBW_MANYLINUX_I686_IMAGE', 'manylinux2010')