From 90ce9d98440ef4f81b92102c4560a07d774910d3 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 16 Dec 2020 16:21:49 +0000 Subject: [PATCH 01/59] Update dependencies and macOS 11 CI on Github --- .github/workflows/test.yml | 3 ++- .../resources/constraints-python27.txt | 8 ++++---- .../resources/constraints-python35.txt | 8 ++++---- .../resources/constraints-python36.txt | 11 +++++----- .../resources/constraints-python37.txt | 11 +++++----- cibuildwheel/resources/constraints.txt | 8 ++++---- .../resources/pinned_docker_images.cfg | 20 +++++++++---------- 7 files changed, 36 insertions(+), 33 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3ab6656c..3422a7a1 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,7 +24,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-18.04, windows-latest, macos-latest] + # os: [ubuntu-latest, windows-latest, macos-11.0] + os: [macos-11.0] python_version: ['3.7'] timeout-minutes: 180 steps: diff --git a/cibuildwheel/resources/constraints-python27.txt b/cibuildwheel/resources/constraints-python27.txt index e387282d..dd4a864b 100644 --- a/cibuildwheel/resources/constraints-python27.txt +++ b/cibuildwheel/resources/constraints-python27.txt @@ -10,17 +10,17 @@ contextlib2==0.6.0.post1 # via importlib-metadata, importlib-resources delocate==0.8.2 # via -r cibuildwheel/resources/constraints.in distlib==0.3.1 # via virtualenv filelock==3.0.12 # via virtualenv -importlib-metadata==2.1.0 # via virtualenv +importlib-metadata==2.1.1 # via virtualenv importlib-resources==3.3.0 # via virtualenv pathlib2==2.3.5 # via importlib-metadata, importlib-resources, virtualenv scandir==1.10.0 # via pathlib2 singledispatch==3.4.0.3 # via importlib-resources six==1.15.0 # via pathlib2, virtualenv typing==3.7.4.3 # via importlib-resources -virtualenv==20.2.1 # via -r cibuildwheel/resources/constraints.in -wheel==0.35.1 # via -r cibuildwheel/resources/constraints.in, delocate +virtualenv==20.2.2 # via -r cibuildwheel/resources/constraints.in +wheel==0.36.2 # via -r cibuildwheel/resources/constraints.in, delocate zipp==1.2.0 # via importlib-metadata, importlib-resources # The following packages are considered to be unsafe in a requirements file: -pip==20.2.4 # via -r cibuildwheel/resources/constraints.in +pip==20.3.3 # via -r cibuildwheel/resources/constraints.in setuptools==44.1.1 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python35.txt b/cibuildwheel/resources/constraints-python35.txt index 4dc2ecd6..c6f210fa 100644 --- a/cibuildwheel/resources/constraints-python35.txt +++ b/cibuildwheel/resources/constraints-python35.txt @@ -8,13 +8,13 @@ appdirs==1.4.4 # via virtualenv delocate==0.8.2 # via -r cibuildwheel/resources/constraints.in distlib==0.3.1 # via virtualenv filelock==3.0.12 # via virtualenv -importlib-metadata==2.1.0 # via virtualenv +importlib-metadata==2.1.1 # via virtualenv importlib-resources==3.2.1 # via virtualenv six==1.15.0 # via virtualenv -virtualenv==20.2.1 # via -r cibuildwheel/resources/constraints.in -wheel==0.35.1 # via -r cibuildwheel/resources/constraints.in, delocate +virtualenv==20.2.2 # via -r cibuildwheel/resources/constraints.in +wheel==0.36.2 # via -r cibuildwheel/resources/constraints.in, delocate zipp==1.2.0 # via importlib-metadata, importlib-resources # The following packages are considered to be unsafe in a requirements file: -pip==20.2.4 # via -r cibuildwheel/resources/constraints.in +pip==20.3.3 # via -r cibuildwheel/resources/constraints.in setuptools==50.3.2 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python36.txt b/cibuildwheel/resources/constraints-python36.txt index 15bca5b6..ea9e0109 100644 --- a/cibuildwheel/resources/constraints-python36.txt +++ b/cibuildwheel/resources/constraints-python36.txt @@ -8,13 +8,14 @@ appdirs==1.4.4 # via virtualenv delocate==0.8.2 # via -r cibuildwheel/resources/constraints.in distlib==0.3.1 # via virtualenv filelock==3.0.12 # via virtualenv -importlib-metadata==3.1.0 # via virtualenv +importlib-metadata==3.3.0 # via virtualenv importlib-resources==3.3.0 # via virtualenv six==1.15.0 # via virtualenv -virtualenv==20.2.1 # via -r cibuildwheel/resources/constraints.in -wheel==0.35.1 # via -r cibuildwheel/resources/constraints.in, delocate +typing-extensions==3.7.4.3 # via importlib-metadata +virtualenv==20.2.2 # via -r cibuildwheel/resources/constraints.in +wheel==0.36.2 # via -r cibuildwheel/resources/constraints.in, delocate zipp==3.4.0 # via importlib-metadata, importlib-resources # The following packages are considered to be unsafe in a requirements file: -pip==20.2.4 # via -r cibuildwheel/resources/constraints.in -setuptools==50.3.2 # via -r cibuildwheel/resources/constraints.in +pip==20.3.3 # via -r cibuildwheel/resources/constraints.in +setuptools==51.0.0 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python37.txt b/cibuildwheel/resources/constraints-python37.txt index cf6d8693..6e0f89f1 100644 --- a/cibuildwheel/resources/constraints-python37.txt +++ b/cibuildwheel/resources/constraints-python37.txt @@ -8,12 +8,13 @@ appdirs==1.4.4 # via virtualenv delocate==0.8.2 # via -r cibuildwheel/resources/constraints.in distlib==0.3.1 # via virtualenv filelock==3.0.12 # via virtualenv -importlib-metadata==3.1.0 # via virtualenv +importlib-metadata==3.3.0 # via virtualenv six==1.15.0 # via virtualenv -virtualenv==20.2.1 # via -r cibuildwheel/resources/constraints.in -wheel==0.35.1 # via -r cibuildwheel/resources/constraints.in, delocate +typing-extensions==3.7.4.3 # via importlib-metadata +virtualenv==20.2.2 # via -r cibuildwheel/resources/constraints.in +wheel==0.36.2 # via -r cibuildwheel/resources/constraints.in, delocate zipp==3.4.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -pip==20.2.4 # via -r cibuildwheel/resources/constraints.in -setuptools==50.3.2 # via -r cibuildwheel/resources/constraints.in +pip==20.3.3 # via -r cibuildwheel/resources/constraints.in +setuptools==51.0.0 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints.txt b/cibuildwheel/resources/constraints.txt index 81a335e9..6d2f7d0b 100644 --- a/cibuildwheel/resources/constraints.txt +++ b/cibuildwheel/resources/constraints.txt @@ -9,9 +9,9 @@ delocate==0.8.2 # via -r cibuildwheel/resources/constraints.in distlib==0.3.1 # via virtualenv filelock==3.0.12 # via virtualenv six==1.15.0 # via virtualenv -virtualenv==20.2.1 # via -r cibuildwheel/resources/constraints.in -wheel==0.35.1 # via -r cibuildwheel/resources/constraints.in, delocate +virtualenv==20.2.2 # via -r cibuildwheel/resources/constraints.in +wheel==0.36.2 # via -r cibuildwheel/resources/constraints.in, delocate # The following packages are considered to be unsafe in a requirements file: -pip==20.2.4 # via -r cibuildwheel/resources/constraints.in -setuptools==50.3.2 # via -r cibuildwheel/resources/constraints.in +pip==20.3.3 # via -r cibuildwheel/resources/constraints.in +setuptools==51.0.0 # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/pinned_docker_images.cfg b/cibuildwheel/resources/pinned_docker_images.cfg index 0646edf8..816ee2be 100644 --- a/cibuildwheel/resources/pinned_docker_images.cfg +++ b/cibuildwheel/resources/pinned_docker_images.cfg @@ -1,22 +1,22 @@ [x86_64] -manylinux1 = quay.io/pypa/manylinux1_x86_64:2020-11-11-0f1f128 -manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2020-12-03-912b0de -manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2020-11-11-bc8ce45 +manylinux1 = quay.io/pypa/manylinux1_x86_64:2020-12-14-7638977 +manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2020-12-15-ba3529a +manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2020-12-15-6cc1890 [i686] -manylinux1 = quay.io/pypa/manylinux1_i686:2020-11-11-0f1f128 -manylinux2010 = quay.io/pypa/manylinux2010_i686:2020-12-03-912b0de -manylinux2014 = quay.io/pypa/manylinux2014_i686:2020-11-11-bc8ce45 +manylinux1 = quay.io/pypa/manylinux1_i686:2020-12-14-7638977 +manylinux2010 = quay.io/pypa/manylinux2010_i686:2020-12-15-ba3529a +manylinux2014 = quay.io/pypa/manylinux2014_i686:2020-12-15-6cc1890 [pypy_x86_64] -manylinux2010 = pypywheels/manylinux2010-pypy_x86_64:2020-11-21-a03b9e9 +manylinux2010 = pypywheels/manylinux2010-pypy_x86_64:2020-12-11-f1e0e80 [aarch64] -manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2020-11-11-bc8ce45 +manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2020-12-15-6cc1890 [ppc64le] -manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2020-11-11-bc8ce45 +manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2020-12-15-6cc1890 [s390x] -manylinux2014 = quay.io/pypa/manylinux2014_s390x:2020-11-11-bc8ce45 +manylinux2014 = quay.io/pypa/manylinux2014_s390x:2020-12-15-6cc1890 From 59801d4c987d24170a1c66193ac53e211d541ff0 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 16 Dec 2020 20:05:31 +0000 Subject: [PATCH 02/59] Disable macOS CPython 3.5 build for now --- cibuildwheel/macos.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 9ffca89d..3d84cdf2 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -35,7 +35,8 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi python_configurations = [ # CPython PythonConfiguration(version='2.7', identifier='cp27-macosx_x86_64', url='https://www.python.org/ftp/python/2.7.18/python-2.7.18-macosx10.9.pkg'), - PythonConfiguration(version='3.5', identifier='cp35-macosx_x86_64', url='https://www.python.org/ftp/python/3.5.4/python-3.5.4-macosx10.6.pkg'), + # TODO: figure out what's going on in CPython 3.5 on macOS 11. or, remove it :) + # PythonConfiguration(version='3.5', identifier='cp35-macosx_x86_64', url='https://www.python.org/ftp/python/3.5.4/python-3.5.4-macosx10.6.pkg'), PythonConfiguration(version='3.6', identifier='cp36-macosx_x86_64', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg'), PythonConfiguration(version='3.7', identifier='cp37-macosx_x86_64', url='https://www.python.org/ftp/python/3.7.8/python-3.7.8-macosx10.9.pkg'), PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.4/python-3.8.4-macosx10.9.pkg'), From 609c0f39862fbb3e751dab254f912f89a87e566f Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Wed, 16 Dec 2020 21:47:08 -0500 Subject: [PATCH 03/59] WIP: try new Python build --- cibuildwheel/macos.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 3d84cdf2..7da855c6 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -40,9 +40,11 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi PythonConfiguration(version='3.6', identifier='cp36-macosx_x86_64', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg'), PythonConfiguration(version='3.7', identifier='cp37-macosx_x86_64', url='https://www.python.org/ftp/python/3.7.8/python-3.7.8-macosx10.9.pkg'), PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.4/python-3.8.4-macosx10.9.pkg'), - PythonConfiguration(version='3.9', identifier='cp39-macosx_x86_64', url='https://www.python.org/ftp/python/3.9.0/python-3.9.0-macosx10.9.pkg'), + # TODO: Find some better way to select Universal2 vs. regular (note that this works on 10.9+, regardless of the name) + PythonConfiguration(version='3.9', identifier='cp39-macosx_x86_64', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg'), # PyPy - PythonConfiguration(version='2.7', identifier='pp27-macosx_x86_64', url='https://downloads.python.org/pypy/pypy2.7-v7.3.3-osx64.tar.bz2'), + # TODO: may not support 11.0 yet + # PythonConfiguration(version='2.7', identifier='pp27-macosx_x86_64', url='https://downloads.python.org/pypy/pypy2.7-v7.3.3-osx64.tar.bz2'), PythonConfiguration(version='3.6', identifier='pp36-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.6-v7.3.3-osx64.tar.bz2'), PythonConfiguration(version='3.7', identifier='pp37-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.7-v7.3.3-osx64.tar.bz2'), ] From 7fb032d70f590b51412c751fc5532d91c74f737e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 18 Dec 2020 12:53:37 +0000 Subject: [PATCH 04/59] Try using _PYTHON_HOST_PLATFORM to create single-arch builds --- cibuildwheel/macos.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 7da855c6..67c4a11d 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -42,6 +42,8 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.4/python-3.8.4-macosx10.9.pkg'), # TODO: Find some better way to select Universal2 vs. regular (note that this works on 10.9+, regardless of the name) PythonConfiguration(version='3.9', identifier='cp39-macosx_x86_64', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg'), + PythonConfiguration(version='3.9', identifier='cp39-macosx_universal2', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg'), + PythonConfiguration(version='3.9', identifier='cp39-macosx_arm64', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg'), # PyPy # TODO: may not support 11.0 yet # PythonConfiguration(version='2.7', identifier='pp27-macosx_x86_64', url='https://downloads.python.org/pypy/pypy2.7-v7.3.3-osx64.tar.bz2'), @@ -178,6 +180,14 @@ def setup_python(python_configuration: PythonConfiguration, # https://github.com/python/cpython/blob/a5ed2fe0eedefa1649aa93ee74a0bafc8e628a10/Lib/_osx_support.py#L260 env.setdefault('ARCHFLAGS', '-arch x86_64') + if python_configuration.version == '3.9': + if python_configuration.identifier.endswith('x86_64'): + env.setdefault('_PYTHON_HOST_PLATFORM', 'macosx-10.9-x86_64') + env.setdefault('ARCHFLAGS', '-arch x86_64') + elif python_configuration.identifier.endswith('arm64'): + env.setdefault('_PYTHON_HOST_PLATFORM', 'macosx-11.0-arm64') + env.setdefault('ARCHFLAGS', '-arch arm64') + log.step('Installing build tools...') call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate', *dependency_constraint_flags], env=env) From 9906cf71053e0249e389b9af9c8b17157b3e070d Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 18 Dec 2020 14:16:28 +0000 Subject: [PATCH 05/59] Fix logger crash --- cibuildwheel/logger.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index a8289e76..97fceccc 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -23,6 +23,8 @@ PLATFORM_IDENTIFIER_DESCIPTIONS = { 'win32': 'Windows 32bit', 'win_amd64': 'Windows 64bit', 'macosx_x86_64': 'macOS x86_64', + 'macosx_universal2': 'macOS Universal 2 - x86_64 and arm64', + 'macosx_arm64': 'macOS arm64 - Apple Silicon', } From 985ef4dfef1da1c807069e67441061cb0149c677 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 18 Dec 2020 11:32:28 -0500 Subject: [PATCH 06/59] wip: drop pypy for now --- cibuildwheel/macos.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 67c4a11d..7140f720 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -47,8 +47,8 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi # PyPy # TODO: may not support 11.0 yet # PythonConfiguration(version='2.7', identifier='pp27-macosx_x86_64', url='https://downloads.python.org/pypy/pypy2.7-v7.3.3-osx64.tar.bz2'), - PythonConfiguration(version='3.6', identifier='pp36-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.6-v7.3.3-osx64.tar.bz2'), - PythonConfiguration(version='3.7', identifier='pp37-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.7-v7.3.3-osx64.tar.bz2'), + # PythonConfiguration(version='3.6', identifier='pp36-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.6-v7.3.3-osx64.tar.bz2'), + # PythonConfiguration(version='3.7', identifier='pp37-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.7-v7.3.3-osx64.tar.bz2'), ] # skip builds as required From 31d0351ddf9235d2c8fcb47934e46a314cccc7d8 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 28 Dec 2020 00:57:53 +0000 Subject: [PATCH 07/59] Upload some sample wheels as artifacts --- .github/workflows/test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3422a7a1..12539e9c 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,6 +53,11 @@ jobs: run: | python bin/sample_build.py + - uses: actions/upload-artifact@v2 + with: + name: sample_wheels + path: wheelhouse + - name: Test cibuildwheel run: | python ./bin/run_tests.py From 2b800bbb3a5f51fbb288f7e55360aeba14a56a5e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 31 Dec 2020 16:40:10 +0000 Subject: [PATCH 08/59] Build some sample wheels --- .github/workflows/test.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 12539e9c..5276be33 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -53,6 +53,11 @@ jobs: run: | python bin/sample_build.py + - name: Get some sample wheels + run: | + python -m test.test_projects test.test_0_basic.basic_project sample_proj + cibuildwheel --output-dir wheelhouse sample_proj + - uses: actions/upload-artifact@v2 with: name: sample_wheels From 76d5f3d3e0a76256564fe8e30e05132e8fe6c848 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 31 Dec 2020 16:48:43 +0000 Subject: [PATCH 09/59] Call delocate with the correct archs --- cibuildwheel/__main__.py | 2 +- cibuildwheel/macos.py | 15 ++++++++++++++- docs/options.md | 5 +++-- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index e171ab20..3040ccb6 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -118,7 +118,7 @@ def main() -> None: if platform == 'linux': repair_command_default = 'auditwheel repair -w {dest_dir} {wheel}' elif platform == 'macos': - repair_command_default = 'delocate-listdeps {wheel} && delocate-wheel --require-archs x86_64 -w {dest_dir} {wheel}' + repair_command_default = 'delocate-listdeps {wheel} && delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}' else: repair_command_default = '' diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 2af1cc8d..efa99a1a 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -250,7 +250,20 @@ def build(options: BuildOptions) -> None: if options.repair_command: log.step('Repairing wheel...') - repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) + + if config.identifier.endswith('universal2'): + delocate_archs = 'x86_64,arm64' + elif config.identifier.endswith('arm64'): + delocate_archs = 'arm64' + else: + delocate_archs = 'x86_64' + + repair_command_prepared = prepare_command( + options.repair_command, + wheel=built_wheel, + dest_dir=repaired_wheel_dir, + delocate_archs=delocate_archs, + ) call(repair_command_prepared, env=env, shell=True) else: shutil.move(str(built_wheel), repaired_wheel_dir) diff --git a/docs/options.md b/docs/options.md index 73c25b3b..ad976cd7 100644 --- a/docs/options.md +++ b/docs/options.md @@ -255,7 +255,7 @@ CIBW_BEFORE_BUILD: "{package}/bin/prepare_for_build.sh" Default: - on Linux: `'auditwheel repair -w {dest_dir} {wheel}'` -- on macOS: `'delocate-listdeps {wheel} && delocate-wheel --require-archs x86_64 -w {dest_dir} {wheel}'` +- on macOS: `'delocate-listdeps {wheel} && delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}'` - on Windows: `''` A shell command to repair a built wheel by copying external library dependencies into the wheel tree and relinking them. @@ -264,7 +264,8 @@ The command is run on each built wheel (except for pure Python ones) before test The following placeholders must be used inside the command and will be replaced by `cibuildwheel`: - `{wheel}` for the absolute path to the built wheel -- `{dest_dir}` for the absolute path of the directory where to create the repaired wheel. +- `{dest_dir}` for the absolute path of the directory where to create the repaired wheel +- `{delocate_archs}` (macOS only) comma-separated list of architectures in the wheel. The command is run in a shell, so you can run multiple commands like `cmd1 && cmd2`. From 4d89e524eb9e0b5021aab0127358a93fa24079e3 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 1 Jan 2021 18:55:50 +0000 Subject: [PATCH 10/59] Add rename hack to make the universal wheel work on both archs --- cibuildwheel/macos.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index efa99a1a..e26538d8 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -270,6 +270,16 @@ def build(options: BuildOptions) -> None: repaired_wheel = next(repaired_wheel_dir.glob('*.whl')) + if repaired_wheel.stem.endswith('_universal2'): + # due to a bug in packaging/tags, this universal wheel isn't + # installable on arm64. so we rename it to add a 11_0 platform + # tag. + # See https://github.com/pypa/packaging/pull/380 and + # https://github.com/pypa/packaging/issues/379 + renamed_wheel = repaired_wheel.parent / f'{repaired_wheel.stem}.macosx_11_0_universal2.whl' + repaired_wheel.rename(renamed_wheel) + repaired_wheel = renamed_wheel + if options.test_command: log.step('Testing wheel...') # set up a virtual environment to install and test from, to make sure From 09fef56a46ba15b4e2fa688da4fdef3bea0cb979 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 1 Jan 2021 18:56:36 +0000 Subject: [PATCH 11/59] Make tests expect arm64 and universal2 --- test/utils.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/test/utils.py b/test/utils.py index 04f3e147..2ddbc813 100644 --- a/test/utils.py +++ b/test/utils.py @@ -10,10 +10,8 @@ import shutil import subprocess import sys from contextlib import contextmanager -from pathlib import Path from tempfile import mkdtemp -IS_WINDOWS_RUNNING_ON_AZURE = Path('C:\\hostedtoolcache').exists() IS_WINDOWS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows' @@ -99,6 +97,10 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, if platform == 'linux': python_abi_tags.append('cp27-cp27mu') # python 2.7 has 2 different ABI on manylinux + if platform == 'macos': + # TODO: perhaps drop Python 3.5 across the board? + python_abi_tags.remove('cp35-cp35m') + wheels = [] for python_abi_tag in python_abi_tags: @@ -123,7 +125,16 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, platform_tags = ['win32'] elif platform == 'macos': - platform_tags = [f'macosx_{macosx_deployment_target.replace(".", "_")}_x86_64'] + if python_abi_tag == 'cp39-cp39': + platform_tags = [ + f'macosx_{macosx_deployment_target.replace(".", "_")}_x86_64', + f'macosx_{macosx_deployment_target.replace(".", "_")}_universal2.macosx_11_0_universal2', + 'macosx_11_0_arm64', + ] + else: + platform_tags = [ + f'macosx_{macosx_deployment_target.replace(".", "_")}_x86_64', + ] else: raise Exception('unsupported platform') From 7cab7125929b69208d582823312869ff0902151f Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 2 Jan 2021 19:25:09 +0000 Subject: [PATCH 12/59] Fix unit test --- unit_test/main_tests/main_options_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unit_test/main_tests/main_options_test.py b/unit_test/main_tests/main_options_test.py index c7e6bfdd..26e50a72 100644 --- a/unit_test/main_tests/main_options_test.py +++ b/unit_test/main_tests/main_options_test.py @@ -94,7 +94,7 @@ def get_default_repair_command(platform): if platform == 'linux': return 'auditwheel repair -w {dest_dir} {wheel}' elif platform == 'macos': - return 'delocate-listdeps {wheel} && delocate-wheel --require-archs x86_64 -w {dest_dir} {wheel}' + return 'delocate-listdeps {wheel} && delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}' elif platform == 'windows': return '' else: From a1ac43e9ded2cc20608a59da3e471ed9ad8ef92f Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 10:23:36 +0000 Subject: [PATCH 13/59] Override buildable archs on macOS 11 --- cibuildwheel/macos.py | 34 +++++++++++++++++++++++++++------- 1 file changed, 27 insertions(+), 7 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 835d73a7..8a70701a 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -1,4 +1,5 @@ import os +import platform import shlex import shutil import subprocess @@ -7,7 +8,7 @@ import tempfile import textwrap from os import PathLike from pathlib import Path -from typing import Dict, List, NamedTuple, Optional, Sequence, Union +from typing import Dict, List, NamedTuple, Optional, Sequence, Tuple, Union, cast from .environment import ParsedEnvironment from .logger import log @@ -26,6 +27,19 @@ def call(args: Union[str, Sequence[Union[str, PathLike]]], env: Optional[Dict[st return subprocess.check_call(args, env=env, cwd=cwd, shell=shell) +def get_macos_version() -> Tuple[int, int]: + ''' + Returns the macOS major/minor version, as a tuple, e.g. (10, 15) or (11, 0) + + These tuples can be used in comparisons, e.g. + (10, 14) <= (11, 0) == True + (11, 2) <= (11, 0) != True + ''' + version_str, _, _ = platform.mac_ver() + version = tuple(map(int, version_str.split(".")[:2])) + return cast(Tuple[int, int], version) + + class PythonConfiguration(NamedTuple): version: str identifier: str @@ -36,22 +50,28 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi python_configurations = [ # CPython PythonConfiguration(version='2.7', identifier='cp27-macosx_x86_64', url='https://www.python.org/ftp/python/2.7.18/python-2.7.18-macosx10.9.pkg'), - # TODO: figure out what's going on in CPython 3.5 on macOS 11. or, remove it :) - # PythonConfiguration(version='3.5', identifier='cp35-macosx_x86_64', url='https://www.python.org/ftp/python/3.5.4/python-3.5.4-macosx10.6.pkg'), + PythonConfiguration(version='3.5', identifier='cp35-macosx_x86_64', url='https://www.python.org/ftp/python/3.5.4/python-3.5.4-macosx10.6.pkg'), PythonConfiguration(version='3.6', identifier='cp36-macosx_x86_64', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg'), PythonConfiguration(version='3.7', identifier='cp37-macosx_x86_64', url='https://www.python.org/ftp/python/3.7.9/python-3.7.9-macosx10.9.pkg'), PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.7/python-3.8.7-macosx10.9.pkg'), - # TODO: Find some better way to select Universal2 vs. regular (note that this works on 10.9+, regardless of the name) PythonConfiguration(version='3.9', identifier='cp39-macosx_x86_64', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg'), PythonConfiguration(version='3.9', identifier='cp39-macosx_universal2', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg'), PythonConfiguration(version='3.9', identifier='cp39-macosx_arm64', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg'), # PyPy # TODO: may not support 11.0 yet - # PythonConfiguration(version='2.7', identifier='pp27-macosx_x86_64', url='https://downloads.python.org/pypy/pypy2.7-v7.3.3-osx64.tar.bz2'), - # PythonConfiguration(version='3.6', identifier='pp36-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.6-v7.3.3-osx64.tar.bz2'), - # PythonConfiguration(version='3.7', identifier='pp37-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.7-v7.3.3-osx64.tar.bz2'), + PythonConfiguration(version='2.7', identifier='pp27-macosx_x86_64', url='https://downloads.python.org/pypy/pypy2.7-v7.3.3-osx64.tar.bz2'), + PythonConfiguration(version='3.6', identifier='pp36-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.6-v7.3.3-osx64.tar.bz2'), + PythonConfiguration(version='3.7', identifier='pp37-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.7-v7.3.3-osx64.tar.bz2'), ] + if get_macos_version() >= (11, 0): + # pypy doesn't work on macOS 11 yet + # See https://foss.heptapod.net/pypy/pypy/-/issues/3314 + python_configurations = [c for c in python_configurations if not c.identifier.startswith('pp')] + + # CPython 3.5 doesn't work on macOS 11 + python_configurations = [c for c in python_configurations if not c.identifier.startswith('cp35')] + # skip builds as required return [c for c in python_configurations if build_selector(c.identifier)] From 46eb28791f473ede2841132854473622e04fb902 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 13:50:21 +0000 Subject: [PATCH 14/59] Refactor for clarity --- cibuildwheel/linux.py | 2 +- cibuildwheel/logger.py | 2 +- cibuildwheel/macos.py | 2 +- cibuildwheel/windows.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 4118fd79..19f36064 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -251,7 +251,7 @@ def build(options: BuildOptions) -> None: docker.copy_out(container_output_dir, options.output_dir) log.step_end() except subprocess.CalledProcessError as error: - log.error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') + log.step_end_with_error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') troubleshoot(options.package_dir, error) exit(1) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index bfb81b3a..4724b605 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -111,7 +111,7 @@ class Logger: self.step_start_time = None - def error(self, error: Union[BaseException, str]) -> None: + def step_end_with_error(self, error: Union[BaseException, str]) -> None: self.step_end(success=False) print() diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index f53b0acd..d43d2954 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -359,5 +359,5 @@ def build(options: BuildOptions) -> None: shutil.move(str(repaired_wheel), options.output_dir) log.build_end() except subprocess.CalledProcessError as error: - log.error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') + log.step_end_with_error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') exit(1) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 3e977e4b..a156db3a 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -330,5 +330,5 @@ def build(options: BuildOptions) -> None: shutil.move(str(repaired_wheel), options.output_dir) log.build_end() except subprocess.CalledProcessError as error: - log.error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') + log.step_end_with_error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') exit(1) From e405f3495efcde3edf1495bd7232d0ee53c89d9c Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 14:29:38 +0000 Subject: [PATCH 15/59] Print warnings when builds are deselected because of build OS --- cibuildwheel/logger.py | 12 ++++++++++++ cibuildwheel/macos.py | 26 +++++++++++++++++++++----- cibuildwheel/util.py | 14 ++++++++++++++ 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index 4724b605..a6c693f3 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -113,6 +113,18 @@ class Logger: def step_end_with_error(self, error: Union[BaseException, str]) -> None: self.step_end(success=False) + self.error(error) + + def warning(self, message: str) -> None: + print() + + if self.fold_mode == 'github': + print(f'::warning::{message}') + else: + c = self.colors + print(f'{c.yellow}Warning{c.end} {message}') + + def error(self, error: Union[BaseException, str]) -> None: print() if self.fold_mode == 'github': diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index d43d2954..0dd5ebac 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -13,7 +13,7 @@ from .environment import ParsedEnvironment from .logger import log from .util import (Architecture, BuildOptions, BuildSelector, NonPlatformWheelError, download, get_build_verbosity_extra_flags, get_pip_script, - install_certifi_script, prepare_command) + install_certifi_script, prepare_command, wrap_text) from .typing import PathOrStr @@ -64,16 +64,32 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi PythonConfiguration(version='3.7', identifier='pp37-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.7-v7.3.3-osx64.tar.bz2'), ] + # skip builds as required + python_configurations = [c for c in python_configurations if build_selector(c.identifier)] + if get_macos_version() >= (11, 0): # pypy doesn't work on macOS 11 yet # See https://foss.heptapod.net/pypy/pypy/-/issues/3314 - python_configurations = [c for c in python_configurations if not c.identifier.startswith('pp')] + if any(c.identifier.startswith('pp') for c in python_configurations): + log.warning(wrap_text(''' + PyPy is currently unsupported when building on macOS 11. To build macOS PyPy wheels, + build on an older OS, such as macOS 10.15. To silence this warning, deselect PyPy by + adding "pp*-macosx*" to your CIBW_SKIP option. + ''')) + python_configurations = [c for c in python_configurations if not c.identifier.startswith('pp')] + + if any(c.identifier.startswith('cp35') for c in python_configurations): + # CPython 3.5 doesn't work on macOS 11 + log.warning(wrap_text(''' + CPython is unsupported when building on macOS 11. To build CPython 3.5 wheels, build + on an older OS, such as macOS 10.15. To silence this warning, deselect CPython 3.5 + by adding "cp35-macosx_x86_64" to your CIBW_SKIP option. + ''')) + python_configurations = [c for c in python_configurations if not c.identifier.startswith('cp35')] - # CPython 3.5 doesn't work on macOS 11 python_configurations = [c for c in python_configurations if not c.identifier.startswith('cp35')] - # skip builds as required - return [c for c in python_configurations if build_selector(c.identifier)] + return python_configurations SYMLINKS_DIR = Path('/tmp/cibw_bin') diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 960b9896..3b4935ff 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -229,3 +229,17 @@ def detect_ci_provider() -> Optional[CIProvider]: return CIProvider.other else: return None + + +def wrap_text(text: str, max_width: int = 80) -> str: + ''' + Formats text, suitable for printing arbitrary long passages to console. + ''' + # remove initial line indent + text = textwrap.dedent(text) + # remove leading/trailing whitespace + text = text.strip() + # remove consecutive whitespace + text = re.sub(r'\s+', ' ', text) + # wrap to max_width + return '\n'.join(textwrap.wrap(text, width=max_width)) From df60457001e50314ac0f4e4c7ae5cd7480578d8a Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 15:14:32 +0000 Subject: [PATCH 16/59] Use the ARCHS option to select universal2/arm64 builds --- cibuildwheel/__main__.py | 2 +- cibuildwheel/macos.py | 31 +++++++++++++++++-------------- cibuildwheel/util.py | 13 +++++++++++++ test/test_macos_archs.py | 25 +++++++++++++++++++++++++ test/utils.py | 34 +++++++++++++++++++++++++++------- 5 files changed, 83 insertions(+), 22 deletions(-) create mode 100644 test/test_macos_archs.py diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index d3ed3492..20ee2baa 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -312,7 +312,7 @@ def print_build_identifiers( elif platform == 'windows': python_configurations = cibuildwheel.windows.get_python_configurations(build_selector) elif platform == 'macos': - python_configurations = cibuildwheel.macos.get_python_configurations(build_selector) + python_configurations = cibuildwheel.macos.get_python_configurations(build_selector, architectures) for config in python_configurations: print(config.identifier) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 0dd5ebac..e12ea648 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -46,7 +46,8 @@ class PythonConfiguration(NamedTuple): url: str -def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfiguration]: +def get_python_configurations(build_selector: BuildSelector, + architectures: List[Architecture]) -> List[PythonConfiguration]: python_configurations = [ # CPython PythonConfiguration(version='2.7', identifier='cp27-macosx_x86_64', url='https://www.python.org/ftp/python/2.7.18/python-2.7.18-macosx10.9.pkg'), @@ -58,19 +59,22 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi PythonConfiguration(version='3.9', identifier='cp39-macosx_universal2', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg'), PythonConfiguration(version='3.9', identifier='cp39-macosx_arm64', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg'), # PyPy - # TODO: may not support 11.0 yet PythonConfiguration(version='2.7', identifier='pp27-macosx_x86_64', url='https://downloads.python.org/pypy/pypy2.7-v7.3.3-osx64.tar.bz2'), PythonConfiguration(version='3.6', identifier='pp36-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.6-v7.3.3-osx64.tar.bz2'), PythonConfiguration(version='3.7', identifier='pp37-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.7-v7.3.3-osx64.tar.bz2'), ] - # skip builds as required + # filter out configs that don't match any of the selected architectures + python_configurations = [c for c in python_configurations + if any(c.identifier.endswith(a.value) for a in architectures)] + + # skip builds as required by BUILD/SKIP python_configurations = [c for c in python_configurations if build_selector(c.identifier)] if get_macos_version() >= (11, 0): - # pypy doesn't work on macOS 11 yet - # See https://foss.heptapod.net/pypy/pypy/-/issues/3314 if any(c.identifier.startswith('pp') for c in python_configurations): + # pypy doesn't work on macOS 11 yet + # See https://foss.heptapod.net/pypy/pypy/-/issues/3314 log.warning(wrap_text(''' PyPy is currently unsupported when building on macOS 11. To build macOS PyPy wheels, build on an older OS, such as macOS 10.15. To silence this warning, deselect PyPy by @@ -81,14 +85,12 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi if any(c.identifier.startswith('cp35') for c in python_configurations): # CPython 3.5 doesn't work on macOS 11 log.warning(wrap_text(''' - CPython is unsupported when building on macOS 11. To build CPython 3.5 wheels, build - on an older OS, such as macOS 10.15. To silence this warning, deselect CPython 3.5 - by adding "cp35-macosx_x86_64" to your CIBW_SKIP option. + CPython 3.5 is unsupported when building on macOS 11. To build CPython 3.5 wheels, + build on an older OS, such as macOS 10.15. To silence this warning, deselect CPython + 3.5 by adding "cp35-macosx_x86_64" to your CIBW_SKIP option. ''')) python_configurations = [c for c in python_configurations if not c.identifier.startswith('cp35')] - python_configurations = [c for c in python_configurations if not c.identifier.startswith('cp35')] - return python_configurations @@ -236,10 +238,11 @@ def setup_python(python_configuration: PythonConfiguration, def build(options: BuildOptions) -> None: - if options.architectures != [Architecture.x86_64]: + allowed_archs = {Architecture.x86_64, Architecture.universal2, Architecture.arm64} + if set(options.architectures) <= allowed_archs: raise ValueError(textwrap.dedent(f''' - Invalid archs option {options.architectures}. macOS only supports x86_64 for the moment. - If you want to set emulation architectures on Linux, use CIBW_ARCHS_LINUX instead. + Invalid archs option {options.architectures}. macOS only supports + these architectures: {', '.join(a.value for a in allowed_archs)}. ''')) temp_dir = Path(tempfile.mkdtemp(prefix='cibuildwheel')) @@ -253,7 +256,7 @@ def build(options: BuildOptions) -> None: before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) call([before_all_prepared], shell=True, env=env) - python_configurations = get_python_configurations(options.build_selector) + python_configurations = get_python_configurations(options.build_selector, options.architectures) for config in python_configurations: log.build_start(config.identifier) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 3b4935ff..3654b51e 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -127,11 +127,17 @@ class DependencyConstraints: class Architecture(Enum): # mac/linux archs x86_64 = 'x86_64' + + # linux archs i686 = 'i686' aarch64 = 'aarch64' ppc64le = 'ppc64le' s390x = 's390x' + # mac archs + universal2 = 'universal2' + arm64 = 'arm64' + # windows archs x86 = 'x86' AMD64 = 'AMD64' @@ -150,11 +156,18 @@ class Architecture(Enum): def auto_archs(platform: str) -> 'List[Architecture]': native_architecture = Architecture(platform_module.machine()) result = [native_architecture] + if platform == 'linux' and native_architecture == Architecture.x86_64: # x86_64 machines can run i686 docker containers result.append(Architecture.i686) + if platform == 'windows' and native_architecture == Architecture.AMD64: result.append(Architecture.x86) + + if platform == 'macos' and native_architecture == Architecture.arm64: + # arm64 can build and test both archs of a universal2 wheel. + result.append(Architecture.universal2) + return result diff --git a/test/test_macos_archs.py b/test/test_macos_archs.py new file mode 100644 index 00000000..cfcaddeb --- /dev/null +++ b/test/test_macos_archs.py @@ -0,0 +1,25 @@ +import pytest + +from . import test_projects, utils + +basic_project = test_projects.new_c_project() + + +def test_cross_compiled_build(tmp_path): + if utils.platform != 'macos': + pytest.skip('this test is only relevant to macos') + + project_dir = tmp_path / 'project' + basic_project.generate(project_dir) + + actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ + 'CIBW_ARCHS': 'x86_64, universal2, arm64', + }) + + expected_wheels = ( + utils.expected_wheels('spam', '0.1.0', machine_arch='x86_64') + + utils.expected_wheels('spam', '0.1.0', machine_arch='arm64') + ) + assert set(actual_wheels) == set(expected_wheels) + +# TODO: add a TEST_COMMAND test when using cross-compiling diff --git a/test/utils.py b/test/utils.py index d8aa50ee..5be952d0 100644 --- a/test/utils.py +++ b/test/utils.py @@ -100,9 +100,11 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, if platform == 'linux': python_abi_tags.append('cp27-cp27mu') # python 2.7 has 2 different ABI on manylinux - if platform == 'macos': - # TODO: perhaps drop Python 3.5 across the board? + if platform == 'macos' and get_macos_version() >= (11, 0): + # CPython 3.5 doesn't work on macOS 11. python_abi_tags.remove('cp35-cp35m') + # pypy not supported on macOS 11. + python_abi_tags = [t for t in python_abi_tags if not t.startswith('pp')] wheels = [] @@ -129,11 +131,17 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, elif platform == 'macos': if python_abi_tag == 'cp39-cp39': - platform_tags = [ - f'macosx_{macosx_deployment_target.replace(".", "_")}_x86_64', - f'macosx_{macosx_deployment_target.replace(".", "_")}_universal2.macosx_11_0_universal2', - 'macosx_11_0_arm64', - ] + if machine_arch == 'x86_64': + platform_tags = [ + f'macosx_{macosx_deployment_target.replace(".", "_")}_x86_64', + ] + elif machine_arch == 'arm64': + platform_tags = [ + f'macosx_{macosx_deployment_target.replace(".", "_")}_universal2.macosx_11_0_universal2', + # macosx_deployment_target is ignored on arm64, because arm64 isn't supported on + # macOS earlier than 11.0 + 'macosx_11_0_arm64', + ] else: platform_tags = [ f'macosx_{macosx_deployment_target.replace(".", "_")}_x86_64', @@ -152,6 +160,18 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, return wheels +def get_macos_version(): + ''' + Returns the macOS major/minor version, as a tuple, e.g. (10, 15) or (11, 0) + + These tuples can be used in comparisons, e.g. + (10, 14) <= (11, 0) == True + (11, 2) <= (11, 0) != True + ''' + version_str, _, _ = pm.mac_ver() + return tuple(map(int, version_str.split(".")[:2])) + + platform = None if 'CIBW_PLATFORM' in os.environ: From 577fd5e8e34ba8a629a551f80dd16181c5b2f13e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 15:58:53 +0000 Subject: [PATCH 17/59] Implement testing for universal2/arm64 wheels --- cibuildwheel/macos.py | 123 +++++++++++++++++++++++++++++------------- 1 file changed, 87 insertions(+), 36 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index e12ea648..6a06c53c 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -7,7 +7,7 @@ import sys import tempfile import textwrap from pathlib import Path -from typing import Dict, List, NamedTuple, Optional, Sequence, Tuple, cast +from typing import Any, Dict, List, NamedTuple, Optional, Sequence, Tuple, cast from .environment import ParsedEnvironment from .logger import log @@ -330,49 +330,100 @@ def build(options: BuildOptions) -> None: repaired_wheel.rename(renamed_wheel) repaired_wheel = renamed_wheel - if options.test_command: - log.step('Testing wheel...') - # set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) - venv_dir = Path(tempfile.mkdtemp()) + log.step_end() - # Use --no-download to ensure determinism by using seed libraries - # built into virtualenv - call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) + machine_arch = platform.machine() + testing_archs: List[str] = [] - virtualenv_env = env.copy() - virtualenv_env['PATH'] = os.pathsep.join([ - str(venv_dir / 'bin'), - virtualenv_env['PATH'], - ]) + if machine_arch == 'x86_64': + if config.identifier.endswith('_arm64'): + log.warning(wrap_text(''' + While arm64 wheels can be built on x86_64, they cannot be tested. The + ability to test the arm64 wheels will be added in a future release of + cibuildwheel, once Apple Silicon CI runners are widely available. + ''')) + testing_archs = [] + elif config.identifier.endswith('_universal2'): + log.warning(wrap_text(''' + While universal2 wheels can be built on x86_64, the arm64 part of them + cannot currently be tested. The ability to test the arm64 part of a + universal2 wheel will be added in a future release of cibuildwheel, once + Apple Silicon CI runners are widely available. + ''')) + testing_archs = ['x86_64'] + else: + testing_archs = ['x86_64'] + elif machine_arch == 'arm64': + if config.identifier.endswith('_x86_64'): + # testing using rosetta2 emulation + testing_archs = ['x86_64'] + elif config.identifier.endswith('_universal2'): + # testing the x86_64 using rosetta2 emulation + testing_archs = ['arm64', 'x86_64'] + else: + testing_archs = ['arm64'] - # check that we are using the Python from the virtual environment - call(['which', 'python'], env=virtualenv_env) + if options.test_command and len(testing_archs) > 0: + for testing_arch in testing_archs: + log.step('Testing wheel...' if testing_arch == machine_arch else f'Testing wheel on {testing_arch}...') - if options.before_test: - before_test_prepared = prepare_command(options.before_test, project='.', package=options.package_dir) - call(before_test_prepared, env=virtualenv_env, shell=True) + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) + venv_dir = Path(tempfile.mkdtemp()) - # install the wheel - call(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env) + arch_prefix = [] + if testing_arch != machine_arch: + if machine_arch == 'arm64' and testing_arch == 'x86_64': + # rosetta2 will provide the emulation with just the arch prefix. + arch_prefix = ['arch', '-x86_64'] + else: + raise RuntimeError("don't know how to emulate {testing_arch} on {machine_arch}") - # test the wheel - if options.test_requires: - call(['pip', 'install'] + options.test_requires, env=virtualenv_env) + # define a custom 'call' function that adds the arch prefix each time + def call_with_arch(args: Sequence[PathOrStr], **kwargs: Any) -> int: + if isinstance(args, str): + args = ' '.join(arch_prefix) + ' ' + args + else: + args = [*arch_prefix, *args] + return call(args, **kwargs) - # run the tests from $HOME, with an absolute path in the command - # (this ensures that Python runs the tests against the installed wheel - # and not the repo code) - test_command_prepared = prepare_command( - options.test_command, - project=Path('.').resolve(), - package=options.package_dir.resolve() - ) - call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True) + # Use --no-download to ensure determinism by using seed libraries + # built into virtualenv + call_with_arch(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) - # clean up - shutil.rmtree(venv_dir) + virtualenv_env = env.copy() + virtualenv_env['PATH'] = os.pathsep.join([ + str(venv_dir / 'bin'), + virtualenv_env['PATH'], + ]) + + # check that we are using the Python from the virtual environment + call_with_arch(['which', 'python'], env=virtualenv_env) + + if options.before_test: + before_test_prepared = prepare_command(options.before_test, project='.', package=options.package_dir) + call_with_arch(before_test_prepared, env=virtualenv_env, shell=True) + + # install the wheel + call_with_arch(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env) + + # test the wheel + if options.test_requires: + call_with_arch(['pip', 'install'] + options.test_requires, env=virtualenv_env) + + # run the tests from $HOME, with an absolute path in the command + # (this ensures that Python runs the tests against the installed wheel + # and not the repo code) + test_command_prepared = prepare_command( + options.test_command, + project=Path('.').resolve(), + package=options.package_dir.resolve() + ) + call_with_arch(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True) + + # clean up + shutil.rmtree(venv_dir) # we're all done here; move it to output (overwrite existing) shutil.move(str(repaired_wheel), options.output_dir) From 914f74a0f03077c503eba0784faf4643adbfd52a Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 16:03:24 +0000 Subject: [PATCH 18/59] Interim fix for archs comparison This will change when #507 is merged --- cibuildwheel/macos.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 6a06c53c..daac4735 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -239,9 +239,9 @@ def setup_python(python_configuration: PythonConfiguration, def build(options: BuildOptions) -> None: allowed_archs = {Architecture.x86_64, Architecture.universal2, Architecture.arm64} - if set(options.architectures) <= allowed_archs: + if any(a not in allowed_archs for a in options.architectures): raise ValueError(textwrap.dedent(f''' - Invalid archs option {options.architectures}. macOS only supports + Invalid archs option {[a.value for a in options.architectures]}. macOS only supports these architectures: {', '.join(a.value for a in allowed_archs)}. ''')) From afd08c11f2177d2ebe317ea282755c946fd81934 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 16:12:24 +0000 Subject: [PATCH 19/59] Maybe Azure secretly supports macOS 11.0? --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index b144c2db..db2a56cb 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -10,7 +10,7 @@ jobs: python ./bin/run_tests.py - job: macos_38 - pool: {vmImage: 'macOS-10.15'} + pool: {vmImage: 'macos-11.0'} steps: - task: UsePythonVersion@0 inputs: From 7a11143be8063ea33d0751c4266209f130159d16 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 16:14:07 +0000 Subject: [PATCH 20/59] Like this? --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index db2a56cb..04dc3c94 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -10,7 +10,7 @@ jobs: python ./bin/run_tests.py - job: macos_38 - pool: {vmImage: 'macos-11.0'} + pool: {vmImage: 'macOS-11.0'} steps: - task: UsePythonVersion@0 inputs: From 16d96fa33d882c9ab0cd3d7e6e6990575c01c840 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 16:18:17 +0000 Subject: [PATCH 21/59] It does not. --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 04dc3c94..b144c2db 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -10,7 +10,7 @@ jobs: python ./bin/run_tests.py - job: macos_38 - pool: {vmImage: 'macOS-11.0'} + pool: {vmImage: 'macOS-10.15'} steps: - task: UsePythonVersion@0 inputs: From 65b530a5dfa1105d005f9d97b56a34217725416d Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 17:01:08 +0000 Subject: [PATCH 22/59] On arm64, build native before cross-compile --- cibuildwheel/macos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index daac4735..dc5044d1 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -56,8 +56,8 @@ def get_python_configurations(build_selector: BuildSelector, PythonConfiguration(version='3.7', identifier='cp37-macosx_x86_64', url='https://www.python.org/ftp/python/3.7.9/python-3.7.9-macosx10.9.pkg'), PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.7/python-3.8.7-macosx10.9.pkg'), PythonConfiguration(version='3.9', identifier='cp39-macosx_x86_64', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg'), - PythonConfiguration(version='3.9', identifier='cp39-macosx_universal2', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg'), PythonConfiguration(version='3.9', identifier='cp39-macosx_arm64', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg'), + PythonConfiguration(version='3.9', identifier='cp39-macosx_universal2', url='https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg'), # PyPy PythonConfiguration(version='2.7', identifier='pp27-macosx_x86_64', url='https://downloads.python.org/pypy/pypy2.7-v7.3.3-osx64.tar.bz2'), PythonConfiguration(version='3.6', identifier='pp36-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.6-v7.3.3-osx64.tar.bz2'), From 121715a56829cabd0fc61e933ea325f6e39b5c47 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 17:01:23 +0000 Subject: [PATCH 23/59] Skip cross-compile test on macOS <11 --- test/test_macos_archs.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test/test_macos_archs.py b/test/test_macos_archs.py index cfcaddeb..990d08e9 100644 --- a/test/test_macos_archs.py +++ b/test/test_macos_archs.py @@ -8,6 +8,8 @@ basic_project = test_projects.new_c_project() def test_cross_compiled_build(tmp_path): if utils.platform != 'macos': pytest.skip('this test is only relevant to macos') + if utils.get_macos_version() < (11, 0): + pytest.skip('this test only works on macOS 11 or greater') project_dir = tmp_path / 'project' basic_project.generate(project_dir) From bf272fe9a6ceca828ff1fdcd9ae81b03b02b2f6f Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 17:01:47 +0000 Subject: [PATCH 24/59] Use same interpreter for sample_build.py (for local testing of arm64) --- bin/sample_build.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/sample_build.py b/bin/sample_build.py index d2adb022..c995defa 100755 --- a/bin/sample_build.py +++ b/bin/sample_build.py @@ -22,4 +22,4 @@ if __name__ == '__main__': options.project_python_path, project_dir ], check=True) - exit(subprocess.run(['cibuildwheel'], cwd=project_dir).returncode) + exit(subprocess.run([sys.executable, '-m', 'cibuildwheel'], cwd=project_dir).returncode) From a6ff55ed5b03cb60be2f45a271d66b86178e21a2 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 17:37:55 +0000 Subject: [PATCH 25/59] Update pip in the test virtualenv --- cibuildwheel/macos.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index dc5044d1..e246b19a 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -401,6 +401,12 @@ def build(options: BuildOptions) -> None: # check that we are using the Python from the virtual environment call_with_arch(['which', 'python'], env=virtualenv_env) + # ensure we have a recent enough pip in the virtualenv + # TODO(joerick): remove this before merge? virtualenv should be bundling a + # recent enough seed version. Waiting on this PR: + # https://github.com/pypa/virtualenv/pull/2036 + call_with_arch(['pip', 'install', 'pip>=20.3.3'], env=virtualenv_env) + if options.before_test: before_test_prepared = prepare_command(options.before_test, project='.', package=options.package_dir) call_with_arch(before_test_prepared, env=virtualenv_env, shell=True) From e215446283aaec979abae665d9becdbcd00b2e08 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 17:40:15 +0000 Subject: [PATCH 26/59] Increase test timeout value --- bin/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/run_tests.py b/bin/run_tests.py index b46bfbe6..9e8e9075 100755 --- a/bin/run_tests.py +++ b/bin/run_tests.py @@ -17,4 +17,4 @@ if __name__ == '__main__': subprocess.check_call(unit_test_args) # run the integration tests - subprocess.check_call([sys.executable, '-m', 'pytest', '-x', '--durations', '0', '--timeout=1200', 'test']) + subprocess.check_call([sys.executable, '-m', 'pytest', '-x', '--durations', '0', '--timeout=2400', 'test']) From 0df5e1295f1512c5641412d8b67c7935e4f80755 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 18:56:07 +0000 Subject: [PATCH 27/59] Only print testing warnings if test command is set --- cibuildwheel/macos.py | 62 +++++++++++++++++++++---------------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index e246b19a..5a1f72be 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -332,38 +332,38 @@ def build(options: BuildOptions) -> None: log.step_end() - machine_arch = platform.machine() - testing_archs: List[str] = [] + if options.test_command: + machine_arch = platform.machine() + testing_archs: List[str] = [] - if machine_arch == 'x86_64': - if config.identifier.endswith('_arm64'): - log.warning(wrap_text(''' - While arm64 wheels can be built on x86_64, they cannot be tested. The - ability to test the arm64 wheels will be added in a future release of - cibuildwheel, once Apple Silicon CI runners are widely available. - ''')) - testing_archs = [] - elif config.identifier.endswith('_universal2'): - log.warning(wrap_text(''' - While universal2 wheels can be built on x86_64, the arm64 part of them - cannot currently be tested. The ability to test the arm64 part of a - universal2 wheel will be added in a future release of cibuildwheel, once - Apple Silicon CI runners are widely available. - ''')) - testing_archs = ['x86_64'] - else: - testing_archs = ['x86_64'] - elif machine_arch == 'arm64': - if config.identifier.endswith('_x86_64'): - # testing using rosetta2 emulation - testing_archs = ['x86_64'] - elif config.identifier.endswith('_universal2'): - # testing the x86_64 using rosetta2 emulation - testing_archs = ['arm64', 'x86_64'] - else: - testing_archs = ['arm64'] - - if options.test_command and len(testing_archs) > 0: + if machine_arch == 'x86_64': + if config.identifier.endswith('_arm64'): + log.warning(wrap_text(''' + While arm64 wheels can be built on x86_64, they cannot be tested. The + ability to test the arm64 wheels will be added in a future release of + cibuildwheel, once Apple Silicon CI runners are widely available. + ''')) + testing_archs = [] + elif config.identifier.endswith('_universal2'): + log.warning(wrap_text(''' + While universal2 wheels can be built on x86_64, the arm64 part of them + cannot currently be tested. The ability to test the arm64 part of a + universal2 wheel will be added in a future release of cibuildwheel, once + Apple Silicon CI runners are widely available. + ''')) + testing_archs = ['x86_64'] + else: + testing_archs = ['x86_64'] + elif machine_arch == 'arm64': + if config.identifier.endswith('_x86_64'): + # testing using rosetta2 emulation + testing_archs = ['x86_64'] + elif config.identifier.endswith('_universal2'): + # testing the x86_64 using rosetta2 emulation + testing_archs = ['arm64', 'x86_64'] + else: + testing_archs = ['arm64'] + for testing_arch in testing_archs: log.step('Testing wheel...' if testing_arch == machine_arch else f'Testing wheel on {testing_arch}...') From ffba89dc6441b3db8c3280ec809d042724b41f22 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 19:02:15 +0000 Subject: [PATCH 28/59] Improve output formatting Github annotations require all output to be on a single line, so let's make the log messages do that --- cibuildwheel/logger.py | 4 ++++ cibuildwheel/macos.py | 12 ++++++------ cibuildwheel/util.py | 8 +++----- 3 files changed, 13 insertions(+), 11 deletions(-) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index a6c693f3..4dd6418a 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -124,6 +124,8 @@ class Logger: c = self.colors print(f'{c.yellow}Warning{c.end} {message}') + print() + def error(self, error: Union[BaseException, str]) -> None: print() @@ -133,6 +135,8 @@ class Logger: c = self.colors print(f'{c.bright_red}Error{c.end} {error}') + print() + def _start_fold_group(self, name: str) -> None: self._end_fold_group() self.active_fold_group_name = name diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 5a1f72be..613ea26c 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -13,7 +13,7 @@ from .environment import ParsedEnvironment from .logger import log from .util import (Architecture, BuildOptions, BuildSelector, NonPlatformWheelError, download, get_build_verbosity_extra_flags, get_pip_script, - install_certifi_script, prepare_command, wrap_text) + install_certifi_script, prepare_command, unwrap) from .typing import PathOrStr @@ -75,7 +75,7 @@ def get_python_configurations(build_selector: BuildSelector, if any(c.identifier.startswith('pp') for c in python_configurations): # pypy doesn't work on macOS 11 yet # See https://foss.heptapod.net/pypy/pypy/-/issues/3314 - log.warning(wrap_text(''' + log.warning(unwrap(''' PyPy is currently unsupported when building on macOS 11. To build macOS PyPy wheels, build on an older OS, such as macOS 10.15. To silence this warning, deselect PyPy by adding "pp*-macosx*" to your CIBW_SKIP option. @@ -84,7 +84,7 @@ def get_python_configurations(build_selector: BuildSelector, if any(c.identifier.startswith('cp35') for c in python_configurations): # CPython 3.5 doesn't work on macOS 11 - log.warning(wrap_text(''' + log.warning(unwrap(''' CPython 3.5 is unsupported when building on macOS 11. To build CPython 3.5 wheels, build on an older OS, such as macOS 10.15. To silence this warning, deselect CPython 3.5 by adding "cp35-macosx_x86_64" to your CIBW_SKIP option. @@ -338,14 +338,14 @@ def build(options: BuildOptions) -> None: if machine_arch == 'x86_64': if config.identifier.endswith('_arm64'): - log.warning(wrap_text(''' + log.warning(unwrap(''' While arm64 wheels can be built on x86_64, they cannot be tested. The ability to test the arm64 wheels will be added in a future release of cibuildwheel, once Apple Silicon CI runners are widely available. ''')) testing_archs = [] elif config.identifier.endswith('_universal2'): - log.warning(wrap_text(''' + log.warning(unwrap(''' While universal2 wheels can be built on x86_64, the arm64 part of them cannot currently be tested. The ability to test the arm64 part of a universal2 wheel will be added in a future release of cibuildwheel, once @@ -363,7 +363,7 @@ def build(options: BuildOptions) -> None: testing_archs = ['arm64', 'x86_64'] else: testing_archs = ['arm64'] - + for testing_arch in testing_archs: log.step('Testing wheel...' if testing_arch == machine_arch else f'Testing wheel on {testing_arch}...') diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 3654b51e..5f19431d 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -244,15 +244,13 @@ def detect_ci_provider() -> Optional[CIProvider]: return None -def wrap_text(text: str, max_width: int = 80) -> str: +def unwrap(text: str) -> str: ''' - Formats text, suitable for printing arbitrary long passages to console. + Unwraps multi-line text to a single line ''' # remove initial line indent text = textwrap.dedent(text) # remove leading/trailing whitespace text = text.strip() # remove consecutive whitespace - text = re.sub(r'\s+', ' ', text) - # wrap to max_width - return '\n'.join(textwrap.wrap(text, width=max_width)) + return re.sub(r'\s+', ' ', text) From e29fab0e2b3e58c1431964546c04a93819b88a00 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 5 Jan 2021 19:25:25 +0000 Subject: [PATCH 29/59] Add test for cross-compile testing behaviour --- test/test_macos_archs.py | 42 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/test/test_macos_archs.py b/test/test_macos_archs.py index 990d08e9..a09ab113 100644 --- a/test/test_macos_archs.py +++ b/test/test_macos_archs.py @@ -1,4 +1,5 @@ import pytest +import platform from . import test_projects, utils @@ -15,13 +16,52 @@ def test_cross_compiled_build(tmp_path): basic_project.generate(project_dir) actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ + 'CIBW_BUILD': 'cp39-*', 'CIBW_ARCHS': 'x86_64, universal2, arm64', }) - expected_wheels = ( + all_macos_wheels = ( utils.expected_wheels('spam', '0.1.0', machine_arch='x86_64') + utils.expected_wheels('spam', '0.1.0', machine_arch='arm64') ) + + # only cpython 3.9 + expected_wheels = [w for w in all_macos_wheels if 'cp39' in w] assert set(actual_wheels) == set(expected_wheels) + +@pytest.mark.parametrize('build_universal2', [False, True]) +def test_cross_compiled_test(tmp_path, capfd, build_universal2): + if utils.platform != 'macos': + pytest.skip('this test is only relevant to macos') + if utils.get_macos_version() < (11, 0): + pytest.skip('this test only works on macOS 11 or greater') + + project_dir = tmp_path / 'project' + basic_project.generate(project_dir) + + actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ + 'CIBW_BUILD': 'cp39-*', + 'CIBW_TEST_COMMAND': '''python -c "import platform; print('running tests on ' + platform.machine())"''', + 'CIBW_ARCHS': 'universal2' if build_universal2 else 'x86_64 arm64', + }) + + captured = capfd.readouterr() + + if platform.machine() == 'x86_64': + # ensure that tests were run on only x86_64 + assert 'running tests on x86_64' in captured.out + assert 'running tests on arm64' not in captured.out + if build_universal2: + assert 'While universal2 wheels can be built on x86_64, the arm64 part of them cannot currently be tested' in captured.out + else: + assert 'While arm64 wheels can be built on x86_64, they cannot be tested' in captured.out + elif platform.machine() == 'arm64': + # ensure that tests were run on both x86_64 and arm64 + assert 'running tests on x86_64' in captured.out + assert 'running tests on arm64' in captured.out + + print(actual_wheels) + + # TODO: add a TEST_COMMAND test when using cross-compiling From e880aca1552c1a1e893de01b3279f3b8385d2220 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 6 Jan 2021 21:33:41 +0000 Subject: [PATCH 30/59] Tidy up --- test/test_macos_archs.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/test/test_macos_archs.py b/test/test_macos_archs.py index a09ab113..2fd21535 100644 --- a/test/test_macos_archs.py +++ b/test/test_macos_archs.py @@ -5,6 +5,11 @@ from . import test_projects, utils basic_project = test_projects.new_c_project() +ALL_MACOS_WHEELS = ( + utils.expected_wheels('spam', '0.1.0', machine_arch='x86_64') + + utils.expected_wheels('spam', '0.1.0', machine_arch='arm64') +) + def test_cross_compiled_build(tmp_path): if utils.platform != 'macos': @@ -20,13 +25,7 @@ def test_cross_compiled_build(tmp_path): 'CIBW_ARCHS': 'x86_64, universal2, arm64', }) - all_macos_wheels = ( - utils.expected_wheels('spam', '0.1.0', machine_arch='x86_64') - + utils.expected_wheels('spam', '0.1.0', machine_arch='arm64') - ) - - # only cpython 3.9 - expected_wheels = [w for w in all_macos_wheels if 'cp39' in w] + expected_wheels = [w for w in ALL_MACOS_WHEELS if 'cp39' in w] assert set(actual_wheels) == set(expected_wheels) @@ -61,7 +60,9 @@ def test_cross_compiled_test(tmp_path, capfd, build_universal2): assert 'running tests on x86_64' in captured.out assert 'running tests on arm64' in captured.out - print(actual_wheels) + if build_universal2: + expected_wheels = [w for w in ALL_MACOS_WHEELS if 'cp39' in w and 'universal2' in w] + else: + expected_wheels = [w for w in ALL_MACOS_WHEELS if 'cp39' in w and 'universal2' not in w] - -# TODO: add a TEST_COMMAND test when using cross-compiling + assert set(actual_wheels) == set(expected_wheels) From 07681239e43cb64881989b5c91c43676d38e6b88 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 6 Jan 2021 21:42:34 +0000 Subject: [PATCH 31/59] Fix allowed archs check --- cibuildwheel/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index d9481d0f..bd549a02 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -262,7 +262,7 @@ PRETTY_NAMES = {'linux': 'Linux', 'macos': 'macOS', 'windows': 'Windows'} ALLOWED_ARCHITECTURES = { 'linux': {Architecture.x86_64, Architecture.i686, Architecture.aarch64, Architecture.ppc64le, Architecture.s390x}, - 'macos': {Architecture.x86_64}, + 'macos': {Architecture.x86_64, Architecture.universal2, Architecture.arm64}, 'windows': {Architecture.AMD64, Architecture.x86}, } From 1606b797496d974763e6bfd952109bbdfada2f58 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 8 Jan 2021 14:28:39 +0000 Subject: [PATCH 32/59] Fix macOS Big Sur version checks --- cibuildwheel/macos.py | 5 ++++- test/test_macos_archs.py | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 65551668..bf25bb59 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -33,6 +33,7 @@ def get_macos_version() -> Tuple[int, int]: These tuples can be used in comparisons, e.g. (10, 14) <= (11, 0) == True + (10, 14) <= (10, 16) == True (11, 2) <= (11, 0) != True ''' version_str, _, _ = platform.mac_ver() @@ -71,7 +72,9 @@ def get_python_configurations(build_selector: BuildSelector, # skip builds as required by BUILD/SKIP python_configurations = [c for c in python_configurations if build_selector(c.identifier)] - if get_macos_version() >= (11, 0): + # When running on macOS 11 in x86_64 mode, the reported OS is the + # non-existent '10.16'. + if get_macos_version() >= (10, 16): if any(c.identifier.startswith('pp') for c in python_configurations): # pypy doesn't work on macOS 11 yet # See https://foss.heptapod.net/pypy/pypy/-/issues/3314 diff --git a/test/test_macos_archs.py b/test/test_macos_archs.py index 2fd21535..5cec82c2 100644 --- a/test/test_macos_archs.py +++ b/test/test_macos_archs.py @@ -14,7 +14,7 @@ ALL_MACOS_WHEELS = ( def test_cross_compiled_build(tmp_path): if utils.platform != 'macos': pytest.skip('this test is only relevant to macos') - if utils.get_macos_version() < (11, 0): + if utils.get_macos_version() < (10, 16): pytest.skip('this test only works on macOS 11 or greater') project_dir = tmp_path / 'project' @@ -33,7 +33,7 @@ def test_cross_compiled_build(tmp_path): def test_cross_compiled_test(tmp_path, capfd, build_universal2): if utils.platform != 'macos': pytest.skip('this test is only relevant to macos') - if utils.get_macos_version() < (11, 0): + if utils.get_macos_version() < (10, 16): pytest.skip('this test only works on macOS 11 or greater') project_dir = tmp_path / 'project' From c4b5b167ca23cbdc4d5e88a6b0a835b3ba03194a Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 8 Jan 2021 14:29:07 +0000 Subject: [PATCH 33/59] Add clarifying comments --- cibuildwheel/macos.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index bf25bb59..80eed9fc 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -228,9 +228,13 @@ def setup_python(python_configuration: PythonConfiguration, if python_configuration.version == '3.9': if python_configuration.identifier.endswith('x86_64'): + # even on the macos11.0 Python installer, on the x86_64 side it's + # compatible back to 10.9. env.setdefault('_PYTHON_HOST_PLATFORM', 'macosx-10.9-x86_64') env.setdefault('ARCHFLAGS', '-arch x86_64') elif python_configuration.identifier.endswith('arm64'): + # macOS 11 is the first OS with arm64 support, so the wheels + # have that as a minimum. env.setdefault('_PYTHON_HOST_PLATFORM', 'macosx-11.0-arm64') env.setdefault('ARCHFLAGS', '-arch arm64') From f43b3f50c5707be11d58f4f33dfb6e07392cffa2 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 8 Jan 2021 14:57:15 +0000 Subject: [PATCH 34/59] Fix warning lines counting as build identifiers in test --- test/utils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/utils.py b/test/utils.py index 5be952d0..2c1025b4 100644 --- a/test/utils.py +++ b/test/utils.py @@ -37,7 +37,10 @@ def cibuildwheel_get_build_identifiers(project_path, env=None): env=env, ) - return cmd_output.strip().split('\n') + lines = cmd_output.strip().split('\n') + + # ignore warning lines in output + return [l for l in lines if l != '' and not l.startswith('Warning ')] def cibuildwheel_run(project_path, package_dir='.', env=None, add_env=None, output_dir=None): From 8512e69733794caced2640bcb4193a7ce5dbb10d Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 8 Jan 2021 14:59:16 +0000 Subject: [PATCH 35/59] Improve expected_wheels correctness --- test/utils.py | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/test/utils.py b/test/utils.py index 2c1025b4..369e95a7 100644 --- a/test/utils.py +++ b/test/utils.py @@ -76,6 +76,19 @@ def cibuildwheel_run(project_path, package_dir='.', env=None, add_env=None, outp return wheels +def _get_arm64_macosx_deployment_target(macosx_deployment_target: str) -> str: + ''' + The first version of macOS that supports arm is 11.0. So the wheel tag + cannot contain an earlier deployment target, even if + MACOSX_DEPLOYMENT_TARGET sets it. + ''' + version_tuple = tuple(map(int, macosx_deployment_target.split('.'))) + if version_tuple <= (11, 0): + return '11.0' + else: + return macosx_deployment_target + + def expected_wheels(package_name, package_version, manylinux_versions=None, macosx_deployment_target='10.9', machine_arch=None): ''' @@ -103,12 +116,17 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, if platform == 'linux': python_abi_tags.append('cp27-cp27mu') # python 2.7 has 2 different ABI on manylinux - if platform == 'macos' and get_macos_version() >= (11, 0): + if platform == 'macos' and get_macos_version() >= (10, 16): # CPython 3.5 doesn't work on macOS 11. + # 10.16 is sometimes reported as the macOS version on macOS 11. python_abi_tags.remove('cp35-cp35m') # pypy not supported on macOS 11. python_abi_tags = [t for t in python_abi_tags if not t.startswith('pp')] + if platform == 'macos' and machine_arch == 'arm64': + # currently, arm64 macs are only supported by cp39 + python_abi_tags = ['cp39-cp39'] + wheels = [] for python_abi_tag in python_abi_tags: @@ -133,18 +151,12 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, platform_tags = ['win32'] elif platform == 'macos': - if python_abi_tag == 'cp39-cp39': - if machine_arch == 'x86_64': - platform_tags = [ - f'macosx_{macosx_deployment_target.replace(".", "_")}_x86_64', - ] - elif machine_arch == 'arm64': - platform_tags = [ - f'macosx_{macosx_deployment_target.replace(".", "_")}_universal2.macosx_11_0_universal2', - # macosx_deployment_target is ignored on arm64, because arm64 isn't supported on - # macOS earlier than 11.0 - 'macosx_11_0_arm64', - ] + if python_abi_tag == 'cp39-cp39' and machine_arch == 'arm64': + arm64_macosx_deployment_target = _get_arm64_macosx_deployment_target(macosx_deployment_target) + platform_tags = [ + f'macosx_{macosx_deployment_target.replace(".", "_")}_universal2.macosx_{arm64_macosx_deployment_target.replace(".", "_")}_universal2', + f'macosx_{arm64_macosx_deployment_target.replace(".", "_")}_arm64', + ] else: platform_tags = [ f'macosx_{macosx_deployment_target.replace(".", "_")}_x86_64', From a99726f477744230c5fe1f6a18a0f3cc2d17a298 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 8 Jan 2021 15:18:37 +0000 Subject: [PATCH 36/59] Neaten comments --- cibuildwheel/macos.py | 4 ++-- test/utils.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 23304844..7aed4c1a 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -81,8 +81,8 @@ def get_python_configurations(build_selector: BuildSelector, # skip builds as required by BUILD/SKIP python_configurations = [c for c in python_configurations if build_selector(c.identifier)] - # When running on macOS 11 in x86_64 mode, the reported OS is the - # non-existent '10.16'. + # When running on macOS 11 and x86_64, the reported OS is '10.16', but + # there is no such OS - it really means macOS 11. if get_macos_version() >= (10, 16): if any(c.identifier.startswith('pp') for c in python_configurations): # pypy doesn't work on macOS 11 yet diff --git a/test/utils.py b/test/utils.py index 369e95a7..e9b48e19 100644 --- a/test/utils.py +++ b/test/utils.py @@ -117,8 +117,8 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, python_abi_tags.append('cp27-cp27mu') # python 2.7 has 2 different ABI on manylinux if platform == 'macos' and get_macos_version() >= (10, 16): - # CPython 3.5 doesn't work on macOS 11. # 10.16 is sometimes reported as the macOS version on macOS 11. + # CPython 3.5 doesn't work on macOS 11. python_abi_tags.remove('cp35-cp35m') # pypy not supported on macOS 11. python_abi_tags = [t for t in python_abi_tags if not t.startswith('pp')] From dbb6d80a7270b0f2c358a94074399ee4ff6b529e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 8 Jan 2021 17:25:07 +0000 Subject: [PATCH 37/59] Route warnings and error messages to stderr, not stdout. --- cibuildwheel/logger.py | 16 ++++------------ test/utils.py | 5 +---- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/cibuildwheel/logger.py b/cibuildwheel/logger.py index ee98feec..60e1b3c2 100644 --- a/cibuildwheel/logger.py +++ b/cibuildwheel/logger.py @@ -116,26 +116,18 @@ class Logger: self.error(error) def warning(self, message: str) -> None: - print() - if self.fold_mode == 'github': - print(f'::warning::{message}') + print(f'::warning::{message}\n', file=sys.stderr) else: c = self.colors - print(f'{c.yellow}Warning{c.end} {message}') - - print() + print(f'{c.yellow}Warning{c.end}: {message}\n', file=sys.stderr) def error(self, error: Union[BaseException, str]) -> None: - print() - if self.fold_mode == 'github': - print(f'::error::{error}') + print(f'::error::{error}\n', file=sys.stderr) else: c = self.colors - print(f'{c.bright_red}Error{c.end} {error}') - - print() + print(f'{c.bright_red}Error{c.end}: {error}\n', file=sys.stderr) def _start_fold_group(self, name: str) -> None: self._end_fold_group() diff --git a/test/utils.py b/test/utils.py index e9b48e19..1539e5ca 100644 --- a/test/utils.py +++ b/test/utils.py @@ -37,10 +37,7 @@ def cibuildwheel_get_build_identifiers(project_path, env=None): env=env, ) - lines = cmd_output.strip().split('\n') - - # ignore warning lines in output - return [l for l in lines if l != '' and not l.startswith('Warning ')] + return cmd_output.strip().split('\n') def cibuildwheel_run(project_path, package_dir='.', env=None, add_env=None, output_dir=None): From af8ab0af2bedd2a78eed6424d67bcf2d4d6b6586 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 9 Jan 2021 01:20:49 +0000 Subject: [PATCH 38/59] Update test to look in stderr for warnings --- test/test_macos_archs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_macos_archs.py b/test/test_macos_archs.py index 590eb5b9..7037a97e 100644 --- a/test/test_macos_archs.py +++ b/test/test_macos_archs.py @@ -53,9 +53,9 @@ def test_cross_compiled_test(tmp_path, capfd, build_universal2): assert 'running tests on x86_64' in captured.out assert 'running tests on arm64' not in captured.out if build_universal2: - assert 'While universal2 wheels can be built on x86_64, the arm64 part of them cannot currently be tested' in captured.out + assert 'While universal2 wheels can be built on x86_64, the arm64 part of them cannot currently be tested' in captured.err else: - assert 'While arm64 wheels can be built on x86_64, they cannot be tested' in captured.out + assert 'While arm64 wheels can be built on x86_64, they cannot be tested' in captured.err elif platform.machine() == 'arm64': # ensure that tests were run on both x86_64 and arm64 assert 'running tests on x86_64' in captured.out From e83b7709296a7f5dc55c869aeea880da2a0e8573 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 9 Jan 2021 13:41:02 +0000 Subject: [PATCH 39/59] Add back other platforms to GHA CI --- .github/workflows/test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 161206c4..f3de75af 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -24,8 +24,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - # os: [ubuntu-latest, windows-latest, macos-11.0] - os: [macos-11.0] + os: [ubuntu-latest, windows-latest, macos-11.0] python_version: ['3.7'] timeout-minutes: 180 steps: From 3f8fcb01ad604d871bcff7db73410c412bc1e800 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 23 Jan 2021 18:15:29 +0000 Subject: [PATCH 40/59] Fix unit tests --- unit_test/main_tests/conftest.py | 3 +++ unit_test/main_tests/main_platform_test.py | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/unit_test/main_tests/conftest.py b/unit_test/main_tests/conftest.py index 3ce025a0..2a0f9bfd 100644 --- a/unit_test/main_tests/conftest.py +++ b/unit_test/main_tests/conftest.py @@ -73,6 +73,9 @@ def platform(request, monkeypatch): else: monkeypatch.setattr(platform_module, 'machine', lambda: 'x86_64') + if platform_value == 'macos': + monkeypatch.setattr(macos, 'get_macos_version', lambda: (11, 1)) + return platform_value diff --git a/unit_test/main_tests/main_platform_test.py b/unit_test/main_tests/main_platform_test.py index a06ffe68..f86de540 100644 --- a/unit_test/main_tests/main_platform_test.py +++ b/unit_test/main_tests/main_platform_test.py @@ -141,4 +141,4 @@ def test_archs_platform_all(platform, intercepted_build_args, monkeypatch): elif platform == 'windows': assert build_options.architectures == {Architecture.x86, Architecture.AMD64} elif platform == 'macos': - assert build_options.architectures == {Architecture.x86_64} + assert build_options.architectures == {Architecture.x86_64, Architecture.arm64, Architecture.universal2} From bd45b1c9011bfdd32119ae39f4c4b7d8c3a1f3d2 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 24 Jan 2021 14:46:46 +0000 Subject: [PATCH 41/59] Remove pinned pip install, virtualenv should bundle this now --- cibuildwheel/macos.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 8635f8ea..3315bd26 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -400,12 +400,6 @@ def build(options: BuildOptions) -> None: # check that we are using the Python from the virtual environment call_with_arch(['which', 'python'], env=virtualenv_env) - # ensure we have a recent enough pip in the virtualenv - # TODO(joerick): remove this before merge? virtualenv should be bundling a - # recent enough seed version. Waiting on this PR: - # https://github.com/pypa/virtualenv/pull/2036 - call_with_arch(['pip', 'install', 'pip>=20.3.3'], env=virtualenv_env) - if options.before_test: before_test_prepared = prepare_command(options.before_test, project='.', package=options.package_dir) call_with_arch(before_test_prepared, env=virtualenv_env, shell=True) From 00eed5aac56c1250fcdf4155b97d2a0e9a4c5dea Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 24 Jan 2021 14:50:47 +0000 Subject: [PATCH 42/59] Fix crashing test --- test/test_dependency_versions.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/test/test_dependency_versions.py b/test/test_dependency_versions.py index 1b415bb4..72c10597 100644 --- a/test/test_dependency_versions.py +++ b/test/test_dependency_versions.py @@ -55,6 +55,13 @@ def test_pinned_versions(tmp_path, python_version): if utils.IS_WINDOWS_RUNNING_ON_TRAVIS and python_version == '2.7': pytest.skip('Windows + Travis CI requires a workaround') + is_running_on_macos_11_or_later = ( + utils.platform == 'macos' and utils.get_macos_version() >= (10, 16) + ) + + if is_running_on_macos_11_or_later and python_version == '3.5': + pytest.skip('CPython 3.5 doesn\'t work on macOS Big Sur+') + project_dir = tmp_path / 'project' project_with_expected_version_checks.generate(project_dir) From 4dddd9e6481c96488f5640f9ed9d988054cfbc37 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 24 Jan 2021 15:08:46 +0000 Subject: [PATCH 43/59] Silence the pip version warnings --- cibuildwheel/linux.py | 1 + cibuildwheel/macos.py | 3 +++ cibuildwheel/windows.py | 2 ++ 3 files changed, 6 insertions(+) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index bde5e34f..5d19dbd2 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -94,6 +94,7 @@ def build(options: BuildOptions) -> None: env = docker.get_environment() env['PATH'] = f'/opt/python/cp38-cp38/bin:{env["PATH"]}' + env['PIP_DISABLE_PIP_VERSION_CHECK'] = '1' env = options.environment.as_dictionary(env, executor=docker.environment_executor) before_all_prepared = prepare_command(options.before_all, project=container_project_path, package=container_package_dir) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 3315bd26..dd1a8743 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -195,6 +195,9 @@ def setup_python(python_configuration: PythonConfiguration, env.pop('__PYVENV_LAUNCHER__', None) env = environment.as_dictionary(prev_environment=env) + # we version pip ourselves, so we don't care about pip version checking + env['PIP_DISABLE_PIP_VERSION_CHECK'] = '1' + # check what version we're on call(['which', 'python'], env=env) call(['python', '--version'], env=env) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 429150eb..45f0141a 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -143,6 +143,8 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain str(installation_path / 'Scripts'), env['PATH'] ]) + env['PIP_DISABLE_PIP_VERSION_CHECK'] = '1' + # update env with results from CIBW_ENVIRONMENT env = environment.as_dictionary(prev_environment=env) From 381ea7dbad70f5767068fbc26485d7d0547c5e34 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 24 Jan 2021 15:48:22 +0000 Subject: [PATCH 44/59] Add docs for CIBW_ARCHS --- docs/options.md | 54 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/docs/options.md b/docs/options.md index 3a1c0cd8..857df08f 100644 --- a/docs/options.md +++ b/docs/options.md @@ -64,7 +64,7 @@ Default: `auto` For `linux` you need Docker running, on macOS or Linux. For `macos`, you need a Mac machine, and note that this script is going to automatically install MacPython on your system, so don't run on your development machine. For `windows`, you need to run in Windows, and `cibuildwheel` will install required versions of Python to `C:\cibw\python` using NuGet. -This option can also be set using the command-line option `--platform`. +This option can also be set using the [command-line option](#command-line) `--platform`. ### `CIBW_BUILD`, `CIBW_SKIP` {: #build-skip} @@ -155,24 +155,36 @@ CIBW_SKIP: pp* } -### `CIBW_ARCHS_LINUX` {: #archs} -> Build non-native architectures +### `CIBW_ARCHS` {: #archs} +> Change the architectures built on your machine by default. -A space-separated list of architectures to build. Use this in conjunction with -emulation, such as that provided by [docker/setup-qemu-action][setup-qemu-action] -or [tonistiigi/binfmt][binfmt], to build architectures other than those your -machine natively supports. +A space-separated list of architectures to build. -Options: `auto` `native` `all` `x86_64` `i686` `aarch64` `ppc64le` `s390x` +On macOS, this option can be used to cross-compile between `x86_64`, +`universal2` and `arm64` for Apple Silicon support. -Default: `auto`, meaning the native archs supported on the build machine. For -example, on an `x86_64` machine, `auto` expands to `x86_64` and `i686`. +On Linux, this option can be used to build non-native architectures under emulation, such as that +provided by [docker/setup-qemu-action][setup-qemu-action] +or [tonistiigi/binfmt][binfmt]. See [this guide](faq.md#automatic-updates) for more information. -`native` will only build on the exact architecture you currently are on; it will -not add `i686` for `x86_64`. +Options: +- Linux: `x86_64` `i686` `aarch64` `ppc64le` `s390x` +- macOS: `x86_64` `arm64` `universal2` +- Windows: `x86` `AMD64` +- `auto`: The default archs for your machine - see the table below. +- `native`: the native arch of the build machine - Matches [`platform.machine()`](https://docs.python.org/3/library/platform.html#platform.machine). +- `all` : expands to all the architectures supported on this OS. You may want + to use [CIBW_BUILD](#build-skip) with this option to target specific + architectures via build selectors. -`all` will expand to all known architectures; remember to use build selectors -to limit builds for each job; this list could grow in the future. +Default: `auto` + +| Runner | `native` | `auto` +|---|---|---|--- +| macOS / Intel | `x86_64` | `x86_64` +| macOS / Apple Silicon | `arm64` | `arm64`, `universal2` +| Linux / Intel | `x86_64` | `x86_64`, `i686` +| Windows / Intel | `AMD64` | `AMD64`, `x86` [setup-qemu-action]: https://github.com/docker/setup-qemu-action [binfmt]: https://hub.docker.com/r/tonistiigi/binfmt @@ -180,10 +192,20 @@ to limit builds for each job; this list could grow in the future. #### Examples ```yaml -# On an intel runner with qemu installed, build Intel and ARM wheels +# Build `universal2` and `arm64` wheels on an Intel runner. +# Note that the `arm64` wheel and the `arm64` part of the `universal2` wheel +# cannot be tested in this configuration. +CIBW_ARCHS_MACOS: "x86_64 universal2 arm64" + +# On an Linux Intel runner with qemu installed, build Intel and ARM wheels CIBW_ARCHS_LINUX: "auto aarch64" ``` +Platform-specific variants also available:
+`CIBW_ARCHS_MACOS` | `CIBW_ARCHS_WINDOWS` | `CIBW_ARCHS_LINUX` + +This option can also be set using the [command-line option](#command-line) `--archs`. + ## Build customization ### `CIBW_ENVIRONMENT` {: #environment} @@ -548,7 +570,7 @@ CIBW_BUILD_VERBOSITY: 1 ``` -## Command line options +## Command line options {: #command-line} ```text usage: cibuildwheel [-h] [--platform {auto,linux,macos,windows}] From 4b491ceb4a1f448594732116522462367e40a11e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 24 Jan 2021 16:07:59 +0000 Subject: [PATCH 45/59] Docs edits --- docs/options.md | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/docs/options.md b/docs/options.md index 857df08f..2058ee40 100644 --- a/docs/options.md +++ b/docs/options.md @@ -163,14 +163,14 @@ A space-separated list of architectures to build. On macOS, this option can be used to cross-compile between `x86_64`, `universal2` and `arm64` for Apple Silicon support. -On Linux, this option can be used to build non-native architectures under emulation, such as that -provided by [docker/setup-qemu-action][setup-qemu-action] -or [tonistiigi/binfmt][binfmt]. See [this guide](faq.md#automatic-updates) for more information. +On Linux, this option can be used to build non-native architectures under +emulation. See [this guide](faq.md#emulation) for more information. Options: + - Linux: `x86_64` `i686` `aarch64` `ppc64le` `s390x` - macOS: `x86_64` `arm64` `universal2` -- Windows: `x86` `AMD64` +- Windows: `AMD64` `x86` - `auto`: The default archs for your machine - see the table below. - `native`: the native arch of the build machine - Matches [`platform.machine()`](https://docs.python.org/3/library/platform.html#platform.machine). - `all` : expands to all the architectures supported on this OS. You may want @@ -180,11 +180,11 @@ Options: Default: `auto` | Runner | `native` | `auto` -|---|---|---|--- -| macOS / Intel | `x86_64` | `x86_64` -| macOS / Apple Silicon | `arm64` | `arm64`, `universal2` -| Linux / Intel | `x86_64` | `x86_64`, `i686` -| Windows / Intel | `AMD64` | `AMD64`, `x86` +|---|---|--- +| Linux / Intel | `x86_64` | `x86_64` `i686` +| Windows / Intel | `AMD64` | `AMD64` `x86` +| macOS / Intel | `x86_64` | `x86_64` +| macOS / Apple Silicon | `arm64` | `arm64` `universal2` [setup-qemu-action]: https://github.com/docker/setup-qemu-action [binfmt]: https://hub.docker.com/r/tonistiigi/binfmt @@ -193,8 +193,8 @@ Default: `auto` ```yaml # Build `universal2` and `arm64` wheels on an Intel runner. -# Note that the `arm64` wheel and the `arm64` part of the `universal2` wheel -# cannot be tested in this configuration. +# Note that the `arm64` wheel and the `arm64` part of the `universal2` +# wheel cannot be tested in this configuration. CIBW_ARCHS_MACOS: "x86_64 universal2 arm64" # On an Linux Intel runner with qemu installed, build Intel and ARM wheels From ef1baac8e24246f53d2a59a1964f10fc197aa478 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 24 Jan 2021 16:12:30 +0000 Subject: [PATCH 46/59] Indent subheads in TOC on all pages --- docs/extra.css | 4 ++++ docs/options.md | 4 ---- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docs/extra.css b/docs/extra.css index 0a4dad6b..e7d035ae 100644 --- a/docs/extra.css +++ b/docs/extra.css @@ -83,5 +83,9 @@ h1, h2, h3, h4, h5, h6 { padding-top: 0.9em; } +.toctree-l3 { + border-left: 10px solid transparent; +} + /* import font awesome 4 for icons */ @import url(https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css); diff --git a/docs/options.md b/docs/options.md index 2058ee40..1b80c58f 100644 --- a/docs/options.md +++ b/docs/options.md @@ -614,10 +614,6 @@ optional arguments: ```