Merge pull request #484 from joerick/macos-universal2

Universal2 wheels on macOS

.github/workflows/test.yml

@@ -52,6 +52,18 @@ jobs:
       run: |
         python bin/sample_build.py
 
+    - name: Get some sample wheels
+      run: |
+        python -m test.test_projects test.test_0_basic.basic_project sample_proj
+        cibuildwheel --output-dir wheelhouse sample_proj
+      env:
+        CIBW_ARCHS_MACOS: x86_64 universal2 arm64
+
+    - uses: actions/upload-artifact@v2
+      with:
+        name: sample_wheels
+        path: wheelhouse
+
     - name: Test cibuildwheel
       run: |
         python ./bin/run_tests.py

bin/run_tests.py

@@ -17,4 +17,4 @@ if __name__ == '__main__':
     subprocess.check_call(unit_test_args)
 
     # run the integration tests
-    subprocess.check_call([sys.executable, '-m', 'pytest', '-x', '--durations', '0', '--timeout=1200', 'test'])
+    subprocess.check_call([sys.executable, '-m', 'pytest', '-x', '--durations', '0', '--timeout=2400', 'test'])

bin/sample_build.py

@@ -22,4 +22,4 @@ if __name__ == '__main__':
         options.project_python_path, project_dir
     ], check=True)
 
-    sys.exit(subprocess.run(['cibuildwheel'], cwd=project_dir).returncode)
+    sys.exit(subprocess.run([sys.executable, '-m', 'cibuildwheel'], cwd=project_dir).returncode)

cibuildwheel/__main__.py

@@ -137,7 +137,7 @@ def main() -> None:
     if platform == 'linux':
         repair_command_default = 'auditwheel repair -w {dest_dir} {wheel}'
     elif platform == 'macos':
-        repair_command_default = 'delocate-listdeps {wheel} && delocate-wheel --require-archs x86_64 -w {dest_dir} {wheel}'
+        repair_command_default = 'delocate-listdeps {wheel} && delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}'
     elif platform == 'windows':
         repair_command_default = ''
     else:
@@ -339,7 +339,7 @@ def get_build_identifiers(
     elif platform == 'windows':
         python_configurations = cibuildwheel.windows.get_python_configurations(build_selector, architectures)
     elif platform == 'macos':
-        python_configurations = cibuildwheel.macos.get_python_configurations(build_selector)
+        python_configurations = cibuildwheel.macos.get_python_configurations(build_selector, architectures)
     else:
         assert_never(platform)
 

cibuildwheel/architecture.py

@@ -15,11 +15,17 @@ class Architecture(Enum):
 
     # mac/linux archs
     x86_64 = 'x86_64'
+
+    # linux archs
     i686 = 'i686'
     aarch64 = 'aarch64'
     ppc64le = 'ppc64le'
     s390x = 's390x'
 
+    # mac archs
+    universal2 = 'universal2'
+    arm64 = 'arm64'
+
     # windows archs
     x86 = 'x86'
     AMD64 = 'AMD64'
@@ -46,11 +52,18 @@ class Architecture(Enum):
     def auto_archs(platform: PlatformName) -> 'Set[Architecture]':
         native_architecture = Architecture(platform_module.machine())
         result = {native_architecture}
+
         if platform == 'linux' and native_architecture == Architecture.x86_64:
             # x86_64 machines can run i686 docker containers
             result.add(Architecture.i686)
+
         if platform == 'windows' and native_architecture == Architecture.AMD64:
             result.add(Architecture.x86)
+
+        if platform == 'macos' and native_architecture == Architecture.arm64:
+            # arm64 can build and test both archs of a universal2 wheel.
+            result.add(Architecture.universal2)
+
         return result
 
     @staticmethod
@@ -58,7 +71,7 @@ class Architecture(Enum):
         if platform == 'linux':
             return {Architecture.x86_64, Architecture.i686, Architecture.aarch64, Architecture.ppc64le, Architecture.s390x}
         elif platform == 'macos':
-            return {Architecture.x86_64}
+            return {Architecture.x86_64, Architecture.arm64, Architecture.universal2}
         elif platform == 'windows':
             return {Architecture.x86, Architecture.AMD64}
         else:

cibuildwheel/linux.py

@@ -94,6 +94,7 @@ def build(options: BuildOptions) -> None:
 
                     env = docker.get_environment()
                     env['PATH'] = f'/opt/python/cp38-cp38/bin:{env["PATH"]}'
+                    env['PIP_DISABLE_PIP_VERSION_CHECK'] = '1'
                     env = options.environment.as_dictionary(env, executor=docker.environment_executor)
 
                     before_all_prepared = prepare_command(options.before_all, project=container_project_path, package=container_package_dir)
@@ -228,7 +229,7 @@ def build(options: BuildOptions) -> None:
                 docker.copy_out(container_output_dir, options.output_dir)
                 log.step_end()
         except subprocess.CalledProcessError as error:
-            log.error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}')
+            log.step_end_with_error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}')
             troubleshoot(options.package_dir, error)
             sys.exit(1)
 

cibuildwheel/logger.py

@@ -23,6 +23,8 @@ PLATFORM_IDENTIFIER_DESCIPTIONS = {
     'win32': 'Windows 32bit',
     'win_amd64': 'Windows 64bit',
     'macosx_x86_64': 'macOS x86_64',
+    'macosx_universal2': 'macOS Universal 2 - x86_64 and arm64',
+    'macosx_arm64': 'macOS arm64 - Apple Silicon',
 }
 
 
@@ -109,15 +111,23 @@ class Logger:
 
             self.step_start_time = None
 
-    def error(self, error: Union[BaseException, str]) -> None:
+    def step_end_with_error(self, error: Union[BaseException, str]) -> None:
         self.step_end(success=False)
-        print()
+        self.error(error)
 
+    def warning(self, message: str) -> None:
         if self.fold_mode == 'github':
-            print(f'::error::{error}')
+            print(f'::warning::{message}\n', file=sys.stderr)
         else:
             c = self.colors
-            print(f'{c.bright_red}Error{c.end} {error}')
+            print(f'{c.yellow}Warning{c.end}: {message}\n', file=sys.stderr)
+
+    def error(self, error: Union[BaseException, str]) -> None:
+        if self.fold_mode == 'github':
+            print(f'::error::{error}\n', file=sys.stderr)
+        else:
+            c = self.colors
+            print(f'{c.bright_red}Error{c.end}: {error}\n', file=sys.stderr)
 
     def _start_fold_group(self, name: str) -> None:
         self._end_fold_group()

cibuildwheel/macos.py

@@ -1,15 +1,18 @@
 import os
+import platform
+import re
 import shlex
 import shutil
 import subprocess
 import sys
 import tempfile
 from pathlib import Path
-from typing import Dict, List, NamedTuple, Optional, Sequence
+from typing import Any, Dict, List, NamedTuple, Optional, Sequence, Set, Tuple, cast
 
+from .architecture import Architecture
 from .environment import ParsedEnvironment
 from .logger import log
-from .typing import PathOrStr
+from .typing import Literal, PathOrStr
 from .util import (
     BuildOptions,
     BuildSelector,
@@ -21,6 +24,7 @@ from .util import (
     prepare_command,
     read_python_configs,
     resources_dir,
+    unwrap,
 )
 
 
@@ -34,21 +38,72 @@ def call(args: Sequence[PathOrStr], env: Optional[Dict[str, str]] = None, cwd: O
     return subprocess.check_call(args, env=env, cwd=cwd, shell=shell)
 
 
+def get_macos_version() -> Tuple[int, int]:
+    '''
+    Returns the macOS major/minor version, as a tuple, e.g. (10, 15) or (11, 0)
+
+    These tuples can be used in comparisons, e.g.
+        (10, 14) <= (11, 0) == True
+        (10, 14) <= (10, 16) == True
+        (11, 2) <= (11, 0) != True
+    '''
+    version_str, _, _ = platform.mac_ver()
+    version = tuple(map(int, version_str.split(".")[:2]))
+    return cast(Tuple[int, int], version)
+
+
+def get_macos_sdks() -> List[str]:
+    output = subprocess.check_output(
+        ['xcodebuild', '-showsdks'],
+        universal_newlines=True,
+    )
+
+    return [m.group(1) for m in re.finditer(r'-sdk (macosx\S+)', output)]
+
+
 class PythonConfiguration(NamedTuple):
     version: str
     identifier: str
     url: str
 
 
-def get_python_configurations(
-        build_selector: BuildSelector) -> List[PythonConfiguration]:
+def get_python_configurations(build_selector: BuildSelector,
+                              architectures: Set[Architecture]) -> List[PythonConfiguration]:
 
     full_python_configs = read_python_configs('macos')
 
     python_configurations = [PythonConfiguration(**item) for item in full_python_configs]
 
-    # skip builds as required
-    return [c for c in python_configurations if build_selector(c.identifier)]
+    # filter out configs that don't match any of the selected architectures
+    python_configurations = [c for c in python_configurations
+                             if any(c.identifier.endswith(a.value) for a in architectures)]
+
+    # skip builds as required by BUILD/SKIP
+    python_configurations = [c for c in python_configurations if build_selector(c.identifier)]
+
+    # When running on macOS 11 and x86_64, the reported OS is '10.16', but
+    # there is no such OS - it really means macOS 11.
+    if get_macos_version() >= (10, 16):
+        if any(c.identifier.startswith('pp') for c in python_configurations):
+            # pypy doesn't work on macOS 11 yet
+            # See https://foss.heptapod.net/pypy/pypy/-/issues/3314
+            log.warning(unwrap('''
+                PyPy is currently unsupported when building on macOS 11. To build macOS PyPy wheels,
+                build on an older OS, such as macOS 10.15. To silence this warning, deselect PyPy by
+                adding "pp*-macosx*" to your CIBW_SKIP option.
+            '''))
+            python_configurations = [c for c in python_configurations if not c.identifier.startswith('pp')]
+
+        if any(c.identifier.startswith('cp35') for c in python_configurations):
+            # CPython 3.5 doesn't work on macOS 11
+            log.warning(unwrap('''
+                CPython 3.5 is unsupported when building on macOS 11. To build CPython 3.5 wheels,
+                build on an older OS, such as macOS 10.15. To silence this warning, deselect CPython
+                3.5 by adding "cp35-macosx_x86_64" to your CIBW_SKIP option.
+            '''))
+            python_configurations = [c for c in python_configurations if not c.identifier.startswith('cp35')]
+
+    return python_configurations
 
 
 SYMLINKS_DIR = Path('/tmp/cibw_bin')
@@ -150,6 +205,9 @@ def setup_python(python_configuration: PythonConfiguration,
     env.pop('__PYVENV_LAUNCHER__', None)
     env = environment.as_dictionary(prev_environment=env)
 
+    # we version pip ourselves, so we don't care about pip version checking
+    env['PIP_DISABLE_PIP_VERSION_CHECK'] = '1'
+
     # check what version we're on
     call(['which', 'python'], env=env)
     call(['python', '--version'], env=env)
@@ -180,6 +238,41 @@ def setup_python(python_configuration: PythonConfiguration,
         # https://github.com/python/cpython/blob/a5ed2fe0eedefa1649aa93ee74a0bafc8e628a10/Lib/_osx_support.py#L260
         env.setdefault('ARCHFLAGS', '-arch x86_64')
 
+    if python_configuration.version == '3.9':
+        if python_configuration.identifier.endswith('x86_64'):
+            # even on the macos11.0 Python installer, on the x86_64 side it's
+            # compatible back to 10.9.
+            env.setdefault('_PYTHON_HOST_PLATFORM', 'macosx-10.9-x86_64')
+            env.setdefault('ARCHFLAGS', '-arch x86_64')
+        elif python_configuration.identifier.endswith('arm64'):
+            # macOS 11 is the first OS with arm64 support, so the wheels
+            # have that as a minimum.
+            env.setdefault('_PYTHON_HOST_PLATFORM', 'macosx-11.0-arm64')
+            env.setdefault('ARCHFLAGS', '-arch arm64')
+        elif python_configuration.identifier.endswith('universal2'):
+            if get_macos_version() < (10, 16):
+                # we can do universal2 builds on macos 10.15, but we need to
+                # set ARCHFLAGS otherwise CPython sets it to `-arch x86_64`
+                env.setdefault('ARCHFLAGS', '-arch arm64 -arch x86_64')
+
+    if python_configuration.identifier.endswith('arm64') or python_configuration.identifier.endswith('universal2'):
+        if get_macos_version() < (10, 16) and 'SDKROOT' not in env:
+            # xcode 12.2 or higher can build arm64 on macos 10.15 or below, but
+            # needs the correct SDK selected.
+            sdks = get_macos_sdks()
+
+            # Different versions of Xcode contain different SDK versions...
+            # we're happy with anything newer than macOS 11.0
+            arm64_compatible_sdks = [s for s in sdks if not s.startswith('macosx10.')]
+
+            if not arm64_compatible_sdks:
+                log.warning(unwrap('''
+                    SDK for building arm64-compatible wheels not found. You need Xcode 12.2 or later
+                    to build universal2 or arm64 wheels.
+                '''))
+            else:
+                env.setdefault('SDKROOT', arm64_compatible_sdks[0])
+
     log.step('Installing build tools...')
     call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate', *dependency_constraint_flags], env=env)
 
@@ -198,7 +291,7 @@ def build(options: BuildOptions) -> None:
             before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir)
             call([before_all_prepared], shell=True, env=env)
 
-        python_configurations = get_python_configurations(options.build_selector)
+        python_configurations = get_python_configurations(options.build_selector, options.architectures)
 
         for config in python_configurations:
             log.build_start(config.identifier)
@@ -242,60 +335,130 @@ def build(options: BuildOptions) -> None:
 
             if options.repair_command:
                 log.step('Repairing wheel...')
-                repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir)
+
+                if config.identifier.endswith('universal2'):
+                    delocate_archs = 'x86_64,arm64'
+                elif config.identifier.endswith('arm64'):
+                    delocate_archs = 'arm64'
+                else:
+                    delocate_archs = 'x86_64'
+
+                repair_command_prepared = prepare_command(
+                    options.repair_command,
+                    wheel=built_wheel,
+                    dest_dir=repaired_wheel_dir,
+                    delocate_archs=delocate_archs,
+                )
                 call(repair_command_prepared, env=env, shell=True)
             else:
                 shutil.move(str(built_wheel), repaired_wheel_dir)
 
             repaired_wheel = next(repaired_wheel_dir.glob('*.whl'))
 
+            log.step_end()
+
             if options.test_command and options.test_selector(config.identifier):
-                log.step('Testing wheel...')
-                # set up a virtual environment to install and test from, to make sure
-                # there are no dependencies that were pulled in at build time.
-                call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env)
-                venv_dir = Path(tempfile.mkdtemp())
+                machine_arch = platform.machine()
+                testing_archs: List[Literal['x86_64', 'arm64']] = []
 
-                # Use --no-download to ensure determinism by using seed libraries
-                # built into virtualenv
-                call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env)
+                if config.identifier.endswith('_arm64'):
+                    testing_archs = ['arm64']
+                elif config.identifier.endswith('_universal2'):
+                    testing_archs = ['x86_64', 'arm64']
+                else:
+                    testing_archs = ['x86_64']
 
-                virtualenv_env = env.copy()
-                virtualenv_env['PATH'] = os.pathsep.join([
-                    str(venv_dir / 'bin'),
-                    virtualenv_env['PATH'],
-                ])
+                for testing_arch in testing_archs:
+                    if config.identifier.endswith('_universal2'):
+                        arch_specific_identifier = f'{config.identifier}:{testing_arch}'
+                        if not options.test_selector(arch_specific_identifier):
+                            continue
 
-                # check that we are using the Python from the virtual environment
-                call(['which', 'python'], env=virtualenv_env)
+                    if machine_arch == 'x86_64' and testing_arch == 'arm64':
+                        if config.identifier.endswith('_arm64'):
+                            log.warning(unwrap('''
+                                While arm64 wheels can be built on x86_64, they cannot be tested. The
+                                ability to test the arm64 wheels will be added in a future release of
+                                cibuildwheel, once Apple Silicon CI runners are widely available. To
+                                silence this warning, set `CIBW_TEST_SKIP: *-macosx_arm64`.
+                            '''))
+                        elif config.identifier.endswith('_universal2'):
+                            log.warning(unwrap('''
+                                While universal2 wheels can be built on x86_64, the arm64 part of them
+                                cannot currently be tested. The ability to test the arm64 part of a
+                                universal2 wheel will be added in a future release of cibuildwheel, once
+                                Apple Silicon CI runners are widely available. To silence this warning,
+                                set `CIBW_TEST_SKIP: *-macosx_universal2:arm64`.
+                            '''))
+                        else:
+                            raise RuntimeError('unreachable')
 
-                if options.before_test:
-                    before_test_prepared = prepare_command(options.before_test, project='.', package=options.package_dir)
-                    call(before_test_prepared, env=virtualenv_env, shell=True)
+                        # skip this test
+                        continue
 
-                # install the wheel
-                call(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env)
+                    log.step('Testing wheel...' if testing_arch == machine_arch else f'Testing wheel on {testing_arch}...')
 
-                # test the wheel
-                if options.test_requires:
-                    call(['pip', 'install'] + options.test_requires, env=virtualenv_env)
+                    # set up a virtual environment to install and test from, to make sure
+                    # there are no dependencies that were pulled in at build time.
+                    call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env)
+                    venv_dir = Path(tempfile.mkdtemp())
 
-                # run the tests from $HOME, with an absolute path in the command
-                # (this ensures that Python runs the tests against the installed wheel
-                # and not the repo code)
-                test_command_prepared = prepare_command(
-                    options.test_command,
-                    project=Path('.').resolve(),
-                    package=options.package_dir.resolve()
-                )
-                call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True)
+                    arch_prefix = []
+                    if testing_arch != machine_arch:
+                        if machine_arch == 'arm64' and testing_arch == 'x86_64':
+                            # rosetta2 will provide the emulation with just the arch prefix.
+                            arch_prefix = ['arch', '-x86_64']
+                        else:
+                            raise RuntimeError("don't know how to emulate {testing_arch} on {machine_arch}")
 
-                # clean up
-                shutil.rmtree(venv_dir)
+                    # define a custom 'call' function that adds the arch prefix each time
+                    def call_with_arch(args: Sequence[PathOrStr], **kwargs: Any) -> int:
+                        if isinstance(args, str):
+                            args = ' '.join(arch_prefix) + ' ' + args
+                        else:
+                            args = [*arch_prefix, *args]
+                        return call(args, **kwargs)
+
+                    # Use --no-download to ensure determinism by using seed libraries
+                    # built into virtualenv
+                    call_with_arch(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env)
+
+                    virtualenv_env = env.copy()
+                    virtualenv_env['PATH'] = os.pathsep.join([
+                        str(venv_dir / 'bin'),
+                        virtualenv_env['PATH'],
+                    ])
+
+                    # check that we are using the Python from the virtual environment
+                    call_with_arch(['which', 'python'], env=virtualenv_env)
+
+                    if options.before_test:
+                        before_test_prepared = prepare_command(options.before_test, project='.', package=options.package_dir)
+                        call_with_arch(before_test_prepared, env=virtualenv_env, shell=True)
+
+                    # install the wheel
+                    call_with_arch(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env)
+
+                    # test the wheel
+                    if options.test_requires:
+                        call_with_arch(['pip', 'install'] + options.test_requires, env=virtualenv_env)
+
+                    # run the tests from $HOME, with an absolute path in the command
+                    # (this ensures that Python runs the tests against the installed wheel
+                    # and not the repo code)
+                    test_command_prepared = prepare_command(
+                        options.test_command,
+                        project=Path('.').resolve(),
+                        package=options.package_dir.resolve()
+                    )
+                    call_with_arch(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True)
+
+                    # clean up
+                    shutil.rmtree(venv_dir)
 
             # we're all done here; move it to output (overwrite existing)
             shutil.move(str(repaired_wheel), options.output_dir)
             log.build_end()
     except subprocess.CalledProcessError as error:
-        log.error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}')
+        log.step_end_with_error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}')
         sys.exit(1)

cibuildwheel/resources/build-platforms.toml

@@ -41,7 +41,9 @@ python_configurations = [
   { identifier = "cp36-macosx_x86_64", version = "3.6", url = "https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg" },
   { identifier = "cp37-macosx_x86_64", version = "3.7", url = "https://www.python.org/ftp/python/3.7.9/python-3.7.9-macosx10.9.pkg" },
   { identifier = "cp38-macosx_x86_64", version = "3.8", url = "https://www.python.org/ftp/python/3.8.7/python-3.8.7-macosx10.9.pkg" },
-  { identifier = "cp39-macosx_x86_64", version = "3.9", url = "https://www.python.org/ftp/python/3.9.1/python-3.9.1-macosx10.9.pkg" },
+  { identifier = "cp39-macosx_x86_64", version = "3.9", url = "https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg" },
+  { identifier = "cp39-macosx_arm64", version = "3.9", url = "https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg" },
+  { identifier = "cp39-macosx_universal2", version = "3.9", url = "https://www.python.org/ftp/python/3.9.1/python-3.9.1-macos11.0.pkg" },
   { identifier = "pp27-macosx_x86_64", version = "2.7", url = "https://downloads.python.org/pypy/pypy2.7-v7.3.3-osx64.tar.bz2" },
   { identifier = "pp36-macosx_x86_64", version = "3.6", url = "https://downloads.python.org/pypy/pypy3.6-v7.3.3-osx64.tar.bz2" },
   { identifier = "pp37-macosx_x86_64", version = "3.7", url = "https://downloads.python.org/pypy/pypy3.7-v7.3.3-osx64.tar.bz2" },

cibuildwheel/util.py

@@ -1,8 +1,8 @@
 import fnmatch
 import itertools
 import os
+import re
 import ssl
-import sys
 import textwrap
 import urllib.request
 from enum import Enum
@@ -18,13 +18,7 @@ from .architecture import Architecture
 from .environment import ParsedEnvironment
 from .typing import PathOrStr, PlatformName
 
-if sys.version_info < (3, 9):
-    from importlib_resources import files
-else:
-    from importlib.resources import files
-
-
-resources_dir = files('cibuildwheel') / 'resources'
+resources_dir = Path(__file__).parent / 'resources'
 get_pip_script = resources_dir / 'get-pip.py'
 install_certifi_script = resources_dir / "install_certifi.py"
 
@@ -228,3 +222,15 @@ def detect_ci_provider() -> Optional[CIProvider]:
         return CIProvider.other
     else:
         return None
+
+
+def unwrap(text: str) -> str:
+    '''
+    Unwraps multi-line text to a single line
+    '''
+    # remove initial line indent
+    text = textwrap.dedent(text)
+    # remove leading/trailing whitespace
+    text = text.strip()
+    # remove consecutive whitespace
+    return re.sub(r'\s+', ' ', text)

cibuildwheel/windows.py

@@ -143,6 +143,8 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain
         str(installation_path / 'Scripts'),
         env['PATH']
     ])
+    env['PIP_DISABLE_PIP_VERSION_CHECK'] = '1'
+
     # update env with results from CIBW_ENVIRONMENT
     env = environment.as_dictionary(prev_environment=env)
 
@@ -327,5 +329,5 @@ def build(options: BuildOptions) -> None:
             shutil.move(str(repaired_wheel), options.output_dir)
             log.build_end()
     except subprocess.CalledProcessError as error:
-        log.error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}')
+        log.step_end_with_error(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}')
         sys.exit(1)

docs/extra.css

@@ -83,5 +83,9 @@ h1, h2, h3, h4, h5, h6 {
     padding-top: 0.9em;
 }
 
+.toctree-l3 {
+  border-left: 10px solid transparent;
+}
+
 /* import font awesome 4 for icons */
 @import url(https://cdnjs.cloudflare.com/ajax/libs/font-awesome/4.7.0/css/font-awesome.min.css);

docs/faq.md

@@ -44,6 +44,31 @@ Linux), and the other architectures are emulated automatically.
 {% include "../examples/github-with-qemu.yml" %}
 ```
 
+### Building Apple Silicon wheels on Intel {: #apple-silicon}
+
+`cibuildwheel` supports cross-compiling `universal2` and `arm64` wheels on `x86_64` runners.
+
+These wheels are not built by default, but can be enabled by setting the [`CIBW_ARCHS_MACOS` option](options.md#archs) to `x86_64 arm64 universal2`. Cross-compilation is provided by the Xcode toolchain.
+
+!!! important
+    When cross-compiling on Intel, it is not possible to test `arm64` and the `arm64` part of a `universal2` wheel.
+
+    `cibuildwheel` will raise a warning to notify you of this - these warnings be be silenced by skipping testing on these platforms: `CIBW_TEST_SKIP: *_arm64 *_universal2:arm64`.
+
+Hopefully, this is a temporary situation. Once we have widely available Apple Silicon CI runners, we can build and test `arm64` and `universal2` wheels more natively. That's why `universal2` wheels are not yet built by default, and require opt-in by setting `CIBW_ARCHS_MACOS`.
+
+!!! note
+    Your runner image needs Xcode Command Line Tools 12.2 or later to build `universal2` and `arm64`.
+
+    So far, only CPython 3.9 supports `universal2` and `arm64` wheels.
+
+Here's an example Github Actions workflow with a job that builds for Apple Silicon:
+
+> .github/workflows/build_macos.yml
+```yml
+{% include "../examples/github-apple-silicon.yml" %}
+```
+
 ### Building packages with optional C extensions
 
 `cibuildwheel` defines the environment variable `CIBUILDWHEEL` to the value `1` allowing projects for which the C extension is optional to make it mandatory when building wheels.

docs/options.md

@@ -64,7 +64,7 @@ Default: `auto`
 
 For `linux` you need Docker running, on macOS or Linux. For `macos`, you need a Mac machine, and note that this script is going to automatically install MacPython on your system, so don't run on your development machine. For `windows`, you need to run in Windows, and `cibuildwheel` will install required versions of Python to `C:\cibw\python` using NuGet.
 
-This option can also be set using the command-line option `--platform`.
+This option can also be set using the [command-line option](#command-line) `--platform`.
 
 
 ### `CIBW_BUILD`, `CIBW_SKIP` {: #build-skip}
@@ -155,24 +155,38 @@ CIBW_SKIP: pp*
   }
 </style>
 
-### `CIBW_ARCHS_LINUX` {: #archs}
-> Build non-native architectures
+### `CIBW_ARCHS` {: #archs}
+> Change the architectures built on your machine by default.
 
-A space-separated list of architectures to build. Use this in conjunction with
-emulation, such as that provided by [docker/setup-qemu-action][setup-qemu-action]
-or [tonistiigi/binfmt][binfmt], to build architectures other than those your
-machine natively supports.
+A space-separated list of architectures to build.
 
-Options: `auto` `native` `all` `x86_64` `i686` `aarch64` `ppc64le` `s390x`
+On macOS, this option can be used to cross-compile between `x86_64`,
+`universal2` and `arm64` for Apple Silicon support.
 
-Default: `auto`, meaning the native archs supported on the build machine. For
-example, on an `x86_64` machine, `auto` expands to `x86_64` and `i686`.
+On Linux, this option can be used to build non-native architectures under
+emulation. See [this guide](faq.md#emulation) for more information.
 
-`native` will only build on the exact architecture you currently are on; it will
-not add `i686` for `x86_64`.
+Options:
 
-`all` will expand to all known architectures; remember to use build selectors
-to limit builds for each job; this list could grow in the future.
+- Linux: `x86_64` `i686` `aarch64` `ppc64le` `s390x`
+- macOS: `x86_64` `arm64` `universal2`
+- Windows: `AMD64` `x86`
+- `auto`: The default archs for your machine - see the table below.
+- `native`: the native arch of the build machine - Matches [`platform.machine()`](https://docs.python.org/3/library/platform.html#platform.machine).
+- `all` : expands to all the architectures supported on this OS. You may want
+  to use [CIBW_BUILD](#build-skip) with this option to target specific
+  architectures via build selectors.
+
+Default: `auto`
+
+| Runner | `native` | `auto`
+|---|---|---
+| Linux / Intel | `x86_64` | `x86_64` `i686`
+| Windows / Intel | `AMD64` | `AMD64` `x86`
+| macOS / Intel | `x86_64` | `x86_64`
+| macOS / Apple Silicon | `arm64` | `arm64` `universal2`
+
+If not listed above, `auto` is the same as `native`.
 
 [setup-qemu-action]: https://github.com/docker/setup-qemu-action
 [binfmt]: https://hub.docker.com/r/tonistiigi/binfmt
@@ -180,10 +194,20 @@ to limit builds for each job; this list could grow in the future.
 #### Examples
 
 ```yaml
-# On an intel runner with qemu installed, build Intel and ARM wheels
+# Build `universal2` and `arm64` wheels on an Intel runner.
+# Note that the `arm64` wheel and the `arm64` part of the `universal2`
+# wheel cannot be tested in this configuration.
+CIBW_ARCHS_MACOS: "x86_64 universal2 arm64"
+
+# On an Linux Intel runner with qemu installed, build Intel and ARM wheels
 CIBW_ARCHS_LINUX: "auto aarch64"
 ```
 
+Platform-specific variants also available:<br/>
+`CIBW_ARCHS_MACOS` | `CIBW_ARCHS_WINDOWS` | `CIBW_ARCHS_LINUX`
+
+This option can also be set using the [command-line option](#command-line) `--archs`.
+
 ## Build customization
 
 ### `CIBW_ENVIRONMENT` {: #environment}
@@ -306,7 +330,7 @@ CIBW_BEFORE_BUILD: "{package}/script/prepare_for_build.sh"
 Default:
 
 - on Linux: `'auditwheel repair -w {dest_dir} {wheel}'`
-- on macOS: `'delocate-listdeps {wheel} && delocate-wheel --require-archs x86_64 -w {dest_dir} {wheel}'`
+- on macOS: `'delocate-listdeps {wheel} && delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}'`
 - on Windows: `''`
 
 A shell command to repair a built wheel by copying external library dependencies into the wheel tree and relinking them.
@@ -315,7 +339,8 @@ The command is run on each built wheel (except for pure Python ones) before test
 The following placeholders must be used inside the command and will be replaced by `cibuildwheel`:
 
 - `{wheel}` for the absolute path to the built wheel
-- `{dest_dir}` for the absolute path of the directory where to create the repaired wheel.
+- `{dest_dir}` for the absolute path of the directory where to create the repaired wheel
+- `{delocate_archs}` (macOS only) comma-separated list of architectures in the wheel.
 
 The command is run in a shell, so you can run multiple commands like `cmd1 && cmd2`.
 
@@ -521,11 +546,16 @@ CIBW_TEST_EXTRAS: test,qt
 
 This will skip testing on any identifiers that match the given skip patterns (see [`CIBW_SKIP`](#build-skip)). This can be used to mask out tests for wheels that have missing dependencies upstream that are slow or hard to build, or to mask up slow tests on emulated architectures.
 
+With macOS `universal2` wheels, you can also skip the the individual archs inside the wheel using an `:arch` suffix. For example, `cp39-macosx_universal2:x86_64` or `cp39-macosx_universal2:arm64`.
+
 #### Examples
 
 ```yaml
 # Will avoid testing on emulated architectures
 CIBW_TEST_SKIP: "*-manylinux_{aarch64,ppc64le,s390x}"
+
+# Skip trying to test arm64 builds on Intel Macs
+CIBW_TEST_SKIP: "*-macosx_arm64 *-macosx_universal2:arm64"
 ```
 
 
@@ -547,7 +577,7 @@ CIBW_BUILD_VERBOSITY: 1
 ```
 
 
-## Command line options
+## Command line options {: #command-line}
 
 ```text
 usage: cibuildwheel [-h] [--platform {auto,linux,macos,windows}]
@@ -591,10 +621,6 @@ optional arguments:
 ```
 
 <style>
-  .toctree-l3 {
-    border-left: 10px solid transparent;
-  }
-
   .options-toc {
     display: grid;
     grid-auto-columns: fit-content(20%) 1fr;

examples/github-apple-silicon.yml

@@ -0,0 +1,20 @@
+name: Build
+
+on: [push, pull_request]
+
+jobs:
+  build_wheels_macos:
+    name: Build wheels on macos-11.0
+    runs-on: macos-11.0
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Build wheels
+        run: |
+          pipx run cibuildwheel==1.8.0 --output-dir wheelhouse
+        env:
+          CIBW_ARCHS_MACOS: x86_64 universal2 arm64
+
+      - uses: actions/upload-artifact@v2
+        with:
+          path: ./wheelhouse/*.whl

setup.cfg

@@ -33,7 +33,6 @@ install_requires =
     certifi
     bracex
     typing_extensions; python_version < '3.8'
-    importlib_resources>=1.4; python_version < '3.9'
 
 [options.package_data]
 cibuildwheel = resources/*

test/test_dependency_versions.py

@@ -55,6 +55,13 @@ def test_pinned_versions(tmp_path, python_version):
     if utils.IS_WINDOWS_RUNNING_ON_TRAVIS and python_version == '2.7':
         pytest.skip('Windows + Travis CI requires a workaround')
 
+    is_running_on_macos_11_or_later = (
+        utils.platform == 'macos' and utils.get_macos_version() >= (10, 16)
+    )
+
+    if is_running_on_macos_11_or_later and python_version == '3.5':
+        pytest.skip('CPython 3.5 doesn\'t work on macOS Big Sur+')
+
     project_dir = tmp_path / 'project'
     project_with_expected_version_checks.generate(project_dir)
 

test/test_macos_archs.py

@@ -0,0 +1,119 @@
+import platform
+import subprocess
+from typing import Tuple
+
+import pytest
+
+from . import test_projects, utils
+
+basic_project = test_projects.new_c_project()
+
+ALL_MACOS_WHEELS = (
+    utils.expected_wheels('spam', '0.1.0', machine_arch='x86_64')
+    + utils.expected_wheels('spam', '0.1.0', machine_arch='arm64')
+)
+
+
+def get_xcode_version() -> Tuple[int, int]:
+    output = subprocess.check_output(
+        ['xcodebuild', '-version'],
+        universal_newlines=True,
+    )
+    lines = output.splitlines()
+    _, version_str = lines[0].split()
+
+    version_parts = version_str.split('.')
+    return (int(version_parts[0]), int(version_parts[1]))
+
+
+def test_cross_compiled_build(tmp_path):
+    if utils.platform != 'macos':
+        pytest.skip('this test is only relevant to macos')
+    if get_xcode_version() < (12, 2):
+        pytest.skip('this test only works with Xcode 12.2 or greater')
+
+    project_dir = tmp_path / 'project'
+    basic_project.generate(project_dir)
+
+    actual_wheels = utils.cibuildwheel_run(project_dir, add_env={
+        'CIBW_BUILD': 'cp39-*',
+        'CIBW_ARCHS': 'x86_64, universal2, arm64',
+    })
+
+    expected_wheels = [w for w in ALL_MACOS_WHEELS if 'cp39' in w]
+    assert set(actual_wheels) == set(expected_wheels)
+
+
+@pytest.mark.parametrize('build_universal2', [False, True])
+def test_cross_compiled_test(tmp_path, capfd, build_universal2):
+    if utils.platform != 'macos':
+        pytest.skip('this test is only relevant to macos')
+    if get_xcode_version() < (12, 2):
+        pytest.skip('this test only works with Xcode 12.2 or greater')
+
+    project_dir = tmp_path / 'project'
+    basic_project.generate(project_dir)
+
+    actual_wheels = utils.cibuildwheel_run(project_dir, add_env={
+        'CIBW_BUILD': 'cp39-*',
+        'CIBW_TEST_COMMAND': '''python -c "import platform; print('running tests on ' + platform.machine())"''',
+        'CIBW_ARCHS': 'universal2' if build_universal2 else 'x86_64 arm64',
+    })
+
+    captured = capfd.readouterr()
+
+    if platform.machine() == 'x86_64':
+        # ensure that tests were run on only x86_64
+        assert 'running tests on x86_64' in captured.out
+        assert 'running tests on arm64' not in captured.out
+        if build_universal2:
+            assert 'While universal2 wheels can be built on x86_64, the arm64 part of them cannot currently be tested' in captured.err
+        else:
+            assert 'While arm64 wheels can be built on x86_64, they cannot be tested' in captured.err
+    elif platform.machine() == 'arm64':
+        # ensure that tests were run on both x86_64 and arm64
+        assert 'running tests on x86_64' in captured.out
+        assert 'running tests on arm64' in captured.out
+
+    if build_universal2:
+        expected_wheels = [w for w in ALL_MACOS_WHEELS if 'cp39' in w and 'universal2' in w]
+    else:
+        expected_wheels = [w for w in ALL_MACOS_WHEELS if 'cp39' in w and 'universal2' not in w]
+
+    assert set(actual_wheels) == set(expected_wheels)
+
+
+@pytest.mark.parametrize('skip_arm64_test', [False, True])
+def test_universal2_testing(tmp_path, capfd, skip_arm64_test):
+    if utils.platform != 'macos':
+        pytest.skip('this test is only relevant to macos')
+    if get_xcode_version() < (12, 2):
+        pytest.skip('this test only works with Xcode 12.2 or greater')
+    if platform.machine() != 'x86_64':
+        pytest.skip('this test only works on x86_64')
+
+    project_dir = tmp_path / 'project'
+    basic_project.generate(project_dir)
+
+    actual_wheels = utils.cibuildwheel_run(project_dir, add_env={
+        'CIBW_BUILD': 'cp39-*',
+        'CIBW_TEST_COMMAND': '''python -c "import platform; print('running tests on ' + platform.machine())"''',
+        'CIBW_ARCHS': 'universal2',
+        'CIBW_TEST_SKIP': '*_universal2:arm64' if skip_arm64_test else '',
+    })
+
+    captured = capfd.readouterr()
+
+    if platform.machine() == 'x86_64':
+        assert 'running tests on x86_64' in captured.out
+        assert 'running tests on arm64' not in captured.out
+
+        warning_message = 'While universal2 wheels can be built on x86_64, the arm64 part of them cannot currently be tested'
+        if skip_arm64_test:
+            assert warning_message not in captured.err
+        else:
+            assert warning_message in captured.err
+
+    expected_wheels = [w for w in ALL_MACOS_WHEELS if 'cp39' in w and 'universal2' in w]
+
+    assert set(actual_wheels) == set(expected_wheels)

test/utils.py

@@ -10,10 +10,8 @@ import shutil
 import subprocess
 import sys
 from contextlib import contextmanager
-from pathlib import Path
 from tempfile import mkdtemp
 
-IS_WINDOWS_RUNNING_ON_AZURE = Path('C:\\hostedtoolcache').exists()
 IS_WINDOWS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows'
 
 
@@ -75,6 +73,19 @@ def cibuildwheel_run(project_path, package_dir='.', env=None, add_env=None, outp
     return wheels
 
 
+def _get_arm64_macosx_deployment_target(macosx_deployment_target: str) -> str:
+    '''
+    The first version of macOS that supports arm is 11.0. So the wheel tag
+    cannot contain an earlier deployment target, even if
+    MACOSX_DEPLOYMENT_TARGET sets it.
+    '''
+    version_tuple = tuple(map(int, macosx_deployment_target.split('.')))
+    if version_tuple <= (11, 0):
+        return '11.0'
+    else:
+        return macosx_deployment_target
+
+
 def expected_wheels(package_name, package_version, manylinux_versions=None,
                     macosx_deployment_target='10.9', machine_arch=None, *,
                     exclude_27=IS_WINDOWS_RUNNING_ON_TRAVIS):
@@ -103,6 +114,17 @@ def expected_wheels(package_name, package_version, manylinux_versions=None,
         if platform == 'linux':
             python_abi_tags.append('cp27-cp27mu')  # python 2.7 has 2 different ABI on manylinux
 
+    if platform == 'macos' and get_macos_version() >= (10, 16):
+        # 10.16 is sometimes reported as the macOS version on macOS 11.
+        # CPython 3.5 doesn't work on macOS 11.
+        python_abi_tags.remove('cp35-cp35m')
+        # pypy not supported on macOS 11.
+        python_abi_tags = [t for t in python_abi_tags if not t.startswith('pp')]
+
+    if platform == 'macos' and machine_arch == 'arm64':
+        # currently, arm64 macs are only supported by cp39
+        python_abi_tags = ['cp39-cp39']
+
     wheels = []
 
     for python_abi_tag in python_abi_tags:
@@ -127,7 +149,16 @@ def expected_wheels(package_name, package_version, manylinux_versions=None,
                 platform_tags = ['win32']
 
         elif platform == 'macos':
-            platform_tags = [f'macosx_{macosx_deployment_target.replace(".", "_")}_x86_64']
+            if python_abi_tag == 'cp39-cp39' and machine_arch == 'arm64':
+                arm64_macosx_deployment_target = _get_arm64_macosx_deployment_target(macosx_deployment_target)
+                platform_tags = [
+                    f'macosx_{macosx_deployment_target.replace(".", "_")}_universal2',
+                    f'macosx_{arm64_macosx_deployment_target.replace(".", "_")}_arm64',
+                ]
+            else:
+                platform_tags = [
+                    f'macosx_{macosx_deployment_target.replace(".", "_")}_x86_64',
+                ]
 
         else:
             raise Exception('unsupported platform')
@@ -143,6 +174,18 @@ def expected_wheels(package_name, package_version, manylinux_versions=None,
     return wheels
 
 
+def get_macos_version():
+    '''
+    Returns the macOS major/minor version, as a tuple, e.g. (10, 15) or (11, 0)
+
+    These tuples can be used in comparisons, e.g.
+        (10, 14) <= (11, 0) == True
+        (11, 2) <= (11, 0) != True
+    '''
+    version_str, _, _ = pm.mac_ver()
+    return tuple(map(int, version_str.split(".")[:2]))
+
+
 platform = None
 
 if 'CIBW_PLATFORM' in os.environ:

unit_test/main_tests/conftest.py

@@ -73,6 +73,9 @@ def platform(request, monkeypatch):
     else:
         monkeypatch.setattr(platform_module, 'machine', lambda: 'x86_64')
 
+    if platform_value == 'macos':
+        monkeypatch.setattr(macos, 'get_macos_version', lambda: (11, 1))
+
     return platform_value
 
 

unit_test/main_tests/main_options_test.py

@@ -102,7 +102,7 @@ def get_default_repair_command(platform):
     if platform == 'linux':
         return 'auditwheel repair -w {dest_dir} {wheel}'
     elif platform == 'macos':
-        return 'delocate-listdeps {wheel} && delocate-wheel --require-archs x86_64 -w {dest_dir} {wheel}'
+        return 'delocate-listdeps {wheel} && delocate-wheel --require-archs {delocate_archs} -w {dest_dir} {wheel}'
     elif platform == 'windows':
         return ''
     else:

unit_test/main_tests/main_platform_test.py

@@ -141,4 +141,4 @@ def test_archs_platform_all(platform, intercepted_build_args, monkeypatch):
     elif platform == 'windows':
         assert build_options.architectures == {Architecture.x86, Architecture.AMD64}
     elif platform == 'macos':
-        assert build_options.architectures == {Architecture.x86_64}
+        assert build_options.architectures == {Architecture.x86_64, Architecture.arm64, Architecture.universal2}