diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5276be33..dd8ea0df 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -66,3 +66,24 @@ jobs: - name: Test cibuildwheel run: | python ./bin/run_tests.py + + test-emulated: + name: Test emulated cibuildwheel using qemu + runs-on: ubuntu-latest + timeout-minutes: 180 + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-python@v2 + - name: Install dependencies + run: | + python -m pip install -r requirements-dev.txt + + - name: Set up QEMU + id: qemu + uses: docker/setup-qemu-action@v1 + with: + platforms: all + + - name: Run the emulation tests + run: | + pytest --run-emulation test/test_emulation.py diff --git a/README.md b/README.md index 1041237c..2bc84a5d 100644 --- a/README.md +++ b/README.md @@ -80,7 +80,7 @@ jobs: name: Install Python - name: Install cibuildwheel - run: python -m pip install cibuildwheel==1.7.2 + run: python -m pip install cibuildwheel==1.7.3 - name: Build wheels run: python -m cibuildwheel --output-dir wheelhouse @@ -129,6 +129,7 @@ Here are some repos that use cibuildwheel. |-----------------------------------|----|----|:------| | [scikit-learn][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | The machine learning library. A complex but clean config using many of cibuildwheel's features to build a large project with Cython and C++ extensions. | | [Matplotlib][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | The venerable Matplotlib, a Python library with C++ portions | +| [psutil][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Cross-platform lib for process and system monitoring in Python | | [twisted-iocpsupport][] | ![github icon][] | ![windows icon][] | A submodule of Twisted that hooks into native C APIs using Cython. | | [scikit-image][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Image processing library. Uses cibuildwheel to build and test a project that uses Cython with platform-native code. | | [websockets][] | ![travisci icon][] | ![apple icon][] ![linux icon][] | Library for building WebSocket servers and clients. Mostly written in Python, with a small C 'speedups' extension module. | @@ -138,12 +139,14 @@ Here are some repos that use cibuildwheel. | [coverage.py][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | The coverage tool for Python | | [River][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | 🌊 Online machine learning in Python | | [PyAV][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Pythonic bindings for FFmpeg's libraries. | -| [google neuroglancer][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | WebGL-based viewer for volumetric data | +| [Dependency Injector][] | ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Dependency injection framework for Python, uses Windows TravisCI | | [aioquic][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | QUIC and HTTP/3 implementation in Python | +| [google neuroglancer][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | WebGL-based viewer for volumetric data | | [AutoPy][] | ![travisci icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Includes a Windows Travis build. | | [pikepdf][] | ![azurepipelines icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | A Python library for reading and writing PDF, powered by qpdf | | [Parselmouth][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | A Python interface to the Praat software package, using pybind11, C++17 and CMake, with the core Praat static library built only once and shared between wheels. | | [python-rapidjson][] | ![travisci icon][] ![gitlab icon][] ![appveyor icon][] | ![windows icon][] ![linux icon][] | Python wrapper around rapidjson | +| [Rtree][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Rtree: spatial index for Python GIS ΒΆ | | [KDEpy][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Kernel Density Estimation in Python | | [pybind11 python_example][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Example pybind11 module built with a Python-based build system | | [pybind11 cmake_example][] | ![github icon][] | ![windows icon][] ![apple icon][] ![linux icon][] | Example pybind11 module built with a CMake-based build system | @@ -163,6 +166,7 @@ Here are some repos that use cibuildwheel. [scikit-learn]: https://github.com/scikit-learn/scikit-learn [Matplotlib]: https://github.com/matplotlib/matplotlib +[psutil]: https://github.com/giampaolo/psutil [twisted-iocpsupport]: https://github.com/twisted/twisted-iocpsupport [scikit-image]: https://github.com/scikit-image/scikit-image [websockets]: https://github.com/aaugustin/websockets @@ -172,12 +176,14 @@ Here are some repos that use cibuildwheel. [coverage.py]: https://github.com/nedbat/coveragepy [River]: https://github.com/online-ml/river [PyAV]: https://github.com/PyAV-Org/PyAV -[google neuroglancer]: https://github.com/google/neuroglancer +[Dependency Injector]: https://github.com/ets-labs/python-dependency-injector [aioquic]: https://github.com/aiortc/aioquic +[google neuroglancer]: https://github.com/google/neuroglancer [AutoPy]: https://github.com/autopilot-rs/autopy [pikepdf]: https://github.com/pikepdf/pikepdf [Parselmouth]: https://github.com/YannickJadoul/Parselmouth [python-rapidjson]: https://github.com/python-rapidjson/python-rapidjson +[Rtree]: https://github.com/Toblerity/rtree [KDEpy]: https://github.com/tommyod/KDEpy [pybind11 python_example]: https://github.com/pybind/python_example [pybind11 cmake_example]: https://github.com/pybind/cmake_example @@ -205,39 +211,42 @@ Here are some repos that use cibuildwheel. [apple icon]: docs/data/readme_icons/apple.svg [linux icon]: docs/data/readme_icons/linux.svg - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + - - - - - - - - + + + + + + + + @@ -261,6 +270,18 @@ Changelog +### 1.7.3 + +_1 January 2021_ + +- πŸ›  Added a patch for Pypy to ensure header files are available for building + in a virtualenv. (#502) +- πŸ›  Some preparatory work towards using cibuildwheel as a Github Action. + Check out + [the FAQ](https://cibuildwheel.readthedocs.io/en/stable/faq/#option-1-github-action) + for information on how to use it. We'll be fully updating the docs to this + approach in a subsequent release (#494) + ### 1.7.2 _21 December 2020_ diff --git a/action.yml b/action.yml new file mode 100644 index 00000000..9d23e023 --- /dev/null +++ b/action.yml @@ -0,0 +1,26 @@ +name: cibuildwheel +description: 'Installs and runs cibuildwheel on the current runner' +inputs: + package-dir: + description: 'Input directory, defaults to "."' + required: false + default: . + output-dir: + description: 'Folder to place the outputs in, defaults to "wheelhouse"' + required: false + default: wheelhouse +branding: + icon: package + color: yellow + +runs: + using: composite + steps: + + # This needs powershell-core because github.action path is a Windows-style + # path on Windows. Powershell-core understands both types of paths. + - run: python -m pip install ${{ github.action_path }} + shell: pwsh + + - run: python -m cibuildwheel ${{ inputs.package-dir }} --output-dir ${{ inputs.output-dir }} + shell: bash diff --git a/bin/bump_version.py b/bin/bump_version.py index 45744d28..2330efda 100755 --- a/bin/bump_version.py +++ b/bin/bump_version.py @@ -11,10 +11,13 @@ import urllib.parse config = [ # file path, version find/replace format - ('cibuildwheel/__init__.py', "__version__ = '{}'"), - ('setup.py', "version='{}'"), ('README.md', "cibuildwheel=={}"), + ('cibuildwheel/__init__.py', "__version__ = '{}'"), + ('docs/faq.md', "cibuildwheel=={}"), + ('docs/faq.md', "cibuildwheel@v{}"), + ('docs/setup.md', "cibuildwheel@v{}"), ('examples/*', "cibuildwheel=={}"), + ('setup.py', "version='{}'"), ] diff --git a/bin/run_example_ci_configs.py b/bin/run_example_ci_configs.py index 33f54de3..9ffe1717 100755 --- a/bin/run_example_ci_configs.py +++ b/bin/run_example_ci_configs.py @@ -8,6 +8,7 @@ import sys import textwrap import time import click +from glob import glob from collections import namedtuple from urllib.parse import quote @@ -30,49 +31,70 @@ def generate_basic_project(path): project.generate(path) -CIService = namedtuple('CIService', 'name src_config_path dst_config_path badge_md') +CIService = namedtuple('CIService', 'name dst_config_path badge_md') services = [ CIService( name='appveyor', - src_config_path='examples/appveyor-minimal.yml', dst_config_path='appveyor.yml', badge_md='[![Build status](https://ci.appveyor.com/api/projects/status/wbsgxshp05tt1tif/branch/{branch}?svg=true)](https://ci.appveyor.com/project/joerick/cibuildwheel/branch/{branch})', ), CIService( name='azure-pipelines', - src_config_path='examples/azure-pipelines-minimal.yml', dst_config_path='azure-pipelines.yml', badge_md='[![Build Status](https://dev.azure.com/joerick0429/cibuildwheel/_apis/build/status/joerick.cibuildwheel?branchName={branch})](https://dev.azure.com/joerick0429/cibuildwheel/_build/latest?definitionId=2&branchName={branch})', ), CIService( - name='circle-ci', - src_config_path='examples/circleci-minimal.yml', + name='circleci', dst_config_path='.circleci/config.yml', badge_md='[![CircleCI](https://circleci.com/gh/joerick/cibuildwheel/tree/{branch_escaped}.svg?style=svg)](https://circleci.com/gh/joerick/cibuildwheel/tree/{branch})', ), CIService( name='github', - src_config_path='examples/github-minimal.yml', dst_config_path='.github/workflows/example.yml', badge_md='[![Build](https://github.com/joerick/cibuildwheel/workflows/Build/badge.svg?branch={branch})](https://github.com/joerick/cibuildwheel/actions)', ), CIService( name='travis-ci', - src_config_path='examples/travis-ci-minimal.yml', dst_config_path='.travis.yml', badge_md='[![Build Status](https://travis-ci.org/joerick/cibuildwheel.svg?branch={branch})](https://travis-ci.org/joerick/cibuildwheel)', ), CIService( name='gitlab', - src_config_path='examples/gitlab-minimal.yml', dst_config_path='.gitlab-ci.yml', badge_md='[![Gitlab](https://gitlab.com/joerick/cibuildwheel/badges/{branch}/pipeline.svg)](https://gitlab.com/joerick/cibuildwheel/-/commits/{branch})' ), ] +def ci_service_for_config_file(config_file): + service_name = Path(config_file).name.rsplit('-', 1)[0] + + for service in services: + if service.name == service_name: + return service + + raise ValueError(f'unknown ci service for config file {config_file}') + + @click.command() -def run_example_ci_configs(): +@click.argument('config_files', nargs=-1, type=click.Path()) +def run_example_ci_configs(config_files=None): + ''' + Test the example configs. If no files are specified, will test + examples/*-minimal.yml + ''' + + if len(config_files) == 0: + config_files = glob('examples/*-minimal.yml') + + # check each CI service has at most 1 config file + configs_by_service = {} + for config_file in config_files: + service = ci_service_for_config_file(config_file) + if service.name in configs_by_service: + raise Exception('You cannot specify more than one config per CI service') + configs_by_service[service.name] = config_file + if git_repo_has_changes(): print('Your git repo has uncommitted changes. Commit or stash before continuing.') exit(1) @@ -91,18 +113,19 @@ def run_example_ci_configs(): example_project = Path('example_root') generate_basic_project(example_project) - for service in services: - src_config_file = Path(service.src_config_path) + for config_file in config_files: + service = ci_service_for_config_file(config_file) + src_config_file = Path(config_file) dst_config_file = example_project / service.dst_config_path dst_config_file.parent.mkdir(parents=True, exist_ok=True) shutil.copyfile(src_config_file, dst_config_file) run(['git', 'add', example_project], check=True) - run(['git', 'commit', '-m', textwrap.dedent(f''' + run(['git', 'commit', '--no-verify', '-m', textwrap.dedent(f''' Test example minimal configs - Testing files: {[s.src_config_path for s in services]} + Testing files: {config_files} Generated from branch: {previous_branch} Time: {timestamp} ''')], check=True) @@ -116,9 +139,10 @@ def run_example_ci_configs(): print('> ') print('> | Service | Config | Status |') print('> |---|---|---|') - for service in services: + for config_file in config_files: + service = ci_service_for_config_file(config_file) badge = service.badge_md.format(branch=branch_name, branch_escaped=quote(branch_name, safe='')) - print(f'> | {service.name} | `{service.src_config_path}` | {badge} |') + print(f'> | {service.name} | `{config_file}` | {badge} |') print('> ') print('> Generated by `bin/run_example_ci_config.py`') print() diff --git a/bin/run_tests.py b/bin/run_tests.py index 17f06aa5..b46bfbe6 100755 --- a/bin/run_tests.py +++ b/bin/run_tests.py @@ -17,4 +17,4 @@ if __name__ == '__main__': subprocess.check_call(unit_test_args) # run the integration tests - subprocess.check_call([sys.executable, '-m', 'pytest', '-x', '--durations', '0', 'test']) + subprocess.check_call([sys.executable, '-m', 'pytest', '-x', '--durations', '0', '--timeout=1200', 'test']) diff --git a/cibuildwheel/__init__.py b/cibuildwheel/__init__.py index 8adfee4f..6da2111e 100644 --- a/cibuildwheel/__init__.py +++ b/cibuildwheel/__init__.py @@ -1 +1 @@ -__version__ = '1.7.2' +__version__ = '1.7.3' diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 3040ccb6..d3ed3492 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -17,6 +17,7 @@ from cibuildwheel.environment import ( parse_environment, ) from cibuildwheel.util import ( + Architecture, BuildOptions, BuildSelector, DependencyConstraints, @@ -67,6 +68,17 @@ def main() -> None: run in Windows, and it will build and test for all versions of Python. Default: auto. ''') + + parser.add_argument('--archs', + default=None, + help=''' + Comma-separated list of CPU architectures to build for. + When set to 'auto', builds the architectures natively supported + on this machine. Set this option to build an architecture + via emulation, for example, using binfmt_misc and QEMU. + Default: auto. + Choices: auto, {} + '''.format(", ".join(a.name for a in Architecture))) parser.add_argument('--output-dir', default=os.environ.get('CIBW_OUTPUT_DIR', 'wheelhouse'), help='Destination folder for the wheels.') @@ -168,8 +180,14 @@ def main() -> None: print('cibuildwheel: Could not find setup.py, setup.cfg or pyproject.toml at root of package', file=sys.stderr) exit(2) + if args.archs is not None: + archs_config_str = args.archs + else: + archs_config_str = get_option_from_environment('CIBW_ARCHS', platform=platform, default='auto') + archs = Architecture.parse_config(archs_config_str, platform=platform) + if args.print_build_identifiers: - print_build_identifiers(platform, build_selector) + print_build_identifiers(platform, build_selector, archs) exit(0) manylinux_images: Optional[Dict[str, str]] = None @@ -202,6 +220,7 @@ def main() -> None: manylinux_images[build_platform] = image build_options = BuildOptions( + architectures=archs, package_dir=package_dir, output_dir=output_dir, test_command=test_command, @@ -284,10 +303,12 @@ def print_preamble(platform: str, build_options: BuildOptions) -> None: print('\nHere we go!\n') -def print_build_identifiers(platform: str, build_selector: BuildSelector) -> None: +def print_build_identifiers( + platform: str, build_selector: BuildSelector, architectures: List[Architecture] +) -> None: python_configurations: List[Any] = [] if platform == 'linux': - python_configurations = cibuildwheel.linux.get_python_configurations(build_selector) + python_configurations = cibuildwheel.linux.get_python_configurations(build_selector, architectures) elif platform == 'windows': python_configurations = cibuildwheel.windows.get_python_configurations(build_selector) elif platform == 'macos': diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index cfd4952c..6a553ef7 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -1,4 +1,3 @@ -import platform import subprocess import sys import textwrap @@ -8,29 +7,10 @@ from typing import List, NamedTuple, Union from .docker_container import DockerContainer from .logger import log -from .util import (BuildOptions, BuildSelector, NonPlatformWheelError, - get_build_verbosity_extra_flags, prepare_command) - - -def matches_platform(identifier: str) -> bool: - pm = platform.machine() - if pm == "x86_64": - # x86_64 machines can run i686 docker containers - if identifier.endswith('x86_64') or identifier.endswith('i686'): - return True - elif pm == "i686": - if identifier.endswith('i686'): - return True - elif pm == "aarch64": - if identifier.endswith('aarch64'): - return True - elif pm == "ppc64le": - if identifier.endswith('ppc64le'): - return True - elif pm == "s390x": - if identifier.endswith('s390x'): - return True - return False +from .util import ( + Architecture, BuildOptions, BuildSelector, NonPlatformWheelError, + get_build_verbosity_extra_flags, prepare_command, +) class PythonConfiguration(NamedTuple): @@ -43,7 +23,9 @@ class PythonConfiguration(NamedTuple): return PurePath(self.path_str) -def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfiguration]: +def get_python_configurations( + build_selector: BuildSelector, architectures: List[Architecture] +) -> List[PythonConfiguration]: python_configurations = [ PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path_str='/opt/python/cp27-cp27m'), PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path_str='/opt/python/cp27-cp27mu'), @@ -78,8 +60,14 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi PythonConfiguration(version='3.8', identifier='cp38-manylinux_s390x', path_str='/opt/python/cp38-cp38'), PythonConfiguration(version='3.9', identifier='cp39-manylinux_s390x', path_str='/opt/python/cp39-cp39'), ] - # skip builds as required - return [c for c in python_configurations if matches_platform(c.identifier) and build_selector(c.identifier)] + + # return all configurations whose arch is in our `architectures` list, + # and match the build/skip rules + return [ + c for c in python_configurations + if any(c.identifier.endswith(arch.value) for arch in architectures) + and build_selector(c.identifier) + ] def build(options: BuildOptions) -> None: @@ -93,7 +81,7 @@ def build(options: BuildOptions) -> None: exit(2) assert options.manylinux_images is not None - python_configurations = get_python_configurations(options.build_selector) + python_configurations = get_python_configurations(options.build_selector, options.architectures) platforms = [ ('cp', 'manylinux_x86_64', options.manylinux_images['x86_64']), ('cp', 'manylinux_i686', options.manylinux_images['i686']), @@ -138,6 +126,16 @@ def build(options: BuildOptions) -> None: log.build_start(config.identifier) dependency_constraint_flags: List[Union[str, PathLike]] = [] + if config.identifier.startswith("pp"): + # Patch PyPy to make sure headers get installed into a venv + patch_version = '_27' if config.version == '2.7' else '' + patch_path = Path(__file__).absolute().parent / 'resources' / f'pypy_venv{patch_version}.patch' + patch_docker_path = PurePath('/pypy_venv.patch') + docker.copy_into(patch_path, patch_docker_path) + try: + docker.call(['patch', '--force', '-p1', '-d', config.path, '-i', patch_docker_path]) + except subprocess.CalledProcessError: + print("PyPy patch not applied", file=sys.stderr) if options.dependency_constraints: constraints_file = options.dependency_constraints.get_for_python_version(config.version) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index e26538d8..835d73a7 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -4,13 +4,14 @@ import shutil import subprocess import sys import tempfile +import textwrap from os import PathLike from pathlib import Path from typing import Dict, List, NamedTuple, Optional, Sequence, Union from .environment import ParsedEnvironment from .logger import log -from .util import (BuildOptions, BuildSelector, NonPlatformWheelError, +from .util import (Architecture, BuildOptions, BuildSelector, NonPlatformWheelError, download, get_build_verbosity_extra_flags, get_pip_script, install_certifi_script, prepare_command) @@ -110,6 +111,10 @@ def install_pypy(version: str, url: str) -> Path: downloaded_tar_bz2 = Path("/tmp") / pypy_tar_bz2 download(url, downloaded_tar_bz2) call(['tar', '-C', '/tmp', '-xf', downloaded_tar_bz2]) + # Patch PyPy to make sure headers get installed into a venv + patch_version = '_27' if version == '2.7' else '' + patch_path = Path(__file__).absolute().parent / 'resources' / f'pypy_venv{patch_version}.patch' + call(['patch', '--force', '-d', installation_path, patch_path]) installation_bin_path = installation_path / 'bin' python_executable = 'pypy3' if version[0] == '3' else 'pypy' @@ -195,6 +200,12 @@ def setup_python(python_configuration: PythonConfiguration, def build(options: BuildOptions) -> None: + if options.architectures != [Architecture.x86_64]: + raise ValueError(textwrap.dedent(f''' + Invalid archs option {options.architectures}. macOS only supports x86_64 for the moment. + If you want to set emulation architectures on Linux, use CIBW_ARCHS_LINUX instead. + ''')) + temp_dir = Path(tempfile.mkdtemp(prefix='cibuildwheel')) built_wheel_dir = temp_dir / 'built_wheel' repaired_wheel_dir = temp_dir / 'repaired_wheel' diff --git a/cibuildwheel/resources/constraints-python27.txt b/cibuildwheel/resources/constraints-python27.txt index eb6d018c..d5d41c2d 100644 --- a/cibuildwheel/resources/constraints-python27.txt +++ b/cibuildwheel/resources/constraints-python27.txt @@ -4,23 +4,52 @@ # # bin/update_dependencies.py # -appdirs==1.4.4 # via virtualenv -configparser==4.0.2 # via importlib-metadata -contextlib2==0.6.0.post1 # via importlib-metadata, importlib-resources -delocate==0.8.2 # via -r cibuildwheel/resources/constraints.in -distlib==0.3.1 # via virtualenv -filelock==3.0.12 # via virtualenv -importlib-metadata==2.1.1 # via virtualenv -importlib-resources==3.3.0 # via virtualenv -pathlib2==2.3.5 # via importlib-metadata, importlib-resources, virtualenv -scandir==1.10.0 # via pathlib2 -singledispatch==3.4.0.3 # via importlib-resources -six==1.15.0 # via pathlib2, virtualenv -typing==3.7.4.3 # via importlib-resources -virtualenv==20.2.2 # via -r cibuildwheel/resources/constraints.in -wheel==0.36.2 # via -r cibuildwheel/resources/constraints.in, delocate -zipp==1.2.0 # via importlib-metadata, importlib-resources +appdirs==1.4.4 + # via virtualenv +configparser==4.0.2 + # via importlib-metadata +contextlib2==0.6.0.post1 + # via + # importlib-metadata + # importlib-resources +delocate==0.8.2 + # via -r cibuildwheel/resources/constraints.in +distlib==0.3.1 + # via virtualenv +filelock==3.0.12 + # via virtualenv +importlib-metadata==2.1.1 + # via virtualenv +importlib-resources==3.3.1 + # via virtualenv +pathlib2==2.3.5 + # via + # importlib-metadata + # importlib-resources + # virtualenv +scandir==1.10.0 + # via pathlib2 +singledispatch==3.4.0.3 + # via importlib-resources +six==1.15.0 + # via + # pathlib2 + # virtualenv +typing==3.7.4.3 + # via importlib-resources +virtualenv==20.2.2 + # via -r cibuildwheel/resources/constraints.in +wheel==0.36.2 + # via + # -r cibuildwheel/resources/constraints.in + # delocate +zipp==1.2.0 + # via + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: -pip==20.3.3 # via -r cibuildwheel/resources/constraints.in -setuptools==44.1.1 # via -r cibuildwheel/resources/constraints.in +pip==20.3.3 + # via -r cibuildwheel/resources/constraints.in +setuptools==44.1.1 + # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python35.txt b/cibuildwheel/resources/constraints-python35.txt index a561e8d3..d51a1299 100644 --- a/cibuildwheel/resources/constraints-python35.txt +++ b/cibuildwheel/resources/constraints-python35.txt @@ -4,17 +4,33 @@ # # bin/update_dependencies.py # -appdirs==1.4.4 # via virtualenv -delocate==0.8.2 # via -r cibuildwheel/resources/constraints.in -distlib==0.3.1 # via virtualenv -filelock==3.0.12 # via virtualenv -importlib-metadata==2.1.1 # via virtualenv -importlib-resources==3.2.1 # via virtualenv -six==1.15.0 # via virtualenv -virtualenv==20.2.2 # via -r cibuildwheel/resources/constraints.in -wheel==0.36.2 # via -r cibuildwheel/resources/constraints.in, delocate -zipp==1.2.0 # via importlib-metadata, importlib-resources +appdirs==1.4.4 + # via virtualenv +delocate==0.8.2 + # via -r cibuildwheel/resources/constraints.in +distlib==0.3.1 + # via virtualenv +filelock==3.0.12 + # via virtualenv +importlib-metadata==2.1.1 + # via virtualenv +importlib-resources==3.2.1 + # via virtualenv +six==1.15.0 + # via virtualenv +virtualenv==20.2.2 + # via -r cibuildwheel/resources/constraints.in +wheel==0.36.2 + # via + # -r cibuildwheel/resources/constraints.in + # delocate +zipp==1.2.0 + # via + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: -pip==20.3.3 # via -r cibuildwheel/resources/constraints.in -setuptools==50.3.2 # via -r cibuildwheel/resources/constraints.in +pip==20.3.3 + # via -r cibuildwheel/resources/constraints.in +setuptools==50.3.2 + # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python36.txt b/cibuildwheel/resources/constraints-python36.txt index 4e782508..2d912194 100644 --- a/cibuildwheel/resources/constraints-python36.txt +++ b/cibuildwheel/resources/constraints-python36.txt @@ -4,18 +4,35 @@ # # bin/update_dependencies.py # -appdirs==1.4.4 # via virtualenv -delocate==0.8.2 # via -r cibuildwheel/resources/constraints.in -distlib==0.3.1 # via virtualenv -filelock==3.0.12 # via virtualenv -importlib-metadata==3.3.0 # via virtualenv -importlib-resources==3.3.0 # via virtualenv -six==1.15.0 # via virtualenv -typing-extensions==3.7.4.3 # via importlib-metadata -virtualenv==20.2.2 # via -r cibuildwheel/resources/constraints.in -wheel==0.36.2 # via -r cibuildwheel/resources/constraints.in, delocate -zipp==3.4.0 # via importlib-metadata, importlib-resources +appdirs==1.4.4 + # via virtualenv +delocate==0.8.2 + # via -r cibuildwheel/resources/constraints.in +distlib==0.3.1 + # via virtualenv +filelock==3.0.12 + # via virtualenv +importlib-metadata==3.3.0 + # via virtualenv +importlib-resources==4.1.1 + # via virtualenv +six==1.15.0 + # via virtualenv +typing-extensions==3.7.4.3 + # via importlib-metadata +virtualenv==20.2.2 + # via -r cibuildwheel/resources/constraints.in +wheel==0.36.2 + # via + # -r cibuildwheel/resources/constraints.in + # delocate +zipp==3.4.0 + # via + # importlib-metadata + # importlib-resources # The following packages are considered to be unsafe in a requirements file: -pip==20.3.3 # via -r cibuildwheel/resources/constraints.in -setuptools==51.1.0 # via -r cibuildwheel/resources/constraints.in +pip==20.3.3 + # via -r cibuildwheel/resources/constraints.in +setuptools==51.1.1 + # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python37.txt b/cibuildwheel/resources/constraints-python37.txt index 9e7b421b..907b755b 100644 --- a/cibuildwheel/resources/constraints-python37.txt +++ b/cibuildwheel/resources/constraints-python37.txt @@ -4,17 +4,31 @@ # # bin/update_dependencies.py # -appdirs==1.4.4 # via virtualenv -delocate==0.8.2 # via -r cibuildwheel/resources/constraints.in -distlib==0.3.1 # via virtualenv -filelock==3.0.12 # via virtualenv -importlib-metadata==3.3.0 # via virtualenv -six==1.15.0 # via virtualenv -typing-extensions==3.7.4.3 # via importlib-metadata -virtualenv==20.2.2 # via -r cibuildwheel/resources/constraints.in -wheel==0.36.2 # via -r cibuildwheel/resources/constraints.in, delocate -zipp==3.4.0 # via importlib-metadata +appdirs==1.4.4 + # via virtualenv +delocate==0.8.2 + # via -r cibuildwheel/resources/constraints.in +distlib==0.3.1 + # via virtualenv +filelock==3.0.12 + # via virtualenv +importlib-metadata==3.3.0 + # via virtualenv +six==1.15.0 + # via virtualenv +typing-extensions==3.7.4.3 + # via importlib-metadata +virtualenv==20.2.2 + # via -r cibuildwheel/resources/constraints.in +wheel==0.36.2 + # via + # -r cibuildwheel/resources/constraints.in + # delocate +zipp==3.4.0 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -pip==20.3.3 # via -r cibuildwheel/resources/constraints.in -setuptools==51.1.0 # via -r cibuildwheel/resources/constraints.in +pip==20.3.3 + # via -r cibuildwheel/resources/constraints.in +setuptools==51.1.1 + # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python38.txt b/cibuildwheel/resources/constraints-python38.txt index 40bcd207..261ebccd 100644 --- a/cibuildwheel/resources/constraints-python38.txt +++ b/cibuildwheel/resources/constraints-python38.txt @@ -4,14 +4,25 @@ # # bin/update_dependencies.py # -appdirs==1.4.4 # via virtualenv -delocate==0.8.2 # via -r cibuildwheel/resources/constraints.in -distlib==0.3.1 # via virtualenv -filelock==3.0.12 # via virtualenv -six==1.15.0 # via virtualenv -virtualenv==20.2.2 # via -r cibuildwheel/resources/constraints.in -wheel==0.36.2 # via -r cibuildwheel/resources/constraints.in, delocate +appdirs==1.4.4 + # via virtualenv +delocate==0.8.2 + # via -r cibuildwheel/resources/constraints.in +distlib==0.3.1 + # via virtualenv +filelock==3.0.12 + # via virtualenv +six==1.15.0 + # via virtualenv +virtualenv==20.2.2 + # via -r cibuildwheel/resources/constraints.in +wheel==0.36.2 + # via + # -r cibuildwheel/resources/constraints.in + # delocate # The following packages are considered to be unsafe in a requirements file: -pip==20.3.3 # via -r cibuildwheel/resources/constraints.in -setuptools==51.1.0 # via -r cibuildwheel/resources/constraints.in +pip==20.3.3 + # via -r cibuildwheel/resources/constraints.in +setuptools==51.1.1 + # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints-python39.txt b/cibuildwheel/resources/constraints-python39.txt index 40bcd207..261ebccd 100644 --- a/cibuildwheel/resources/constraints-python39.txt +++ b/cibuildwheel/resources/constraints-python39.txt @@ -4,14 +4,25 @@ # # bin/update_dependencies.py # -appdirs==1.4.4 # via virtualenv -delocate==0.8.2 # via -r cibuildwheel/resources/constraints.in -distlib==0.3.1 # via virtualenv -filelock==3.0.12 # via virtualenv -six==1.15.0 # via virtualenv -virtualenv==20.2.2 # via -r cibuildwheel/resources/constraints.in -wheel==0.36.2 # via -r cibuildwheel/resources/constraints.in, delocate +appdirs==1.4.4 + # via virtualenv +delocate==0.8.2 + # via -r cibuildwheel/resources/constraints.in +distlib==0.3.1 + # via virtualenv +filelock==3.0.12 + # via virtualenv +six==1.15.0 + # via virtualenv +virtualenv==20.2.2 + # via -r cibuildwheel/resources/constraints.in +wheel==0.36.2 + # via + # -r cibuildwheel/resources/constraints.in + # delocate # The following packages are considered to be unsafe in a requirements file: -pip==20.3.3 # via -r cibuildwheel/resources/constraints.in -setuptools==51.1.0 # via -r cibuildwheel/resources/constraints.in +pip==20.3.3 + # via -r cibuildwheel/resources/constraints.in +setuptools==51.1.1 + # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/constraints.txt b/cibuildwheel/resources/constraints.txt index 40bcd207..261ebccd 100644 --- a/cibuildwheel/resources/constraints.txt +++ b/cibuildwheel/resources/constraints.txt @@ -4,14 +4,25 @@ # # bin/update_dependencies.py # -appdirs==1.4.4 # via virtualenv -delocate==0.8.2 # via -r cibuildwheel/resources/constraints.in -distlib==0.3.1 # via virtualenv -filelock==3.0.12 # via virtualenv -six==1.15.0 # via virtualenv -virtualenv==20.2.2 # via -r cibuildwheel/resources/constraints.in -wheel==0.36.2 # via -r cibuildwheel/resources/constraints.in, delocate +appdirs==1.4.4 + # via virtualenv +delocate==0.8.2 + # via -r cibuildwheel/resources/constraints.in +distlib==0.3.1 + # via virtualenv +filelock==3.0.12 + # via virtualenv +six==1.15.0 + # via virtualenv +virtualenv==20.2.2 + # via -r cibuildwheel/resources/constraints.in +wheel==0.36.2 + # via + # -r cibuildwheel/resources/constraints.in + # delocate # The following packages are considered to be unsafe in a requirements file: -pip==20.3.3 # via -r cibuildwheel/resources/constraints.in -setuptools==51.1.0 # via -r cibuildwheel/resources/constraints.in +pip==20.3.3 + # via -r cibuildwheel/resources/constraints.in +setuptools==51.1.1 + # via -r cibuildwheel/resources/constraints.in diff --git a/cibuildwheel/resources/pinned_docker_images.cfg b/cibuildwheel/resources/pinned_docker_images.cfg index ec1b0fd3..9f46370d 100644 --- a/cibuildwheel/resources/pinned_docker_images.cfg +++ b/cibuildwheel/resources/pinned_docker_images.cfg @@ -1,22 +1,22 @@ [x86_64] -manylinux1 = quay.io/pypa/manylinux1_x86_64:2020-12-18-c48c073 -manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2020-12-19-8df9e2d -manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2020-12-18-cb453dc +manylinux1 = quay.io/pypa/manylinux1_x86_64:2020-12-30-e2b3664 +manylinux2010 = quay.io/pypa/manylinux2010_x86_64:2020-12-30-3161d69 +manylinux2014 = quay.io/pypa/manylinux2014_x86_64:2020-12-30-68d3257 [i686] -manylinux1 = quay.io/pypa/manylinux1_i686:2020-12-18-c48c073 -manylinux2010 = quay.io/pypa/manylinux2010_i686:2020-12-19-8df9e2d -manylinux2014 = quay.io/pypa/manylinux2014_i686:2020-12-18-cb453dc +manylinux1 = quay.io/pypa/manylinux1_i686:2020-12-30-e2b3664 +manylinux2010 = quay.io/pypa/manylinux2010_i686:2020-12-30-3161d69 +manylinux2014 = quay.io/pypa/manylinux2014_i686:2020-12-30-68d3257 [pypy_x86_64] manylinux2010 = pypywheels/manylinux2010-pypy_x86_64:2020-12-11-f1e0e80 [aarch64] -manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2020-12-18-cb453dc +manylinux2014 = quay.io/pypa/manylinux2014_aarch64:2020-12-30-68d3257 [ppc64le] -manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2020-12-18-cb453dc +manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2020-12-30-68d3257 [s390x] -manylinux2014 = quay.io/pypa/manylinux2014_s390x:2020-12-18-cb453dc +manylinux2014 = quay.io/pypa/manylinux2014_s390x:2020-12-30-68d3257 diff --git a/cibuildwheel/resources/pypy_venv.patch b/cibuildwheel/resources/pypy_venv.patch new file mode 100644 index 00000000..49207b1a --- /dev/null +++ b/cibuildwheel/resources/pypy_venv.patch @@ -0,0 +1,36 @@ +diff --git a/lib-python/3/sysconfig.py b/lib-python/3/sysconfig.py +index 0d08cde66b4150e1772c13fdc6fe34d079c2b10d..30c77d165a2505d98bc7660ff0cc3b728f56c682 100644 +--- a/lib-python/3/sysconfig.py ++++ b/lib-python/3/sysconfig.py +@@ -618,9 +618,6 @@ def get_config_vars(*args): + import _osx_support + _osx_support.customize_config_vars(_CONFIG_VARS) + +- _CONFIG_VARS['INCLUDEPY'] = os.path.join(_CONFIG_VARS['prefix'], +- 'include') +- + if args: + vals = [] + for name in args: +diff --git a/lib_pypy/_sysconfigdata.py b/lib_pypy/_sysconfigdata.py +index f1cca41a4b7553819bb1a2a63af1c1a9b8591ce9..a34216e8e12b82abd72fabdd7793b3e8bb19bbcf 100644 +--- a/lib_pypy/_sysconfigdata.py ++++ b/lib_pypy/_sysconfigdata.py +@@ -5,6 +5,7 @@ from distutils.spawn import find_executable + + so_ext = _imp.extension_suffixes()[0] + ++mybase = os.path.dirname(os.path.dirname(__file__)) + + build_time_vars = { + # SOABI is PEP 3149 compliant, but CPython3 has so_ext.split('.')[1] +@@ -25,7 +26,8 @@ build_time_vars = { + 'AR': "ar", + 'ARFLAGS': "rc", + 'EXE': "", +- 'LIBDIR': os.path.join(sys.prefix, 'bin'), ++ 'LIBDIR': os.path.join(mybase, 'lib'), ++ 'INCLUDEPY': os.path.join(mybase, 'include'), + } + + if find_executable("gcc"): diff --git a/cibuildwheel/resources/pypy_venv_27.patch b/cibuildwheel/resources/pypy_venv_27.patch new file mode 100644 index 00000000..4dd31537 --- /dev/null +++ b/cibuildwheel/resources/pypy_venv_27.patch @@ -0,0 +1,27 @@ +--- a/lib-python/2.7/sysconfig.py ++++ b/lib-python/2.7/sysconfig.py +@@ -553,9 +553,6 @@ def get_config_vars(*args): + if type_ == imp.C_EXTENSION: + _CONFIG_VARS['SOABI'] = suffix.split('.')[1] + break +- _CONFIG_VARS['INCLUDEPY'] = os.path.join(_CONFIG_VARS['prefix'], +- 'include') +- + if args: + vals = [] + for name in args: +--- a/lib_pypy/_sysconfigdata.py ++++ b/lib_pypy/_sysconfigdata.py +@@ -1,5 +1,11 @@ +-import imp ++import imp, os ++ ++mybase = os.path.dirname(os.path.dirname(__file__)) + + build_time_vars = { + "SO": [s[0] for s in imp.get_suffixes() if s[2] == imp.C_EXTENSION][0] + } ++ ++build_time_vars['LIBDIR'] = os.path.join(mybase, 'lib') ++build_time_vars['INCLUDEPY'] = os.path.join(mybase, 'include') ++ diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 87ce5d17..70eaa44d 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -1,4 +1,6 @@ import os +import platform as platform_module +import re import ssl import textwrap import urllib.request @@ -121,10 +123,45 @@ class DependencyConstraints: return f'{self.__class__.__name__}{self.base_file_path!r})' +class Architecture(Enum): + # mac/linux archs + x86_64 = 'x86_64' + i686 = 'i686' + aarch64 = 'aarch64' + ppc64le = 'ppc64le' + s390x = 's390x' + + # windows archs + x86 = 'x86' + AMD64 = 'AMD64' + + @staticmethod + def parse_config(config: str, platform: str) -> 'List[Architecture]': + result = [] + for arch_str in re.split(r'[\s,]+', config): + if arch_str == 'auto': + result += Architecture.auto_archs(platform=platform) + else: + result.append(Architecture(arch_str)) + return result + + @staticmethod + def auto_archs(platform: str) -> 'List[Architecture]': + native_architecture = Architecture(platform_module.machine()) + result = [native_architecture] + if platform == 'linux' and native_architecture == Architecture.x86_64: + # x86_64 machines can run i686 docker containers + result.append(Architecture.i686) + if platform == 'windows' and native_architecture == Architecture.AMD64: + result.append(Architecture.x86) + return result + + class BuildOptions(NamedTuple): package_dir: Path output_dir: Path build_selector: BuildSelector + architectures: List[Architecture] environment: ParsedEnvironment before_all: str before_build: Optional[str] @@ -164,7 +201,7 @@ def strtobool(val: str) -> bool: return False -class CIProvider(str, Enum): +class CIProvider(Enum): travis_ci = 'travis' appveyor = 'appveyor' circle_ci = 'circle_ci' diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 11dbdcec..3abb61dd 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -3,6 +3,7 @@ import shutil import subprocess import sys import tempfile +import textwrap from os import PathLike from pathlib import Path from typing import Dict, List, NamedTuple, Optional, Sequence, Union @@ -12,7 +13,7 @@ import toml from .environment import ParsedEnvironment from .logger import log -from .util import (BuildOptions, BuildSelector, NonPlatformWheelError, +from .util import (Architecture, BuildOptions, BuildSelector, NonPlatformWheelError, download, get_build_verbosity_extra_flags, get_pip_script, prepare_command) @@ -201,6 +202,13 @@ def pep_518_cp35_workaround(package_dir: Path, env: Dict[str, str]) -> None: def build(options: BuildOptions) -> None: + if options.architectures != [Architecture.AMD64, Architecture.x86]: + raise ValueError(textwrap.dedent(f''' + Invalid archs option {options.architectures}. Windows only supports 'amd64,x86' for the + moment. If you want to set emulation architectures on Linux, use CIBW_ARCHS_LINUX + instead. + ''')) + temp_dir = Path(tempfile.mkdtemp(prefix='cibuildwheel')) built_wheel_dir = temp_dir / 'built_wheel' repaired_wheel_dir = temp_dir / 'repaired_wheel' diff --git a/docs/data/projects.yml b/docs/data/projects.yml index 366e96cd..19acaf94 100644 --- a/docs/data/projects.yml +++ b/docs/data/projects.yml @@ -213,3 +213,19 @@ ci: [github, travisci] os: [windows, apple, linux] notes: Supports full range of wheels, including PyPy and alternate archs. + +- name: Rtree + gh: Toblerity/rtree + ci: [github] + os: [windows, apple, linux] + +- name: psutil + gh: giampaolo/psutil + ci: [github] + os: [windows, apple, linux] + +- name: Dependency Injector + gh: ets-labs/python-dependency-injector + ci: [travisci] + os: [windows, apple, linux] + notes: Dependency injection framework for Python, uses Windows TravisCI diff --git a/docs/data/readme_icons/apple.svg b/docs/data/readme_icons/apple.svg index f8780367..abef0307 100644 --- a/docs/data/readme_icons/apple.svg +++ b/docs/data/readme_icons/apple.svg @@ -1 +1 @@ -Apple icon \ No newline at end of file +Apple icon \ No newline at end of file diff --git a/docs/data/readme_icons/travisci.svg b/docs/data/readme_icons/travisci.svg index 3efbee2a..e2dfd05c 100644 --- a/docs/data/readme_icons/travisci.svg +++ b/docs/data/readme_icons/travisci.svg @@ -1 +1 @@ -Travis CI icon \ No newline at end of file +Travis CI icon \ No newline at end of file diff --git a/docs/faq.md b/docs/faq.md index 2ff777f6..676d8f45 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -24,6 +24,26 @@ Linux wheels are built in the [`manylinux` docker images](https://github.com/pyp - Alternative dockers images can be specified with the `CIBW_MANYLINUX_X86_64_IMAGE`, `CIBW_MANYLINUX_I686_IMAGE`, and `CIBW_MANYLINUX_PYPY_X86_64_IMAGE` options to allow for a custom, preconfigured build environment for the Linux builds. See [options](options.md#manylinux-image) for more details. +### Building non-native architectures using emulation {: #emulation} + +cibuildwheel supports building non-native architectures on Linux, via +emulation through the binfmt_misc kernel feature. The easiest way to use this +is via the [docker/setup-qemu-action][setup-qemu-action] on Github Actions or +[tonistiigi/binfmt][binfmt]. + +[setup-qemu-action]: https://github.com/docker/setup-qemu-action +[binfmt]: https://hub.docker.com/r/tonistiigi/binfmt + +Check out the following config for an example of how to set it up on Github +Actions. Once QEMU is set up and registered, you just need to set the +`CIBW_ARCHS_LINUX` environment variable (or use the `--archs` option on +Linux), and the other architectures are emulated automatically. + +> .github/workflows/build.yml +```yaml +{% include "../examples/github-with-qemu.yml" %} +``` + ### Building packages with optional C extensions `cibuildwheel` defines the environment variable `CIBUILDWHEEL` to the value `1` allowing projects for which the C extension is optional to make it mandatory when building wheels. @@ -81,3 +101,56 @@ To add the `/d2FH4-` flag to a standard `setup.py` using `setuptools`, the `extr ``` To investigate the dependencies of a C extension (i.e., the `.pyd` file, a DLL in disguise) on Windows, [Dependency Walker](http://www.dependencywalker.com/) is a great tool. + + +### Automatic updates {: #automatic-updates} + +Selecting a moving target (like the latest release) is generally a bad idea in CI. If something breaks, you can't tell whether it was your code or an upstream update that caused the breakage, and in a worse-case scenario, it could occur during a release. +There are two suggested methods for keeping cibuildwheel up to date that instead involve scheduled pull requests using GitHub's dependabot. + +#### Option 1: GitHub Action + +If you use GitHub Actions for builds, you can use cibuildwheel as an action: + +```yaml +uses: joerick/cibuildwheel@v1.7.3 +``` + +This is a composite step that just installs and runs cibuildwheel. You can set command-line options as `with:` parameters, and use `env:` as normal. + +Then, your `dependabot.yml` file could look like this: + +```yaml +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "weekly" + ignore: + # Optional: Official actions have moving tags like v1; + # if you use those, you don't need updates. + - dependency-name: "actions/*" +``` + +#### Option 2: Requirement files + +The second option, and the only one that supports other CI systems, is using a `requirements-*.txt` file. The file should have a distinct name and have only one entry: + +```bash +# requirements-cibw.txt +cibuildwheel==1.7.3 +``` + +Then your install step would have `python -m pip install -r requirements-cibw.txt` in it. Your `dependabot.yml` file could look like this: + +```yaml +version: 2 +updates: + - package-ecosystem: "pip" + directory: "/" + schedule: + interval: "daily" +``` + +This will also try to update other pins in all requirement files, so be sure you want to do that. The only control you have over the files used is via the directory option. diff --git a/docs/options.md b/docs/options.md index ad976cd7..ccfc4665 100644 --- a/docs/options.md +++ b/docs/options.md @@ -155,6 +155,28 @@ CIBW_SKIP: pp* } +### `CIBW_ARCHS_LINUX` {: #archs} +> Build non-native architectures + +A space-separated list of architectures to build. Use this in conjunction with +emulation, such as that provided by [docker/setup-qemu-action][setup-qemu-action] +or [tonistiigi/binfmt][binfmt], to build architectures other than those your +machine natively supports. + +Options: `auto` `x86_64` `i686` `aarch64` `ppc64le` `s390x` + +Default: `auto`, meaning the native archs supported on the build machine. For +example, on an `x86_64` machine, `auto` expands to `x86_64` and `i686`. + +[setup-qemu-action]: https://github.com/docker/setup-qemu-action +[binfmt]: https://hub.docker.com/r/tonistiigi/binfmt + +#### Examples + +```yaml +# On an intel runner with qemu installed, build Intel and ARM wheels +CIBW_ARCHS_LINUX: "auto aarch64" +``` ## Build customization @@ -489,7 +511,8 @@ CIBW_BUILD_VERBOSITY: 1 ```text usage: cibuildwheel [-h] [--platform {auto,linux,macos,windows}] - [--output-dir OUTPUT_DIR] [--print-build-identifiers] + [--archs ARCHS] [--output-dir OUTPUT_DIR] + [--print-build-identifiers] [package_dir] Build wheels for all the platforms. @@ -511,6 +534,12 @@ optional arguments: don't run on your development machine. For "windows", you need to run in Windows, and it will build and test for all versions of Python. Default: auto. + --archs ARCHS Comma-separated list of CPU architectures to build + for. If unspecified, builds the architectures natively + supported on this machine. Set this option to build an + architecture via emulation, for example, using + binfmt_misc and qemu. Default: auto Choices: auto, + x86_64, i686, aarch64, ppc64le, s390x, x86, AMD64 --output-dir OUTPUT_DIR Destination folder for the wheels. --print-build-identifiers diff --git a/docs/setup.md b/docs/setup.md index c454d877..7245a9ff 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -17,6 +17,8 @@ For more info on this file, check out the [docs](https://help.github.com/en/acti [`examples/github-deploy.yml`](https://github.com/joerick/cibuildwheel/blob/master/examples/github-deploy.yml) extends this minimal example with a demonstration of how to automatically upload the built wheels to PyPI. +You can also use cibuildwheel directly as an action with `uses: joerick/cibuildwheel@v1.7.3`; this combines the download and run steps into a single action, and command line arguments are available via `with:`. This makes it easy to manage cibuildwheel updates via normal actions update mechanisms like dependabot, see [Automatic updates](faq.md#automatic-updates). + # Azure Pipelines [linux/mac/windows] {: #azure-pipelines} To build Linux, Mac, and Windows wheels on Azure Pipelines, create a `azure-pipelines.yml` file in your repo. diff --git a/examples/appveyor-minimal.yml b/examples/appveyor-minimal.yml index 806a9859..fd490ee2 100644 --- a/examples/appveyor-minimal.yml +++ b/examples/appveyor-minimal.yml @@ -12,7 +12,7 @@ stack: python 3.7 init: - cmd: set PATH=C:\Python37;C:\Python37\Scripts;%PATH% -install: python -m pip install cibuildwheel==1.7.2 +install: python -m pip install cibuildwheel==1.7.3 build_script: python -m cibuildwheel --output-dir wheelhouse diff --git a/examples/azure-pipelines-minimal.yml b/examples/azure-pipelines-minimal.yml index 7ce34e40..8043a861 100644 --- a/examples/azure-pipelines-minimal.yml +++ b/examples/azure-pipelines-minimal.yml @@ -6,7 +6,7 @@ jobs: - bash: | set -o errexit python3 -m pip install --upgrade pip - pip3 install cibuildwheel==1.7.2 + pip3 install cibuildwheel==1.7.3 displayName: Install dependencies - bash: cibuildwheel --output-dir wheelhouse . displayName: Build wheels @@ -20,7 +20,7 @@ jobs: - bash: | set -o errexit python3 -m pip install --upgrade pip - python3 -m pip install cibuildwheel==1.7.2 + python3 -m pip install cibuildwheel==1.7.3 displayName: Install dependencies - bash: cibuildwheel --output-dir wheelhouse . displayName: Build wheels @@ -36,7 +36,7 @@ jobs: - bash: | set -o errexit python -m pip install --upgrade pip - pip install cibuildwheel==1.7.2 + pip install cibuildwheel==1.7.3 displayName: Install dependencies - bash: cibuildwheel --output-dir wheelhouse . displayName: Build wheels diff --git a/examples/circleci-minimal.yml b/examples/circleci-minimal.yml index 116dede7..5e09945d 100644 --- a/examples/circleci-minimal.yml +++ b/examples/circleci-minimal.yml @@ -11,7 +11,7 @@ jobs: - run: name: Build the Linux wheels. command: | - pip3 install --user cibuildwheel==1.7.2 + pip3 install --user cibuildwheel==1.7.3 cibuildwheel --output-dir wheelhouse - store_artifacts: path: wheelhouse/ @@ -25,7 +25,7 @@ jobs: - run: name: Build the OS X wheels. command: | - pip3 install cibuildwheel==1.7.2 + pip3 install cibuildwheel==1.7.3 cibuildwheel --output-dir wheelhouse - store_artifacts: path: wheelhouse/ diff --git a/examples/github-deploy.yml b/examples/github-deploy.yml index 43abcd66..0f636cc7 100644 --- a/examples/github-deploy.yml +++ b/examples/github-deploy.yml @@ -16,7 +16,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-18.04, windows-latest, macos-latest] + os: [ubuntu-18.04, windows-2019, macos-10.15] steps: - uses: actions/checkout@v2 @@ -28,7 +28,7 @@ jobs: - name: Install cibuildwheel run: | - python -m pip install cibuildwheel==1.7.2 + python -m pip install cibuildwheel==1.7.3 - name: Install Visual C++ for Python 2.7 if: runner.os == 'Windows' diff --git a/examples/github-minimal.yml b/examples/github-minimal.yml index 29600b35..ad8886c4 100644 --- a/examples/github-minimal.yml +++ b/examples/github-minimal.yml @@ -8,7 +8,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-18.04, windows-latest, macos-latest] + os: [ubuntu-18.04, windows-2019, macos-10.15] steps: - uses: actions/checkout@v2 @@ -20,7 +20,7 @@ jobs: - name: Install cibuildwheel run: | - python -m pip install cibuildwheel==1.7.2 + python -m pip install cibuildwheel==1.7.3 - name: Install Visual C++ for Python 2.7 if: runner.os == 'Windows' diff --git a/examples/github-with-qemu.yml b/examples/github-with-qemu.yml new file mode 100644 index 00000000..95a980f2 --- /dev/null +++ b/examples/github-with-qemu.yml @@ -0,0 +1,46 @@ +name: Build + +on: [push, pull_request] + +jobs: + build_wheels: + name: Build wheels on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: [ubuntu-18.04, windows-2019, macos-10.15] + + steps: + - uses: actions/checkout@v2 + + - uses: actions/setup-python@v2 + name: Install Python + with: + python-version: '3.7' + + - name: Install cibuildwheel + run: | + python -m pip install cibuildwheel==1.7.1 + + - name: Install Visual C++ for Python 2.7 + if: runner.os == 'Windows' + run: | + choco install vcpython27 -f -y + + - name: Set up QEMU + if: runner.os == 'Linux' + uses: docker/setup-qemu-action@v1 + with: + platforms: all + + - name: Build wheels + run: | + python -m cibuildwheel --output-dir wheelhouse + env: + # configure cibuildwheel to build native archs ('auto'), and some + # emulated ones + CIBW_ARCHS_LINUX: auto aarch64 ppc64le s390x + + - uses: actions/upload-artifact@v2 + with: + path: ./wheelhouse/*.whl diff --git a/examples/travis-ci-deploy.yml b/examples/travis-ci-deploy.yml index 402a95e1..73299b8b 100644 --- a/examples/travis-ci-deploy.yml +++ b/examples/travis-ci-deploy.yml @@ -20,7 +20,7 @@ jobs: - ln -s /c/Python38/python.exe /c/Python38/python3.exe install: - - python3 -m pip install cibuildwheel==1.7.2 + - python3 -m pip install cibuildwheel==1.7.3 script: # build the wheels, put them into './wheelhouse' diff --git a/examples/travis-ci-minimal.yml b/examples/travis-ci-minimal.yml index d9e1b64f..080b0e5c 100644 --- a/examples/travis-ci-minimal.yml +++ b/examples/travis-ci-minimal.yml @@ -26,7 +26,7 @@ jobs: - ln -s /c/Python38/python.exe /c/Python38/python3.exe install: - - python3 -m pip install cibuildwheel==1.7.2 + - python3 -m pip install cibuildwheel==1.7.3 script: # build the wheels, put them into './wheelhouse' diff --git a/examples/travis-ci-test-and-deploy.yml b/examples/travis-ci-test-and-deploy.yml index d1cfba18..58db2b02 100644 --- a/examples/travis-ci-test-and-deploy.yml +++ b/examples/travis-ci-test-and-deploy.yml @@ -55,7 +55,7 @@ jobs: - stage: deploy name: Build and deploy Linux wheels services: docker - install: python3 -m pip install cibuildwheel==1.7.2 + install: python3 -m pip install cibuildwheel==1.7.3 script: python3 -m cibuildwheel --output-dir wheelhouse after_success: | python3 -m pip install twine @@ -65,7 +65,7 @@ jobs: name: Build and deploy macOS wheels os: osx language: shell - install: python3 -m pip install cibuildwheel==1.7.2 + install: python3 -m pip install cibuildwheel==1.7.3 script: python3 -m cibuildwheel --output-dir wheelhouse after_success: | python3 -m pip install twine @@ -75,7 +75,7 @@ jobs: name: Build and deploy Windows wheels os: windows language: shell - install: python3 -m pip install cibuildwheel==1.7.2 + install: python3 -m pip install cibuildwheel==1.7.3 script: python3 -m cibuildwheel --output-dir wheelhouse after_success: | python3 -m pip install twine diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 00000000..285367ae --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,7 @@ +[build-system] +requires = [ + "setuptools>=42", + "wheel" +] + +build-backend = "setuptools.build_meta" diff --git a/requirements-dev.txt b/requirements-dev.txt index bfe96baa..304cf212 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,6 @@ -e . pytest +pytest-timeout mkdocs==1.0.4 mkdocs-include-markdown-plugin==2.1.1 pymdown-extensions diff --git a/setup.py b/setup.py index 204e7ccd..5e276555 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ long_description = (this_directory / 'README.md').read_text(encoding='utf-8') setup( name='cibuildwheel', - version='1.7.2', + version='1.7.3', install_requires=['bashlex!=0.13', 'toml', 'certifi'], description="Build Python wheels on CI with minimal configuration.", long_description=long_description, diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 00000000..410c0707 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,21 @@ +import pytest + + +def pytest_addoption(parser): + parser.addoption( + "--run-emulation", action="store_true", default=False, help="run emulation tests" + ) + + +def pytest_configure(config): + config.addinivalue_line("markers", "emulation: mark test requiring qemu binfmt_misc to run") + + +def pytest_collection_modifyitems(config, items): + if config.getoption("--run-emulation"): + # --run-emulation given in cli: do not skip emulation tests + return + skip_emulation = pytest.mark.skip(reason="need --run-emulation option to run") + for item in items: + if "emulation" in item.keywords: + item.add_marker(skip_emulation) diff --git a/test/test_before_test.py b/test/test_before_test.py index d0dad59d..bb87bc59 100644 --- a/test/test_before_test.py +++ b/test/test_before_test.py @@ -37,13 +37,15 @@ class TestBeforeTest(TestCase): def test(tmp_path): project_dir = tmp_path / 'project' before_test_project.generate(project_dir) + test_project_dir = project_dir / 'dependency' + test_projects.new_c_project().generate(test_project_dir) # build the wheels actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ # write python version information to a temporary file, this is # checked in setup.py - 'CIBW_BEFORE_TEST': '''python -c "import sys; open('/tmp/pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('/tmp/pythonprefix.txt', 'w').write(sys.prefix)"''', - 'CIBW_BEFORE_TEST_WINDOWS': '''python -c "import sys; open('c:\\pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('c:\\pythonprefix.txt', 'w').write(sys.prefix)"''', + 'CIBW_BEFORE_TEST': '''python -c "import sys; open('/tmp/pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('/tmp/pythonprefix.txt', 'w').write(sys.prefix)" && python -m pip install {project}/dependency''', + 'CIBW_BEFORE_TEST_WINDOWS': '''python -c "import sys; open('c:\\pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('c:\\pythonprefix.txt', 'w').write(sys.prefix)" && python -m pip install {project}/dependency''', 'CIBW_TEST_REQUIRES': 'nose', # the 'false ||' bit is to ensure this command runs in a shell on # mac/linux. diff --git a/test/test_emulation.py b/test/test_emulation.py new file mode 100644 index 00000000..e486327c --- /dev/null +++ b/test/test_emulation.py @@ -0,0 +1,53 @@ +import subprocess +import pytest +from . import utils +from . import test_projects + + +project_with_a_test = test_projects.new_c_project() + +project_with_a_test.files['test/spam_test.py'] = r''' +import spam + +def test_spam(): + assert spam.system('python -c "exit(0)"') == 0 + assert spam.system('python -c "exit(1)"') != 0 +''' + + +@pytest.mark.emulation +def test(tmp_path): + project_dir = tmp_path / 'project' + project_with_a_test.generate(project_dir) + + # build and test the wheels + actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ + 'CIBW_TEST_REQUIRES': 'pytest', + 'CIBW_TEST_COMMAND': 'pytest {project}/test', + 'CIBW_ARCHS': 'aarch64 ppc64le s390x', + }) + + # also check that we got the right wheels + expected_wheels = ( + utils.expected_wheels('spam', '0.1.0', machine_arch='aarch64') + + utils.expected_wheels('spam', '0.1.0', machine_arch='ppc64le') + + utils.expected_wheels('spam', '0.1.0', machine_arch='s390x') + ) + assert set(actual_wheels) == set(expected_wheels) + + +def test_setting_arch_on_other_platforms(tmp_path, capfd): + if utils.platform == 'linux': + pytest.skip('this test checks the behaviour on platforms other than linux') + + project_dir = tmp_path / 'project' + project_with_a_test.generate(project_dir) + + # build and test the wheels + with pytest.raises(subprocess.CalledProcessError): + utils.cibuildwheel_run(project_dir, add_env={ + 'CIBW_ARCHS': 'aarch64', + }) + + captured = capfd.readouterr() + assert "Invalid archs option" in captured.err diff --git a/test/test_testing.py b/test/test_testing.py index 6136c4aa..ea5122c4 100644 --- a/test/test_testing.py +++ b/test/test_testing.py @@ -64,6 +64,10 @@ class TestSpam(TestCase): bits = struct.calcsize("P") * 8 if bits == 32: self.assertEqual(platform.machine(), "i686") + + def test_time_to_remove_the_pypy_venv_patch(self): + if sys.platform == "darwin": + assert not hasattr(sys, "pypy_version_info") or sys.pypy_version_info < (7,3,4) ''' diff --git a/test/utils.py b/test/utils.py index 2ddbc813..d8aa50ee 100644 --- a/test/utils.py +++ b/test/utils.py @@ -74,7 +74,7 @@ def cibuildwheel_run(project_path, package_dir='.', env=None, add_env=None, outp def expected_wheels(package_name, package_version, manylinux_versions=None, - macosx_deployment_target='10.9'): + macosx_deployment_target='10.9', machine_arch=None): ''' Returns a list of expected wheels from a run of cibuildwheel. ''' @@ -83,15 +83,18 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, # {python tag} and {abi tag} are closely related to the python interpreter used to build the wheel # so we'll merge them below as python_abi_tag + if machine_arch is None: + machine_arch = pm.machine() + if manylinux_versions is None: - if pm.machine() == 'x86_64': + if machine_arch == 'x86_64': manylinux_versions = ['manylinux1', 'manylinux2010'] else: manylinux_versions = ['manylinux2014'] python_abi_tags = ['cp35-cp35m', 'cp36-cp36m', 'cp37-cp37m', 'cp38-cp38', 'cp39-cp39'] - if pm.machine() in ['x86_64', 'AMD64', 'x86']: + if machine_arch in ['x86_64', 'AMD64', 'x86']: python_abi_tags += ['cp27-cp27m', 'pp27-pypy_73', 'pp36-pypy36_pp73', 'pp37-pypy37_pp73'] if platform == 'linux': @@ -107,9 +110,9 @@ def expected_wheels(package_name, package_version, manylinux_versions=None, platform_tags = [] if platform == 'linux': - architectures = [pm.machine()] + architectures = [machine_arch] - if pm.machine() == 'x86_64' and python_abi_tag.startswith('cp'): + if machine_arch == 'x86_64' and python_abi_tag.startswith('cp'): architectures.append('i686') platform_tags = [ diff --git a/unit_test/main_tests/conftest.py b/unit_test/main_tests/conftest.py index 3f684b6c..8a165537 100644 --- a/unit_test/main_tests/conftest.py +++ b/unit_test/main_tests/conftest.py @@ -70,5 +70,14 @@ def platform(request, monkeypatch): @pytest.fixture def intercepted_build_args(platform, monkeypatch): intercepted = ArgsInterceptor() - monkeypatch.setattr(globals()[platform], 'build', intercepted) + + if platform == 'linux': + monkeypatch.setattr(linux, 'build', intercepted) + elif platform == 'macos': + monkeypatch.setattr(macos, 'build', intercepted) + elif platform == 'windows': + monkeypatch.setattr(windows, 'build', intercepted) + else: + raise ValueError(f'unknown platform value: {platform}') + return intercepted diff --git a/unit_test/main_tests/main_platform_test.py b/unit_test/main_tests/main_platform_test.py index 004855e8..9ac84ea8 100644 --- a/unit_test/main_tests/main_platform_test.py +++ b/unit_test/main_tests/main_platform_test.py @@ -1,3 +1,5 @@ +import platform as platform_module +from cibuildwheel.util import Architecture import sys import pytest @@ -64,3 +66,48 @@ def test_platform_environment(platform, intercepted_build_args, monkeypatch): main() assert intercepted_build_args.args[0].package_dir == MOCK_PACKAGE_DIR + + +def test_archs_default(platform, intercepted_build_args, monkeypatch): + monkeypatch.setattr(platform_module, 'machine', lambda: 'x86_64') + + main() + build_options = intercepted_build_args.args[0] + + if platform == 'linux': + assert build_options.architectures == [Architecture.x86_64, Architecture.i686] + else: + assert build_options.architectures == [Architecture.x86_64] + + +@pytest.mark.parametrize('use_env_var', [False, True]) +def test_archs_argument(platform, intercepted_build_args, monkeypatch, use_env_var): + monkeypatch.setattr(platform_module, 'machine', lambda: 'x86_64') + if use_env_var: + monkeypatch.setenv('CIBW_ARCHS', 'ppc64le') + else: + monkeypatch.setenv('CIBW_ARCHS', 'unused') + monkeypatch.setattr(sys, 'argv', sys.argv + ['--archs', 'ppc64le']) + + main() + build_options = intercepted_build_args.args[0] + + assert build_options.architectures == [Architecture.ppc64le] + + +def test_archs_platform_specific(platform, intercepted_build_args, monkeypatch): + monkeypatch.setattr(platform_module, 'machine', lambda: 'x86_64') + monkeypatch.setenv('CIBW_ARCHS', 'unused') + monkeypatch.setenv('CIBW_ARCHS_LINUX', 'ppc64le') + monkeypatch.setenv('CIBW_ARCHS_WINDOWS', 'x86') + monkeypatch.setenv('CIBW_ARCHS_MACOS', 'x86_64') + + main() + build_options = intercepted_build_args.args[0] + + if platform == 'linux': + assert build_options.architectures == [Architecture.ppc64le] + elif platform == 'windows': + assert build_options.architectures == [Architecture.x86] + elif platform == 'macos': + assert build_options.architectures == [Architecture.x86_64]