From ff9508a3e16ed89451be7759cfa97328240663d7 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 16 Mar 2026 19:24:46 +0000 Subject: [PATCH] [2.x] Update virtualenv URLs to avoid blob URLs (#2775) * [2.x] Update virtualenv URLs to avoid blob URLs It looks like many people are hitting the HTTP 429 for old versions of cibuildwheel. These hosted versions of virtualenv should work, let's see... * Write the correct version strings * ci: some updates based on main branch Signed-off-by: Henry Schreiner * ci: move to using latest images Signed-off-by: Henry Schreiner * chore: fix up style checks Signed-off-by: Henry Schreiner * ci: newer azure images too Signed-off-by: Henry Schreiner * tests: use new SSL test from main Signed-off-by: Henry Schreiner * Drop `build`'s installation from Pyodide build tools --------- Signed-off-by: Henry Schreiner Co-authored-by: Henry Schreiner Co-authored-by: Agriya Khetarpal <74401230+agriyakhetarpal@users.noreply.github.com> --- .github/workflows/release.yml | 10 +++- .github/workflows/test.yml | 61 +++++++++++--------- .github/workflows/update-dependencies.yml | 57 ------------------ .github/workflows/update-major-minor-tag.yml | 4 +- .gitlab-ci.yml | 2 +- .pre-commit-config.yaml | 2 +- .readthedocs.yml | 2 +- README.md | 2 +- azure-pipelines.yml | 6 +- cibuildwheel/pyodide.py | 1 - cibuildwheel/resources/virtualenv.toml | 4 +- docs/changelog.md | 4 +- docs/faq.md | 4 +- docs/setup.md | 8 +-- examples/github-deploy.yml | 4 +- examples/github-minimal.yml | 4 +- examples/github-with-qemu.yml | 4 +- examples/gitlab-minimal.yml | 2 +- noxfile.py | 2 +- pyproject.toml | 1 + test/test_ssl.py | 28 ++++++--- 21 files changed, 91 insertions(+), 121 deletions(-) delete mode 100644 .github/workflows/update-dependencies.yml diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 8f97b1c9..2e8824a7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,11 +7,15 @@ on: types: - published +permissions: {} + jobs: dist: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 + with: + persist-credentials: false - uses: hynek/build-and-inspect-python-package@v2 @@ -27,13 +31,13 @@ jobs: attestations: write steps: - - uses: actions/download-artifact@v4 + - uses: actions/download-artifact@v8 with: name: Packages path: dist - name: Generate artifact attestation for sdist and wheel - uses: actions/attest-build-provenance@ef244123eb79f2f7a7e75d99086184180e6d0018 # v1.4.4 + uses: actions/attest-build-provenance@a2bbfa25375fe432b6a289bc6b6cd05ecd0c4c32 # v4.1.0 with: subject-path: "dist/cibuildwheel-*" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 95ca185b..565674c8 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -12,6 +12,8 @@ on: workflow_dispatch: # allow manual runs on branches without a PR +permissions: {} + concurrency: group: test-${{ github.ref }} cancel-in-progress: true @@ -21,14 +23,14 @@ jobs: name: Linters (mypy, flake8, etc.) runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 id: python with: python-version: "3.x" - - uses: pre-commit/action@v3.0.1 - - name: Check manifest - run: pipx run --python "${{ steps.python.outputs.python-path }}" nox -s check_manifest + - uses: j178/prek-action@v1 - name: PyLint checks run: pipx run --python "${{ steps.python.outputs.python-path }}" nox -s pylint -- --output-format=github @@ -38,21 +40,23 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-13, macos-14] + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-15-intel, macos-latest] python_version: ['3.13'] include: - os: ubuntu-latest python_version: '3.8' timeout-minutes: 180 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 name: Install Python ${{ matrix.python_version }} with: python-version: ${{ matrix.python_version }} allow-prereleases: true - - uses: astral-sh/setup-uv@v3 + - uses: astral-sh/setup-uv@v7 # free some space to prevent reaching GHA disk space limits - name: Clean docker images @@ -64,7 +68,7 @@ jobs: # for oci_container unit tests - name: Set up QEMU if: runner.os == 'Linux' - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 - name: Install dependencies run: | @@ -118,7 +122,7 @@ jobs: test $(find wheelhouse_only -name '*.whl' | wc -l) -eq 1 test $(find wheelhouse_config_file -name '*.whl' | wc -l) -eq 1 - - uses: actions/upload-artifact@v4 + - uses: actions/upload-artifact@v7 with: name: cibw-wheels-${{ matrix.os }}-${{ strategy.job-index }} path: wheelhouse/*.whl @@ -134,17 +138,19 @@ jobs: outputs: archs: ${{ steps.archs.outputs.archs }} steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 with: python-version: "3.x" - - uses: astral-sh/setup-uv@v3 + - uses: astral-sh/setup-uv@v7 - name: Install dependencies run: uv sync --no-dev --group test - name: Get qemu emulated architectures id: archs run: | - OUTPUT=$(uv run python -c "from json import dumps; from test.utils import EMULATED_ARCHS; print(dumps(EMULATED_ARCHS))") + OUTPUT=$(uv run --no-sync python -c "from json import dumps; from test.utils import EMULATED_ARCHS; print(dumps(EMULATED_ARCHS))") echo "${OUTPUT}" echo "archs=${OUTPUT}" >> "$GITHUB_OUTPUT" @@ -157,32 +163,36 @@ jobs: matrix: arch: ${{ fromJSON(needs.emulated-archs.outputs.archs) }} steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 with: python-version: "3.x" - - uses: astral-sh/setup-uv@v3 + - uses: astral-sh/setup-uv@v7 - name: Install dependencies run: uv sync --no-dev --group test - name: Set up QEMU - uses: docker/setup-qemu-action@v3 + uses: docker/setup-qemu-action@v4 - name: Run the emulation tests - run: uv run pytest --run-emulation ${{ matrix.arch }} test/test_emulation.py + run: uv run --no-sync pytest --run-emulation ${{ matrix.arch }} test/test_emulation.py test-pyodide: name: Test cibuildwheel building Pyodide wheels needs: lint - runs-on: ubuntu-24.04 + runs-on: ubuntu-latest timeout-minutes: 180 steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 + - uses: actions/checkout@v6 + with: + persist-credentials: false + - uses: actions/setup-python@v6 name: Install Python 3.12 with: python-version: '3.12' - - uses: astral-sh/setup-uv@v3 + - uses: astral-sh/setup-uv@v7 - name: Install dependencies run: uv sync --no-dev --group test @@ -200,7 +210,6 @@ jobs: CIBW_PLATFORM: pyodide - name: Run tests with 'CIBW_PLATFORM' set to 'pyodide' - run: | - uv run ./bin/run_tests.py + run: uv run --no-sync ./bin/run_tests.py env: CIBW_PLATFORM: pyodide diff --git a/.github/workflows/update-dependencies.yml b/.github/workflows/update-dependencies.yml deleted file mode 100644 index 465056c5..00000000 --- a/.github/workflows/update-dependencies.yml +++ /dev/null @@ -1,57 +0,0 @@ -name: Update dependencies - -on: - pull_request: - paths: - - '.github/workflows/update-dependencies.yml' - - 'bin/update_pythons.py' - - 'bin/update_docker.py' - - 'bin/update_virtualenv.py' - - 'bin/projects.py' - - 'docs/data/projects.yml' - - 'noxfile.py' - workflow_dispatch: - schedule: - - cron: '0 6 * * 1' # "At 06:00 on Monday." - -jobs: - update-dependencies: - name: Update dependencies - if: github.repository_owner == 'pypa' || github.event_name != 'schedule' - runs-on: ubuntu-latest - steps: - - # we use this step to grab a Github App auth token, so that PRs generated by this workflow - # run the GHA tests. - - uses: actions/create-github-app-token@v1 - id: generate-token - if: github.ref == 'refs/heads/main' && github.repository == 'pypa/cibuildwheel' - with: - app_id: ${{ secrets.CIBUILDWHEEL_BOT_APP_ID }} - private_key: ${{ secrets.CIBUILDWHEEL_BOT_APP_PRIVATE_KEY }} - - - uses: actions/checkout@v4 - - - uses: wntrblm/nox@2024.10.09 - - - name: "Run update: dependencies" - run: nox --force-color -s update_constraints - - name: "Run update: python configs" - run: nox --force-color -s update_pins - - name: "Run update: docs user projects" - run: nox --force-color -s update_proj -- --auth=${{ secrets.GITHUB_TOKEN }} - - - name: Create Pull Request - if: github.ref == 'refs/heads/main' && github.repository == 'pypa/cibuildwheel' - uses: peter-evans/create-pull-request@v7 - with: - commit-message: Update dependencies - title: '[Bot] Update dependencies' - body: | - Update the versions of our dependencies. - - PR generated by "Update dependencies" [workflow](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}). - branch: update-dependencies-pr - sign-commits: true - token: ${{ steps.generate-token.outputs.token }} - delete-branch: true diff --git a/.github/workflows/update-major-minor-tag.yml b/.github/workflows/update-major-minor-tag.yml index 9958811b..d08a3d7d 100644 --- a/.github/workflows/update-major-minor-tag.yml +++ b/.github/workflows/update-major-minor-tag.yml @@ -21,7 +21,9 @@ jobs: permissions: contents: write steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6 + with: + persist-credentials: false - name: Update the ${{ env.TAG_NAME }} tag id: update-major-minor-tag diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ecf0361f..e72ef5e1 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -34,7 +34,7 @@ windows: - saas-windows-medium-amd64 macos: - image: macos-14-xcode-15 + image: macos-latest-xcode-15 variables: PYTEST_ADDOPTS: -k "unit_test or test_0_basic" --suppress-no-test-exit-code script: diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 4c0e3e99..8d169873 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -34,7 +34,7 @@ repos: - orjson - packaging - pygithub - - pytest + - pytest<9 - rich - tomli - tomli_w diff --git a/.readthedocs.yml b/.readthedocs.yml index 60724f36..84c6e5c8 100644 --- a/.readthedocs.yml +++ b/.readthedocs.yml @@ -3,7 +3,7 @@ version: 2 build: - os: ubuntu-22.04 + os: ubuntu-24.04 tools: python: "3.12" commands: diff --git a/README.md b/README.md index 9a3c0ab2..4934d13f 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-13, macos-latest] + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-15-intel, macos-latest] steps: - uses: actions/checkout@v4 diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6541244c..3eba0043 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -7,7 +7,7 @@ pr: jobs: - job: linux_38 timeoutInMinutes: 120 - pool: {vmImage: 'Ubuntu-20.04'} + pool: {vmImage: 'ubuntu-latest'} steps: - task: UsePythonVersion@0 inputs: @@ -19,7 +19,7 @@ jobs: python ./bin/run_tests.py - job: macos_38 - pool: {vmImage: 'macOS-13'} + pool: {vmImage: 'macos-latest'} steps: - task: UsePythonVersion@0 inputs: @@ -30,7 +30,7 @@ jobs: python ./bin/run_tests.py --num-processes 2 - job: windows_38 - pool: {vmImage: 'windows-2019'} + pool: {vmImage: 'windows-latest'} timeoutInMinutes: 180 steps: - task: UsePythonVersion@0 diff --git a/cibuildwheel/pyodide.py b/cibuildwheel/pyodide.py index 19d47e5e..3ec6159b 100644 --- a/cibuildwheel/pyodide.py +++ b/cibuildwheel/pyodide.py @@ -165,7 +165,6 @@ def setup_python( "install", "--upgrade", "auditwheel-emscripten", - "build[virtualenv]", "pyodide-build", *dependency_constraint_flags, env=env, diff --git a/cibuildwheel/resources/virtualenv.toml b/cibuildwheel/resources/virtualenv.toml index 09e96e30..d6c59ac8 100644 --- a/cibuildwheel/resources/virtualenv.toml +++ b/cibuildwheel/resources/virtualenv.toml @@ -1,2 +1,2 @@ -py36 = { version = "20.21.1", url = "https://github.com/pypa/get-virtualenv/blob/20.21.1/public/virtualenv.pyz?raw=true" } -default = { version = "20.30.0", url = "https://github.com/pypa/get-virtualenv/blob/20.30.0/public/virtualenv.pyz?raw=true" } +py36 = { version = "20.17.1", url = "https://bootstrap.pypa.io/virtualenv/3.6/virtualenv.pyz" } +default = { version = "20.26.6", url = "https://bootstrap.pypa.io/virtualenv/3.7/virtualenv.pyz" } diff --git a/docs/changelog.md b/docs/changelog.md index a633ea80..5c70e022 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -185,7 +185,7 @@ _12 May 2024_ _11 March 2024_ - 🌟 Adds the ability to inherit configuration in TOML overrides. This makes certain configurations much simpler. If you're overriding an option like `before-build` or `environment`, and you just want to add an extra command or environment variable, you can just append (or prepend) to the previous config. See [the docs](https://cibuildwheel.pypa.io/en/stable/options/#inherit) for more information. (#1730) -- 🌟 Adds official support for native `arm64` macOS GitHub runners. To use them, just specify `macos-14` as an `os` of your job in your workflow file. You can also keep `macos-13` in your build matrix to build `x86_64`. Check out the new [GitHub Actions example config](https://cibuildwheel.pypa.io/en/stable/setup/#github-actions). +- 🌟 Adds official support for native `arm64` macOS GitHub runners. To use them, just specify `macos-latest` as an `os` of your job in your workflow file. You can also keep `macos-15-intel` in your build matrix to build `x86_64`. Check out the new [GitHub Actions example config](https://cibuildwheel.pypa.io/en/stable/setup/#github-actions). - ✨ You no longer need to specify `--platform` to run cibuildwheel locally! Instead it will detect your platform automatically. This was a safety feature, no longer necessary. (#1727) - 🛠 Removed setuptools and wheel pinned versions. This only affects old-style projects without a `pyproject.toml`, projects with `pyproject.toml` are already getting fresh versions of their `build-system.requires` installed into an isolated environment. (#1725) - 🛠 Improve how the GitHub Action passes arguments (#1757) @@ -202,7 +202,7 @@ _11 March 2024_ _30 January 2024_ - 🐛 Fix an incompatibility with the GitHub Action and new GitHub Runner images for Windows that bundle Powershell 7.3+ (#1741) -- 🛠 Preliminary support for new `macos-14` arm64 runners (#1743) +- 🛠 Preliminary support for new `macos-latest` arm64 runners (#1743) ### v2.16.4 diff --git a/docs/faq.md b/docs/faq.md index 82a6ccc8..4682a4c2 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -72,7 +72,7 @@ See [GitHub issue 1333](https://github.com/pypa/cibuildwheel/issues/1333) for mo It's easiest to build `x86_64` wheels on `x86_64` runners, and `arm64` wheels on `arm64` runners. -On GitHub Actions, `macos-14` runners are `arm64`, and `macos-13` runners are `x86_64`. So all you need to do is ensure both are in your build matrix. +On GitHub Actions, `macos-latest` runners are `arm64`, and `macos-15-intel` runners are `x86_64`. So all you need to do is ensure both are in your build matrix. #### Cross-compiling @@ -363,7 +363,7 @@ If you're building on an arm64 runner, you might notice something strange about This is fine for simple C extensions, but for more complicated builds on arm64 it becomes an issue. -So, if you want to build macOS arm64 wheels on an arm64 runner (e.g., `macos-14`) on Python 3.8, before invoking cibuildwheel, you should install a native arm64 Python 3.8 interpreter on the runner: +So, if you want to build macOS arm64 wheels on an arm64 runner (e.g., `macos-latest`) on Python 3.8, before invoking cibuildwheel, you should install a native arm64 Python 3.8 interpreter on the runner: !!! tab "GitHub Actions" diff --git a/docs/setup.md b/docs/setup.md index 4c743f98..b4d0cf52 100644 --- a/docs/setup.md +++ b/docs/setup.md @@ -154,8 +154,8 @@ To build Linux, Mac, and Windows wheels using GitHub Actions, create a `.github/ runs-on: ${{ matrix.os }} strategy: matrix: - # macos-13 is an intel runner, macos-14 is apple silicon - os: [ubuntu-latest, windows-latest, macos-13, macos-14] + # macos-15-intel is an intel runner, macos-latest is apple silicon + os: [ubuntu-latest, windows-latest, macos-15-intel, macos-latest] steps: - uses: actions/checkout@v4 @@ -188,8 +188,8 @@ To build Linux, Mac, and Windows wheels using GitHub Actions, create a `.github/ runs-on: ${{ matrix.os }} strategy: matrix: - # macos-13 is an intel runner, macos-14 is apple silicon - os: [ubuntu-latest, windows-latest, macos-13, macos-14] + # macos-15-intel is an intel runner, macos-latest is apple silicon + os: [ubuntu-latest, windows-latest, macos-15-intel, macos-latest] steps: - uses: actions/checkout@v4 diff --git a/examples/github-deploy.yml b/examples/github-deploy.yml index bb4747d5..885df49d 100644 --- a/examples/github-deploy.yml +++ b/examples/github-deploy.yml @@ -16,8 +16,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - # macos-13 is an intel runner, macos-14 is apple silicon - os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-13, macos-14] + # macos-15-intel is an intel runner, macos-latest is apple silicon + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-15-intel, macos-latest] steps: - uses: actions/checkout@v4 diff --git a/examples/github-minimal.yml b/examples/github-minimal.yml index 839100bf..45c15b21 100644 --- a/examples/github-minimal.yml +++ b/examples/github-minimal.yml @@ -8,8 +8,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - # macos-13 is an intel runner, macos-14 is apple silicon - os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-13, macos-14] + # macos-15-intel is an intel runner, macos-latest is apple silicon + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-15-intel, macos-latest] steps: - uses: actions/checkout@v4 diff --git a/examples/github-with-qemu.yml b/examples/github-with-qemu.yml index 3c6ee1ab..8ba27e3e 100644 --- a/examples/github-with-qemu.yml +++ b/examples/github-with-qemu.yml @@ -8,8 +8,8 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - # macos-13 is an intel runner, macos-14 is apple silicon - os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-13, macos-14] + # macos-15-intel is an intel runner, macos-latest is apple silicon + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest, macos-15-intel, macos-latest] steps: - uses: actions/checkout@v4 diff --git a/examples/gitlab-minimal.yml b/examples/gitlab-minimal.yml index 7441a0e6..e1f54e59 100644 --- a/examples/gitlab-minimal.yml +++ b/examples/gitlab-minimal.yml @@ -33,7 +33,7 @@ windows: - saas-windows-medium-amd64 macos: - image: macos-14-xcode-15 + image: macos-latest-xcode-15 before_script: - python3 -m pip install cibuildwheel==2.23.3 script: diff --git a/noxfile.py b/noxfile.py index c1f52f4c..790de730 100644 --- a/noxfile.py +++ b/noxfile.py @@ -15,7 +15,7 @@ nox.options.default_venv_backend = "uv|virtualenv" DIR = Path(__file__).parent.resolve() -def install_and_run(session: nox.Session, script: str, *args: str, **kwargs: Any) -> str | None: +def install_and_run(session: nox.Session, script: str, *args: str, **kwargs: Any) -> Any: deps = nox.project.load_toml(script)["dependencies"] session.install(*deps) return session.run("python", script, *args, **kwargs) diff --git a/pyproject.toml b/pyproject.toml index b8a180b2..5dd46f8a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -87,6 +87,7 @@ test = [ "jinja2", "pytest-timeout", "pytest-xdist", + "pytest-rerunfailures!=16.0", "pytest>=6", "setuptools", "tomli_w", diff --git a/test/test_ssl.py b/test/test_ssl.py index 9c35650e..60e7ffef 100644 --- a/test/test_ssl.py +++ b/test/test_ssl.py @@ -1,25 +1,37 @@ -from __future__ import annotations - import textwrap +import pytest + from . import test_projects, utils project_with_ssl_tests = test_projects.new_c_project( setup_py_add=textwrap.dedent( r""" - import ssl + import ssl, time, urllib.request, urllib.error - from urllib.request import urlopen + def check_https(context=None): + # google hosts this endpoint that returns a 204 No Content, it's used for + # connectivity checks in Android & Chrome + url = "https://google.com/generate_204" - context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) - data = urlopen("https://www.nist.gov", context=context) - data = urlopen("https://raw.githubusercontent.com/pypa/cibuildwheel/main/CI.md", context=context) - data = urlopen("https://raw.githubusercontent.com/pypa/cibuildwheel/main/CI.md") + for i in range(5): + try: + urllib.request.urlopen(url, context=context, timeout=5) + return + except (OSError, urllib.error.URLError) as e: + print(f"Attempt {i+1}: Could not connect to {url}: {e}") + time.sleep(2 ** i) # Backoff: 1s, 2s, 4s, 8s... + + raise ConnectionError(f"Could not connect to {url} after retries.") + + check_https() + check_https(context=ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)) """ ) ) +@pytest.mark.flaky(reruns=2) def test(tmp_path): # this test checks that SSL is working in the build environment using # some checks in setup.py.