From 21e9c897ec5e8ea7d53ced2edf37922fcdc5bc30 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 21 May 2021 17:45:43 -0400 Subject: [PATCH] feat: add nox runner (#661) --- .github/workflows/update-dependencies.yml | 15 +-- .gitignore | 3 + bin/update_dependencies.py | 154 ++++------------------ bin/update_docker.py | 86 ++++++++++++ docs/contributing.md | 23 +++- noxfile.py | 93 +++++++++++++ 6 files changed, 235 insertions(+), 139 deletions(-) create mode 100755 bin/update_docker.py create mode 100644 noxfile.py diff --git a/.github/workflows/update-dependencies.yml b/.github/workflows/update-dependencies.yml index 60c7ce15..57a64468 100644 --- a/.github/workflows/update-dependencies.yml +++ b/.github/workflows/update-dependencies.yml @@ -11,21 +11,12 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - - - uses: actions/setup-python@v2 - name: Install Python 3.9 - with: - python-version: 3.9 - architecture: x64 - - - name: Install dependencies - run: python -m pip install ".[dev]" + - uses: excitedleigh/setup-nox@v2.0.0 - name: "Run update: dependencies" - run: python ./bin/update_dependencies.py - + run: nox --force-color -s update_constraints - name: "Run update: python configs" - run: python ./bin/update_pythons.py --force + run: nox --force-color -s update_pins # we use this step to grab a Github App auth token, so that PRs generated by this workflow # run the GHA tests. diff --git a/.gitignore b/.gitignore index b8f29c5e..98559d4f 100644 --- a/.gitignore +++ b/.gitignore @@ -114,3 +114,6 @@ site/ # Virtual environments venv* + +# PyCharm +.idea/ diff --git a/bin/update_dependencies.py b/bin/update_dependencies.py index 2aa2b2fd..1a997e4c 100755 --- a/bin/update_dependencies.py +++ b/bin/update_dependencies.py @@ -1,141 +1,43 @@ #!/usr/bin/env python3 -from __future__ import annotations +# This file supports 3.6+ -import configparser import os import shutil import subprocess import sys -from typing import NamedTuple +from pathlib import Path -import requests +DIR = Path(__file__).parent.resolve() +RESOURCES = DIR.parent / "cibuildwheel/resources" -os.chdir(os.path.dirname(__file__)) -os.chdir("..") +python_version = "".join(str(v) for v in sys.version_info[:2]) + +env = os.environ.copy() # CUSTOM_COMPILE_COMMAND is a pip-compile option that tells users how to # regenerate the constraints files -os.environ["CUSTOM_COMPILE_COMMAND"] = "bin/update_dependencies.py" +env["CUSTOM_COMPILE_COMMAND"] = "bin/update_dependencies.py" -PYTHON_VERSIONS = ["36", "37", "38", "39"] +if python_version == "36": + # Bug with click and Python 3.6 + env["LC_ALL"] = "C.UTF-8" + env["LANG"] = "C.UTF-8" -if "--no-docker" in sys.argv: - for python_version in PYTHON_VERSIONS: - subprocess.run( - [ - f"./env{python_version}/bin/pip-compile", - "--allow-unsafe", - "--upgrade", - "cibuildwheel/resources/constraints.in", - f"--output-file=cibuildwheel/resources/constraints-python{python_version}.txt", - ], - check=True, - ) -else: - image_runner = "quay.io/pypa/manylinux2010_x86_64:latest" - subprocess.run(["docker", "pull", image_runner], check=True) - for python_version in PYTHON_VERSIONS: - abi_flags = "" if int(python_version) >= 38 else "m" - python_path = f"/opt/python/cp{python_version}-cp{python_version}{abi_flags}/bin/" - command = ( - f"{python_path}pip install pip-tools && " - f"{python_path}pip-compile --allow-unsafe --upgrade " - "cibuildwheel/resources/constraints.in " - f"--output-file cibuildwheel/resources/constraints-python{python_version}.txt" - ) - subprocess.run( - [ - "docker", - "run", - "--rm", - "--env=CUSTOM_COMPILE_COMMAND", - f"--volume={os.getcwd()}:/volume", - "--workdir=/volume", - image_runner, - "bash", - "-c", - command, - ], - check=True, - ) - -# default constraints.txt -shutil.copyfile( - f"cibuildwheel/resources/constraints-python{PYTHON_VERSIONS[-1]}.txt", - "cibuildwheel/resources/constraints.txt", +subprocess.run( + [ + "pip-compile", + "--allow-unsafe", + "--upgrade", + f"{RESOURCES}/constraints.in", + f"--output-file={RESOURCES}/constraints-python{python_version}.txt", + ], + check=True, + env=env, ) - -class Image(NamedTuple): - manylinux_version: str - platform: str - image_name: str - tag: str | None - - -images = [ - Image("manylinux1", "x86_64", "quay.io/pypa/manylinux1_x86_64", None), - Image("manylinux1", "i686", "quay.io/pypa/manylinux1_i686", None), - # 2010 images - Image("manylinux2010", "x86_64", "quay.io/pypa/manylinux2010_x86_64", None), - Image("manylinux2010", "i686", "quay.io/pypa/manylinux2010_i686", None), - Image("manylinux2010", "pypy_x86_64", "pypywheels/manylinux2010-pypy_x86_64", None), - # 2014 images - Image("manylinux2014", "x86_64", "quay.io/pypa/manylinux2014_x86_64", None), - Image("manylinux2014", "i686", "quay.io/pypa/manylinux2014_i686", None), - Image("manylinux2014", "aarch64", "quay.io/pypa/manylinux2014_aarch64", None), - Image("manylinux2014", "ppc64le", "quay.io/pypa/manylinux2014_ppc64le", None), - Image("manylinux2014", "s390x", "quay.io/pypa/manylinux2014_s390x", None), - # 2_24 images - Image("manylinux_2_24", "x86_64", "quay.io/pypa/manylinux_2_24_x86_64", None), - Image("manylinux_2_24", "i686", "quay.io/pypa/manylinux_2_24_i686", None), - Image("manylinux_2_24", "aarch64", "quay.io/pypa/manylinux_2_24_aarch64", None), - Image("manylinux_2_24", "ppc64le", "quay.io/pypa/manylinux_2_24_ppc64le", None), - Image("manylinux_2_24", "s390x", "quay.io/pypa/manylinux_2_24_s390x", None), -] - -config = configparser.ConfigParser() - -for image in images: - # get the tag name whose digest matches 'latest' - if image.tag is not None: - # image has been pinned, do not update - tag_name = image.tag - elif image.image_name.startswith("quay.io/"): - _, _, repository_name = image.image_name.partition("/") - response = requests.get( - f"https://quay.io/api/v1/repository/{repository_name}?includeTags=true" - ) - response.raise_for_status() - repo_info = response.json() - tags_dict = repo_info["tags"] - - latest_tag = tags_dict.pop("latest") - # find the tag whose manifest matches 'latest' - tag_name = next( - name - for (name, info) in tags_dict.items() - if info["manifest_digest"] == latest_tag["manifest_digest"] - ) - else: - response = requests.get(f"https://hub.docker.com/v2/repositories/{image.image_name}/tags") - response.raise_for_status() - tags = response.json()["results"] - - latest_tag = next(tag for tag in tags if tag["name"] == "latest") - # i don't know what it would mean to have multiple images per tag - assert len(latest_tag["images"]) == 1 - digest = latest_tag["images"][0]["digest"] - - pinned_tag = next( - tag for tag in tags if tag != latest_tag and tag["images"][0]["digest"] == digest - ) - tag_name = pinned_tag["name"] - - if not config.has_section(image.platform): - config[image.platform] = {} - - config[image.platform][image.manylinux_version] = f"{image.image_name}:{tag_name}" - -with open("cibuildwheel/resources/pinned_docker_images.cfg", "w") as f: - config.write(f) +# default constraints.txt +if python_version == "39": + shutil.copyfile( + RESOURCES / f"constraints-python{python_version}.txt", + RESOURCES / "constraints.txt", + ) diff --git a/bin/update_docker.py b/bin/update_docker.py new file mode 100755 index 00000000..22034aae --- /dev/null +++ b/bin/update_docker.py @@ -0,0 +1,86 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import configparser +from pathlib import Path +from typing import NamedTuple + +import requests + +DIR = Path(__file__).parent.resolve() +RESOURCES = DIR.parent / "cibuildwheel/resources" + + +class Image(NamedTuple): + manylinux_version: str + platform: str + image_name: str + tag: str | None # Set this to pin the image + + +images = [ + Image("manylinux1", "x86_64", "quay.io/pypa/manylinux1_x86_64", None), + Image("manylinux1", "i686", "quay.io/pypa/manylinux1_i686", None), + # 2010 images + Image("manylinux2010", "x86_64", "quay.io/pypa/manylinux2010_x86_64", None), + Image("manylinux2010", "i686", "quay.io/pypa/manylinux2010_i686", None), + Image("manylinux2010", "pypy_x86_64", "pypywheels/manylinux2010-pypy_x86_64", None), + # 2014 images + Image("manylinux2014", "x86_64", "quay.io/pypa/manylinux2014_x86_64", None), + Image("manylinux2014", "i686", "quay.io/pypa/manylinux2014_i686", None), + Image("manylinux2014", "aarch64", "quay.io/pypa/manylinux2014_aarch64", None), + Image("manylinux2014", "ppc64le", "quay.io/pypa/manylinux2014_ppc64le", None), + Image("manylinux2014", "s390x", "quay.io/pypa/manylinux2014_s390x", None), + # 2_24 images + Image("manylinux_2_24", "x86_64", "quay.io/pypa/manylinux_2_24_x86_64", None), + Image("manylinux_2_24", "i686", "quay.io/pypa/manylinux_2_24_i686", None), + Image("manylinux_2_24", "aarch64", "quay.io/pypa/manylinux_2_24_aarch64", None), + Image("manylinux_2_24", "ppc64le", "quay.io/pypa/manylinux_2_24_ppc64le", None), + Image("manylinux_2_24", "s390x", "quay.io/pypa/manylinux_2_24_s390x", None), +] + +config = configparser.ConfigParser() + +for image in images: + # get the tag name whose digest matches 'latest' + if image.tag is not None: + # image has been pinned, do not update + tag_name = image.tag + elif image.image_name.startswith("quay.io/"): + _, _, repository_name = image.image_name.partition("/") + response = requests.get( + f"https://quay.io/api/v1/repository/{repository_name}?includeTags=true" + ) + response.raise_for_status() + repo_info = response.json() + tags_dict = repo_info["tags"] + + latest_tag = tags_dict.pop("latest") + # find the tag whose manifest matches 'latest' + tag_name = next( + name + for (name, info) in tags_dict.items() + if info["manifest_digest"] == latest_tag["manifest_digest"] + ) + else: + response = requests.get(f"https://hub.docker.com/v2/repositories/{image.image_name}/tags") + response.raise_for_status() + tags = response.json()["results"] + + latest_tag = next(tag for tag in tags if tag["name"] == "latest") + # i don't know what it would mean to have multiple images per tag + assert len(latest_tag["images"]) == 1 + digest = latest_tag["images"][0]["digest"] + + pinned_tag = next( + tag for tag in tags if tag != latest_tag and tag["images"][0]["digest"] == digest + ) + tag_name = pinned_tag["name"] + + if not config.has_section(image.platform): + config[image.platform] = {} + + config[image.platform][image.manylinux_version] = f"{image.image_name}:{tag_name}" + +with open(RESOURCES / "pinned_docker_images.cfg", "w") as f: + config.write(f) diff --git a/docs/contributing.md b/docs/contributing.md index 50242765..60b8f825 100644 --- a/docs/contributing.md +++ b/docs/contributing.md @@ -34,11 +34,32 @@ So, if we can, I'd like to improve the experience on errors as well. In [this](h Maintainer notes ---------------- +### Nox support + +Most developer tasks have a nox interface. This allows you to very simply run tasks without worrying about setting up a development environment (as shown below). This is a slower than setting up a development environment and reusing it, but has the (important) benefit of being highly reproducible; an earlier run does not affect a current run, or anything else on your machine. + +Install [nox](https://nox.thea.codes); homebrew is recommend on macOS, otherwise, pipx is a great choice - in fact, you can use `pipx run nox` and avoid installing completely. + +You can see a list of sessions by typing `nox -l`; here are a few common ones: + +```console +nox -s lint # Run the linters (default) +nox -s tests # Run the tests (default) +nox -s docs -- serve # Build and serve the documentation +nox -s build # Make SDist and wheel +``` + +More advanced users can run the update scripts. `update_pins` should work directly, but `update_constraints` needs all versions of Python installed. If you don't want to do that locally, a fast way to run it to use docker to run nox: + +```console +docker run --rm -it -v $PWD:/src quay.io/pypa/manylinux2010_x86_64:latest pipx run nox -f src/noxfile.py -s update_constraints +``` + ### Local testing You should run: -```python +```console python3 -m venv venv . venv/bin/activate pip install -e .[dev] diff --git a/noxfile.py b/noxfile.py new file mode 100644 index 00000000..90f3ceaf --- /dev/null +++ b/noxfile.py @@ -0,0 +1,93 @@ +import shutil +import sys +from pathlib import Path + +import nox + +nox.options.sessions = ["lint", "tests"] + +PYTHON_ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9"] + +DIR = Path(__file__).parent.resolve() + + +@nox.session +def tests(session): + """ + Run the unit and regular tests. + """ + unit_test_args = ["--run-docker"] if sys.platform.startswith("linux") else [] + session.install("-e", ".[test]") + session.run("pytest", "unit_test", *unit_test_args) + session.run("pytest", "test", "-x", "--durations", "0", "--timeout=2400", "test") + + +@nox.session +def lint(session): + """ + Run the linter. + """ + session.install("pre-commit") + session.run("pre-commit", "run", "--all-files") + + +@nox.session(python=PYTHON_ALL_VERSIONS) +def update_constraints(session): + """ + Update the dependencies inplace. + """ + session.install("requests", "pip-tools") + session.run("python", "bin/update_dependencies.py") + + +@nox.session +def update_pins(session): + """ + Update the python and docker pins version inplace. + """ + session.install("-e", ".[dev]") + session.run("python", "bin/update_pythons.py", "--force") + session.run("python", "bin/update_docker.py") + + +@nox.session +def update_proj(session): + """ + Update the README inplace. + """ + session.install("-e", ".[dev]") + session.run("./bin/projects.py", "docs/data/projects.yml", "--readme=README.md") + + +@nox.session +def docs(session): + """ + Build the docs. + """ + session.install("-e", ".[docs]") + + if session.posargs: + if "serve" in session.posargs: + session.run("mkdocs", "serve") + else: + print("Unrecognized args, use 'serve'") + else: + session.run("mkdocs", "build") + + +@nox.session +def build(session): + """ + Build an SDist and wheel. + """ + + build_p = DIR.joinpath("build") + if build_p.exists(): + shutil.rmtree(build_p) + + dist_p = DIR.joinpath("dist") + if dist_p.exists(): + shutil.rmtree(dist_p) + + session.install("build") + session.run("python", "-m", "build")