feat: add nox runner (#661)

.github/workflows/update-dependencies.yml

@@ -11,21 +11,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-
+    - uses: excitedleigh/setup-nox@v2.0.0
-    - uses: actions/setup-python@v2
-      name: Install Python 3.9
-      with:
-        python-version: 3.9
-        architecture: x64
-
-    - name: Install dependencies
-      run: python -m pip install ".[dev]"
 
     - name: "Run update: dependencies"
-      run: python ./bin/update_dependencies.py
+      run: nox --force-color -s update_constraints
-
     - name: "Run update: python configs"
-      run: python ./bin/update_pythons.py --force
+      run: nox --force-color -s update_pins
 
     # we use this step to grab a Github App auth token, so that PRs generated by this workflow
     # run the GHA tests.

.gitignore

@@ -114,3 +114,6 @@ site/
 
 # Virtual environments
 venv*
+
+# PyCharm
+.idea/

bin/update_dependencies.py

@@ -1,141 +1,43 @@
 #!/usr/bin/env python3
-from __future__ import annotations
+# This file supports 3.6+
 
-import configparser
 import os
 import shutil
 import subprocess
 import sys
-from typing import NamedTuple
+from pathlib import Path
 
-import requests
+DIR = Path(__file__).parent.resolve()
+RESOURCES = DIR.parent / "cibuildwheel/resources"
 
-os.chdir(os.path.dirname(__file__))
+python_version = "".join(str(v) for v in sys.version_info[:2])
-os.chdir("..")
+
+env = os.environ.copy()
 
 # CUSTOM_COMPILE_COMMAND is a pip-compile option that tells users how to
 # regenerate the constraints files
-os.environ["CUSTOM_COMPILE_COMMAND"] = "bin/update_dependencies.py"
+env["CUSTOM_COMPILE_COMMAND"] = "bin/update_dependencies.py"
 
-PYTHON_VERSIONS = ["36", "37", "38", "39"]
+if python_version == "36":
+    # Bug with click and Python 3.6
+    env["LC_ALL"] = "C.UTF-8"
+    env["LANG"] = "C.UTF-8"
 
-if "--no-docker" in sys.argv:
+subprocess.run(
-    for python_version in PYTHON_VERSIONS:
+    [
-        subprocess.run(
+        "pip-compile",
-            [
+        "--allow-unsafe",
-                f"./env{python_version}/bin/pip-compile",
+        "--upgrade",
-                "--allow-unsafe",
+        f"{RESOURCES}/constraints.in",
-                "--upgrade",
+        f"--output-file={RESOURCES}/constraints-python{python_version}.txt",
-                "cibuildwheel/resources/constraints.in",
+    ],
-                f"--output-file=cibuildwheel/resources/constraints-python{python_version}.txt",
+    check=True,
-            ],
+    env=env,
-            check=True,
-        )
-else:
-    image_runner = "quay.io/pypa/manylinux2010_x86_64:latest"
-    subprocess.run(["docker", "pull", image_runner], check=True)
-    for python_version in PYTHON_VERSIONS:
-        abi_flags = "" if int(python_version) >= 38 else "m"
-        python_path = f"/opt/python/cp{python_version}-cp{python_version}{abi_flags}/bin/"
-        command = (
-            f"{python_path}pip install pip-tools && "
-            f"{python_path}pip-compile --allow-unsafe --upgrade "
-            "cibuildwheel/resources/constraints.in "
-            f"--output-file cibuildwheel/resources/constraints-python{python_version}.txt"
-        )
-        subprocess.run(
-            [
-                "docker",
-                "run",
-                "--rm",
-                "--env=CUSTOM_COMPILE_COMMAND",
-                f"--volume={os.getcwd()}:/volume",
-                "--workdir=/volume",
-                image_runner,
-                "bash",
-                "-c",
-                command,
-            ],
-            check=True,
-        )
-
-# default constraints.txt
-shutil.copyfile(
-    f"cibuildwheel/resources/constraints-python{PYTHON_VERSIONS[-1]}.txt",
-    "cibuildwheel/resources/constraints.txt",
 )
 
-
+# default constraints.txt
-class Image(NamedTuple):
+if python_version == "39":
-    manylinux_version: str
+    shutil.copyfile(
-    platform: str
+        RESOURCES / f"constraints-python{python_version}.txt",
-    image_name: str
+        RESOURCES / "constraints.txt",
-    tag: str | None
+    )
-
-
-images = [
-    Image("manylinux1", "x86_64", "quay.io/pypa/manylinux1_x86_64", None),
-    Image("manylinux1", "i686", "quay.io/pypa/manylinux1_i686", None),
-    # 2010 images
-    Image("manylinux2010", "x86_64", "quay.io/pypa/manylinux2010_x86_64", None),
-    Image("manylinux2010", "i686", "quay.io/pypa/manylinux2010_i686", None),
-    Image("manylinux2010", "pypy_x86_64", "pypywheels/manylinux2010-pypy_x86_64", None),
-    # 2014 images
-    Image("manylinux2014", "x86_64", "quay.io/pypa/manylinux2014_x86_64", None),
-    Image("manylinux2014", "i686", "quay.io/pypa/manylinux2014_i686", None),
-    Image("manylinux2014", "aarch64", "quay.io/pypa/manylinux2014_aarch64", None),
-    Image("manylinux2014", "ppc64le", "quay.io/pypa/manylinux2014_ppc64le", None),
-    Image("manylinux2014", "s390x", "quay.io/pypa/manylinux2014_s390x", None),
-    # 2_24 images
-    Image("manylinux_2_24", "x86_64", "quay.io/pypa/manylinux_2_24_x86_64", None),
-    Image("manylinux_2_24", "i686", "quay.io/pypa/manylinux_2_24_i686", None),
-    Image("manylinux_2_24", "aarch64", "quay.io/pypa/manylinux_2_24_aarch64", None),
-    Image("manylinux_2_24", "ppc64le", "quay.io/pypa/manylinux_2_24_ppc64le", None),
-    Image("manylinux_2_24", "s390x", "quay.io/pypa/manylinux_2_24_s390x", None),
-]
-
-config = configparser.ConfigParser()
-
-for image in images:
-    # get the tag name whose digest matches 'latest'
-    if image.tag is not None:
-        # image has been pinned, do not update
-        tag_name = image.tag
-    elif image.image_name.startswith("quay.io/"):
-        _, _, repository_name = image.image_name.partition("/")
-        response = requests.get(
-            f"https://quay.io/api/v1/repository/{repository_name}?includeTags=true"
-        )
-        response.raise_for_status()
-        repo_info = response.json()
-        tags_dict = repo_info["tags"]
-
-        latest_tag = tags_dict.pop("latest")
-        # find the tag whose manifest matches 'latest'
-        tag_name = next(
-            name
-            for (name, info) in tags_dict.items()
-            if info["manifest_digest"] == latest_tag["manifest_digest"]
-        )
-    else:
-        response = requests.get(f"https://hub.docker.com/v2/repositories/{image.image_name}/tags")
-        response.raise_for_status()
-        tags = response.json()["results"]
-
-        latest_tag = next(tag for tag in tags if tag["name"] == "latest")
-        # i don't know what it would mean to have multiple images per tag
-        assert len(latest_tag["images"]) == 1
-        digest = latest_tag["images"][0]["digest"]
-
-        pinned_tag = next(
-            tag for tag in tags if tag != latest_tag and tag["images"][0]["digest"] == digest
-        )
-        tag_name = pinned_tag["name"]
-
-    if not config.has_section(image.platform):
-        config[image.platform] = {}
-
-    config[image.platform][image.manylinux_version] = f"{image.image_name}:{tag_name}"
-
-with open("cibuildwheel/resources/pinned_docker_images.cfg", "w") as f:
-    config.write(f)

bin/update_docker.py

@@ -0,0 +1,86 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import configparser
+from pathlib import Path
+from typing import NamedTuple
+
+import requests
+
+DIR = Path(__file__).parent.resolve()
+RESOURCES = DIR.parent / "cibuildwheel/resources"
+
+
+class Image(NamedTuple):
+    manylinux_version: str
+    platform: str
+    image_name: str
+    tag: str | None  # Set this to pin the image
+
+
+images = [
+    Image("manylinux1", "x86_64", "quay.io/pypa/manylinux1_x86_64", None),
+    Image("manylinux1", "i686", "quay.io/pypa/manylinux1_i686", None),
+    # 2010 images
+    Image("manylinux2010", "x86_64", "quay.io/pypa/manylinux2010_x86_64", None),
+    Image("manylinux2010", "i686", "quay.io/pypa/manylinux2010_i686", None),
+    Image("manylinux2010", "pypy_x86_64", "pypywheels/manylinux2010-pypy_x86_64", None),
+    # 2014 images
+    Image("manylinux2014", "x86_64", "quay.io/pypa/manylinux2014_x86_64", None),
+    Image("manylinux2014", "i686", "quay.io/pypa/manylinux2014_i686", None),
+    Image("manylinux2014", "aarch64", "quay.io/pypa/manylinux2014_aarch64", None),
+    Image("manylinux2014", "ppc64le", "quay.io/pypa/manylinux2014_ppc64le", None),
+    Image("manylinux2014", "s390x", "quay.io/pypa/manylinux2014_s390x", None),
+    # 2_24 images
+    Image("manylinux_2_24", "x86_64", "quay.io/pypa/manylinux_2_24_x86_64", None),
+    Image("manylinux_2_24", "i686", "quay.io/pypa/manylinux_2_24_i686", None),
+    Image("manylinux_2_24", "aarch64", "quay.io/pypa/manylinux_2_24_aarch64", None),
+    Image("manylinux_2_24", "ppc64le", "quay.io/pypa/manylinux_2_24_ppc64le", None),
+    Image("manylinux_2_24", "s390x", "quay.io/pypa/manylinux_2_24_s390x", None),
+]
+
+config = configparser.ConfigParser()
+
+for image in images:
+    # get the tag name whose digest matches 'latest'
+    if image.tag is not None:
+        # image has been pinned, do not update
+        tag_name = image.tag
+    elif image.image_name.startswith("quay.io/"):
+        _, _, repository_name = image.image_name.partition("/")
+        response = requests.get(
+            f"https://quay.io/api/v1/repository/{repository_name}?includeTags=true"
+        )
+        response.raise_for_status()
+        repo_info = response.json()
+        tags_dict = repo_info["tags"]
+
+        latest_tag = tags_dict.pop("latest")
+        # find the tag whose manifest matches 'latest'
+        tag_name = next(
+            name
+            for (name, info) in tags_dict.items()
+            if info["manifest_digest"] == latest_tag["manifest_digest"]
+        )
+    else:
+        response = requests.get(f"https://hub.docker.com/v2/repositories/{image.image_name}/tags")
+        response.raise_for_status()
+        tags = response.json()["results"]
+
+        latest_tag = next(tag for tag in tags if tag["name"] == "latest")
+        # i don't know what it would mean to have multiple images per tag
+        assert len(latest_tag["images"]) == 1
+        digest = latest_tag["images"][0]["digest"]
+
+        pinned_tag = next(
+            tag for tag in tags if tag != latest_tag and tag["images"][0]["digest"] == digest
+        )
+        tag_name = pinned_tag["name"]
+
+    if not config.has_section(image.platform):
+        config[image.platform] = {}
+
+    config[image.platform][image.manylinux_version] = f"{image.image_name}:{tag_name}"
+
+with open(RESOURCES / "pinned_docker_images.cfg", "w") as f:
+    config.write(f)

docs/contributing.md

@@ -34,11 +34,32 @@ So, if we can, I'd like to improve the experience on errors as well. In [this](h
 Maintainer notes
 ----------------
 
+### Nox support
+
+Most developer tasks have a nox interface. This allows you to very simply run tasks without worrying about setting up a development environment (as shown below). This is a slower than setting up a development environment and reusing it, but has the (important) benefit of being highly reproducible; an earlier run does not affect a current run, or anything else on your machine.
+
+Install [nox](https://nox.thea.codes); homebrew is recommend on macOS, otherwise, pipx is a great choice - in fact, you can use `pipx run nox` and avoid installing completely.
+
+You can see a list of sessions by typing `nox -l`; here are a few common ones:
+
+```console
+nox -s lint          # Run the linters (default)
+nox -s tests         # Run the tests   (default)
+nox -s docs -- serve # Build and serve the documentation
+nox -s build         # Make SDist and wheel
+```
+
+More advanced users can run the update scripts. `update_pins` should work directly, but `update_constraints` needs all versions of Python installed. If you don't want to do that locally, a fast way to run it to use docker to run nox:
+
+```console
+docker run --rm -it -v $PWD:/src quay.io/pypa/manylinux2010_x86_64:latest pipx run nox -f src/noxfile.py -s update_constraints
+```
+
 ### Local testing
 
 You should run:
 
-```python
+```console
 python3 -m venv venv
 . venv/bin/activate
 pip install -e .[dev]

noxfile.py

@@ -0,0 +1,93 @@
+import shutil
+import sys
+from pathlib import Path
+
+import nox
+
+nox.options.sessions = ["lint", "tests"]
+
+PYTHON_ALL_VERSIONS = ["3.6", "3.7", "3.8", "3.9"]
+
+DIR = Path(__file__).parent.resolve()
+
+
+@nox.session
+def tests(session):
+    """
+    Run the unit and regular tests.
+    """
+    unit_test_args = ["--run-docker"] if sys.platform.startswith("linux") else []
+    session.install("-e", ".[test]")
+    session.run("pytest", "unit_test", *unit_test_args)
+    session.run("pytest", "test", "-x", "--durations", "0", "--timeout=2400", "test")
+
+
+@nox.session
+def lint(session):
+    """
+    Run the linter.
+    """
+    session.install("pre-commit")
+    session.run("pre-commit", "run", "--all-files")
+
+
+@nox.session(python=PYTHON_ALL_VERSIONS)
+def update_constraints(session):
+    """
+    Update the dependencies inplace.
+    """
+    session.install("requests", "pip-tools")
+    session.run("python", "bin/update_dependencies.py")
+
+
+@nox.session
+def update_pins(session):
+    """
+    Update the python and docker pins version inplace.
+    """
+    session.install("-e", ".[dev]")
+    session.run("python", "bin/update_pythons.py", "--force")
+    session.run("python", "bin/update_docker.py")
+
+
+@nox.session
+def update_proj(session):
+    """
+    Update the README inplace.
+    """
+    session.install("-e", ".[dev]")
+    session.run("./bin/projects.py", "docs/data/projects.yml", "--readme=README.md")
+
+
+@nox.session
+def docs(session):
+    """
+    Build the docs.
+    """
+    session.install("-e", ".[docs]")
+
+    if session.posargs:
+        if "serve" in session.posargs:
+            session.run("mkdocs", "serve")
+        else:
+            print("Unrecognized args, use 'serve'")
+    else:
+        session.run("mkdocs", "build")
+
+
+@nox.session
+def build(session):
+    """
+    Build an SDist and wheel.
+    """
+
+    build_p = DIR.joinpath("build")
+    if build_p.exists():
+        shutil.rmtree(build_p)
+
+    dist_p = DIR.joinpath("dist")
+    if dist_p.exists():
+        shutil.rmtree(dist_p)
+
+    session.install("build")
+    session.run("python", "-m", "build")