Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: support pypa build (#521)

Browse Source 
* feat: support pypa build

fix: update `cibuildwheel.linux.troubleshoot` to know about build

Workaround issue with PyPy venv module by installing build[virtualenv]

fix: test_dependency_constraints_file when using build

fix: test/test_pep518.py

fix: use `strtobool` to parse `CIBW_PYPA_BUILD`

fix: update `cibuildwheel.linux.troubleshoot` to know about `python -m pip wheel`

test: use `build_mode` in `test/test_dependency_versions.py` tests

* refactor: use build-backend instead

* Apply suggestions from code review

Co-authored-by: Joe Rickerby <joerick@mac.com>

* refactor: use literal types for build_frontend

* refactor(tests): build_mode -> build_frontend_env

* Update test/test_before_build.py

Co-authored-by: Joe Rickerby <joerick@mac.com>
This commit is contained in:
Henry Schreiner
2021-06-23 10:47:18 -04:00
committed by GitHub
co-authored by Joe Rickerby
parent 9e1369e07d
commit 515e9be2c4
13 changed files with 301 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cibuildwheel/__main__.py
+13
View File
@@ -20,6 +20,7 @@ from cibuildwheel.options import ConfigOptions
from cibuildwheel.projectfiles import get_requires_python_str
from cibuildwheel.typing import PLATFORMS, PlatformName, assert_never
from cibuildwheel.util import (
BuildFrontend,
BuildOptions,
BuildSelector,
DependencyConstraints,
@@ -184,6 +185,7 @@ def main() -> None:
archs_config_str = args.archs or options("archs", sep=" ")
build_frontend_str = options("build-frontend", env_plat=False)
environment_config = options("environment", table={"item": '{k}="{v}"', "sep": " "})
before_all = options("before-all", sep=" && ")
before_build = options("before-build", sep=" && ")
@@ -200,6 +202,16 @@ def main() -> None:
os.environ.get("CIBW_PRERELEASE_PYTHONS", "0")
)
build_frontend: BuildFrontend
if build_frontend_str == "build":
build_frontend = "build"
elif build_frontend_str == "pip":
build_frontend = "pip"
else:
msg = f"cibuildwheel: Unrecognised build frontend '{build_frontend}', only 'pip' and 'build' are supported"
print(msg, file=sys.stderr)
sys.exit(2)
package_files = {"setup.py", "setup.cfg", "pyproject.toml"}
if not any(package_dir.joinpath(name).exists() for name in package_files):
@@ -308,6 +320,7 @@ def main() -> None:
environment=environment,
dependency_constraints=dependency_constraints,
manylinux_images=manylinux_images or None,
build_frontend=build_frontend,
)
# Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print'
cibuildwheel/linux.py
+39 -16
View File
@@ -7,7 +7,7 @@ from typing import List, NamedTuple, Set
from .architecture import Architecture
from .docker_container import DockerContainer
from .logger import log
from .typing import PathOrStr
from .typing import PathOrStr, assert_never
from .util import (
BuildOptions,
BuildSelector,
@@ -145,7 +145,7 @@ def build(options: BuildOptions) -> None:
env, executor=docker.environment_executor
)
# check config python and pip are still on PATH
# check config python is still on PATH
which_python = docker.call(
["which", "python"], env=env, capture_output=True
).strip()
@@ -180,18 +180,38 @@ def build(options: BuildOptions) -> None:
docker.call(["rm", "-rf", built_wheel_dir])
docker.call(["mkdir", "-p", built_wheel_dir])
docker.call(
[
"pip",
"wheel",
container_package_dir,
"--wheel-dir",
built_wheel_dir,
"--no-deps",
*get_build_verbosity_extra_flags(options.build_verbosity),
],
env=env,
)
verbosity_flags = get_build_verbosity_extra_flags(options.build_verbosity)
if options.build_frontend == "pip":
docker.call(
[
"python",
"-m",
"pip",
"wheel",
container_package_dir,
f"--wheel-dir={built_wheel_dir}",
"--no-deps",
*verbosity_flags,
],
env=env,
)
elif options.build_frontend == "build":
config_setting = " ".join(verbosity_flags)
docker.call(
[
"python",
"-m",
"build",
container_package_dir,
"--wheel",
f"--outdir={built_wheel_dir}",
f"--config-setting={config_setting}",
],
env=env,
)
else:
assert_never(options.build_frontend)
built_wheel = docker.glob(built_wheel_dir, "*.whl")[0]
@@ -291,8 +311,11 @@ def build(options: BuildOptions) -> None:
def troubleshoot(package_dir: Path, error: Exception) -> None:
if isinstance(error, subprocess.CalledProcessError) and error.cmd[0:2] == ["pip", "wheel"]:
# the 'pip wheel' step failed.
if isinstance(error, subprocess.CalledProcessError) and (
error.cmd[0:4] == ["python", "-m", "pip", "wheel"]
or error.cmd[0:3] == ["python", "-m", "build"]
):
# the wheel build step failed
print("Checking for common errors...")
so_files = list(package_dir.glob("**/*.so"))
cibuildwheel/macos.py
+78 -28
View File
@@ -12,13 +12,15 @@ from typing import Any, Dict, List, NamedTuple, Optional, Sequence, Set, Tuple,
from .architecture import Architecture
from .environment import ParsedEnvironment
from .logger import log
from .typing import Literal, PathOrStr
from .typing import Literal, PathOrStr, assert_never
from .util import (
BuildFrontend,
BuildOptions,
BuildSelector,
NonPlatformWheelError,
download,
get_build_verbosity_extra_flags,
get_pip_version,
install_certifi_script,
prepare_command,
read_python_configs,
@@ -178,7 +180,9 @@ def setup_python(
python_configuration: PythonConfiguration,
dependency_constraint_flags: Sequence[PathOrStr],
environment: ParsedEnvironment,
build_frontend: BuildFrontend,
) -> Dict[str, str]:
implementation_id = python_configuration.identifier.split("-")[0]
log.step(f"Installing Python {implementation_id}...")
@@ -308,18 +312,33 @@ def setup_python(
env.setdefault("SDKROOT", arm64_compatible_sdks[0])
log.step("Installing build tools...")
call(
[
"pip",
"install",
"--upgrade",
"setuptools",
"wheel",
"delocate",
*dependency_constraint_flags,
],
env=env,
)
if build_frontend == "pip":
call(
[
"pip",
"install",
"--upgrade",
"setuptools",
"wheel",
"delocate",
*dependency_constraint_flags,
],
env=env,
)
elif build_frontend == "build":
call(
[
"pip",
"install",
"--upgrade",
"delocate",
"build[virtualenv]",
*dependency_constraint_flags,
],
env=env,
)
else:
assert_never(build_frontend)
return env
@@ -356,7 +375,12 @@ def build(options: BuildOptions) -> None:
options.dependency_constraints.get_for_python_version(config.version),
]
env = setup_python(config, dependency_constraint_flags, options.environment)
env = setup_python(
config,
dependency_constraint_flags,
options.environment,
options.build_frontend,
)
if options.before_build:
log.step("Running before_build...")
@@ -370,20 +394,46 @@ def build(options: BuildOptions) -> None:
shutil.rmtree(built_wheel_dir)
built_wheel_dir.mkdir(parents=True)
# Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
# see https://github.com/pypa/cibuildwheel/pull/369
call(
[
"pip",
"wheel",
options.package_dir.resolve(),
"--wheel-dir",
built_wheel_dir,
"--no-deps",
*get_build_verbosity_extra_flags(options.build_verbosity),
],
env=env,
)
verbosity_flags = get_build_verbosity_extra_flags(options.build_verbosity)
if options.build_frontend == "pip":
# Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
# see https://github.com/pypa/cibuildwheel/pull/369
call(
[
"python",
"-m",
"pip",
"wheel",
options.package_dir.resolve(),
f"--wheel-dir={built_wheel_dir}",
"--no-deps",
*verbosity_flags,
],
env=env,
)
elif options.build_frontend == "build":
config_setting = " ".join(verbosity_flags)
build_env = env.copy()
if options.dependency_constraints:
build_env["PIP_CONSTRAINT"] = str(
options.dependency_constraints.get_for_python_version(config.version)
)
build_env["VIRTUALENV_PIP"] = get_pip_version(env)
call(
[
"python",
"-m",
"build",
options.package_dir,
"--wheel",
f"--outdir={built_wheel_dir}",
f"--config-setting={config_setting}",
],
env=build_env,
)
else:
assert_never(options.build_frontend)
built_wheel = next(built_wheel_dir.glob("*.whl"))
cibuildwheel/resources/defaults.toml
+1
View File
@@ -4,6 +4,7 @@ skip = ""
test-skip = ""
archs = ["auto"]
build-frontend = "pip"
dependency-versions = "pinned"
environment = {}
build-verbosity = ""
cibuildwheel/util.py
+21 -1
View File
@@ -4,6 +4,8 @@ import itertools
import os
import re
import ssl
import subprocess
import sys
import textwrap
import time
import urllib.request
@@ -20,12 +22,14 @@ from packaging.version import Version
from .architecture import Architecture
from .environment import ParsedEnvironment
from .typing import PathOrStr, PlatformName
from .typing import Literal, PathOrStr, PlatformName
resources_dir = Path(__file__).parent / "resources"
install_certifi_script = resources_dir / "install_certifi.py"
BuildFrontend = Literal["pip", "build"]
def prepare_command(command: str, **kwargs: PathOrStr) -> str:
"""
@@ -218,6 +222,7 @@ class BuildOptions(NamedTuple):
test_requires: List[str]
test_extras: str
build_verbosity: int
build_frontend: BuildFrontend
class NonPlatformWheelError(Exception):
@@ -300,3 +305,18 @@ def print_new_wheels(msg: str, output_dir: Path) -> Iterator[None]:
s = time.time() - start_time
m = s / 60
print(msg.format(n=n, s=s, m=m), *sorted(f" {f.name}" for f in new_contents), sep="\n")
def get_pip_version(env: Dict[str, str]) -> str:
# we use shell=True here for windows, even though we don't need a shell due to a bug
# https://bugs.python.org/issue8557
shell = sys.platform.startswith("win")
versions_output_text = subprocess.check_output(
["python", "-m", "pip", "freeze", "--all"], universal_newlines=True, shell=shell, env=env
)
(pip_version,) = (
version[5:]
for version in versions_output_text.strip().splitlines()
if version.startswith("pip==")
)
return pip_version
cibuildwheel/windows.py
+72 -20
View File
@@ -10,13 +10,15 @@ from zipfile import ZipFile
from .architecture import Architecture
from .environment import ParsedEnvironment
from .logger import log
from .typing import PathOrStr
from .typing import PathOrStr, assert_never
from .util import (
BuildFrontend,
BuildOptions,
BuildSelector,
NonPlatformWheelError,
download,
get_build_verbosity_extra_flags,
get_pip_version,
prepare_command,
read_python_configs,
)
@@ -114,7 +116,9 @@ def setup_python(
python_configuration: PythonConfiguration,
dependency_constraint_flags: Sequence[PathOrStr],
environment: ParsedEnvironment,
build_frontend: BuildFrontend,
) -> Dict[str, str]:
nuget = Path("C:\\cibw\\nuget.exe")
if not nuget.exists():
log.step("Downloading nuget...")
@@ -214,10 +218,26 @@ def setup_python(
sys.exit(1)
call(["pip", "--version"], env=env)
call(
["pip", "install", "--upgrade", "setuptools", "wheel", *dependency_constraint_flags],
env=env,
)
if build_frontend == "pip":
call(
[
"pip",
"install",
"--upgrade",
"setuptools",
"wheel",
*dependency_constraint_flags,
],
env=env,
)
elif build_frontend == "build":
call(
["pip", "install", "--upgrade", "build[virtualenv]", *dependency_constraint_flags],
env=env,
)
else:
assert_never(build_frontend)
return env
@@ -251,7 +271,12 @@ def build(options: BuildOptions) -> None:
]
# install Python
env = setup_python(config, dependency_constraint_flags, options.environment)
env = setup_python(
config,
dependency_constraint_flags,
options.environment,
options.build_frontend,
)
# run the before_build command
if options.before_build:
@@ -265,20 +290,47 @@ def build(options: BuildOptions) -> None:
if built_wheel_dir.exists():
shutil.rmtree(built_wheel_dir)
built_wheel_dir.mkdir(parents=True)
# Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
# see https://github.com/pypa/cibuildwheel/pull/369
call(
[
"pip",
"wheel",
options.package_dir.resolve(),
"-w",
built_wheel_dir,
"--no-deps",
*get_build_verbosity_extra_flags(options.build_verbosity),
],
env=env,
)
verbosity_flags = get_build_verbosity_extra_flags(options.build_verbosity)
if options.build_frontend == "pip":
# Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
# see https://github.com/pypa/cibuildwheel/pull/369
call(
[
"python",
"-m",
"pip",
"wheel",
options.package_dir.resolve(),
f"--wheel-dir={built_wheel_dir}",
"--no-deps",
*get_build_verbosity_extra_flags(options.build_verbosity),
],
env=env,
)
elif options.build_frontend == "build":
config_setting = " ".join(verbosity_flags)
build_env = env.copy()
if options.dependency_constraints:
build_env["PIP_CONSTRAINT"] = str(
options.dependency_constraints.get_for_python_version(config.version)
)
build_env["VIRTUALENV_PIP"] = get_pip_version(env)
call(
[
"python",
"-m",
"build",
options.package_dir,
"--wheel",
f"--outdir={built_wheel_dir}",
f"--config-setting={config_setting}",
],
env=build_env,
)
else:
assert_never(options.build_frontend)
built_wheel = next(built_wheel_dir.glob("*.whl"))
docs/options.md
+39
View File
@@ -435,6 +435,45 @@ This option can also be set using the [command-line option](#command-line) `--pr
## Build customization
### `CIBW_BUILD_FRONTEND` {: #build-frontend}
> Set the tool to use to build, either "pip" (default for now) or "build"
Choose which build backend to use. Can either be "pip", which will run
`python -m pip wheel`, or "build", which will run `python -m build --wheel`.
!!! tip
Until v2.0.0, [pip] was the only way to build wheels, and is still the
default. However, we expect that at some point in the future, cibuildwheel
will change the default to [build], in line with the PyPA's recommendation.
If you want to try `build` before this, you can use this option.
[pip]: https://pip.pypa.io/en/stable/cli/pip_wheel/
[build]: https://github.com/pypa/build/
#### Examples
!!! tab examples "Environment variables"
```yaml
# Switch to using build
CIBW_BUILD_FRONTEND: "build"
# Ensure pip is used even if the default changes in the future
CIBW_BUILD_FRONTEND: "pip"
```
!!! tab examples "pyproject.toml"
```toml
[tool.cibuildwheel]
# Switch to using build
build-frontend = "build"
# Ensure pip is used even if the default changes in the future
build-frontend = "pip"
```
### `CIBW_ENVIRONMENT` {: #environment}
> Set environment variables needed during the build
test/conftest.py
+11 -2
View File
@@ -1,7 +1,9 @@
from typing import Dict
import pytest
def pytest_addoption(parser):
def pytest_addoption(parser) -> None:
parser.addoption(
"--run-emulation", action="store_true", default=False, help="run emulation tests"
)
@@ -11,7 +13,7 @@ def pytest_configure(config):
config.addinivalue_line("markers", "emulation: mark test requiring qemu binfmt_misc to run")
def pytest_collection_modifyitems(config, items):
def pytest_collection_modifyitems(config, items) -> None:
if config.getoption("--run-emulation"):
# --run-emulation given in cli: do not skip emulation tests
return
@@ -19,3 +21,10 @@ def pytest_collection_modifyitems(config, items):
for item in items:
if "emulation" in item.keywords:
item.add_marker(skip_emulation)
@pytest.fixture(
params=[{"CIBW_BUILD_FRONTEND": "pip"}, {"CIBW_BUILD_FRONTEND": "build"}], ids=["pip", "build"]
)
def build_frontend_env(request) -> Dict[str, str]:
return request.param # type: ignore
test/test_0_basic.py
+2 -2
View File
@@ -18,12 +18,12 @@ basic_project = test_projects.new_c_project(
)
def test(tmp_path):
def test(tmp_path, build_frontend_env):
project_dir = tmp_path / "project"
basic_project.generate(project_dir)
# build the wheels
actual_wheels = utils.cibuildwheel_run(project_dir)
actual_wheels = utils.cibuildwheel_run(project_dir, add_env=build_frontend_env)
# check that the expected wheels are produced
expected_wheels = utils.expected_wheels("spam", "0.1.0")
test/test_before_build.py
+6 -1
View File
@@ -25,8 +25,13 @@ project_with_before_build_asserts = test_projects.new_c_project(
stored_executable = f.read()
print('stored_executable', stored_executable)
print('sys.executable', sys.executable)
# windows/mac are case insensitive
assert os.path.realpath(stored_executable).lower() == os.path.realpath(sys.executable).lower()
stored_path = os.path.realpath(stored_executable).lower()
current_path = os.path.realpath(sys.executable).lower()
# TODO: This is not valid in an virtual environment
assert stored_path == current_path, '{0} != {1}'.format(stored_path, current_path)
"""
)
)
test/test_dependency_versions.py
+6 -3
View File
@@ -48,7 +48,7 @@ def get_versions_from_constraint_file(constraint_file):
@pytest.mark.parametrize("python_version", ["3.6", "3.8", "3.9"])
def test_pinned_versions(tmp_path, python_version):
def test_pinned_versions(tmp_path, python_version, build_frontend_env):
if utils.platform == "linux":
pytest.skip("linux doesn't pin individual tool versions, it pins manylinux images instead")
@@ -85,6 +85,7 @@ def test_pinned_versions(tmp_path, python_version):
add_env={
"CIBW_BUILD": build_pattern,
"CIBW_ENVIRONMENT": cibw_environment_option,
**build_frontend_env,
},
)
@@ -107,7 +108,7 @@ def test_pinned_versions(tmp_path, python_version):
assert set(actual_wheels) == set(expected_wheels)
def test_dependency_constraints_file(tmp_path):
def test_dependency_constraints_file(tmp_path, build_frontend_env):
if utils.platform == "linux":
pytest.skip("linux doesn't pin individual tool versions, it pins manylinux images instead")
@@ -118,7 +119,7 @@ def test_dependency_constraints_file(tmp_path):
"pip": "20.0.2",
"setuptools": "53.0.0",
"wheel": "0.34.2",
"virtualenv": "20.0.10",
"virtualenv": "20.0.35",
}
constraints_file = tmp_path / "constraints.txt"
@@ -129,6 +130,7 @@ def test_dependency_constraints_file(tmp_path):
setuptools=={setuptools}
wheel=={wheel}
virtualenv=={virtualenv}
importlib-metadata<3,>=0.12; python_version < "3.8"
""".format(
**tool_versions
)
@@ -149,6 +151,7 @@ def test_dependency_constraints_file(tmp_path):
add_env={
"CIBW_ENVIRONMENT": cibw_environment_option,
"CIBW_DEPENDENCY_VERSIONS": str(constraints_file),
**build_frontend_env,
},
)
test/test_pep518.py
+11 -4
View File
@@ -1,4 +1,3 @@
import os
import textwrap
from . import test_projects, utils
@@ -33,13 +32,13 @@ build-backend = "setuptools.build_meta"
"""
def test_pep518(tmp_path):
def test_pep518(tmp_path, build_frontend_env):
project_dir = tmp_path / "project"
basic_project.generate(project_dir)
# build the wheels
actual_wheels = utils.cibuildwheel_run(project_dir)
actual_wheels = utils.cibuildwheel_run(project_dir, add_env=build_frontend_env)
# check that the expected wheels are produced
expected_wheels = utils.expected_wheels("spam", "0.1.0")
@@ -50,4 +49,12 @@ def test_pep518(tmp_path):
assert not (project_dir / "42").exists()
assert not (project_dir / "4.1.2").exists()
assert len(os.listdir(project_dir)) == len(basic_project.files)
# pypa/build creates a "build" folder & a "*.egg-info" folder for the wheel being built,
# this should be harmless so remove them
contents = [
item
for item in project_dir.iterdir()
if item.name != "build" and not item.name.endswith(".egg-info")
]
assert len(contents) == len(basic_project.files)
test/test_troubleshooting.py
+2 -2
View File
@@ -16,7 +16,7 @@ raise Exception('this build will fail')
"""
def test_failed_project_with_so_files(tmp_path, capfd):
def test_failed_project_with_so_files(tmp_path, capfd, build_frontend_env):
if utils.platform != "linux":
pytest.skip("this test is only relevant to the linux build")
@@ -24,7 +24,7 @@ def test_failed_project_with_so_files(tmp_path, capfd):
so_file_project.generate(project_dir)
with pytest.raises(subprocess.CalledProcessError):
utils.cibuildwheel_run(project_dir)
utils.cibuildwheel_run(project_dir, add_env=build_frontend_env)
captured = capfd.readouterr()
print("out", captured.out)