Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: Use utils.call & utils.shell in macos.py & windows.py (#978)

Browse Source
This commit is contained in:
Matthieu Darbois
2022-01-05 02:59:24 +01:00
committed by GitHub
parent 73b7de0a75
commit 9e18cb8656
3 changed files with 196 additions and 206 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cibuildwheel/macos.py
+69 -108
View File
@@ -1,13 +1,12 @@
import os
import platform
import re
import shlex
import shutil
import subprocess
import sys
import tempfile
from pathlib import Path
from typing import Any, Dict, List, NamedTuple, Optional, Sequence, Set, Tuple, cast
from typing import Any, Dict, List, NamedTuple, Sequence, Set, Tuple, cast
from .architecture import Architecture
from .environment import ParsedEnvironment
@@ -18,31 +17,18 @@ from .util import (
BuildFrontend,
BuildSelector,
NonPlatformWheelError,
call,
download,
get_build_verbosity_extra_flags,
get_pip_version,
install_certifi_script,
prepare_command,
read_python_configs,
shell,
unwrap,
)
def call(
args: Sequence[PathOrStr],
env: Optional[Dict[str, str]] = None,
cwd: Optional[str] = None,
shell: bool = False,
) -> None:
# print the command executing for the logs
if shell:
print(f"+ {args}")
else:
print("+ " + " ".join(shlex.quote(str(a)) for a in args))
subprocess.run(args, env=env, cwd=cwd, shell=shell, check=True)
def get_macos_version() -> Tuple[int, int]:
"""
Returns the macOS major/minor version, as a tuple, e.g. (10, 15) or (11, 0)
@@ -58,13 +44,7 @@ def get_macos_version() -> Tuple[int, int]:
def get_macos_sdks() -> List[str]:
output = subprocess.run(
["xcodebuild", "-showsdks"],
universal_newlines=True,
check=True,
stdout=subprocess.PIPE,
).stdout
output = call("xcodebuild", "-showsdks", capture_stdout=True)
return [m.group(1) for m in re.finditer(r"-sdk (macosx\S+)", output)]
@@ -114,9 +94,7 @@ def make_symlinks(installation_bin_path: Path, python_executable: str, pip_execu
def install_cpython(version: str, url: str) -> Path:
installed_system_packages = subprocess.run(
["pkgutil", "--pkgs"], universal_newlines=True, check=True, stdout=subprocess.PIPE
).stdout.splitlines()
installed_system_packages = call("pkgutil", "--pkgs", capture_stdout=True).splitlines()
# if this version of python isn't installed, get it from python.org and install
python_package_identifier = f"org.python.Python.PythonFramework-{version}"
@@ -127,10 +105,10 @@ def install_cpython(version: str, url: str) -> Path:
# download the pkg
download(url, Path("/tmp/Python.pkg"))
# install
call(["sudo", "installer", "-pkg", "/tmp/Python.pkg", "-target", "/"])
call("sudo", "installer", "-pkg", "/tmp/Python.pkg", "-target", "/")
env = os.environ.copy()
env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
call([str(installation_bin_path / python_executable), str(install_certifi_script)], env=env)
call(str(installation_bin_path / python_executable), str(install_certifi_script), env=env)
pip_executable = "pip3"
make_symlinks(installation_bin_path, python_executable, pip_executable)
@@ -147,7 +125,7 @@ def install_pypy(version: str, url: str) -> Path:
if not installation_path.exists():
downloaded_tar_bz2 = Path("/tmp") / pypy_tar_bz2
download(url, downloaded_tar_bz2)
call(["tar", "-C", "/tmp", "-xf", downloaded_tar_bz2])
call("tar", "-C", "/tmp", "-xf", downloaded_tar_bz2)
installation_bin_path = installation_path / "bin"
python_executable = "pypy3"
@@ -203,20 +181,18 @@ def setup_python(
requires_reinstall = not (installation_bin_path / "pip").exists()
if requires_reinstall:
# maybe pip isn't installed at all. ensurepip resolves that.
call(["python", "-m", "ensurepip"], env=env, cwd="/tmp")
call("python", "-m", "ensurepip", env=env, cwd="/tmp")
# upgrade pip to the version matching our constraints
# if necessary, reinstall it to ensure that it's available on PATH as 'pip'
call(
[
"python",
"-m",
"pip",
"install",
"--force-reinstall" if requires_reinstall else "--upgrade",
"pip",
*dependency_constraint_flags,
],
"python",
"-m",
"pip",
"install",
"--force-reinstall" if requires_reinstall else "--upgrade",
"pip",
*dependency_constraint_flags,
env=env,
cwd="/tmp",
)
@@ -226,11 +202,9 @@ def setup_python(
# check what pip version we're on
assert (installation_bin_path / "pip").exists()
call(["which", "pip"], env=env)
call(["pip", "--version"], env=env)
which_pip = subprocess.run(
["which", "pip"], env=env, universal_newlines=True, check=True, stdout=subprocess.PIPE
).stdout.strip()
call("which", "pip", env=env)
call("pip", "--version", env=env)
which_pip = call("which", "pip", env=env, capture_stdout=True).strip()
if which_pip != "/tmp/cibw_bin/pip":
print(
"cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.",
@@ -239,11 +213,9 @@ def setup_python(
sys.exit(1)
# check what Python version we're on
call(["which", "python"], env=env)
call(["python", "--version"], env=env)
which_python = subprocess.run(
["which", "python"], env=env, universal_newlines=True, check=True, stdout=subprocess.PIPE
).stdout.strip()
call("which", "python", env=env)
call("python", "--version", env=env)
which_python = call("which", "python", env=env, capture_stdout=True).strip()
if which_python != "/tmp/cibw_bin/python":
print(
"cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.",
@@ -298,27 +270,23 @@ def setup_python(
log.step("Installing build tools...")
if build_frontend == "pip":
call(
[
"pip",
"install",
"--upgrade",
"setuptools",
"wheel",
"delocate",
*dependency_constraint_flags,
],
"pip",
"install",
"--upgrade",
"setuptools",
"wheel",
"delocate",
*dependency_constraint_flags,
env=env,
)
elif build_frontend == "build":
call(
[
"pip",
"install",
"--upgrade",
"delocate",
"build[virtualenv]",
*dependency_constraint_flags,
],
"pip",
"install",
"--upgrade",
"delocate",
"build[virtualenv]",
*dependency_constraint_flags,
env=env,
)
else:
@@ -347,7 +315,7 @@ def build(options: Options) -> None:
before_all_prepared = prepare_command(
before_all_options.before_all, project=".", package=before_all_options.package_dir
)
call([before_all_prepared], shell=True, env=env)
shell(before_all_prepared, env=env)
for config in python_configurations:
build_options = options.build_options(config.identifier)
@@ -375,7 +343,7 @@ def build(options: Options) -> None:
before_build_prepared = prepare_command(
build_options.before_build, project=".", package=build_options.package_dir
)
call(before_build_prepared, env=env, shell=True)
shell(before_build_prepared, env=env)
log.step("Building wheel...")
if built_wheel_dir.exists():
@@ -388,16 +356,14 @@ def build(options: Options) -> None:
# Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
# see https://github.com/pypa/cibuildwheel/pull/369
call(
[
"python",
"-m",
"pip",
"wheel",
build_options.package_dir.resolve(),
f"--wheel-dir={built_wheel_dir}",
"--no-deps",
*verbosity_flags,
],
"python",
"-m",
"pip",
"wheel",
build_options.package_dir.resolve(),
f"--wheel-dir={built_wheel_dir}",
"--no-deps",
*verbosity_flags,
env=env,
)
elif build_options.build_frontend == "build":
@@ -410,15 +376,13 @@ def build(options: Options) -> None:
build_env["PIP_CONSTRAINT"] = constraint_path.as_uri()
build_env["VIRTUALENV_PIP"] = get_pip_version(env)
call(
[
"python",
"-m",
"build",
build_options.package_dir,
"--wheel",
f"--outdir={built_wheel_dir}",
f"--config-setting={config_setting}",
],
"python",
"-m",
"build",
build_options.package_dir,
"--wheel",
f"--outdir={built_wheel_dir}",
f"--config-setting={config_setting}",
env=build_env,
)
else:
@@ -449,7 +413,7 @@ def build(options: Options) -> None:
dest_dir=repaired_wheel_dir,
delocate_archs=delocate_archs,
)
call(repair_command_prepared, env=env, shell=True)
shell(repair_command_prepared, env=env)
else:
shutil.move(str(built_wheel), repaired_wheel_dir)
@@ -514,7 +478,7 @@ def build(options: Options) -> None:
# set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
call(["pip", "install", "virtualenv", *dependency_constraint_flags], env=env)
call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env)
venv_dir = Path(tempfile.mkdtemp())
arch_prefix = []
@@ -528,18 +492,16 @@ def build(options: Options) -> None:
)
# define a custom 'call' function that adds the arch prefix each time
def call_with_arch(args: Sequence[PathOrStr], **kwargs: Any) -> None:
if isinstance(args, str):
args = " ".join(arch_prefix) + " " + args
else:
args = [*arch_prefix, *args]
call(args, **kwargs)
def call_with_arch(*args: PathOrStr, **kwargs: Any) -> None:
call(*arch_prefix, *args, **kwargs)
def shell_with_arch(command: str, **kwargs: Any) -> None:
command = " ".join(arch_prefix) + " " + command
shell(command, **kwargs)
# Use --no-download to ensure determinism by using seed libraries
# built into virtualenv
call_with_arch(
["python", "-m", "virtualenv", "--no-download", venv_dir], env=env
)
call_with_arch("python", "-m", "virtualenv", "--no-download", venv_dir, env=env)
virtualenv_env = env.copy()
virtualenv_env["PATH"] = os.pathsep.join(
@@ -550,7 +512,7 @@ def build(options: Options) -> None:
)
# check that we are using the Python from the virtual environment
call_with_arch(["which", "python"], env=virtualenv_env)
call_with_arch("which", "python", env=virtualenv_env)
if build_options.before_test:
before_test_prepared = prepare_command(
@@ -558,18 +520,20 @@ def build(options: Options) -> None:
project=".",
package=build_options.package_dir,
)
call_with_arch(before_test_prepared, env=virtualenv_env, shell=True)
shell_with_arch(before_test_prepared, env=virtualenv_env)
# install the wheel
call_with_arch(
["pip", "install", f"{repaired_wheel}{build_options.test_extras}"],
"pip",
"install",
f"{repaired_wheel}{build_options.test_extras}",
env=virtualenv_env,
)
# test the wheel
if build_options.test_requires:
call_with_arch(
["pip", "install"] + build_options.test_requires, env=virtualenv_env
"pip", "install", *build_options.test_requires, env=virtualenv_env
)
# run the tests from $HOME, with an absolute path in the command
@@ -580,11 +544,8 @@ def build(options: Options) -> None:
project=Path(".").resolve(),
package=build_options.package_dir.resolve(),
)
call_with_arch(
test_command_prepared,
cwd=os.environ["HOME"],
env=virtualenv_env,
shell=True,
shell_with_arch(
test_command_prepared, cwd=os.environ["HOME"], env=virtualenv_env
)
# clean up
cibuildwheel/util.py
+69 -6
View File
@@ -3,6 +3,7 @@ import fnmatch
import itertools
import os
import re
import shlex
import ssl
import subprocess
import sys
@@ -12,7 +13,18 @@ import urllib.request
from enum import Enum
from pathlib import Path
from time import sleep
from typing import Any, Dict, Iterable, Iterator, List, NamedTuple, Optional, TextIO
from typing import (
Any,
Dict,
Iterable,
Iterator,
List,
NamedTuple,
Optional,
TextIO,
cast,
overload,
)
import bracex
import certifi
@@ -47,6 +59,60 @@ MUSLLINUX_ARCHS = (
"s390x",
)
IS_WIN = sys.platform.startswith("win")
@overload
def call(
*args: PathOrStr,
env: Optional[Dict[str, str]] = None,
cwd: Optional[PathOrStr] = None,
capture_stdout: Literal[False] = ...,
) -> None:
...
@overload
def call(
*args: PathOrStr,
env: Optional[Dict[str, str]] = None,
cwd: Optional[PathOrStr] = None,
capture_stdout: Literal[True],
) -> str:
...
def call(
*args: PathOrStr,
env: Optional[Dict[str, str]] = None,
cwd: Optional[PathOrStr] = None,
capture_stdout: bool = False,
) -> Optional[str]:
"""
Run subprocess.run, but print the commands first. Takes the commands as
*args. Uses shell=True on Windows due to a bug. Also converts to
Paths to strings, due to Windows behavior at least on older Pythons.
https://bugs.python.org/issue8557
"""
args_ = [str(arg) for arg in args]
# print the command executing for the logs
print("+ " + " ".join(shlex.quote(a) for a in args_))
kwargs: Dict[str, Any] = {}
if capture_stdout:
kwargs["universal_newlines"] = True
kwargs["stdout"] = subprocess.PIPE
result = subprocess.run(args_, check=True, shell=IS_WIN, env=env, cwd=cwd, **kwargs)
if not capture_stdout:
return None
return cast(str, result.stdout)
def shell(
command: str, env: Optional[Dict[str, str]] = None, cwd: Optional[PathOrStr] = None
) -> None:
print(f"+ {command}")
subprocess.run(command, env=env, cwd=cwd, shell=True, check=True)
def format_safe(template: str, **kwargs: Any) -> str:
"""
@@ -376,11 +442,8 @@ def print_new_wheels(msg: str, output_dir: Path) -> Iterator[None]:
def get_pip_version(env: Dict[str, str]) -> str:
# we use shell=True here for windows, even though we don't need a shell due to a bug
# https://bugs.python.org/issue8557
shell = sys.platform.startswith("win")
versions_output_text = subprocess.check_output(
["python", "-m", "pip", "freeze", "--all"], universal_newlines=True, shell=shell, env=env
versions_output_text = call(
"python", "-m", "pip", "freeze", "--all", capture_stdout=True, env=env
)
(pip_version,) = (
version[5:]
cibuildwheel/windows.py
+58 -92
View File
@@ -18,32 +18,18 @@ from .util import (
BuildFrontend,
BuildSelector,
NonPlatformWheelError,
call,
download,
get_build_verbosity_extra_flags,
get_pip_version,
prepare_command,
read_python_configs,
shell,
)
CIBW_INSTALL_PATH = Path("C:\\cibw")
def call(
args: Sequence[PathOrStr], env: Optional[Dict[str, str]] = None, cwd: Optional[PathOrStr] = None
) -> None:
print("+ " + " ".join(str(a) for a in args))
# we use shell=True here, even though we don't need a shell due to a bug
# https://bugs.python.org/issue8557
subprocess.run([str(a) for a in args], env=env, cwd=cwd, shell=True, check=True)
def shell(
command: str, env: Optional[Dict[str, str]] = None, cwd: Optional[PathOrStr] = None
) -> None:
print(f"+ {command}")
subprocess.run(command, env=env, cwd=cwd, shell=True, check=True)
def get_nuget_args(version: str, arch: str) -> List[str]:
platform_suffix = {"32": "x86", "64": "", "ARM64": "arm64"}
python_name = "python" + platform_suffix[arch]
@@ -94,7 +80,7 @@ def extract_zip(zip_src: Path, dest: Path) -> None:
def install_cpython(version: str, arch: str, nuget: Path) -> Path:
nuget_args = get_nuget_args(version, arch)
installation_path = Path(nuget_args[-1]) / (nuget_args[0] + "." + version) / "tools"
call([nuget, "install", *nuget_args])
call(nuget, "install", *nuget_args)
# "python3" is not included in the vanilla nuget package,
# though it can be present if modified (like on Azure).
if not (installation_path / "python3.exe").exists():
@@ -165,7 +151,7 @@ def setup_python(
if requires_reinstall:
# maybe pip isn't installed at all. ensurepip resolves that.
call(["python", "-m", "ensurepip"], env=env, cwd=CIBW_INSTALL_PATH)
call("python", "-m", "ensurepip", env=env, cwd=CIBW_INSTALL_PATH)
# pip older than 21.3 builds executables such as pip.exe for x64 platform.
# The first re-install of pip updates pip module but builds pip.exe using
@@ -175,16 +161,14 @@ def setup_python(
# pip versions newer than 21.3.
if python_configuration.arch == "ARM64" and Version(get_pip_version(env)) < Version("21.3"):
call(
[
"python",
"-m",
"pip",
"install",
"--force-reinstall",
"--upgrade",
"pip",
*dependency_constraint_flags,
],
"python",
"-m",
"pip",
"install",
"--force-reinstall",
"--upgrade",
"pip",
*dependency_constraint_flags,
env=env,
cwd=CIBW_INSTALL_PATH,
)
@@ -192,15 +176,13 @@ def setup_python(
# upgrade pip to the version matching our constraints
# if necessary, reinstall it to ensure that it's available on PATH as 'pip.exe'
call(
[
"python",
"-m",
"pip",
"install",
"--force-reinstall" if requires_reinstall else "--upgrade",
"pip",
*dependency_constraint_flags,
],
"python",
"-m",
"pip",
"install",
"--force-reinstall" if requires_reinstall else "--upgrade",
"pip",
*dependency_constraint_flags,
env=env,
cwd=CIBW_INSTALL_PATH,
)
@@ -209,20 +191,10 @@ def setup_python(
env = environment.as_dictionary(prev_environment=env)
# check what Python version we're on
call(["where", "python"], env=env)
call(["python", "--version"], env=env)
call(["python", "-c", "\"import struct; print(struct.calcsize('P') * 8)\""], env=env)
where_python = (
subprocess.run(
["where", "python"],
env=env,
universal_newlines=True,
check=True,
stdout=subprocess.PIPE,
)
.stdout.splitlines()[0]
.strip()
)
call("where", "python", env=env)
call("python", "--version", env=env)
call("python", "-c", "\"import struct; print(struct.calcsize('P') * 8)\"", env=env)
where_python = call("where", "python", env=env, capture_stdout=True).splitlines()[0].strip()
if where_python != str(installation_path / "python.exe"):
print(
"cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.",
@@ -232,13 +204,7 @@ def setup_python(
# check what pip version we're on
assert (installation_path / "Scripts" / "pip.exe").exists()
where_pip = (
subprocess.run(
["where", "pip"], env=env, universal_newlines=True, check=True, stdout=subprocess.PIPE
)
.stdout.splitlines()[0]
.strip()
)
where_pip = call("where", "pip", env=env, capture_stdout=True).splitlines()[0].strip()
if where_pip.strip() != str(installation_path / "Scripts" / "pip.exe"):
print(
"cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.",
@@ -246,23 +212,25 @@ def setup_python(
)
sys.exit(1)
call(["pip", "--version"], env=env)
call("pip", "--version", env=env)
if build_frontend == "pip":
call(
[
"pip",
"install",
"--upgrade",
"setuptools",
"wheel",
*dependency_constraint_flags,
],
"pip",
"install",
"--upgrade",
"setuptools",
"wheel",
*dependency_constraint_flags,
env=env,
)
elif build_frontend == "build":
call(
["pip", "install", "--upgrade", "build[virtualenv]", *dependency_constraint_flags],
"pip",
"install",
"--upgrade",
"build[virtualenv]",
*dependency_constraint_flags,
env=env,
)
else:
@@ -330,16 +298,14 @@ def build(options: Options) -> None:
# Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
# see https://github.com/pypa/cibuildwheel/pull/369
call(
[
"python",
"-m",
"pip",
"wheel",
options.globals.package_dir.resolve(),
f"--wheel-dir={built_wheel_dir}",
"--no-deps",
*get_build_verbosity_extra_flags(build_options.build_verbosity),
],
"python",
"-m",
"pip",
"wheel",
options.globals.package_dir.resolve(),
f"--wheel-dir={built_wheel_dir}",
"--no-deps",
*get_build_verbosity_extra_flags(build_options.build_verbosity),
env=env,
)
elif build_options.build_frontend == "build":
@@ -364,15 +330,13 @@ def build(options: Options) -> None:
build_env["PIP_CONSTRAINT"] = str(constraints_path)
build_env["VIRTUALENV_PIP"] = get_pip_version(env)
call(
[
"python",
"-m",
"build",
build_options.package_dir,
"--wheel",
f"--outdir={built_wheel_dir}",
f"--config-setting={config_setting}",
],
"python",
"-m",
"build",
build_options.package_dir,
"--wheel",
f"--outdir={built_wheel_dir}",
f"--config-setting={config_setting}",
env=build_env,
)
else:
@@ -403,12 +367,12 @@ def build(options: Options) -> None:
log.step("Testing wheel...")
# set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
call(["pip", "install", "virtualenv", *dependency_constraint_flags], env=env)
call("pip", "install", "virtualenv", *dependency_constraint_flags, env=env)
venv_dir = Path(tempfile.mkdtemp())
# Use --no-download to ensure determinism by using seed libraries
# built into virtualenv
call(["python", "-m", "virtualenv", "--no-download", venv_dir], env=env)
call("python", "-m", "virtualenv", "--no-download", venv_dir, env=env)
virtualenv_env = env.copy()
virtualenv_env["PATH"] = os.pathsep.join(
@@ -419,7 +383,7 @@ def build(options: Options) -> None:
)
# check that we are using the Python from the virtual environment
call(["where", "python"], env=virtualenv_env)
call("where", "python", env=virtualenv_env)
if build_options.before_test:
before_test_prepared = prepare_command(
@@ -431,13 +395,15 @@ def build(options: Options) -> None:
# install the wheel
call(
["pip", "install", str(repaired_wheel) + build_options.test_extras],
"pip",
"install",
str(repaired_wheel) + build_options.test_extras,
env=virtualenv_env,
)
# test the wheel
if build_options.test_requires:
call(["pip", "install"] + build_options.test_requires, env=virtualenv_env)
call("pip", "install", *build_options.test_requires, env=virtualenv_env)
# run the tests from c:\, with an absolute path in the command
# (this ensures that Python runs the tests against the installed wheel