Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: use single entry for SDist builds

Browse Source 
Signed-off-by: Henry Schreiner <henryschreineriii@gmail.com>
This commit is contained in:
Henry Schreiner
2022-04-27 09:37:17 -04:00
parent 91fae8268e
commit 958a7c32c1
11 changed files with 103 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cibuildwheel/__main__.py
+60 -9
View File
@@ -2,6 +2,8 @@ import argparse
import os
import shutil
import sys
import tarfile
import tempfile
import textwrap
from pathlib import Path
from tempfile import mkdtemp
@@ -20,15 +22,14 @@ from cibuildwheel.util import (
CIBW_CACHE_PATH,
BuildSelector,
Unbuffered,
chdir,
detect_ci_provider,
format_safe,
)
def main() -> None:
platform: PlatformName
parser = argparse.ArgumentParser(
prog="cibuildwheel",
description="Build wheels for all the platforms.",
epilog="""
Most options are supplied via environment variables or in
@@ -66,6 +67,7 @@ def main() -> None:
parser.add_argument(
"--output-dir",
type=Path,
help="Destination folder for the wheels. Default: wheelhouse.",
)
@@ -74,19 +76,24 @@ def main() -> None:
default="",
help="""
TOML config file. Default: "", meaning {package}/pyproject.toml,
if it exists.
if it exists. To refer to a project inside your project, use {package}
or {project}.
""",
)
parser.add_argument(
"package_dir",
default=".",
default=Path("."),
type=Path,
nargs="?",
help="""
Path to the package that you want wheels for. Must be a subdirectory of
the working directory. When set, the working directory is still
considered the 'project' and is copied into the Docker container on
Linux. Default: the working directory.
Path to the package that you want wheels for. Must be a
subdirectory of the working directory. When set, the working
directory is still considered the 'project' and is copied into the
Docker container on Linux. Default: the working directory. This can
also be a tar.gz file - if it is, then --config-file and
--output-dir are relative to the current directory, and other paths
are relative to the expanded SDist directory.
""",
)
@@ -110,6 +117,50 @@ def main() -> None:
args = parser.parse_args(namespace=CommandLineArguments())
# These are always relative to the base directory, even in SDist builds
args.package_dir = args.package_dir.resolve()
args.output_dir = Path(
args.output_dir
if args.output_dir is not None
else os.environ.get("CIBW_OUTPUT_DIR", "wheelhouse")
).resolve()
# Standard builds if a directory or non-existent path is given
if not args.package_dir.is_file() and not args.package_dir.name.endswith("tar.gz"):
build_in_directory(args)
return
if not args.package_dir.name.endswith("tar.gz"):
raise SystemExit("Must be a tar.gz file if a file is given.")
# Tarfile builds require extraction and changing the directory
with tempfile.TemporaryDirectory(prefix="cibw-sdist-") as temp_dir_str:
temp_dir = Path(temp_dir_str)
with tarfile.open(args.package_dir) as tar:
tar.extractall(path=temp_dir)
# The extract directory is now the project dir
try:
(project_dir,) = temp_dir.iterdir()
except ValueError:
raise SystemExit("invalid sdist: didn't contain a single dir") from None
args.package_dir = project_dir.resolve()
if args.config_file:
# expand the placeholders if they're used
config_file_path = format_safe(
args.config_file,
project=project_dir,
package=project_dir,
)
args.config_file = str(Path(config_file_path).resolve())
with chdir(temp_dir):
build_in_directory(args)
def build_in_directory(args: CommandLineArguments) -> None:
if args.platform != "auto":
platform = args.platform
else:
cibuildwheel/from_sdist.py
-103
View File
@@ -1,103 +0,0 @@
import argparse
import subprocess
import sys
import tarfile
import tempfile
import textwrap
from pathlib import Path
from cibuildwheel.util import format_safe
def main() -> None:
parser = argparse.ArgumentParser(
prog="cibuildwheel-from-sdist",
description=textwrap.dedent(
"""
Build wheels from an sdist archive.
Extracts the sdist to a temp dir and calls cibuildwheel on the
resulting package directory. Note that cibuildwheel will be
invoked with its working directory as the package directory, so
options aside from --output-dir and --config-file are relative to
the package directory.
""",
),
epilog="Any further arguments will be passed on to cibuildwheel.",
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument(
"--output-dir",
default="wheelhouse",
help="""
Destination folder for the wheels. Default: wheelhouse.
""",
)
parser.add_argument(
"--config-file",
default="",
help="""
TOML config file. To refer to a file inside the sdist, use the
`{project}` or `{package}` placeholder. e.g. `--config-file
{project}/config/cibuildwheel.toml` Default: "", meaning the
pyproject.toml inside the sdist, if it exists.
""",
)
parser.add_argument(
"package",
help="""
Path to the sdist archive that you want wheels for. Must be a
tar.gz archive file.
""",
)
args, passthrough_args = parser.parse_known_args()
output_dir = Path(args.output_dir).resolve()
with tempfile.TemporaryDirectory(prefix="cibw-sdist-") as temp_dir_str:
temp_dir = Path(temp_dir_str)
with tarfile.open(args.package) as tar:
tar.extractall(path=temp_dir)
temp_dir_contents = list(temp_dir.iterdir())
if len(temp_dir_contents) != 1 or not temp_dir_contents[0].is_dir():
exit("invalid sdist: didn't contain a single dir")
project_dir = temp_dir_contents[0]
if args.config_file:
# expand the placeholders if they're used
config_file_path = format_safe(
args.config_file,
project=project_dir,
package=project_dir,
)
config_file = Path(config_file_path).resolve()
else:
config_file = None
exit(
subprocess.call(
[
sys.executable,
"-m",
"cibuildwheel",
*(["--config-file", str(config_file)] if config_file else []),
"--output-dir",
output_dir,
*passthrough_args,
".",
],
cwd=project_dir,
)
)
if __name__ == "__main__":
main()
cibuildwheel/options.py
+5 -8
View File
@@ -46,9 +46,9 @@ from .util import (
class CommandLineArguments:
platform: Literal["auto", "linux", "macos", "windows"]
archs: Optional[str]
output_dir: Optional[str]
output_dir: Optional[Path]
config_file: str
package_dir: str
package_dir: Path
print_build_identifiers: bool
allow_empty: bool
prerelease_pythons: bool
@@ -361,12 +361,9 @@ class Options:
@property
def globals(self) -> GlobalOptions:
args = self.command_line_arguments
package_dir = Path(args.package_dir)
output_dir = Path(
args.output_dir
if args.output_dir is not None
else os.environ.get("CIBW_OUTPUT_DIR", "wheelhouse")
)
assert args.output_dir is not None, "Must be resolved"
package_dir = args.package_dir
output_dir = args.output_dir
build_config = self.reader.get("build", env_plat=False, sep=" ") or "*"
skip_config = self.reader.get("skip", env_plat=False, sep=" ")
cibuildwheel/util.py
+17 -2
View File
@@ -19,13 +19,14 @@ from typing import (
Any,
ClassVar,
Dict,
Generator,
Iterable,
Iterator,
List,
NamedTuple,
Optional,
Sequence,
TextIO,
Union,
cast,
overload,
)
@@ -58,6 +59,7 @@ __all__ = [
"selector_matches",
"strtobool",
"cached_property",
"chdir",
]
resources_dir: Final = Path(__file__).parent / "resources"
@@ -414,7 +416,7 @@ def unwrap(text: str) -> str:
@contextlib.contextmanager
def print_new_wheels(msg: str, output_dir: Path) -> Iterator[None]:
def print_new_wheels(msg: str, output_dir: Path) -> Generator[None, None, None]:
"""
Prints the new items in a directory upon exiting. The message to display
can include {n} for number of wheels, {s} for total number of seconds,
@@ -570,3 +572,16 @@ if sys.version_info >= (3, 8):
from functools import cached_property
else:
from .functools_cached_property_38 import cached_property
# Can be replaced by contextlib.chdir in Python 3.11
@contextlib.contextmanager
def chdir(new_path: Union[Path, str]) -> Generator[None, None, None]:
"""Non thread-safe context manager to change the current working directory."""
cwd = os.getcwd()
try:
os.chdir(new_path)
yield
finally:
os.chdir(cwd)
setup.cfg
-1
View File
@@ -51,7 +51,6 @@ include =
[options.entry_points]
console_scripts =
cibuildwheel = cibuildwheel.__main__:main
cibuildwheel-from-sdist = cibuildwheel.from_sdist:main
[options.package_data]
cibuildwheel = resources/*
test/test_from_sdist.py
+8 -7
View File
@@ -18,7 +18,8 @@ def make_sdist(project: TestProject, working_dir: Path) -> Path:
sdist_dir = working_dir / "sdist"
subprocess.run(
[sys.executable, "-m", "build", "--sdist", "--outdir", sdist_dir, project_dir], check=True
[sys.executable, "-m", "build", "--sdist", "--outdir", str(sdist_dir), str(project_dir)],
check=True,
)
return next(sdist_dir.glob("*.tar.gz"))
@@ -35,11 +36,11 @@ def cibuildwheel_from_sdist_run(sdist_path, add_env=None, config_file=None):
[
sys.executable,
"-m",
"cibuildwheel.from_sdist",
"cibuildwheel",
*(["--config-file", config_file] if config_file else []),
"--output-dir",
tmp_output_dir,
sdist_path,
str(tmp_output_dir),
str(sdist_path),
],
env=env,
check=True,
@@ -92,7 +93,7 @@ def test_external_config_file_argument(tmp_path, capfd):
actual_wheels = cibuildwheel_from_sdist_run(
sdist_path,
add_env={"CIBW_BUILD": "cp39-*"},
config_file=config_file,
config_file=str(config_file),
)
# check that the expected wheels are produced
@@ -186,8 +187,8 @@ def test_argument_passthrough(tmp_path, capfd):
[
sys.executable,
"-m",
"cibuildwheel.from_sdist",
sdist_path,
"cibuildwheel",
str(sdist_path),
"--platform",
"linux",
"--archs",
unit_test/conftest.py
+1 -1
View File
@@ -32,7 +32,7 @@ def fake_package_dir(monkeypatch):
real_path_exists = Path.exists
def mock_path_exists(path):
if path == MOCK_PACKAGE_DIR / "setup.py":
if str(path).endswith(str(MOCK_PACKAGE_DIR / "setup.py")):
return True
else:
return real_path_exists(path)
unit_test/main_tests/main_options_test.py
+3 -3
View File
@@ -24,13 +24,13 @@ def test_output_dir(platform, intercepted_build_args, monkeypatch):
main()
assert intercepted_build_args.args[0].globals.output_dir == OUTPUT_DIR
assert intercepted_build_args.args[0].globals.output_dir == OUTPUT_DIR.resolve()
def test_output_dir_default(platform, intercepted_build_args, monkeypatch):
main()
assert intercepted_build_args.args[0].globals.output_dir == Path("wheelhouse")
assert intercepted_build_args.args[0].globals.output_dir == Path("wheelhouse").resolve()
@pytest.mark.parametrize("also_set_environment", [False, True])
@@ -43,7 +43,7 @@ def test_output_dir_argument(also_set_environment, platform, intercepted_build_a
main()
assert intercepted_build_args.args[0].globals.output_dir == OUTPUT_DIR
assert intercepted_build_args.args[0].globals.output_dir == OUTPUT_DIR.resolve()
def test_build_selector(platform, intercepted_build_args, monkeypatch, allow_empty):
unit_test/main_tests/main_platform_test.py
+2 -2
View File
@@ -60,14 +60,14 @@ def test_platform_argument(platform, intercepted_build_args, monkeypatch):
options = intercepted_build_args.args[0]
assert options.globals.package_dir == MOCK_PACKAGE_DIR
assert options.globals.package_dir == MOCK_PACKAGE_DIR.resolve()
def test_platform_environment(platform, intercepted_build_args, monkeypatch):
main()
options = intercepted_build_args.args[0]
assert options.globals.package_dir == MOCK_PACKAGE_DIR
assert options.globals.package_dir == MOCK_PACKAGE_DIR.resolve()
def test_archs_default(platform, intercepted_build_args, monkeypatch):
unit_test/options_test.py
+3 -3
View File
@@ -34,7 +34,7 @@ def test_options_1(tmp_path, monkeypatch):
f.write(PYPROJECT_1)
args = get_default_command_line_arguments()
args.package_dir = str(tmp_path)
args.package_dir = tmp_path
monkeypatch.setattr(platform_module, "machine", lambda: "x86_64")
@@ -77,7 +77,7 @@ def test_passthrough(tmp_path, monkeypatch):
f.write(PYPROJECT_1)
args = get_default_command_line_arguments()
args.package_dir = str(tmp_path)
args.package_dir = tmp_path
monkeypatch.setattr(platform_module, "machine", lambda: "x86_64")
monkeypatch.setenv("EXAMPLE_ENV", "ONE")
@@ -105,7 +105,7 @@ def test_passthrough(tmp_path, monkeypatch):
)
def test_passthrough_evil(tmp_path, monkeypatch, env_var_value):
args = get_default_command_line_arguments()
args.package_dir = str(tmp_path)
args.package_dir = tmp_path
monkeypatch.setattr(platform_module, "machine", lambda: "x86_64")
monkeypatch.setenv("CIBW_ENVIRONMENT_PASS_LINUX", "ENV_VAR")
unit_test/utils.py
+4 -2
View File
@@ -1,3 +1,5 @@
from pathlib import Path
from cibuildwheel.options import CommandLineArguments
@@ -8,8 +10,8 @@ def get_default_command_line_arguments() -> CommandLineArguments:
defaults.allow_empty = False
defaults.archs = None
defaults.config_file = ""
defaults.output_dir = None
defaults.package_dir = "."
defaults.output_dir = Path("wheelhouse") # This must be resolved from "None" before passing
defaults.package_dir = Path(".")
defaults.prerelease_pythons = False
defaults.print_build_identifiers = False