Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Lots more naming changes to reflect that containers are not necessarily 'docker'

Browse Source
This commit is contained in:
Joe Rickerby
2022-06-27 18:30:49 +01:00
parent 1d2e5d5d8f
commit d66c1e92bf
13 changed files with 129 additions and 124 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cibuildwheel/__main__.py
+2 -3
View File
@@ -46,7 +46,7 @@ def main() -> None:
auto-detected platform or to run cibuildwheel on your development
machine. Specifying "macos" or "windows" only works on that
operating system, but "linux" works on all three, as long as
Docker is installed. Default: auto.
Docker/Podman is installed. Default: auto.
""",
)
@@ -91,7 +91,7 @@ def main() -> None:
Path to the package that you want wheels for. Default: the working
directory. Can be a directory inside the working directory, or an
sdist. When set to a directory, the working directory is still
considered the 'project' and is copied into the Docker container
considered the 'project' and is copied into the build container
on Linux. When set to a tar.gz sdist file, --config-file
and --output-dir are relative to the current directory, and other
paths are relative to the expanded SDist directory.
@@ -208,7 +208,6 @@ def build_in_directory(args: CommandLineArguments) -> None:
sys.exit(0)
# Add CIBUILDWHEEL environment variable
# This needs to be passed on to the docker container in linux.py
os.environ["CIBUILDWHEEL"] = "1"
# Python is buffering by default when running on the CI platforms, giving problems interleaving subprocess call output with unflushed calls to 'print'
cibuildwheel/architecture.py
+1 -1
View File
@@ -59,7 +59,7 @@ class Architecture(Enum):
result = {native_architecture}
if platform == "linux" and native_architecture == Architecture.x86_64:
# x86_64 machines can run i686 docker containers
# x86_64 machines can run i686 containers
result.add(Architecture.i686)
if platform == "windows" and native_architecture == Architecture.AMD64:
cibuildwheel/linux.py
+44 -42
View File
@@ -5,8 +5,8 @@ from pathlib import Path, PurePath, PurePosixPath
from typing import Iterator, List, NamedTuple, Set, Tuple
from .architecture import Architecture
from .docker_container import OCIContainer
from .logger import log
from .oci_container import OCIContainer
from .options import Options
from .typing import OrderedDict, PathOrStr, assert_never
from .util import (
@@ -79,7 +79,7 @@ def get_build_steps(
) -> Iterator[BuildStep]:
"""
Groups PythonConfigurations into BuildSteps. Each BuildStep represents a
separate Docker container.
separate container instance.
"""
steps = OrderedDict[Tuple[str, str, str], BuildStep]()
@@ -103,18 +103,18 @@ def get_build_steps(
yield from steps.values()
def build_on_docker(
def build_in_container(
*,
options: Options,
platform_configs: List[PythonConfiguration],
docker: OCIContainer,
container: OCIContainer,
container_project_path: PurePath,
container_package_dir: PurePath,
) -> None:
container_output_dir = PurePosixPath("/output")
log.step("Copying project into Docker...")
docker.copy_into(Path.cwd(), container_project_path)
log.step("Copying project into container...")
container.copy_into(Path.cwd(), container_project_path)
before_all_options_identifier = platform_configs[0].identifier
before_all_options = options.build_options(before_all_options_identifier)
@@ -122,11 +122,11 @@ def build_on_docker(
if before_all_options.before_all:
log.step("Running before_all...")
env = docker.get_environment()
env = container.get_environment()
env["PATH"] = f'/opt/python/cp38-cp38/bin:{env["PATH"]}'
env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
env = before_all_options.environment.as_dictionary(
env, executor=docker.environment_executor
env, executor=container.environment_executor
)
before_all_prepared = prepare_command(
@@ -134,7 +134,7 @@ def build_on_docker(
project=container_project_path,
package=container_package_dir,
)
docker.call(["sh", "-c", before_all_prepared], env=env)
container.call(["sh", "-c", before_all_prepared], env=env)
built_wheels: List[PurePosixPath] = []
@@ -150,21 +150,21 @@ def build_on_docker(
)
container_constraints_file = PurePath("/constraints.txt")
docker.copy_into(constraints_file, container_constraints_file)
container.copy_into(constraints_file, container_constraints_file)
dependency_constraint_flags = ["-c", container_constraints_file]
log.step("Setting up build environment...")
env = docker.get_environment()
env = container.get_environment()
# put this config's python top of the list
python_bin = config.path / "bin"
env["PATH"] = f'{python_bin}:{env["PATH"]}'
env = build_options.environment.as_dictionary(env, executor=docker.environment_executor)
env = build_options.environment.as_dictionary(env, executor=container.environment_executor)
# check config python is still on PATH
which_python = docker.call(["which", "python"], env=env, capture_output=True).strip()
which_python = container.call(["which", "python"], env=env, capture_output=True).strip()
if PurePosixPath(which_python) != python_bin / "python":
print(
"cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.",
@@ -172,7 +172,7 @@ def build_on_docker(
)
sys.exit(1)
which_pip = docker.call(["which", "pip"], env=env, capture_output=True).strip()
which_pip = container.call(["which", "pip"], env=env, capture_output=True).strip()
if PurePosixPath(which_pip) != python_bin / "pip":
print(
"cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.",
@@ -196,19 +196,19 @@ def build_on_docker(
project=container_project_path,
package=container_package_dir,
)
docker.call(["sh", "-c", before_build_prepared], env=env)
container.call(["sh", "-c", before_build_prepared], env=env)
log.step("Building wheel...")
temp_dir = PurePosixPath("/tmp/cibuildwheel")
built_wheel_dir = temp_dir / "built_wheel"
docker.call(["rm", "-rf", built_wheel_dir])
docker.call(["mkdir", "-p", built_wheel_dir])
container.call(["rm", "-rf", built_wheel_dir])
container.call(["mkdir", "-p", built_wheel_dir])
verbosity_flags = get_build_verbosity_extra_flags(build_options.build_verbosity)
if build_options.build_frontend == "pip":
docker.call(
container.call(
[
"python",
"-m",
@@ -223,7 +223,7 @@ def build_on_docker(
)
elif build_options.build_frontend == "build":
config_setting = " ".join(verbosity_flags)
docker.call(
container.call(
[
"python",
"-m",
@@ -238,11 +238,11 @@ def build_on_docker(
else:
assert_never(build_options.build_frontend)
built_wheel = docker.glob(built_wheel_dir, "*.whl")[0]
built_wheel = container.glob(built_wheel_dir, "*.whl")[0]
repaired_wheel_dir = temp_dir / "repaired_wheel"
docker.call(["rm", "-rf", repaired_wheel_dir])
docker.call(["mkdir", "-p", repaired_wheel_dir])
container.call(["rm", "-rf", repaired_wheel_dir])
container.call(["mkdir", "-p", repaired_wheel_dir])
if built_wheel.name.endswith("none-any.whl"):
raise NonPlatformWheelError()
@@ -252,21 +252,23 @@ def build_on_docker(
repair_command_prepared = prepare_command(
build_options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir
)
docker.call(["sh", "-c", repair_command_prepared], env=env)
container.call(["sh", "-c", repair_command_prepared], env=env)
else:
docker.call(["mv", built_wheel, repaired_wheel_dir])
container.call(["mv", built_wheel, repaired_wheel_dir])
repaired_wheels = docker.glob(repaired_wheel_dir, "*.whl")
repaired_wheels = container.glob(repaired_wheel_dir, "*.whl")
if build_options.test_command and build_options.test_selector(config.identifier):
log.step("Testing wheel...")
# set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
docker.call(["pip", "install", "virtualenv", *dependency_constraint_flags], env=env)
venv_dir = PurePath(docker.call(["mktemp", "-d"], capture_output=True).strip()) / "venv"
container.call(["pip", "install", "virtualenv", *dependency_constraint_flags], env=env)
venv_dir = (
PurePath(container.call(["mktemp", "-d"], capture_output=True).strip()) / "venv"
)
docker.call(["python", "-m", "virtualenv", "--no-download", venv_dir], env=env)
container.call(["python", "-m", "virtualenv", "--no-download", venv_dir], env=env)
virtualenv_env = env.copy()
virtualenv_env["PATH"] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}"
@@ -277,7 +279,7 @@ def build_on_docker(
project=container_project_path,
package=container_package_dir,
)
docker.call(["sh", "-c", before_test_prepared], env=virtualenv_env)
container.call(["sh", "-c", before_test_prepared], env=virtualenv_env)
# Install the wheel we just built
# Note: If auditwheel produced two wheels, it's because the earlier produced wheel
@@ -286,14 +288,14 @@ def build_on_docker(
# different external shared libraries. so it doesn't matter which one we run the tests on.
# Let's just pick the first one.
wheel_to_test = repaired_wheels[0]
docker.call(
container.call(
["pip", "install", str(wheel_to_test) + build_options.test_extras],
env=virtualenv_env,
)
# Install any requirements to run the tests
if build_options.test_requires:
docker.call(["pip", "install", *build_options.test_requires], env=virtualenv_env)
container.call(["pip", "install", *build_options.test_requires], env=virtualenv_env)
# Run the tests from a different directory
test_command_prepared = prepare_command(
@@ -301,15 +303,15 @@ def build_on_docker(
project=container_project_path,
package=container_package_dir,
)
docker.call(["sh", "-c", test_command_prepared], cwd="/root", env=virtualenv_env)
container.call(["sh", "-c", test_command_prepared], cwd="/root", env=virtualenv_env)
# clean up test environment
docker.call(["rm", "-rf", venv_dir])
container.call(["rm", "-rf", venv_dir])
# move repaired wheels to output
if abi3_wheel is None:
docker.call(["mkdir", "-p", container_output_dir])
docker.call(["mv", *repaired_wheels, container_output_dir])
container.call(["mkdir", "-p", container_output_dir])
container.call(["mv", *repaired_wheels, container_output_dir])
built_wheels.extend(
container_output_dir / repaired_wheel.name for repaired_wheel in repaired_wheels
)
@@ -318,7 +320,7 @@ def build_on_docker(
log.step("Copying wheels back to host...")
# copy the output back into the host
docker.copy_out(container_output_dir, options.globals.output_dir)
container.copy_out(container_output_dir, options.globals.output_dir)
log.step_end()
@@ -358,21 +360,21 @@ def build(options: Options, tmp_path: Path) -> None: # pylint: disable=unused-a
for build_step in get_build_steps(options, python_configurations):
try:
ids_to_build = [x.identifier for x in build_step.platform_configs]
log.step(
f"Starting Docker image {build_step.container_image} for {', '.join(ids_to_build)}..."
)
log.step(f"Starting container image {build_step.container_image}...")
print(f"info: This container will host the build for {', '.join(ids_to_build)}...")
with OCIContainer(
image=build_step.container_image,
simulate_32_bit=build_step.platform_tag.endswith("i686"),
cwd=container_project_path,
engine=options.globals.container_engine,
) as docker:
) as container:
build_on_docker(
build_in_container(
options=options,
platform_configs=build_step.platform_configs,
docker=docker,
container=container,
container_project_path=container_project_path,
container_package_dir=container_package_dir,
)
cibuildwheel/docker_container.py → cibuildwheel/oci_container.py
+10 -8
View File
@@ -56,7 +56,7 @@ class OCIContainer:
engine: ContainerEngine = "docker",
):
if not image:
raise ValueError("Must have a non-empty docker image to run.")
raise ValueError("Must have a non-empty image to run.")
self.image = image
self.simulate_32_bit = simulate_32_bit
@@ -175,17 +175,19 @@ class OCIContainer:
f"cat > {shell_quote(to_path)}",
],
stdin=subprocess.PIPE,
) as docker:
docker.stdin = cast(IO[bytes], docker.stdin)
) as exec_process:
exec_process.stdin = cast(IO[bytes], exec_process.stdin)
with open(from_path, "rb") as from_file:
shutil.copyfileobj(from_file, docker.stdin)
shutil.copyfileobj(from_file, exec_process.stdin)
docker.stdin.close()
docker.wait()
exec_process.stdin.close()
exec_process.wait()
if docker.returncode:
raise subprocess.CalledProcessError(docker.returncode, docker.args, None, None)
if exec_process.returncode:
raise subprocess.CalledProcessError(
exec_process.returncode, exec_process.args, None, None
)
def copy_out(self, from_path: PurePath, to_path: Path) -> None:
# note: we assume from_path is a dir
cibuildwheel/options.py
+1 -1
View File
@@ -27,8 +27,8 @@ else:
from packaging.specifiers import SpecifierSet
from .architecture import Architecture
from .docker_container import ContainerEngine
from .environment import EnvironmentParseError, ParsedEnvironment, parse_environment
from .oci_container import ContainerEngine
from .projectfiles import get_requires_python_str
from .typing import PLATFORMS, Literal, PlatformName, TypedDict
from .util import (