refactor: reduce linux indentation with nicer breakup
This commit is contained in:
Henry Schreiner
+198
-204
@@ -87,6 +87,201 @@ def get_linux_platforms(
|
|||||||
yield platform_configs, platform_tag, docker_image
|
yield platform_configs, platform_tag, docker_image
|
||||||
|
|
||||||
|
|
||||||
|
def build_on_docker(
|
||||||
|
options: BuildOptions,
|
||||||
|
platform_configs: List[PythonConfiguration],
|
||||||
|
docker: DockerContainer,
|
||||||
|
container_project_path: PurePath,
|
||||||
|
container_package_dir: PurePath,
|
||||||
|
) -> None:
|
||||||
|
container_output_dir = PurePath("/output")
|
||||||
|
|
||||||
|
log.step("Copying project into Docker...")
|
||||||
|
docker.copy_into(Path.cwd(), container_project_path)
|
||||||
|
|
||||||
|
if options.before_all:
|
||||||
|
log.step("Running before_all...")
|
||||||
|
|
||||||
|
env = docker.get_environment()
|
||||||
|
env["PATH"] = f'/opt/python/cp38-cp38/bin:{env["PATH"]}'
|
||||||
|
env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
|
||||||
|
env = options.environment.as_dictionary(env, executor=docker.environment_executor)
|
||||||
|
|
||||||
|
before_all_prepared = prepare_command(
|
||||||
|
options.before_all,
|
||||||
|
project=container_project_path,
|
||||||
|
package=container_package_dir,
|
||||||
|
)
|
||||||
|
docker.call(["sh", "-c", before_all_prepared], env=env)
|
||||||
|
|
||||||
|
for config in platform_configs:
|
||||||
|
log.build_start(config.identifier)
|
||||||
|
|
||||||
|
dependency_constraint_flags: List[PathOrStr] = []
|
||||||
|
|
||||||
|
if options.dependency_constraints:
|
||||||
|
constraints_file = options.dependency_constraints.get_for_python_version(config.version)
|
||||||
|
container_constraints_file = PurePath("/constraints.txt")
|
||||||
|
|
||||||
|
docker.copy_into(constraints_file, container_constraints_file)
|
||||||
|
dependency_constraint_flags = ["-c", container_constraints_file]
|
||||||
|
|
||||||
|
log.step("Setting up build environment...")
|
||||||
|
|
||||||
|
env = docker.get_environment()
|
||||||
|
|
||||||
|
# put this config's python top of the list
|
||||||
|
python_bin = config.path / "bin"
|
||||||
|
env["PATH"] = f'{python_bin}:{env["PATH"]}'
|
||||||
|
|
||||||
|
env = options.environment.as_dictionary(env, executor=docker.environment_executor)
|
||||||
|
|
||||||
|
# check config python is still on PATH
|
||||||
|
which_python = docker.call(["which", "python"], env=env, capture_output=True).strip()
|
||||||
|
if PurePath(which_python) != python_bin / "python":
|
||||||
|
print(
|
||||||
|
"cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
which_pip = docker.call(["which", "pip"], env=env, capture_output=True).strip()
|
||||||
|
if PurePath(which_pip) != python_bin / "pip":
|
||||||
|
print(
|
||||||
|
"cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.",
|
||||||
|
file=sys.stderr,
|
||||||
|
)
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
if options.before_build:
|
||||||
|
log.step("Running before_build...")
|
||||||
|
before_build_prepared = prepare_command(
|
||||||
|
options.before_build,
|
||||||
|
project=container_project_path,
|
||||||
|
package=container_package_dir,
|
||||||
|
)
|
||||||
|
docker.call(["sh", "-c", before_build_prepared], env=env)
|
||||||
|
|
||||||
|
log.step("Building wheel...")
|
||||||
|
|
||||||
|
temp_dir = PurePath("/tmp/cibuildwheel")
|
||||||
|
built_wheel_dir = temp_dir / "built_wheel"
|
||||||
|
docker.call(["rm", "-rf", built_wheel_dir])
|
||||||
|
docker.call(["mkdir", "-p", built_wheel_dir])
|
||||||
|
|
||||||
|
verbosity_flags = get_build_verbosity_extra_flags(options.build_verbosity)
|
||||||
|
|
||||||
|
if options.build_frontend == "pip":
|
||||||
|
docker.call(
|
||||||
|
[
|
||||||
|
"python",
|
||||||
|
"-m",
|
||||||
|
"pip",
|
||||||
|
"wheel",
|
||||||
|
container_package_dir,
|
||||||
|
f"--wheel-dir={built_wheel_dir}",
|
||||||
|
"--no-deps",
|
||||||
|
*verbosity_flags,
|
||||||
|
],
|
||||||
|
env=env,
|
||||||
|
)
|
||||||
|
elif options.build_frontend == "build":
|
||||||
|
config_setting = " ".join(verbosity_flags)
|
||||||
|
docker.call(
|
||||||
|
[
|
||||||
|
"python",
|
||||||
|
"-m",
|
||||||
|
"build",
|
||||||
|
container_package_dir,
|
||||||
|
"--wheel",
|
||||||
|
f"--outdir={built_wheel_dir}",
|
||||||
|
f"--config-setting={config_setting}",
|
||||||
|
],
|
||||||
|
env=env,
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
assert_never(options.build_frontend)
|
||||||
|
|
||||||
|
built_wheel = docker.glob(built_wheel_dir, "*.whl")[0]
|
||||||
|
|
||||||
|
repaired_wheel_dir = temp_dir / "repaired_wheel"
|
||||||
|
docker.call(["rm", "-rf", repaired_wheel_dir])
|
||||||
|
docker.call(["mkdir", "-p", repaired_wheel_dir])
|
||||||
|
|
||||||
|
if built_wheel.name.endswith("none-any.whl"):
|
||||||
|
raise NonPlatformWheelError()
|
||||||
|
|
||||||
|
if options.repair_command:
|
||||||
|
log.step("Repairing wheel...")
|
||||||
|
repair_command_prepared = prepare_command(
|
||||||
|
options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir
|
||||||
|
)
|
||||||
|
docker.call(["sh", "-c", repair_command_prepared], env=env)
|
||||||
|
else:
|
||||||
|
docker.call(["mv", built_wheel, repaired_wheel_dir])
|
||||||
|
|
||||||
|
repaired_wheels = docker.glob(repaired_wheel_dir, "*.whl")
|
||||||
|
|
||||||
|
if options.test_command and options.test_selector(config.identifier):
|
||||||
|
log.step("Testing wheel...")
|
||||||
|
|
||||||
|
# set up a virtual environment to install and test from, to make sure
|
||||||
|
# there are no dependencies that were pulled in at build time.
|
||||||
|
docker.call(["pip", "install", "virtualenv", *dependency_constraint_flags], env=env)
|
||||||
|
venv_dir = PurePath(docker.call(["mktemp", "-d"], capture_output=True).strip()) / "venv"
|
||||||
|
|
||||||
|
docker.call(["python", "-m", "virtualenv", "--no-download", venv_dir], env=env)
|
||||||
|
|
||||||
|
virtualenv_env = env.copy()
|
||||||
|
virtualenv_env["PATH"] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}"
|
||||||
|
|
||||||
|
if options.before_test:
|
||||||
|
before_test_prepared = prepare_command(
|
||||||
|
options.before_test,
|
||||||
|
project=container_project_path,
|
||||||
|
package=container_package_dir,
|
||||||
|
)
|
||||||
|
docker.call(["sh", "-c", before_test_prepared], env=virtualenv_env)
|
||||||
|
|
||||||
|
# Install the wheel we just built
|
||||||
|
# Note: If auditwheel produced two wheels, it's because the earlier produced wheel
|
||||||
|
# conforms to multiple manylinux standards. These multiple versions of the wheel are
|
||||||
|
# functionally the same, differing only in name, wheel metadata, and possibly include
|
||||||
|
# different external shared libraries. so it doesn't matter which one we run the tests on.
|
||||||
|
# Let's just pick the first one.
|
||||||
|
wheel_to_test = repaired_wheels[0]
|
||||||
|
docker.call(
|
||||||
|
["pip", "install", str(wheel_to_test) + options.test_extras],
|
||||||
|
env=virtualenv_env,
|
||||||
|
)
|
||||||
|
|
||||||
|
# Install any requirements to run the tests
|
||||||
|
if options.test_requires:
|
||||||
|
docker.call(["pip", "install", *options.test_requires], env=virtualenv_env)
|
||||||
|
|
||||||
|
# Run the tests from a different directory
|
||||||
|
test_command_prepared = prepare_command(
|
||||||
|
options.test_command,
|
||||||
|
project=container_project_path,
|
||||||
|
package=container_package_dir,
|
||||||
|
)
|
||||||
|
docker.call(["sh", "-c", test_command_prepared], cwd="/root", env=virtualenv_env)
|
||||||
|
|
||||||
|
# clean up test environment
|
||||||
|
docker.call(["rm", "-rf", venv_dir])
|
||||||
|
|
||||||
|
# move repaired wheels to output
|
||||||
|
docker.call(["mkdir", "-p", container_output_dir])
|
||||||
|
docker.call(["mv", *repaired_wheels, container_output_dir])
|
||||||
|
|
||||||
|
log.build_end()
|
||||||
|
|
||||||
|
log.step("Copying wheels back to host...")
|
||||||
|
# copy the output back into the host
|
||||||
|
docker.copy_out(container_output_dir, options.output_dir)
|
||||||
|
log.step_end()
|
||||||
|
|
||||||
|
|
||||||
def build(options: BuildOptions) -> None:
|
def build(options: BuildOptions) -> None:
|
||||||
try:
|
try:
|
||||||
# check docker is installed
|
# check docker is installed
|
||||||
@@ -111,7 +306,6 @@ def build(options: BuildOptions) -> None:
|
|||||||
|
|
||||||
container_project_path = PurePath("/project")
|
container_project_path = PurePath("/project")
|
||||||
container_package_dir = container_project_path / abs_package_dir.relative_to(cwd)
|
container_package_dir = container_project_path / abs_package_dir.relative_to(cwd)
|
||||||
container_output_dir = PurePath("/output")
|
|
||||||
|
|
||||||
for platform_configs, platform_tag, docker_image in get_linux_platforms(
|
for platform_configs, platform_tag, docker_image in get_linux_platforms(
|
||||||
options, python_configurations
|
options, python_configurations
|
||||||
@@ -123,210 +317,10 @@ def build(options: BuildOptions) -> None:
|
|||||||
simulate_32_bit=platform_tag.endswith("i686"),
|
simulate_32_bit=platform_tag.endswith("i686"),
|
||||||
cwd=container_project_path,
|
cwd=container_project_path,
|
||||||
) as docker:
|
) as docker:
|
||||||
|
build_on_docker(
|
||||||
|
options, platform_configs, docker, container_project_path, container_package_dir
|
||||||
|
)
|
||||||
|
|
||||||
log.step("Copying project into Docker...")
|
|
||||||
docker.copy_into(Path.cwd(), container_project_path)
|
|
||||||
|
|
||||||
if options.before_all:
|
|
||||||
log.step("Running before_all...")
|
|
||||||
|
|
||||||
env = docker.get_environment()
|
|
||||||
env["PATH"] = f'/opt/python/cp38-cp38/bin:{env["PATH"]}'
|
|
||||||
env["PIP_DISABLE_PIP_VERSION_CHECK"] = "1"
|
|
||||||
env = options.environment.as_dictionary(
|
|
||||||
env, executor=docker.environment_executor
|
|
||||||
)
|
|
||||||
|
|
||||||
before_all_prepared = prepare_command(
|
|
||||||
options.before_all,
|
|
||||||
project=container_project_path,
|
|
||||||
package=container_package_dir,
|
|
||||||
)
|
|
||||||
docker.call(["sh", "-c", before_all_prepared], env=env)
|
|
||||||
|
|
||||||
for config in platform_configs:
|
|
||||||
log.build_start(config.identifier)
|
|
||||||
|
|
||||||
dependency_constraint_flags: List[PathOrStr] = []
|
|
||||||
|
|
||||||
if options.dependency_constraints:
|
|
||||||
constraints_file = options.dependency_constraints.get_for_python_version(
|
|
||||||
config.version
|
|
||||||
)
|
|
||||||
container_constraints_file = PurePath("/constraints.txt")
|
|
||||||
|
|
||||||
docker.copy_into(constraints_file, container_constraints_file)
|
|
||||||
dependency_constraint_flags = ["-c", container_constraints_file]
|
|
||||||
|
|
||||||
log.step("Setting up build environment...")
|
|
||||||
|
|
||||||
env = docker.get_environment()
|
|
||||||
|
|
||||||
# put this config's python top of the list
|
|
||||||
python_bin = config.path / "bin"
|
|
||||||
env["PATH"] = f'{python_bin}:{env["PATH"]}'
|
|
||||||
|
|
||||||
env = options.environment.as_dictionary(
|
|
||||||
env, executor=docker.environment_executor
|
|
||||||
)
|
|
||||||
|
|
||||||
# check config python is still on PATH
|
|
||||||
which_python = docker.call(
|
|
||||||
["which", "python"], env=env, capture_output=True
|
|
||||||
).strip()
|
|
||||||
if PurePath(which_python) != python_bin / "python":
|
|
||||||
print(
|
|
||||||
"cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.",
|
|
||||||
file=sys.stderr,
|
|
||||||
)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
which_pip = docker.call(["which", "pip"], env=env, capture_output=True).strip()
|
|
||||||
if PurePath(which_pip) != python_bin / "pip":
|
|
||||||
print(
|
|
||||||
"cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.",
|
|
||||||
file=sys.stderr,
|
|
||||||
)
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
if options.before_build:
|
|
||||||
log.step("Running before_build...")
|
|
||||||
before_build_prepared = prepare_command(
|
|
||||||
options.before_build,
|
|
||||||
project=container_project_path,
|
|
||||||
package=container_package_dir,
|
|
||||||
)
|
|
||||||
docker.call(["sh", "-c", before_build_prepared], env=env)
|
|
||||||
|
|
||||||
log.step("Building wheel...")
|
|
||||||
|
|
||||||
temp_dir = PurePath("/tmp/cibuildwheel")
|
|
||||||
built_wheel_dir = temp_dir / "built_wheel"
|
|
||||||
docker.call(["rm", "-rf", built_wheel_dir])
|
|
||||||
docker.call(["mkdir", "-p", built_wheel_dir])
|
|
||||||
|
|
||||||
verbosity_flags = get_build_verbosity_extra_flags(options.build_verbosity)
|
|
||||||
|
|
||||||
if options.build_frontend == "pip":
|
|
||||||
docker.call(
|
|
||||||
[
|
|
||||||
"python",
|
|
||||||
"-m",
|
|
||||||
"pip",
|
|
||||||
"wheel",
|
|
||||||
container_package_dir,
|
|
||||||
f"--wheel-dir={built_wheel_dir}",
|
|
||||||
"--no-deps",
|
|
||||||
*verbosity_flags,
|
|
||||||
],
|
|
||||||
env=env,
|
|
||||||
)
|
|
||||||
elif options.build_frontend == "build":
|
|
||||||
config_setting = " ".join(verbosity_flags)
|
|
||||||
docker.call(
|
|
||||||
[
|
|
||||||
"python",
|
|
||||||
"-m",
|
|
||||||
"build",
|
|
||||||
container_package_dir,
|
|
||||||
"--wheel",
|
|
||||||
f"--outdir={built_wheel_dir}",
|
|
||||||
f"--config-setting={config_setting}",
|
|
||||||
],
|
|
||||||
env=env,
|
|
||||||
)
|
|
||||||
else:
|
|
||||||
assert_never(options.build_frontend)
|
|
||||||
|
|
||||||
built_wheel = docker.glob(built_wheel_dir, "*.whl")[0]
|
|
||||||
|
|
||||||
repaired_wheel_dir = temp_dir / "repaired_wheel"
|
|
||||||
docker.call(["rm", "-rf", repaired_wheel_dir])
|
|
||||||
docker.call(["mkdir", "-p", repaired_wheel_dir])
|
|
||||||
|
|
||||||
if built_wheel.name.endswith("none-any.whl"):
|
|
||||||
raise NonPlatformWheelError()
|
|
||||||
|
|
||||||
if options.repair_command:
|
|
||||||
log.step("Repairing wheel...")
|
|
||||||
repair_command_prepared = prepare_command(
|
|
||||||
options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir
|
|
||||||
)
|
|
||||||
docker.call(["sh", "-c", repair_command_prepared], env=env)
|
|
||||||
else:
|
|
||||||
docker.call(["mv", built_wheel, repaired_wheel_dir])
|
|
||||||
|
|
||||||
repaired_wheels = docker.glob(repaired_wheel_dir, "*.whl")
|
|
||||||
|
|
||||||
if options.test_command and options.test_selector(config.identifier):
|
|
||||||
log.step("Testing wheel...")
|
|
||||||
|
|
||||||
# set up a virtual environment to install and test from, to make sure
|
|
||||||
# there are no dependencies that were pulled in at build time.
|
|
||||||
docker.call(
|
|
||||||
["pip", "install", "virtualenv", *dependency_constraint_flags], env=env
|
|
||||||
)
|
|
||||||
venv_dir = (
|
|
||||||
PurePath(docker.call(["mktemp", "-d"], capture_output=True).strip())
|
|
||||||
/ "venv"
|
|
||||||
)
|
|
||||||
|
|
||||||
docker.call(
|
|
||||||
["python", "-m", "virtualenv", "--no-download", venv_dir], env=env
|
|
||||||
)
|
|
||||||
|
|
||||||
virtualenv_env = env.copy()
|
|
||||||
virtualenv_env["PATH"] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}"
|
|
||||||
|
|
||||||
if options.before_test:
|
|
||||||
before_test_prepared = prepare_command(
|
|
||||||
options.before_test,
|
|
||||||
project=container_project_path,
|
|
||||||
package=container_package_dir,
|
|
||||||
)
|
|
||||||
docker.call(["sh", "-c", before_test_prepared], env=virtualenv_env)
|
|
||||||
|
|
||||||
# Install the wheel we just built
|
|
||||||
# Note: If auditwheel produced two wheels, it's because the earlier produced wheel
|
|
||||||
# conforms to multiple manylinux standards. These multiple versions of the wheel are
|
|
||||||
# functionally the same, differing only in name, wheel metadata, and possibly include
|
|
||||||
# different external shared libraries. so it doesn't matter which one we run the tests on.
|
|
||||||
# Let's just pick the first one.
|
|
||||||
wheel_to_test = repaired_wheels[0]
|
|
||||||
docker.call(
|
|
||||||
["pip", "install", str(wheel_to_test) + options.test_extras],
|
|
||||||
env=virtualenv_env,
|
|
||||||
)
|
|
||||||
|
|
||||||
# Install any requirements to run the tests
|
|
||||||
if options.test_requires:
|
|
||||||
docker.call(
|
|
||||||
["pip", "install", *options.test_requires], env=virtualenv_env
|
|
||||||
)
|
|
||||||
|
|
||||||
# Run the tests from a different directory
|
|
||||||
test_command_prepared = prepare_command(
|
|
||||||
options.test_command,
|
|
||||||
project=container_project_path,
|
|
||||||
package=container_package_dir,
|
|
||||||
)
|
|
||||||
docker.call(
|
|
||||||
["sh", "-c", test_command_prepared], cwd="/root", env=virtualenv_env
|
|
||||||
)
|
|
||||||
|
|
||||||
# clean up test environment
|
|
||||||
docker.call(["rm", "-rf", venv_dir])
|
|
||||||
|
|
||||||
# move repaired wheels to output
|
|
||||||
docker.call(["mkdir", "-p", container_output_dir])
|
|
||||||
docker.call(["mv", *repaired_wheels, container_output_dir])
|
|
||||||
|
|
||||||
log.build_end()
|
|
||||||
|
|
||||||
log.step("Copying wheels back to host...")
|
|
||||||
# copy the output back into the host
|
|
||||||
docker.copy_out(container_output_dir, options.output_dir)
|
|
||||||
log.step_end()
|
|
||||||
except subprocess.CalledProcessError as error:
|
except subprocess.CalledProcessError as error:
|
||||||
log.step_end_with_error(
|
log.step_end_with_error(
|
||||||
f"Command {error.cmd} failed with code {error.returncode}. {error.stdout}"
|
f"Command {error.cmd} failed with code {error.returncode}. {error.stdout}"
|
||||||
|
|||||||
Reference in New Issue
Block a user