Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove the linux bash script, replaced with a docker exec approach

Browse Source
This commit is contained in:
Joe Rickerby
2020-06-23 19:46:23 +01:00
parent 666699a51e
commit e417791ae4
4 changed files with 261 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cibuildwheel/bashlex_eval.py
+13 -4
View File
@@ -1,17 +1,26 @@
import shlex
import subprocess
from typing import Dict, NamedTuple
from typing import Dict, NamedTuple, Callable, Optional
import bashlex # type: ignore
# a function that takes a shell command and the environment, and returns the result
EnvironmentExecutor = Callable[[str, Dict[str, str]], str]
def local_environment_executor(command: str, env: Dict[str, str]) -> str:
return subprocess.check_output(shlex.split(command), env=env, universal_newlines=True)
class NodeExecutionContext(NamedTuple):
environment: Dict[str, str]
input: str
executor: EnvironmentExecutor
def evaluate(value: str, environment: Dict[str, str]) -> str:
def evaluate(value: str, environment: Dict[str, str], executor: Optional[EnvironmentExecutor] = None) -> str:
if not value:
# empty string evaluates to empty string
# (but trips up bashlex)
@@ -26,7 +35,7 @@ def evaluate(value: str, environment: Dict[str, str]) -> str:
return evaluate_node(
value_word_node,
context=NodeExecutionContext(environment=environment, input=value)
context=NodeExecutionContext(environment=environment, input=value, executor=executor or local_environment_executor)
)
@@ -67,7 +76,7 @@ def evaluate_word_node(node: bashlex.ast.node, context: NodeExecutionContext) ->
def evaluate_command_node(node: bashlex.ast.node, context: NodeExecutionContext) -> str:
words = [evaluate_node(part, context=context) for part in node.parts]
command = ' '.join(words)
return subprocess.check_output(shlex.split(command), env=context.environment, universal_newlines=True)
return context.executor(command, context.environment)
def evaluate_parameter_node(node: bashlex.ast.node, context: NodeExecutionContext) -> str:
cibuildwheel/environment.py
+7 -5
View File
@@ -1,6 +1,6 @@
import bashlex # type: ignore
from typing import Dict, List, Mapping
from typing import Dict, List, Mapping, Optional
from . import bashlex_eval
@@ -46,9 +46,9 @@ class EnvironmentAssignment:
self.name = name
self.value = value
def evaluated_value(self, environment: Dict[str, str]) -> str:
def evaluated_value(self, environment: Dict[str, str], executor: Optional[bashlex_eval.EnvironmentExecutor] = None) -> str:
'''Returns the value of this assignment, as evaluated in the environment'''
return bashlex_eval.evaluate(self.value, environment=environment)
return bashlex_eval.evaluate(self.value, environment=environment, executor=executor)
def as_shell_assignment(self) -> str:
return f'export {self.name}={self.value}'
@@ -61,11 +61,13 @@ class ParsedEnvironment:
def __init__(self, assignments: List[EnvironmentAssignment]):
self.assignments = assignments
def as_dictionary(self, prev_environment: Mapping[str, str]) -> Dict[str, str]:
def as_dictionary(self,
prev_environment: Mapping[str, str],
executor: Optional[bashlex_eval.EnvironmentExecutor] = None) -> Dict[str, str]:
environment = dict(**prev_environment)
for assignment in self.assignments:
value = assignment.evaluated_value(environment=environment)
value = assignment.evaluated_value(environment=environment, executor=executor)
environment[assignment.name] = value
return environment
cibuildwheel/linux.py
+239 -206
View File
@@ -1,27 +1,16 @@
import os
import json
import platform
import shlex
import subprocess
import sys
import textwrap
import uuid
from os import PathLike
from pathlib import Path, PurePath
from typing import Dict, List, NamedTuple, Optional, Union, Sequence
from typing import List, NamedTuple, Optional, Union
from .util import (
BuildOptions,
BuildSelector,
get_build_verbosity_extra_flags,
prepare_command,
)
def call(args: List[str], input: Optional[Union[str, bytes]] = None, universal_newlines: bool = False) -> None:
print('+ ' + ' '.join(shlex.quote(a) for a in args))
subprocess.run(
args, input=input, universal_newlines=universal_newlines, check=True
)
from .util import (BuildOptions, BuildSelector,
get_build_verbosity_extra_flags, prepare_command)
def matches_platform(identifier: str) -> bool:
@@ -48,37 +37,41 @@ def matches_platform(identifier: str) -> bool:
class PythonConfiguration(NamedTuple):
version: str
identifier: str
path: str
path_str: str
@property
def path(self):
return PurePath(self.path_str)
def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfiguration]:
python_configurations = [
PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path='/opt/python/cp27-cp27m'),
PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path='/opt/python/cp27-cp27mu'),
PythonConfiguration(version='3.5', identifier='cp35-manylinux_x86_64', path='/opt/python/cp35-cp35m'),
PythonConfiguration(version='3.6', identifier='cp36-manylinux_x86_64', path='/opt/python/cp36-cp36m'),
PythonConfiguration(version='3.7', identifier='cp37-manylinux_x86_64', path='/opt/python/cp37-cp37m'),
PythonConfiguration(version='3.8', identifier='cp38-manylinux_x86_64', path='/opt/python/cp38-cp38'),
PythonConfiguration(version='2.7', identifier='cp27-manylinux_i686', path='/opt/python/cp27-cp27m'),
PythonConfiguration(version='2.7', identifier='cp27-manylinux_i686', path='/opt/python/cp27-cp27mu'),
PythonConfiguration(version='3.5', identifier='cp35-manylinux_i686', path='/opt/python/cp35-cp35m'),
PythonConfiguration(version='3.6', identifier='cp36-manylinux_i686', path='/opt/python/cp36-cp36m'),
PythonConfiguration(version='3.7', identifier='cp37-manylinux_i686', path='/opt/python/cp37-cp37m'),
PythonConfiguration(version='3.8', identifier='cp38-manylinux_i686', path='/opt/python/cp38-cp38'),
PythonConfiguration(version='2.7', identifier='pp27-manylinux_x86_64', path='/opt/python/pp27-pypy_73'),
PythonConfiguration(version='3.6', identifier='pp36-manylinux_x86_64', path='/opt/python/pp36-pypy36_pp73'),
PythonConfiguration(version='3.5', identifier='cp35-manylinux_aarch64', path='/opt/python/cp35-cp35m'),
PythonConfiguration(version='3.6', identifier='cp36-manylinux_aarch64', path='/opt/python/cp36-cp36m'),
PythonConfiguration(version='3.7', identifier='cp37-manylinux_aarch64', path='/opt/python/cp37-cp37m'),
PythonConfiguration(version='3.8', identifier='cp38-manylinux_aarch64', path='/opt/python/cp38-cp38'),
PythonConfiguration(version='3.5', identifier='cp35-manylinux_ppc64le', path='/opt/python/cp35-cp35m'),
PythonConfiguration(version='3.6', identifier='cp36-manylinux_ppc64le', path='/opt/python/cp36-cp36m'),
PythonConfiguration(version='3.7', identifier='cp37-manylinux_ppc64le', path='/opt/python/cp37-cp37m'),
PythonConfiguration(version='3.8', identifier='cp38-manylinux_ppc64le', path='/opt/python/cp38-cp38'),
PythonConfiguration(version='3.5', identifier='cp35-manylinux_s390x', path='/opt/python/cp35-cp35m'),
PythonConfiguration(version='3.6', identifier='cp36-manylinux_s390x', path='/opt/python/cp36-cp36m'),
PythonConfiguration(version='3.7', identifier='cp37-manylinux_s390x', path='/opt/python/cp37-cp37m'),
PythonConfiguration(version='3.8', identifier='cp38-manylinux_s390x', path='/opt/python/cp38-cp38'),
PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path_str='/opt/python/cp27-cp27m'),
PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path_str='/opt/python/cp27-cp27mu'),
PythonConfiguration(version='3.5', identifier='cp35-manylinux_x86_64', path_str='/opt/python/cp35-cp35m'),
PythonConfiguration(version='3.6', identifier='cp36-manylinux_x86_64', path_str='/opt/python/cp36-cp36m'),
PythonConfiguration(version='3.7', identifier='cp37-manylinux_x86_64', path_str='/opt/python/cp37-cp37m'),
PythonConfiguration(version='3.8', identifier='cp38-manylinux_x86_64', path_str='/opt/python/cp38-cp38'),
PythonConfiguration(version='2.7', identifier='cp27-manylinux_i686', path_str='/opt/python/cp27-cp27m'),
PythonConfiguration(version='2.7', identifier='cp27-manylinux_i686', path_str='/opt/python/cp27-cp27mu'),
PythonConfiguration(version='3.5', identifier='cp35-manylinux_i686', path_str='/opt/python/cp35-cp35m'),
PythonConfiguration(version='3.6', identifier='cp36-manylinux_i686', path_str='/opt/python/cp36-cp36m'),
PythonConfiguration(version='3.7', identifier='cp37-manylinux_i686', path_str='/opt/python/cp37-cp37m'),
PythonConfiguration(version='3.8', identifier='cp38-manylinux_i686', path_str='/opt/python/cp38-cp38'),
PythonConfiguration(version='2.7', identifier='pp27-manylinux_x86_64', path_str='/opt/python/pp27-pypy_73'),
PythonConfiguration(version='3.6', identifier='pp36-manylinux_x86_64', path_str='/opt/python/pp36-pypy36_pp73'),
PythonConfiguration(version='3.5', identifier='cp35-manylinux_aarch64', path_str='/opt/python/cp35-cp35m'),
PythonConfiguration(version='3.6', identifier='cp36-manylinux_aarch64', path_str='/opt/python/cp36-cp36m'),
PythonConfiguration(version='3.7', identifier='cp37-manylinux_aarch64', path_str='/opt/python/cp37-cp37m'),
PythonConfiguration(version='3.8', identifier='cp38-manylinux_aarch64', path_str='/opt/python/cp38-cp38'),
PythonConfiguration(version='3.5', identifier='cp35-manylinux_ppc64le', path_str='/opt/python/cp35-cp35m'),
PythonConfiguration(version='3.6', identifier='cp36-manylinux_ppc64le', path_str='/opt/python/cp36-cp36m'),
PythonConfiguration(version='3.7', identifier='cp37-manylinux_ppc64le', path_str='/opt/python/cp37-cp37m'),
PythonConfiguration(version='3.8', identifier='cp38-manylinux_ppc64le', path_str='/opt/python/cp38-cp38'),
PythonConfiguration(version='3.5', identifier='cp35-manylinux_s390x', path_str='/opt/python/cp35-cp35m'),
PythonConfiguration(version='3.6', identifier='cp36-manylinux_s390x', path_str='/opt/python/cp36-cp36m'),
PythonConfiguration(version='3.7', identifier='cp37-manylinux_s390x', path_str='/opt/python/cp37-cp37m'),
PythonConfiguration(version='3.8', identifier='cp38-manylinux_s390x', path_str='/opt/python/cp38-cp38'),
]
# skip builds as required
return [c for c in python_configurations if matches_platform(c.identifier) and build_selector(c.identifier)]
@@ -111,201 +104,133 @@ def build(options: BuildOptions) -> None:
raise Exception('package_dir must be inside the working directory')
container_package_dir = PurePath('/project') / abs_package_dir.relative_to(cwd)
container_output_dir = PurePath('/output')
for implementation, platform_tag, docker_image in platforms:
platform_configs = [c for c in python_configurations if c.identifier.startswith(implementation) and c.identifier.endswith(platform_tag)]
if not platform_configs:
continue
shell_cmd = ['linux32', '/bin/bash'] if platform_tag.endswith("i686") else ['/bin/bash']
container_name = f'cibuildwheel-{uuid.uuid4()}'
call(['docker', 'create',
'--env', 'CIBUILDWHEEL',
'--name', container_name,
'-i',
'-v', '/:/host', # ignored on CircleCI
docker_image,
'/bin/bash'])
try:
call(['docker', 'cp', '.', container_name + ':/project'])
with DockerContainer(docker_image, simulate_32_bit=platform_tag.endswith('i686')) as docker:
docker.copy_into(Path.cwd(), Path('/project'))
call(['docker', 'start', container_name])
if options.before_all:
env = docker.get_environment()
env['PATH'] = f'/opt/python/cp38-cp38:{env["PATH"]}'
env = options.environment.as_dictionary(env, executor=docker.environment_executor)
if options.before_all:
call(
['docker', 'exec', '-i', container_name] + shell_cmd,
universal_newlines=True,
input='''
PS4=' + '
before_all_prepared = prepare_command(options.before_all, project='/project', package=container_package_dir)
docker.call(['sh', '-c', before_all_prepared], env=env)
set -o errexit
set -o xtrace
for config in platform_configs:
dependency_constraint_flags: List[Union[str, PathLike]] = []
# add a modern Python interpreter to PATH so it can be used by BEFORE_ALL
# commands
export PATH=/opt/python/cp38-cp38:$PATH
if options.dependency_constraints:
constraints_file = options.dependency_constraints.get_for_python_version(config.version)
container_constraints_file = PurePath('/constraints.txt')
{environment_exports}
docker.copy_into(constraints_file, container_constraints_file)
dependency_constraint_flags = ['-c', container_constraints_file]
sh -c {before_all}
env = docker.get_environment()
'''.format(
environment_exports='\n'.join(options.environment.as_shell_commands()),
before_all=shlex.quote(prepare_command(options.before_all, project='/project', package=container_package_dir))
# put this config's python top of the list
python_bin = config.path / 'bin'
env['PATH'] = f'{str(python_bin)}:{env["PATH"]}'
env = options.environment.as_dictionary(env, executor=docker.environment_executor)
# check config python and pip are still on PATH
which_python = docker.call(['which', 'python'], env=env, capture_output=True).strip()
if PurePath(which_python) != python_bin / 'python':
print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr)
exit(1)
which_pip = docker.call(['which', 'pip'], env=env, capture_output=True).strip()
if PurePath(which_pip) != python_bin / 'pip':
print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr)
exit(1)
if options.before_build:
before_build_prepared = prepare_command(options.before_build, project='/project', package=container_package_dir)
docker.call(['sh', '-c', before_build_prepared], env=env)
temp_dir = PurePath('/tmp/cibuildwheel')
built_wheel_dir = temp_dir / 'built_wheel'
docker.call(['rm', '-rf', built_wheel_dir])
docker.call(['mkdir', '-p', built_wheel_dir])
docker.call(
[
'pip', 'wheel',
container_package_dir,
'-w', built_wheel_dir,
'--no-deps',
*get_build_verbosity_extra_flags(options.build_verbosity)
],
env=env,
)
)
for config in platform_configs:
if options.dependency_constraints:
constraints_file = options.dependency_constraints.get_for_python_version(config.version)
built_wheel = docker.glob(built_wheel_dir / '*.whl')[0]
# `docker cp` causes 'no space left on device' error when
# a container is running and the host filesystem is
# mounted. https://github.com/moby/moby/issues/38995
# Use `docker exec` instead.
with open(constraints_file, 'rb') as f:
call(
['docker', 'exec', '-i', container_name, 'sh', '-c', 'cat > /constraints.txt'],
input=f.read(),
)
repaired_wheel_dir = temp_dir / 'repaired_wheel'
docker.call(['rm', '-rf', repaired_wheel_dir])
docker.call(['mkdir', '-p', repaired_wheel_dir])
call(
['docker', 'exec', '-i', container_name] + shell_cmd,
universal_newlines=True,
input='''
# give xtrace output an extra level of indent inside docker
PS4=' + '
if built_wheel.name.endswith('none-any.whl') or not options.repair_command:
docker.call(['mv', built_wheel, repaired_wheel_dir])
else:
repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir)
docker.call(['sh', '-c', repair_command_prepared], env=env)
set -o errexit
set -o xtrace
mkdir -p /output
cd /project
repaired_wheels = docker.glob(repaired_wheel_dir / '*.whl')
PYBIN="{config_python_bin}"
if options.test_command:
# set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
docker.call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env)
venv_dir = PurePath(docker.call(['mktemp', '-d'], capture_output=True).strip()) / 'venv'
export PATH="$PYBIN:$PATH"
{environment_exports}
docker.call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env)
# check the active python and pip are in PYBIN
if [ "$(which pip)" != "$PYBIN/pip" ]; then
echo "cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it."
exit 1
fi
if [ "$(which python)" != "$PYBIN/python" ]; then
echo "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it."
exit 1
fi
virtualenv_env = env.copy()
virtualenv_env['PATH'] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}"
if [ ! -z {before_build} ]; then
sh -c {before_build}
fi
if options.before_test:
before_test_prepared = prepare_command(options.before_test, project='/project', package=container_package_dir)
docker.call(['sh', '-c', before_test_prepared], env=virtualenv_env)
# Build the wheel
rm -rf /tmp/built_wheel
mkdir /tmp/built_wheel
pip wheel {package_dir} -w /tmp/built_wheel --no-deps {build_verbosity_flag}
built_wheel=(/tmp/built_wheel/*.whl)
# Install the wheel we just built
# Note: If auditwheel produced two wheels, it's because the earlier produced wheel
# conforms to multiple manylinux standards. These multiple versions of the wheel are
# functionally the same, differing only in name, wheel metadata, and possibly include
# different external shared libraries. so it doesn't matter which one we run the tests on.
# Let's just pick the first one.
wheel_to_test = repaired_wheels[0]
docker.call(['pip', 'install', str(wheel_to_test) + options.test_extras], env=virtualenv_env)
# repair the wheel
rm -rf /tmp/repaired_wheels
mkdir /tmp/repaired_wheels
# NOTE: 'built_wheel' here is a bash array of glob matches; "$built_wheel" returns
# the first element
if [[ "$built_wheel" == *none-any.whl ]] || [ -z {repair_command} ]; then
# pure Python wheel or empty repair command
mv "$built_wheel" /tmp/repaired_wheels
else
sh -c {repair_command} repair_command "$built_wheel"
fi
repaired_wheels=(/tmp/repaired_wheels/*.whl)
# Install any requirements to run the tests
if options.test_requires:
docker.call(['pip', 'install', *options.test_requires], env=virtualenv_env)
if [ ! -z {test_command} ]; then
# Set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
pip install {dependency_install_flags} virtualenv
venv_dir=`mktemp -d`/venv
python -m virtualenv --no-download "$venv_dir"
# Run the tests from a different directory
test_command_prepared = prepare_command(options.test_command, project='/project', package=container_package_dir)
docker.call(['sh', '-c', test_command_prepared], cwd='/root', env=virtualenv_env)
# run the tests in a subshell to keep that `activate`
# script from polluting the env
(
source "$venv_dir/bin/activate"
# clean up test environment
docker.call(['rm', '-rf', venv_dir])
echo "Running tests using `which python`"
# move repaired wheels to output
docker.call(['mkdir', '-p', container_output_dir])
docker.call(['mv', *repaired_wheels, container_output_dir])
if [ ! -z {before_test} ]; then
sh -c {before_test}
fi
# Install the wheel we just built
# Note: If auditwheel produced two wheels, it's because the earlier produced wheel
# conforms to multiple manylinux standards. These multiple versions of the wheel are
# functionally the same, differing only in name, wheel metadata, and possibly include
# different external shared libraries. so it doesn't matter which one we run the tests on.
# Let's just pick the first one.
pip install "${{repaired_wheels[0]}}"{test_extras}
# Install any requirements to run the tests
if [ ! -z "{test_requires}" ]; then
pip install {test_requires}
fi
# Run the tests from a different directory
pushd $HOME
sh -c {test_command}
popd
)
# exit if tests failed (needed for older bash versions)
if [ $? -ne 0 ]; then
exit 1;
fi
# clean up
rm -rf "$venv_dir"
fi
# we're all done here; move it to output
mv "${{repaired_wheels[@]}}" /output
for repaired_wheel in "${{repaired_wheels[@]}}"; do
chown {uid}:{gid} "/output/$(basename "$repaired_wheel")"
done
'''.format(
config_python_bin=config.path + '/bin',
package_dir=container_package_dir,
test_requires=' '.join(options.test_requires),
test_extras=options.test_extras,
test_command=shlex.quote(
prepare_command(options.test_command, project='/project', package=container_package_dir) if options.test_command else ''
),
before_build=shlex.quote(
prepare_command(options.before_build, project='/project', package=container_package_dir) if options.before_build else ''
),
build_verbosity_flag=' '.join(get_build_verbosity_extra_flags(options.build_verbosity)),
repair_command=shlex.quote(
prepare_command(options.repair_command, wheel='"$1"', dest_dir='/tmp/repaired_wheels') if options.repair_command else ''
),
environment_exports='\n'.join(options.environment.as_shell_commands()),
uid=os.getuid(),
gid=os.getgid(),
before_test=shlex.quote(
prepare_command(options.before_test, project='/project', package=container_package_dir) if options.before_test else ''
),
dependency_install_flags='-c /constraints.txt' if options.dependency_constraints else '',
)
)
# copy the output back into the host
call(['docker', 'cp',
container_name + ':/output/.',
str(options.output_dir.resolve())])
# copy the output back into the host
docker.copy_out(container_output_dir, options.output_dir)
except subprocess.CalledProcessError as error:
print(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}')
troubleshoot(options.package_dir, error)
exit(1)
finally:
# Still gets executed, even when 'exit(1)' gets called
call(['docker', 'rm', '--force', '-v', container_name])
def troubleshoot(package_dir: Path, error: Exception) -> None:
@@ -328,3 +253,111 @@ def troubleshoot(package_dir: Path, error: Exception) -> None:
print(' Files detected:')
print('\n'.join([f' {f}' for f in so_files]))
print('')
class DockerContainer:
UTILITY_PYTHON = '/opt/python/cp38-cp38/bin/python'
def __init__(self, docker_image, simulate_32_bit=False):
self.docker_image = docker_image
self.simulate_32_bit = simulate_32_bit
def __enter__(self) -> 'DockerContainer':
self.container_name = f'cibuildwheel-{uuid.uuid4()}'
subprocess.run(
[
'docker', 'create',
'--env', 'CIBUILDWHEEL',
'--name', self.container_name,
'-i',
'-v', '/:/host', # ignored on CircleCI
self.docker_image,
'/bin/bash'
],
check=True,
)
subprocess.run(['docker', 'start', self.container_name], check=True)
return self
def __exit__(self, exc_type, exc_val, exc_tb):
subprocess.run(['docker', 'rm', '--force', '-v', self.container_name])
self.container_name = None
def copy_into(self, from_path: Path, to_path: PurePath) -> None:
# `docker cp` causes 'no space left on device' error when
# a container is running and the host filesystem is
# mounted. https://github.com/moby/moby/issues/38995
# Use `docker exec` instead.
if from_path.is_dir():
self.call(['mkdir', '-p', to_path])
subprocess.run(
f'tar cf - . | docker exec -i {self.container_name} tar -xC {to_path} -f -',
shell=True,
check=True,
cwd=from_path)
else:
subprocess.run(
f'cat {from_path} | docker exec -i {self.container_name} sh -c "cat > {to_path}"',
shell=True,
check=True)
def copy_out(self, from_path: PurePath, to_path: Path) -> None:
# note: we assume from_path is a dir
to_path.mkdir(parents=True, exist_ok=True)
subprocess.run(
f'docker exec -i -w {from_path} {self.container_name} tar cf - . | tar -xf -',
shell=True,
check=True,
cwd=to_path
)
def glob(self, pattern: PurePath) -> List[PurePath]:
path_strs = json.loads(self.call([
self.UTILITY_PYTHON,
'-c',
f'import sys, json, glob; json.dump(glob.glob({str(pattern)!r}), sys.stdout)'
], capture_output=True))
return [PurePath(p) for p in path_strs]
def call(self, args: Sequence[Union[str, PathLike]], env: Dict[str, str] = {},
capture_output=False, cwd: Optional[Union[str, PathLike]] = None) -> str:
# ensure paths are converted to strs
args = [str(a) for a in args]
if self.simulate_32_bit:
args = ['linux32', *args]
options = ['-i']
for key, value in env.items():
options += ['--env', f'{key}={value}']
if cwd:
options += ['-w', str(cwd)]
print('+ ' + ' '.join(shlex.quote(str(a)) for a in args))
process = subprocess.run(
['docker', 'exec', *options, self.container_name, *args],
check=True,
universal_newlines=True,
stdout=subprocess.PIPE if capture_output else None
)
if capture_output:
return process.stdout
else:
return ''
def get_environment(self) -> Dict[str, str]:
return json.loads(self.call([
self.UTILITY_PYTHON,
'-c',
'import sys, json, os; json.dump(os.environ.copy(), sys.stdout)'
], capture_output=True))
def environment_executor(self, command: str, environment: Dict[str, str]) -> str:
# used as an EnvironmentExecutor to evaluate commands and capture output
return self.call(shlex.split(command), env=environment)
requirements-dev.txt
+2
View File
@@ -6,3 +6,5 @@ pymdown-extensions
pip-tools
requests
click
mypy
typing-extensions