Use a long-running remote bash shell, rather than docker exec

cibuildwheel/linux.py

@@ -1,3 +1,4 @@
+import io
 import json
 import platform
 import shlex
@@ -7,7 +8,8 @@ import textwrap
 import uuid
 from os import PathLike
 from pathlib import Path, PurePath
-from typing import Dict, List, NamedTuple, Optional, Union, Sequence
+from typing import (IO, Dict, List, NamedTuple, Optional, Sequence, TextIO,
+                    Union)
 
 from .util import (BuildOptions, BuildSelector,
                    get_build_verbosity_extra_flags, prepare_command)
@@ -258,12 +260,17 @@ def troubleshoot(package_dir: Path, error: Exception) -> None:
 class DockerContainer:
     UTILITY_PYTHON = '/opt/python/cp38-cp38/bin/python'
 
-    def __init__(self, docker_image, simulate_32_bit=False):
+    process: subprocess.Popen
+    bash_stdin: IO[str]
+    bash_stdout: IO[str]
+
+    def __init__(self, docker_image: str, simulate_32_bit=False):
         self.docker_image = docker_image
         self.simulate_32_bit = simulate_32_bit
 
     def __enter__(self) -> 'DockerContainer':
         self.container_name = f'cibuildwheel-{uuid.uuid4()}'
+        shell_args = ['linux32', '/bin/bash'] if self.simulate_32_bit else ['/bin/bash']
         subprocess.run(
             [
                 'docker', 'create',
@@ -272,14 +279,34 @@ class DockerContainer:
                 '-i',
                 '-v', '/:/host',  # ignored on CircleCI
                 self.docker_image,
-                '/bin/bash'
+                *shell_args
             ],
             check=True,
         )
-        subprocess.run(['docker', 'start', self.container_name], check=True)
+        process = subprocess.Popen(
+            [
+                'docker', 'start',
+                '--attach', '--interactive',
+                self.container_name,
+            ],
+            encoding='utf8',
+            stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
+            # make the input buffer large enough to carry a lot of environment
+            # variables. We choose 256kB.
+            bufsize=262144,
+        )
+        self.process = process
+        assert process.stdin and process.stdout
+        self.bash_stdin = process.stdin
+        self.bash_stdout = process.stdout
         return self
 
     def __exit__(self, exc_type, exc_val, exc_tb):
+        self.bash_stdin.close()
+        self.process.terminate()
+        self.process.wait()
+
         subprocess.run(['docker', 'rm', '--force', '-v', self.container_name])
         self.container_name = None
 
@@ -323,33 +350,58 @@ class DockerContainer:
 
     def call(self, args: Sequence[Union[str, PathLike]], env: Dict[str, str] = {},
              capture_output=False, cwd: Optional[Union[str, PathLike]] = None) -> str:
-        # ensure paths are converted to strs
+        env_exports = '\n'.join(f'export {k}={v}' for k, v in env.items())
-        args = [str(a) for a in args]
+        chdir = f'cd {cwd}' if cwd else ''
+        command = ' '.join(shlex.quote(str(a)) for a in args)
+        end_of_message = str(uuid.uuid4())
 
-        if self.simulate_32_bit:
+        # log the command we're executing
-            args = ['linux32', *args]
+        print(f'    + {command}')
 
-        options = ['-i']
+        # Write a command to the remote shell. First we write the
-
+        # environment variables, exported inside the subshell. We change the
-        for key, value in env.items():
+        # cwd, if that's required. Then, the command is written. Finally, the
-            options += ['--env', f'{key}={value}']
+        # remote shell is told to write a footer - this will show up in the
-
+        # output so we know when to stop reading, and will include the
-        if cwd:
+        # returncode of `command`.
-            options += ['-w', str(cwd)]
+        self.bash_stdin.write(f'''(
-
+            {env_exports}
-        print('+ ' + ' '.join(shlex.quote(str(a)) for a in args))
+            {chdir}
-
+            {command}
-        process = subprocess.run(
+            printf "%04d%s\n" $? {end_of_message}
-            ['docker', 'exec', *options, self.container_name, *args],
-            check=True,
-            universal_newlines=True,
-            stdout=subprocess.PIPE if capture_output else None
         )
+        ''')
+        self.bash_stdin.flush()
 
         if capture_output:
-            return process.stdout
+            output_io: TextIO = io.StringIO()
         else:
-            return ''
+            output_io = sys.stdout
+
+        while True:
+            line = self.bash_stdout.readline()
+
+            if line.endswith(end_of_message+'\n'):
+                footer_offset = (
+                    len(line)
+                    - 1  # newline character
+                    - len(end_of_message)  # delimiter
+                    - 4  # 4 returncode decimals
+                )
+                returncode_str = line[footer_offset:footer_offset+4]
+                returncode = int(returncode_str)
+                # add the last line to output, without the footer
+                output_io.write(line[0:footer_offset])
+                break
+            else:
+                output_io.write(line)
+
+        output = output_io.getvalue() if isinstance(output_io, io.StringIO) else None
+
+        if returncode != 0:
+            raise subprocess.CalledProcessError(returncode, args, output)
+
+        return output if output else ''
 
     def get_environment(self) -> Dict[str, str]:
         return json.loads(self.call([

setup.cfg

@@ -1,5 +1,5 @@
 [flake8]
-ignore = E501,W503,E741
+ignore = E501,W503,E741,E226
 application-import-names = cibuildwheel
 exclude =
   cibuildwheel/resources/,