Bump version: v1.5.5

Fix cwd in Linux docker images

.github/workflows/test.yml

@@ -17,6 +17,7 @@ jobs:
       matrix:
         os: [ubuntu-18.04, windows-latest, macos-latest]
         python_version: ['3.7']
+    timeout-minutes: 180
     steps:
     - uses: actions/checkout@v2
     - uses: actions/setup-python@v2

README.md

@@ -79,7 +79,7 @@ env:
     # Note: TWINE_PASSWORD is set to a PyPI API token in Travis settings
 
 install:
-  - python3 -m pip install cibuildwheel==1.5.3
+  - python3 -m pip install cibuildwheel==1.5.5
 
 script:
   # build the wheels, put them into './wheelhouse'
@@ -152,6 +152,22 @@ This is similar to static linking, so it might have some licence implications. C
 Changelog
 =========
 
+### 1.5.5
+
+_22 July 2020_
+
+- 🐛 Fix a bug that would cause command substitutions in CIBW_ENVIRONMENT to
+  produce no output on Linux (#411)
+- 🐛 Fix regression (introduced in 1.5.3) which caused BEFORE_BUILD and
+  BEFORE_ALL to be executed in the wrong directory (#410)
+
+### 1.5.4
+
+_19 June 2020_
+
+- 🐛 Fix a bug that would cause command substitutions in CIBW_ENVIRONMENT
+  variables to not interpret quotes in commands correctly (#406, #408)
+
 ### 1.5.3
 
 _19 July 2020_

azure-pipelines.yml

@@ -21,6 +21,7 @@ jobs:
 
 - job: windows_36
   pool: {vmImage: 'vs2017-win2016'}
+  timeoutInMinutes: 180
   steps:
     - task: UsePythonVersion@0
       inputs:
@@ -33,6 +34,7 @@ jobs:
 
 - job: windows_38
   pool: {vmImage: 'vs2017-win2016'}
+  timeoutInMinutes: 180
   steps:
     - task: UsePythonVersion@0
       inputs:

cibuildwheel/__init__.py

@@ -1 +1 @@
-__version__ = '1.5.3'
+__version__ = '1.5.5'

cibuildwheel/bashlex_eval.py

@@ -1,15 +1,14 @@
-import shlex
 import subprocess
 from typing import Callable, Dict, List, NamedTuple, Optional, Sequence
 
 import bashlex  # type: ignore
 
-# a function that takes a shell command and the environment, and returns the result
-EnvironmentExecutor = Callable[[str, Dict[str, str]], str]
+# a function that takes a command and the environment, and returns the result
+EnvironmentExecutor = Callable[[List[str], Dict[str, str]], str]
 
 
-def local_environment_executor(command: str, env: Dict[str, str]) -> str:
-    return subprocess.check_output(shlex.split(command), env=env, universal_newlines=True)
+def local_environment_executor(command: List[str], env: Dict[str, str]) -> str:
+    return subprocess.check_output(command, env=env, universal_newlines=True)
 
 
 class NodeExecutionContext(NamedTuple):
@@ -97,8 +96,7 @@ def evaluate_nodes_as_compound_command(nodes: Sequence[bashlex.ast.node], contex
 
 
 def evaluate_nodes_as_simple_command(nodes: List[bashlex.ast.node], context: NodeExecutionContext):
-    words = [evaluate_node(part, context=context) for part in nodes]
-    command = ' '.join(words)
+    command = [evaluate_node(part, context=context) for part in nodes]
     return context.executor(command, context.environment)
 
 

cibuildwheel/docker_container.py

@@ -27,12 +27,14 @@ class DockerContainer:
     bash_stdin: IO[bytes]
     bash_stdout: IO[bytes]
 
-    def __init__(self, docker_image: str, simulate_32_bit=False):
+    def __init__(self, docker_image: str, simulate_32_bit: bool = False, cwd: Optional[Union[str, PathLike]] = None):
         self.docker_image = docker_image
         self.simulate_32_bit = simulate_32_bit
+        self.cwd = cwd
 
     def __enter__(self) -> 'DockerContainer':
         self.name = f'cibuildwheel-{uuid.uuid4()}'
+        cwd_args = ['-w', str(self.cwd)] if self.cwd else []
         shell_args = ['linux32', '/bin/bash'] if self.simulate_32_bit else ['/bin/bash']
         subprocess.run(
             [
@@ -41,6 +43,7 @@ class DockerContainer:
                 '--name', self.name,
                 '-i',
                 '-v', '/:/host',  # ignored on CircleCI
+                *cwd_args,
                 self.docker_image,
                 *shell_args
             ],
@@ -180,9 +183,9 @@ class DockerContainer:
             'import sys, json, os; json.dump(os.environ.copy(), sys.stdout)'
         ], capture_output=True))
 
-    def environment_executor(self, command: str, environment: Dict[str, str]) -> str:
+    def environment_executor(self, command: List[str], environment: Dict[str, str]) -> str:
         # used as an EnvironmentExecutor to evaluate commands and capture output
-        return self.call(shlex.split(command), env=environment)
+        return self.call(command, env=environment, capture_output=True)
 
 
 def shell_quote(path: PurePath) -> str:

cibuildwheel/linux.py

@@ -110,8 +110,8 @@ def build(options: BuildOptions) -> None:
             continue
 
         try:
-            with DockerContainer(docker_image, simulate_32_bit=platform_tag.endswith('i686')) as docker:
-                docker.copy_into(Path.cwd(), Path('/project'))
+            with DockerContainer(docker_image, simulate_32_bit=platform_tag.endswith('i686'), cwd='/project') as docker:
+                docker.copy_into(Path.cwd(), PurePath('/project'))
 
                 if options.before_all:
                     env = docker.get_environment()

docs/deliver-to-pypi.md

@@ -33,4 +33,4 @@ Obviously, manual steps are for chumps, so we can automate this a little by usin
 
 If you don't need much control over the release of a package, you can set up cibuildwheel to deliver the wheels straight to PyPI. This doesn't require anycloud storage to work - you just need to bump the version and tag it.
 
-[`examples/travis-ci-deploy.yml`](https://github.com/joerick/cibuildwheel/blob/master/examples/travis-ci-deploy.yml) and [`examples/github-deploy.yml`](https://github.com/joerick/cibuildwheel/blob/master/examples/github-deploy.yml) are example configurations that automatocially upload wheels to PyPI. Also check out [this example repo](https://github.com/joerick/cibuildwheel-autopypi-example) for more detailed instructions on how to set this up.
+[`examples/travis-ci-deploy.yml`](https://github.com/joerick/cibuildwheel/blob/master/examples/travis-ci-deploy.yml) and [`examples/github-deploy.yml`](https://github.com/joerick/cibuildwheel/blob/master/examples/github-deploy.yml) are example configurations that automatically upload wheels to PyPI. Also check out [this example repo](https://github.com/joerick/cibuildwheel-autopypi-example) for more detailed instructions on how to set this up.

examples/appveyor-minimal.yml

@@ -12,7 +12,7 @@ stack: python 3.7
 init:
 - cmd: set PATH=C:\Python37;C:\Python37\Scripts;%PATH%
 
-install: python -m pip install cibuildwheel==1.5.3
+install: python -m pip install cibuildwheel==1.5.5
 
 build_script: python -m cibuildwheel --output-dir wheelhouse
 

examples/azure-pipelines-minimal.yml

@@ -5,7 +5,7 @@ jobs:
     - task: UsePythonVersion@0
     - bash: |
         python3 -m pip install --upgrade pip
-        pip3 install cibuildwheel==1.5.3
+        pip3 install cibuildwheel==1.5.5
         cibuildwheel --output-dir wheelhouse .
     - task: PublishBuildArtifacts@1
       inputs: {pathtoPublish: 'wheelhouse'}
@@ -16,7 +16,7 @@ jobs:
     - task: UsePythonVersion@0
     - bash: |
         python3 -m pip install --upgrade pip
-        pip3 install cibuildwheel==1.5.3
+        pip3 install cibuildwheel==1.5.5
         cibuildwheel --output-dir wheelhouse .
     - task: PublishBuildArtifacts@1
       inputs: {pathtoPublish: 'wheelhouse'}
@@ -29,7 +29,7 @@ jobs:
       displayName: Install Visual C++ for Python 2.7
     - bash: |
         python -m pip install --upgrade pip
-        pip install cibuildwheel==1.5.3
+        pip install cibuildwheel==1.5.5
         cibuildwheel --output-dir wheelhouse .
     - task: PublishBuildArtifacts@1
       inputs: {pathtoPublish: 'wheelhouse'}

examples/circleci-minimal.yml

@@ -11,7 +11,7 @@ jobs:
       - run:
           name: Build the Linux wheels.
           command: |
-            pip3 install --user cibuildwheel==1.5.3
+            pip3 install --user cibuildwheel==1.5.5
             cibuildwheel --output-dir wheelhouse
       - store_artifacts:
           path: wheelhouse/
@@ -25,7 +25,7 @@ jobs:
       - run:
           name: Build the OS X wheels.
           command: |
-            pip3 install --user cibuildwheel==1.5.3
+            pip3 install --user cibuildwheel==1.5.5
             cibuildwheel --output-dir wheelhouse
       - store_artifacts:
           path: wheelhouse/

examples/github-deploy.yml

@@ -28,7 +28,7 @@ jobs:
 
       - name: Install cibuildwheel
         run: |
-          python -m pip install cibuildwheel==1.5.3
+          python -m pip install cibuildwheel==1.5.5
 
       - name: Install Visual C++ for Python 2.7
         if: runner.os == 'Windows'

examples/github-minimal.yml

@@ -20,7 +20,7 @@ jobs:
 
       - name: Install cibuildwheel
         run: |
-          python -m pip install cibuildwheel==1.5.3
+          python -m pip install cibuildwheel==1.5.5
 
       - name: Install Visual C++ for Python 2.7
         if: runner.os == 'Windows'

examples/travis-ci-deploy.yml

@@ -25,7 +25,7 @@ env:
     # Note: TWINE_PASSWORD is set to a PyPI API token in Travis settings
 
 install:
-  - python3 -m pip install cibuildwheel==1.5.3
+  - python3 -m pip install cibuildwheel==1.5.5
 
 script:
   # build the wheels, put them into './wheelhouse'

examples/travis-ci-minimal.yml

@@ -26,7 +26,7 @@ jobs:
         - ln -s /c/Python38/python.exe /c/Python38/python3.exe
 
 install:
-  - python3 -m pip install cibuildwheel==1.5.3
+  - python3 -m pip install cibuildwheel==1.5.5
 
 script:
   # build the wheels, put them into './wheelhouse'

examples/travis-ci-test-and-deploy.yml

@@ -55,7 +55,7 @@ jobs:
     - stage: deploy
       name: Build and deploy Linux wheels
       services: docker
-      install: python3 -m pip install cibuildwheel==1.5.3
+      install: python3 -m pip install cibuildwheel==1.5.5
       script: python3 -m cibuildwheel --output-dir wheelhouse
       after_success: |
         python3 -m pip install twine
@@ -65,7 +65,7 @@ jobs:
       name: Build and deploy macOS wheels
       os: osx
       language: shell
-      install: python3 -m pip install cibuildwheel==1.5.3
+      install: python3 -m pip install cibuildwheel==1.5.5
       script: python3 -m cibuildwheel --output-dir wheelhouse
       after_success: |
         python3 -m pip install twine
@@ -75,7 +75,7 @@ jobs:
       name: Build and deploy Windows wheels
       os: windows
       language: shell
-      install: python3 -m pip install cibuildwheel==1.5.3
+      install: python3 -m pip install cibuildwheel==1.5.5
       script: python3 -m cibuildwheel --output-dir wheelhouse
       after_success: |
         python3 -m pip install twine

setup.py

@@ -11,7 +11,7 @@ long_description = (this_directory / 'README.md').read_text(encoding='utf-8')
 
 setup(
     name='cibuildwheel',
-    version='1.5.3',
+    version='1.5.5',
     install_requires=['bashlex!=0.13', 'toml'],
     description="Build Python wheels on CI with minimal configuration.",
     long_description=long_description,

test/test_before_all.py

@@ -1,4 +1,5 @@
-import os
+import pytest
+import subprocess
 import textwrap
 
 from . import utils
@@ -21,7 +22,7 @@ def test(tmp_path):
     project_dir = tmp_path / 'project'
     project_with_before_build_asserts.generate(project_dir)
 
-    with open(os.path.join(project_dir, "text_info.txt"), mode='w') as ff:
+    with (project_dir / 'text_info.txt').open(mode='w') as ff:
         print("dummy text", file=ff)
 
     # build the wheels
@@ -33,6 +34,30 @@ def test(tmp_path):
     })
 
     # also check that we got the right wheels
-    os.remove(os.path.join(project_dir, "text_info.txt"))
+    (project_dir / 'text_info.txt').unlink()
+    expected_wheels = utils.expected_wheels('spam', '0.1.0')
+    assert set(actual_wheels) == set(expected_wheels)
+
+
+def test_failing_command(tmp_path):
+    project_dir = tmp_path / 'project'
+    test_projects.new_c_project().generate(project_dir)
+
+    with pytest.raises(subprocess.CalledProcessError):
+        utils.cibuildwheel_run(project_dir, add_env={
+            'CIBW_BEFORE_ALL': 'false',
+            'CIBW_BEFORE_ALL_WINDOWS': 'exit /b 1',
+        })
+
+
+def test_cwd(tmp_path):
+    project_dir = tmp_path / 'project'
+    test_projects.new_c_project().generate(project_dir)
+
+    actual_wheels = utils.cibuildwheel_run(project_dir, add_env={
+        'CIBW_BEFORE_ALL': f'''python -c "import os; assert os.getcwd() == {str(project_dir)!r}"''',
+        'CIBW_BEFORE_ALL_LINUX': '''python -c "import os; assert os.getcwd() == '/project'"''',
+    })
+
     expected_wheels = utils.expected_wheels('spam', '0.1.0')
     assert set(actual_wheels) == set(expected_wheels)

test/test_before_build.py

@@ -1,3 +1,5 @@
+import pytest
+import subprocess
 import textwrap
 
 from . import utils
@@ -32,14 +34,41 @@ def test(tmp_path):
     project_dir = tmp_path / 'project'
     project_with_before_build_asserts.generate(project_dir)
 
+    before_build = ('''python -c "import sys; open('{output_dir}pythonversion.txt', 'w').write(sys.version)" && '''
+                    '''python -c "import sys; open('{output_dir}pythonexecutable.txt', 'w').write(sys.executable)"''')
+
     # build the wheels
     actual_wheels = utils.cibuildwheel_run(project_dir, add_env={
         # write python version information to a temporary file, this is
         # checked in setup.py
-        'CIBW_BEFORE_BUILD': '''python -c "import sys; open('/tmp/pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('/tmp/pythonexecutable.txt', 'w').write(sys.executable)"''',
-        'CIBW_BEFORE_BUILD_WINDOWS': '''python -c "import sys; open('c:\\pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('c:\\pythonexecutable.txt', 'w').write(sys.executable)"''',
+        'CIBW_BEFORE_BUILD': before_build.format(output_dir='/tmp/'),
+        'CIBW_BEFORE_BUILD_WINDOWS': before_build.format(output_dir=r'c:\\'),
     })
 
     # also check that we got the right wheels
     expected_wheels = utils.expected_wheels('spam', '0.1.0')
     assert set(actual_wheels) == set(expected_wheels)
+
+
+def test_failing_command(tmp_path):
+    project_dir = tmp_path / 'project'
+    test_projects.new_c_project().generate(project_dir)
+
+    with pytest.raises(subprocess.CalledProcessError):
+        utils.cibuildwheel_run(project_dir, add_env={
+            'CIBW_BEFORE_BUILD': 'false',
+            'CIBW_BEFORE_BUILD_WINDOWS': 'exit /b 1',
+        })
+
+
+def test_cwd(tmp_path):
+    project_dir = tmp_path / 'project'
+    test_projects.new_c_project().generate(project_dir)
+
+    actual_wheels = utils.cibuildwheel_run(project_dir, add_env={
+        'CIBW_BEFORE_BUILD': f'''python -c "import os; assert os.getcwd() == {str(project_dir)!r}"''',
+        'CIBW_BEFORE_BUILD_LINUX': '''python -c "import os; assert os.getcwd() == '/project'"''',
+    })
+
+    expected_wheels = utils.expected_wheels('spam', '0.1.0')
+    assert set(actual_wheels) == set(expected_wheels)

test/test_environment.py

@@ -13,12 +13,15 @@ project_with_environment_asserts = test_projects.new_c_project(
         # explode if environment isn't correct, as set in CIBW_ENVIRONMENT
         CIBW_TEST_VAR = os.environ.get("CIBW_TEST_VAR")
         CIBW_TEST_VAR_2 = os.environ.get("CIBW_TEST_VAR_2")
+        CIBW_TEST_VAR_3 = os.environ.get("CIBW_TEST_VAR_3")
         PATH = os.environ.get("PATH")
 
         if CIBW_TEST_VAR != "a b c":
             raise Exception('CIBW_TEST_VAR should equal "a b c". It was "%s"' % CIBW_TEST_VAR)
         if CIBW_TEST_VAR_2 != "1":
             raise Exception('CIBW_TEST_VAR_2 should equal "1". It was "%s"' % CIBW_TEST_VAR_2)
+        if CIBW_TEST_VAR_3 != "test string 3":
+            raise Exception('CIBW_TEST_VAR_3 should equal "test string 3". It was "%s"' % CIBW_TEST_VAR_3)
         if "/opt/cibw_test_path" not in PATH:
             raise Exception('PATH should contain "/opt/cibw_test_path". It was "%s"' % PATH)
         if "$PATH" in PATH:

unit_test/docker_container_test.py

@@ -8,6 +8,7 @@ from pathlib import Path, PurePath
 import pytest
 
 from cibuildwheel.docker_container import DockerContainer
+from cibuildwheel.environment import EnvironmentAssignment
 
 # for these tests we use manylinux2014 images, because they're available on
 # multi architectures and include python3.8
@@ -40,6 +41,13 @@ def test_environment():
         assert container.call(['sh', '-c', 'echo $TEST_VAR'], env={'TEST_VAR': '1'}, capture_output=True) == '1\n'
 
 
+@pytest.mark.docker
+def test_cwd():
+    with DockerContainer(DEFAULT_IMAGE, cwd='/cibuildwheel/working_directory') as container:
+        assert container.call(['pwd'], capture_output=True) == '/cibuildwheel/working_directory\n'
+        assert container.call(['pwd'], capture_output=True, cwd='/opt') == '/opt\n'
+
+
 @pytest.mark.docker
 def test_container_removed():
     with DockerContainer(DEFAULT_IMAGE) as container:
@@ -145,3 +153,10 @@ def test_dir_operations(tmp_path: Path):
         container.copy_out(dst_dir, new_test_dir)
 
         assert test_binary_data == (new_test_dir / 'test.dat').read_bytes()
+
+
+@pytest.mark.docker
+def test_environment_executor():
+    with DockerContainer(DEFAULT_IMAGE) as container:
+        assignment = EnvironmentAssignment("TEST=$(echo 42)")
+        assert assignment.evaluated_value({}, container.environment_executor) == "42"

unit_test/environment_test.py

@@ -40,7 +40,7 @@ def test_inheritance():
 
 
 def test_shell_eval():
-    environment_recipe = parse_environment('VAR="$(echo "a test" string)"')
+    environment_recipe = parse_environment('VAR="$(echo "a   test" string)"')
 
     env_copy = os.environ.copy()
     env_copy.pop('VAR', None)
@@ -50,8 +50,8 @@ def test_shell_eval():
     )
     environment_cmds = environment_recipe.as_shell_commands()
 
-    assert environment_dict['VAR'] == 'a test string'
-    assert environment_cmds == ['export VAR="$(echo "a test" string)"']
+    assert environment_dict['VAR'] == 'a   test string'
+    assert environment_cmds == ['export VAR="$(echo "a   test" string)"']
 
 
 def test_shell_eval_and_env():