Make shell evals work and add tests for that

cibuildwheel/bashlex_eval.py

@@ -1,39 +1,67 @@
 import subprocess, shlex
+from collections import namedtuple
+import bashlex
+
+NodeExecutionContext = namedtuple('NodeExecutionContext', ['environment', 'input'])
+
+def evaluate(value, environment):
+    if not value:
+        # empty string evaluates to empty string
+        # (but trips up bashlex)
+        return ''
+
+    command_node = bashlex.parsesingle(value)
+
+    if len(command_node.parts) != 1:
+        raise ValueError('"%s" has too many parts' % value)
+
+    value_word_node = command_node.parts[0]
+    
+    return evaluate_node(
+        value_word_node, 
+        context=NodeExecutionContext(environment=environment, input=value)
+    )
 
 
-def evaluate_node(node, environment):
+def evaluate_node(node, context):
     if node.kind == 'word':
-        return evaluate_word_node(node, environment=environment)
+        return evaluate_word_node(node, context=context)
     elif node.kind == 'commandsubstitution':
-        return evaluate_command_node(node.command, environment=environment)
+        return evaluate_command_node(node.command, context=context)
     elif node.kind == 'parameter':
-        return evaluate_parameter_node(node, environment=environment)
+        return evaluate_parameter_node(node, context=context)
     else:
         raise ValueError('Unsupported bash construct: "%s"' % node.word)
 
 
-def evaluate_word_node(node, environment):
+def evaluate_word_node(node, context):
-    letters = list(node.word)
+    word_start = node.pos[0]
+    word_end = node.pos[1]
+    word_string = context.input[word_start:word_end]
+    letters = list(word_string)
 
     for part in node.parts:
-        part_start = part.pos[0]
+        part_start = part.pos[0] - word_start
-        part_end = part.pos[1]
+        part_end = part.pos[1] - word_start
 
         # Set all the characters in the part to None
         for i in range(part_start, part_end):
             letters[i] = None
 
-        letters[part_start] = evaluate_node(part, environment=environment)
+        letters[part_start] = evaluate_node(part, context=context)
 
     # remove the None letters and concat
-    return ''.join(l for l in letters if l is not None)
+    value = ''.join(l for l in letters if l is not None)
+
+    # apply bash-like quotes/whitespace treatment
+    return ' '.join(word.strip() for word in shlex.split(value))
 
 
-def evaluate_command_node(node, environment):
+def evaluate_command_node(node, context):
-    words = [evaluate_node(part, environment=environment) for part in node.parts]
+    words = [evaluate_node(part, context=context) for part in node.parts]
     command = ' '.join(words)
-    return subprocess.check_output(shlex.split(command), env=environment)
+    return subprocess.check_output(shlex.split(command), env=context.environment)
 
 
-def evaluate_parameter_node(node, environment):
+def evaluate_parameter_node(node, context):
-    return environment.get(node.value, '')
+    return context.environment.get(node.value, '')

cibuildwheel/environment.py

@@ -35,22 +35,9 @@ class EnvironmentAssignment(object):
         self.name = name
         self.value = value
 
-        if value:
-            command_node = bashlex.parsesingle(value)
-
-            if len(command_node.parts) != 1:
-                raise ValueError('"%s" has too many parts' % value)
-
-            self.value_word_node = command_node.parts[0]
-        else:
-            self.value_word_node = None
-
     def evaluated_value(self, environment):
         '''Returns the value of this assignment, as evaluated in the environment'''
-        if self.value_word_node:
+        return bashlex_eval.evaluate(self.value, environment=environment)
-            return bashlex_eval.evaluate_node(self.value_word_node, environment=environment)
-        else:
-            return ''
 
     def as_shell_assignment(self):
         return 'export %s=%s' % (self.name, self.value)

test/05_environment/environment.json

@@ -1,3 +1,3 @@
 {
-    "CIBW_ENVIRONMENT": "CIBW_TEST_VAR=\"a b c\" CIBW_TEST_VAR_2=1 PATH=$PATH:/opt/cibw_test_path"
+    "CIBW_ENVIRONMENT": "CIBW_TEST_VAR=\"a b c\" CIBW_TEST_VAR_2=1 CIBW_TEST_VAR_3=\"$(echo 'test string 3')\" PATH=$PATH:/opt/cibw_test_path "
 }

unit_test/environment_test.py

@@ -34,6 +34,28 @@ def test_inheritance():
     assert environment_dict == {'PATH': '/usr/bin:/usr/local/bin'}
     assert environment_cmds == ['export PATH=$PATH:/usr/local/bin']
 
+def test_shell_eval():
+    environment_recipe = parse_environment('VAR="$(echo "a test" string)"')
+
+    environment_dict = environment_recipe.as_dictionary(
+        prev_environment={}
+    )
+    environment_cmds = environment_recipe.as_shell_commands()
+
+    assert environment_dict == {'VAR': 'a test string'}
+    assert environment_cmds == ['export VAR="$(echo "a test" string)"']
+
+def test_shell_eval_and_env():
+    environment_recipe = parse_environment('VAR="$(echo "$PREV_VAR" string)"')
+
+    environment_dict = environment_recipe.as_dictionary(
+        prev_environment={'PREV_VAR': '1 2 3'}
+    )
+    environment_cmds = environment_recipe.as_shell_commands()
+
+    assert environment_dict == {'PREV_VAR': '1 2 3', 'VAR': '1 2 3 string'}
+    assert environment_cmds == ['export VAR="$(echo "$PREV_VAR" string)"']
+
 def test_empty_var():
     environment_recipe = parse_environment('CFLAGS=')