From f56ddda15739578b7190dae75bf0b543058de885 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 2 Sep 2017 22:09:32 +0100 Subject: [PATCH] Make shell evals work and add tests for that --- cibuildwheel/bashlex_eval.py | 58 +++++++++++++++++++++------- cibuildwheel/environment.py | 15 +------ test/05_environment/environment.json | 2 +- unit_test/environment_test.py | 22 +++++++++++ 4 files changed, 67 insertions(+), 30 deletions(-) diff --git a/cibuildwheel/bashlex_eval.py b/cibuildwheel/bashlex_eval.py index 4437990b..90a399fe 100644 --- a/cibuildwheel/bashlex_eval.py +++ b/cibuildwheel/bashlex_eval.py @@ -1,39 +1,67 @@ import subprocess, shlex +from collections import namedtuple +import bashlex + +NodeExecutionContext = namedtuple('NodeExecutionContext', ['environment', 'input']) + +def evaluate(value, environment): + if not value: + # empty string evaluates to empty string + # (but trips up bashlex) + return '' + + command_node = bashlex.parsesingle(value) + + if len(command_node.parts) != 1: + raise ValueError('"%s" has too many parts' % value) + + value_word_node = command_node.parts[0] + + return evaluate_node( + value_word_node, + context=NodeExecutionContext(environment=environment, input=value) + ) -def evaluate_node(node, environment): +def evaluate_node(node, context): if node.kind == 'word': - return evaluate_word_node(node, environment=environment) + return evaluate_word_node(node, context=context) elif node.kind == 'commandsubstitution': - return evaluate_command_node(node.command, environment=environment) + return evaluate_command_node(node.command, context=context) elif node.kind == 'parameter': - return evaluate_parameter_node(node, environment=environment) + return evaluate_parameter_node(node, context=context) else: raise ValueError('Unsupported bash construct: "%s"' % node.word) -def evaluate_word_node(node, environment): - letters = list(node.word) +def evaluate_word_node(node, context): + word_start = node.pos[0] + word_end = node.pos[1] + word_string = context.input[word_start:word_end] + letters = list(word_string) for part in node.parts: - part_start = part.pos[0] - part_end = part.pos[1] + part_start = part.pos[0] - word_start + part_end = part.pos[1] - word_start # Set all the characters in the part to None for i in range(part_start, part_end): letters[i] = None - letters[part_start] = evaluate_node(part, environment=environment) + letters[part_start] = evaluate_node(part, context=context) # remove the None letters and concat - return ''.join(l for l in letters if l is not None) + value = ''.join(l for l in letters if l is not None) + + # apply bash-like quotes/whitespace treatment + return ' '.join(word.strip() for word in shlex.split(value)) -def evaluate_command_node(node, environment): - words = [evaluate_node(part, environment=environment) for part in node.parts] +def evaluate_command_node(node, context): + words = [evaluate_node(part, context=context) for part in node.parts] command = ' '.join(words) - return subprocess.check_output(shlex.split(command), env=environment) + return subprocess.check_output(shlex.split(command), env=context.environment) -def evaluate_parameter_node(node, environment): - return environment.get(node.value, '') +def evaluate_parameter_node(node, context): + return context.environment.get(node.value, '') diff --git a/cibuildwheel/environment.py b/cibuildwheel/environment.py index 5b5f45e1..249dcc3a 100644 --- a/cibuildwheel/environment.py +++ b/cibuildwheel/environment.py @@ -35,22 +35,9 @@ class EnvironmentAssignment(object): self.name = name self.value = value - if value: - command_node = bashlex.parsesingle(value) - - if len(command_node.parts) != 1: - raise ValueError('"%s" has too many parts' % value) - - self.value_word_node = command_node.parts[0] - else: - self.value_word_node = None - def evaluated_value(self, environment): '''Returns the value of this assignment, as evaluated in the environment''' - if self.value_word_node: - return bashlex_eval.evaluate_node(self.value_word_node, environment=environment) - else: - return '' + return bashlex_eval.evaluate(self.value, environment=environment) def as_shell_assignment(self): return 'export %s=%s' % (self.name, self.value) diff --git a/test/05_environment/environment.json b/test/05_environment/environment.json index c97c84d7..f7c9b46c 100644 --- a/test/05_environment/environment.json +++ b/test/05_environment/environment.json @@ -1,3 +1,3 @@ { - "CIBW_ENVIRONMENT": "CIBW_TEST_VAR=\"a b c\" CIBW_TEST_VAR_2=1 PATH=$PATH:/opt/cibw_test_path" + "CIBW_ENVIRONMENT": "CIBW_TEST_VAR=\"a b c\" CIBW_TEST_VAR_2=1 CIBW_TEST_VAR_3=\"$(echo 'test string 3')\" PATH=$PATH:/opt/cibw_test_path " } diff --git a/unit_test/environment_test.py b/unit_test/environment_test.py index a1556c87..6b538a4d 100644 --- a/unit_test/environment_test.py +++ b/unit_test/environment_test.py @@ -34,6 +34,28 @@ def test_inheritance(): assert environment_dict == {'PATH': '/usr/bin:/usr/local/bin'} assert environment_cmds == ['export PATH=$PATH:/usr/local/bin'] +def test_shell_eval(): + environment_recipe = parse_environment('VAR="$(echo "a test" string)"') + + environment_dict = environment_recipe.as_dictionary( + prev_environment={} + ) + environment_cmds = environment_recipe.as_shell_commands() + + assert environment_dict == {'VAR': 'a test string'} + assert environment_cmds == ['export VAR="$(echo "a test" string)"'] + +def test_shell_eval_and_env(): + environment_recipe = parse_environment('VAR="$(echo "$PREV_VAR" string)"') + + environment_dict = environment_recipe.as_dictionary( + prev_environment={'PREV_VAR': '1 2 3'} + ) + environment_cmds = environment_recipe.as_shell_commands() + + assert environment_dict == {'PREV_VAR': '1 2 3', 'VAR': '1 2 3 string'} + assert environment_cmds == ['export VAR="$(echo "$PREV_VAR" string)"'] + def test_empty_var(): environment_recipe = parse_environment('CFLAGS=')