Merge branch 'master' into python3-ci

* master: (28 commits)
  Update README to combine environment sections
  Update README
  os.path.samefile is not available on windows.
  Add shell-style execution on the Mac
  Use os.path.samefile to get around filename case insensitivity
  Add test for before_build executing in a shell
  Fix some linter warnings
  Add CIBW_ENVIRONMENT to README and reorder options
  Fix empty env string case
  Fix string splitting for the env string
  Making call from run_tests to run_test subprocess.check_call
  Fixing 'ValueError: Attempted relative import in non-package'
  Update README.md
  Reducing code duplication in bin/run_test.py and bin/run_tests.py
  Semicolons need quoting
  Ignore vscode project settings
  Windows uses a different pathsep so have to use that in the test
  Make shell evals work and add tests for that
  Use parse_environment in the main code
  Run the unit tests in CI
  ...

# Conflicts:
#	.travis.yml
#	appveyor.yml
#	run_tests.py

cibuildwheel/__main__.py

@@ -3,6 +3,7 @@ import argparse, os, subprocess, sys, textwrap
 
 import cibuildwheel
 import cibuildwheel.linux, cibuildwheel.windows, cibuildwheel.macos
+from cibuildwheel.environment import parse_environment, EnvironmentParseError
 from cibuildwheel.util import BuildSkipper
 
 def get_option_from_environment(option_name, platform=None):
@@ -72,9 +73,22 @@ def main():
     project_dir = args.project_dir
     before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform)
     skip_config = os.environ.get('CIBW_SKIP', '')
+    environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform) or ''
+
+    try:
+        environment = parse_environment(environment_config)
+    except (EnvironmentParseError, ValueError) as e:
+        print('cibuildwheel: Malformed environment option "%s"' % environment_config, file=sys.stderr)
+        import traceback
+        traceback.print_exc(None, sys.stderr)
+        exit(2)
 
     skip = BuildSkipper(skip_config)
 
+    # Add CIBUILDWHEEL environment variable
+    # This needs to be passed on to the docker container in linux.py
+    os.environ['CIBUILDWHEEL'] = '1'
+
     try:
         project_setup_py = os.path.join(project_dir, 'setup.py')
         name_output = subprocess.check_output([sys.executable, project_setup_py, '--name'],
@@ -103,12 +117,13 @@ def main():
         test_requires=test_requires,
         before_build=before_build,
         skip=skip,
+        environment=environment,
     )
 
     print_preamble(platform, build_options)
 
     if not os.path.exists(output_dir):
-        os.mkdir(output_dir)
+        os.makedirs(output_dir)
 
     if platform == 'linux':
         cibuildwheel.linux.build(**build_options)

cibuildwheel/bashlex_eval.py

@@ -0,0 +1,67 @@
+import subprocess, shlex
+from collections import namedtuple
+import bashlex
+
+NodeExecutionContext = namedtuple('NodeExecutionContext', ['environment', 'input'])
+
+def evaluate(value, environment):
+    if not value:
+        # empty string evaluates to empty string
+        # (but trips up bashlex)
+        return ''
+
+    command_node = bashlex.parsesingle(value)
+
+    if len(command_node.parts) != 1:
+        raise ValueError('"%s" has too many parts' % value)
+
+    value_word_node = command_node.parts[0]
+    
+    return evaluate_node(
+        value_word_node, 
+        context=NodeExecutionContext(environment=environment, input=value)
+    )
+
+
+def evaluate_node(node, context):
+    if node.kind == 'word':
+        return evaluate_word_node(node, context=context)
+    elif node.kind == 'commandsubstitution':
+        return evaluate_command_node(node.command, context=context)
+    elif node.kind == 'parameter':
+        return evaluate_parameter_node(node, context=context)
+    else:
+        raise ValueError('Unsupported bash construct: "%s"' % node.word)
+
+
+def evaluate_word_node(node, context):
+    word_start = node.pos[0]
+    word_end = node.pos[1]
+    word_string = context.input[word_start:word_end]
+    letters = list(word_string)
+
+    for part in node.parts:
+        part_start = part.pos[0] - word_start
+        part_end = part.pos[1] - word_start
+
+        # Set all the characters in the part to None
+        for i in range(part_start, part_end):
+            letters[i] = None
+
+        letters[part_start] = evaluate_node(part, context=context)
+
+    # remove the None letters and concat
+    value = ''.join(l for l in letters if l is not None)
+
+    # apply bash-like quotes/whitespace treatment
+    return ' '.join(word.strip() for word in shlex.split(value))
+
+
+def evaluate_command_node(node, context):
+    words = [evaluate_node(part, context=context) for part in node.parts]
+    command = ' '.join(words)
+    return subprocess.check_output(shlex.split(command), env=context.environment)
+
+
+def evaluate_parameter_node(node, context):
+    return context.environment.get(node.value, '')

cibuildwheel/environment.py

@@ -0,0 +1,80 @@
+import bashlex
+from . import bashlex_eval
+
+
+class EnvironmentParseError(Exception):
+    pass
+
+
+def parse_environment(env_string):
+    env_items = split_env_items(env_string)
+    assignments = [EnvironmentAssignment(item) for item in env_items]
+    return ParsedEnvironment(assignments=assignments)
+
+
+def split_env_items(env_string):
+    '''Splits space-separated variable assignments into a list of individual assignments.
+
+    >>> split_env_items('VAR=abc')
+    ['VAR=abc']
+    >>> split_env_items('VAR="a string" THING=3')
+    ['VAR="a string"', 'THING=3']
+    >>> split_env_items('VAR="a string" THING=\\'single "quotes"\\'')
+    ['VAR="a string"', 'THING=\\'single "quotes"\\'']
+    >>> split_env_items('VAR="dont \\\\"forget\\\\" about backslashes"')
+    ['VAR="dont \\\\"forget\\\\" about backslashes"']
+    >>> split_env_items('PATH="$PATH;/opt/cibw_test_path"')
+    ['PATH="$PATH;/opt/cibw_test_path"']
+    >>> split_env_items('PATH2="something with spaces"')
+    ['PATH2="something with spaces"']
+    '''
+    if not env_string:
+        return []
+
+    command_node = bashlex.parsesingle(env_string)
+    result = []
+
+    for word_node in command_node.parts:
+        part_string = env_string[word_node.pos[0]:word_node.pos[1]]
+        result.append(part_string)
+
+    return result
+
+
+class EnvironmentAssignment(object):
+    def __init__(self, assignment):
+        name, equals, value = assignment.partition('=')
+        if not equals:
+            raise EnvironmentParseError(assignment)
+        self.name = name
+        self.value = value
+
+    def evaluated_value(self, environment):
+        '''Returns the value of this assignment, as evaluated in the environment'''
+        return bashlex_eval.evaluate(self.value, environment=environment)
+
+    def as_shell_assignment(self):
+        return 'export %s=%s' % (self.name, self.value)
+
+    def __repr__(self):
+        return '%s=%s' % (self.name, self.value)
+
+
+class ParsedEnvironment(object):
+    def __init__(self, assignments):
+        self.assignments = assignments
+
+    def as_dictionary(self, prev_environment):
+        environment = prev_environment.copy()
+
+        for assignment in self.assignments:
+            value = assignment.evaluated_value(environment=environment)
+            environment[assignment.name] = value
+
+        return environment
+
+    def as_shell_commands(self):
+        return [a.as_shell_assignment() for a in self.assignments]
+
+    def __repr__(self):
+        return 'ParsedEnvironment(%r)' % [repr(a) for a in self.assignments]

cibuildwheel/linux.py

@@ -9,7 +9,7 @@ except ImportError:
     from pipes import quote as shlex_quote
 
 
-def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, skip):
+def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, skip, environment):
     try:
         subprocess.check_call(['docker', '--version'])
     except:
@@ -52,6 +52,8 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be
             set -o xtrace
             cd /project
 
+            {environment_exports}
+
             for PYBIN in {pybin_paths}; do
                 # Setup
                 rm -rf /tmp/built_wheel
@@ -106,11 +108,14 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be
             before_build=shlex_quote(
                 prepare_command(before_build, python='python', pip='pip') if before_build else ''
             ),
+            environment_exports='\n'.join(environment.as_shell_commands()),
         )
 
         docker_process = subprocess.Popen([
                 'docker',
                 'run',
+                '--env',
+                'CIBUILDWHEEL',
                 '--rm',
                 '-i',
                 '-v', '%s:/project' % os.path.abspath(project_dir),

cibuildwheel/macos.py

@@ -10,7 +10,7 @@ except ImportError:
 from .util import prepare_command
 
 
-def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, skip):
+def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, skip, environment):
     PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'url'])
     python_configurations = [
         PythonConfiguration(version='2.7', identifier='cp27-macosx_10_6_intel', url='https://www.python.org/ftp/python/2.7.13/python-2.7.13-macosx10.6.pkg'),
@@ -19,10 +19,14 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be
         PythonConfiguration(version='3.6', identifier='cp36-macosx_10_6_intel', url='https://www.python.org/ftp/python/3.6.0/python-3.6.0-macosx10.6.pkg'),
     ]
 
-    def shell(args, env=None, cwd=None):
+    def call(args, env=None, cwd=None, shell=False):
         # print the command executing for the logs
-        print('+ ' + ' '.join(shlex_quote(a) for a in args))
-        return subprocess.check_call(args, env=env, cwd=cwd)
+        if shell:
+            print('+ %s' % args)
+        else:
+            print('+ ' + ' '.join(shlex_quote(a) for a in args))
+
+        return subprocess.check_call(args, env=env, cwd=cwd, shell=shell)
 
     for config in python_configurations:
         if skip(config.identifier):
@@ -30,28 +34,29 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be
             continue
 
         # download the pkg
-        shell(['curl', '-L', '-o', '/tmp/Python.pkg', config.url])
+        call(['curl', '-L', '-o', '/tmp/Python.pkg', config.url])
         # install
-        shell(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/'])
+        call(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/'])
 
         env = os.environ.copy()
         env['PATH'] = os.pathsep.join([
             '/Library/Frameworks/Python.framework/Versions/%s/bin' % config.version,
             env['PATH'],
         ])
+        env = environment.as_dictionary(prev_environment=env)
 
         python = 'python3' if config.version[0] == '3' else 'python2'
         pip = 'pip3' if config.version[0] == '3' else 'pip2'
 
         # check what version we're on
-        shell(['which', python], env=env)
-        shell([python, '--version'], env=env)
+        call(['which', python], env=env)
+        call([python, '--version'], env=env)
 
         # install pip & wheel
-        shell([python, '-m', 'ensurepip', '--upgrade'], env=env)
-        shell([pip, '--version'], env=env)
-        shell([pip, 'install', 'wheel'], env=env)
-        shell([pip, 'install', 'delocate'], env=env)
+        call([python, '-m', 'ensurepip', '--upgrade'], env=env)
+        call([pip, '--version'], env=env)
+        call([pip, 'install', 'wheel'], env=env)
+        call([pip, 'install', 'delocate'], env=env)
 
         # setup dirs
         if os.path.exists('/tmp/built_wheel'):
@@ -64,10 +69,10 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be
         # run the before_build command
         if before_build:
             before_build_prepared = prepare_command(before_build, python=python, pip=pip)
-            shell(shlex.split(before_build_prepared), env=env)
+            call(before_build_prepared, env=env, shell=True)
 
         # build the wheel
-        shell([pip, 'wheel', project_dir, '-w', '/tmp/built_wheel', '--no-deps'], env=env)
+        call([pip, 'wheel', project_dir, '-w', '/tmp/built_wheel', '--no-deps'], env=env)
         built_wheel = glob('/tmp/built_wheel/*.whl')[0]
 
         if built_wheel.endswith('none-any.whl'):
@@ -75,24 +80,24 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be
             shutil.move(built_wheel, '/tmp/delocated_wheel')
         else:
             # list the dependencies
-            shell(['delocate-listdeps', built_wheel], env=env)
+            call(['delocate-listdeps', built_wheel], env=env)
             # rebuild the wheel with shared libraries included and place in output dir
-            shell(['delocate-wheel', '-w', '/tmp/delocated_wheel', built_wheel], env=env)
+            call(['delocate-wheel', '-w', '/tmp/delocated_wheel', built_wheel], env=env)
         delocated_wheel = glob('/tmp/delocated_wheel/*.whl')[0]
 
         # install the wheel
-        shell([pip, 'install', delocated_wheel], env=env)
+        call([pip, 'install', delocated_wheel], env=env)
 
         # test the wheel
         if test_requires:
-            shell([pip, 'install'] + test_requires, env=env)
+            call([pip, 'install'] + test_requires, env=env)
         if test_command:
             # run the tests from $HOME, with an absolute path in the command
             # (this ensures that Python runs the tests against the installed wheel
             # and not the repo code)
             abs_project_dir = os.path.abspath(project_dir)
             test_command_absolute = test_command.format(project=abs_project_dir)
-            shell(shlex.split(test_command_absolute), cwd=os.environ['HOME'], env=env)
+            call(shlex.split(test_command_absolute), cwd=os.environ['HOME'], env=env)
 
         # we're all done here; move it to output
         shutil.move(delocated_wheel, output_dir)

cibuildwheel/windows.py

@@ -10,7 +10,7 @@ from glob import glob
 from .util import prepare_command, Unbuffered
 
 
-def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, skip):
+def build(project_dir, package_name, output_dir, test_command, test_requires, before_build, skip, environment):
     # Python under AppVeyor/Windows seems to be buffering by default, giving problems interleaving subprocess call output with unflushed calls to 'print'
     sys.stdout.flush()
     sys.stdout = Unbuffered(sys.stdout)
@@ -64,6 +64,7 @@ def build(project_dir, package_name, output_dir, test_command, test_requires, be
             os.path.join(config.path, 'Scripts'),
             env['PATH']
         ])
+        env = environment.as_dictionary(prev_environment=env)
 
         # for the logs - check we're running the right version of python
         shell(['python', '--version'], env=env)