From 0597088f64b7ac51940986c12c04127143a1ddfb Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Fri, 29 May 2020 17:24:01 -0400 Subject: [PATCH 001/117] Adding PEP 518 test --- cibuildwheel/windows.py | 19 ++++++++++++++++++ setup.py | 2 +- test/test_pep518.py | 44 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 test/test_pep518.py diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 9f33ad91..db0a063c 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -5,6 +5,7 @@ import sys import tempfile from glob import glob from zipfile import ZipFile +import toml from typing import Dict, List, Optional, NamedTuple @@ -148,6 +149,24 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain shell(['pip', '--version'], env=env) shell(['pip', 'install', '--upgrade', 'setuptools', 'wheel'] + dependency_constraint_flags, env=env) + # Python 3.5 PEP 518 hack (see https://github.com/pypa/pip/issues/8392#issuecomment-639563494) + # Basically, nuget's Python is an embedded Python distribution, which is not supported by pip. + # Before version 3.6, there was no way to disable the "embedded" behavior, including the ignoring + # of environment variables, including the ones pip uses to setup PEP 518 builds. + # + # The fix here is as suggested in that issue; we manually setup the PEP 518 requirements. Since we + # are in a fresh environment (except for pinned cibuildweel dependencies), the build is already + # mostly "isolated". + if python_configuration.version.startswith('3.5') and os.path.exists('pyproject.toml'): + data = toml.load('pyproject.toml') + requirements = ( + data['build-system'].get('requires', []) + if 'build-system' in data + else [] + ) + if requirements: + shell(['pip', 'install'] + requirements + dependency_constraint_flags, env=env) + return env diff --git a/setup.py b/setup.py index 3e4c90cb..68dbff3a 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ with io.open(os.path.join(this_directory, 'README.md'), encoding='utf-8') as f: setup( name='cibuildwheel', version='1.4.2', - install_requires=['bashlex!=0.13'], + install_requires=['bashlex!=0.13', 'toml'], description="Build Python wheels on CI with minimal configuration.", long_description=long_description, long_description_content_type='text/markdown', diff --git a/test/test_pep518.py b/test/test_pep518.py new file mode 100644 index 00000000..7d5f5081 --- /dev/null +++ b/test/test_pep518.py @@ -0,0 +1,44 @@ +import textwrap +from . import test_projects +from . import utils + +basic_project = test_projects.new_c_project( + setup_py_add=textwrap.dedent( + """ + # Will fail if PEP 518 does work + import requests + assert requests.__version__ == "2.23.0", "Requests found but wrong version ({0})".format(requests.__version__) + + # Just making sure environment is still set + import os + if os.environ.get("CIBUILDWHEEL", "0") != "1": + raise Exception("CIBUILDWHEEL environment variable is not set to 1") + """ + ) +) + +basic_project.files[ + "pyproject.toml" +] = """ +[build-system] +requires = [ + "setuptools>=42", + "wheel", + "requests==2.23.0" +] + +build-backend = "setuptools.build_meta" +""" + + +def test_pep518(tmp_path): + + project_dir = tmp_path / "project" + basic_project.generate(project_dir) + + # build the wheels + actual_wheels = utils.cibuildwheel_run(project_dir) + + # check that the expected wheels are produced + expected_wheels = utils.expected_wheels("spam", "0.1.0") + assert set(actual_wheels) == set(expected_wheels) From 901ac79dbcd3b96d44a92f6a64a11c961ffec632 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sun, 14 Jun 2020 11:25:14 -0400 Subject: [PATCH 002/117] review: Address review suggestion --- cibuildwheel/windows.py | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index db0a063c..6a62b63d 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -149,25 +149,32 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain shell(['pip', '--version'], env=env) shell(['pip', 'install', '--upgrade', 'setuptools', 'wheel'] + dependency_constraint_flags, env=env) - # Python 3.5 PEP 518 hack (see https://github.com/pypa/pip/issues/8392#issuecomment-639563494) - # Basically, nuget's Python is an embedded Python distribution, which is not supported by pip. - # Before version 3.6, there was no way to disable the "embedded" behavior, including the ignoring - # of environment variables, including the ones pip uses to setup PEP 518 builds. - # - # The fix here is as suggested in that issue; we manually setup the PEP 518 requirements. Since we - # are in a fresh environment (except for pinned cibuildweel dependencies), the build is already - # mostly "isolated". - if python_configuration.version.startswith('3.5') and os.path.exists('pyproject.toml'): - data = toml.load('pyproject.toml') + return env + + +def pep_518_cp35_workaround(package_dir: str, env: Dict[str, str]) -> None: + """ + Python 3.5 PEP 518 hack (see https://github.com/pypa/pip/issues/8392#issuecomment-639563494) + Basically, nuget's Python is an embedded Python distribution, which is not supported by pip. + Before version 3.6, there was no way to disable the "embedded" behavior, including the ignoring + of environment variables, including the ones pip uses to setup PEP 518 builds. + + The fix here is as suggested in that issue; we manually setup the PEP 518 requirements. Since we + are in a fresh environment (except for pinned cibuildweel dependencies), the build is already + mostly "isolated". + """ + + pyproject_path = os.path.join(package_dir, 'pyproject.toml') + + if os.path.exists(pyproject_path): + data = toml.load(pyproject_path) requirements = ( data['build-system'].get('requires', []) if 'build-system' in data else [] ) if requirements: - shell(['pip', 'install'] + requirements + dependency_constraint_flags, env=env) - - return env + shell(['pip', 'install'] + requirements, env=env) def build(options: BuildOptions) -> None: @@ -195,6 +202,11 @@ def build(options: BuildOptions) -> None: before_build_prepared = prepare_command(options.before_build, project='.', package=options.package_dir) shell([before_build_prepared], env=env) + # activate the PEP 518 patch if on Windows Python 3.5 + # (will only have an effect if PEP 517 builds are used): + if config.version.startswith('3.5'): + pep_518_cp35_workaround(options.package_dir, env) + # build the wheel if os.path.exists(built_wheel_dir): shutil.rmtree(built_wheel_dir) From a7f73f159365ef5948dfcca1a4dff3828b5e2e40 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Mon, 15 Jun 2020 01:53:31 +0200 Subject: [PATCH 003/117] Use pathlib.Path in main cibuildwheel files --- cibuildwheel/__main__.py | 18 ++++---- cibuildwheel/linux.py | 20 ++++----- cibuildwheel/macos.py | 97 +++++++++++++++++++++------------------- cibuildwheel/util.py | 44 +++++++++--------- cibuildwheel/windows.py | 95 ++++++++++++++++++++------------------- 5 files changed, 139 insertions(+), 135 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 40241ac1..fd4c418e 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -4,6 +4,7 @@ import sys import textwrap import traceback from configparser import ConfigParser +from pathlib import Path from typing import Any, Dict, List, Optional, overload @@ -114,8 +115,8 @@ def main() -> None: file=sys.stderr) exit(2) - package_dir = args.package_dir - output_dir = args.output_dir + package_dir = Path(args.package_dir) + output_dir = Path(args.output_dir) if platform == 'linux': repair_command_default = 'auditwheel repair -w {dest_dir} {wheel}' @@ -149,7 +150,8 @@ def main() -> None: elif dependency_versions == 'latest': dependency_constraints = None else: - dependency_constraints = DependencyConstraints(dependency_versions) + dependency_versions_path = Path(dependency_versions) + dependency_constraints = DependencyConstraints(dependency_versions_path) if test_extras: test_extras = f'[{test_extras}]' @@ -163,7 +165,7 @@ def main() -> None: # This needs to be passed on to the docker container in linux.py os.environ['CIBUILDWHEEL'] = '1' - if not any(os.path.exists(os.path.join(package_dir, name)) + if not any((package_dir / name).exists() for name in ["setup.py", "setup.cfg", "pyproject.toml"]): print('cibuildwheel: Could not find setup.py, setup.cfg or pyproject.toml at root of package', file=sys.stderr) exit(2) @@ -174,9 +176,7 @@ def main() -> None: manylinux_images: Optional[Dict[str, str]] = None if platform == 'linux': - pinned_docker_images_file = os.path.join( - os.path.dirname(__file__), 'resources', 'pinned_docker_images.cfg' - ) + pinned_docker_images_file = Path(__file__).parent / 'resources' / 'pinned_docker_images.cfg' all_pinned_docker_images = ConfigParser() all_pinned_docker_images.read(pinned_docker_images_file) # all_pinned_docker_images looks like a dict of dicts, e.g. @@ -224,8 +224,8 @@ def main() -> None: print_preamble(platform, build_options) - if not os.path.exists(output_dir): - os.makedirs(output_dir) + if not output_dir.exists(): + output_dir.mkdir(parents=True) if platform == 'linux': cibuildwheel.linux.build(build_options) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 0058c2db..cdaef019 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -5,6 +5,7 @@ import subprocess import sys import textwrap import uuid +from pathlib import Path from typing import List, NamedTuple, Optional, Union @@ -104,10 +105,12 @@ def build(options: BuildOptions) -> None: ('pp', 'manylinux_x86_64', options.manylinux_images['pypy_x86_64']), ] - if not os.path.realpath(options.package_dir).startswith(os.path.realpath('.')): + pwd = Path().resolve() + abs_package_dir = options.package_dir.resolve() + if pwd != abs_package_dir and pwd not in abs_package_dir.parents: raise Exception('package_dir must be inside the working directory') - container_package_dir = os.path.join('/project', os.path.relpath(options.package_dir, '.')) + container_package_dir = Path('/project') / abs_package_dir.relative_to(pwd) for implementation, platform_tag, docker_image in platforms: platform_configs = [c for c in python_configurations if c.identifier.startswith(implementation) and c.identifier.endswith(platform_tag)] @@ -272,7 +275,7 @@ def build(options: BuildOptions) -> None: # copy the output back into the host call(['docker', 'cp', container_name + ':/output/.', - os.path.abspath(options.output_dir)]) + str(options.output_dir.resolve())]) except subprocess.CalledProcessError as error: troubleshoot(options.package_dir, error) exit(1) @@ -281,16 +284,11 @@ def build(options: BuildOptions) -> None: call(['docker', 'rm', '--force', '-v', container_name]) -def troubleshoot(package_dir: str, error: Exception) -> None: +def troubleshoot(package_dir: Path, error: Exception) -> None: if (isinstance(error, subprocess.CalledProcessError) and 'exec' in error.cmd): # the bash script failed print('Checking for common errors...') - so_files = [] - for root, dirs, files in os.walk(package_dir): - for name in files: - _, ext = os.path.splitext(name) - if ext == '.so': - so_files.append(os.path.join(root, name)) + so_files = list(package_dir.glob('**/*.so')) if so_files: print(textwrap.dedent(''' @@ -304,5 +302,5 @@ def troubleshoot(package_dir: str, error: Exception) -> None: ''')) print(' Files detected:') - print('\n'.join([' ' + f for f in so_files])) + print('\n'.join([' ' + str(f) for f in so_files])) print('') diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 69a06c04..a4d611c4 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -5,6 +5,7 @@ import subprocess import sys import tempfile from glob import glob +from pathlib import Path from typing import Dict, List, Optional, NamedTuple, Union @@ -24,7 +25,7 @@ def call(args: Union[str, List[str]], env: Optional[Dict[str, str]] = None, cwd: if shell: print(f'+ {args}') else: - print('+ ' + ' '.join(shlex.quote(a) for a in args)) + print('+ ' + ' '.join(shlex.quote(str(a)) for a in args)) return subprocess.check_call(args, env=env, cwd=cwd, shell=shell) @@ -52,41 +53,41 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi return [c for c in python_configurations if build_selector(c.identifier)] -SYMLINKS_DIR = '/tmp/cibw_bin' +SYMLINKS_DIR = Path('/tmp/cibw_bin') -def make_symlinks(installation_bin_path: str, python_executable: str, pip_executable: str) -> None: - assert os.path.exists(os.path.join(installation_bin_path, python_executable)) +def make_symlinks(installation_bin_path: Path, python_executable: str, pip_executable: str) -> None: + assert (installation_bin_path / python_executable).exists() # Python bin folders on Mac don't symlink `python3` to `python`, and neither # does PyPy for `pypy` or `pypy3`, so we do that so `python` and `pip` always # point to the active configuration. - if os.path.exists(SYMLINKS_DIR): + if SYMLINKS_DIR.exists(): shutil.rmtree(SYMLINKS_DIR) - os.makedirs(SYMLINKS_DIR) + SYMLINKS_DIR.mkdir(parents=True) - os.symlink(os.path.join(installation_bin_path, python_executable), os.path.join(SYMLINKS_DIR, 'python')) - os.symlink(os.path.join(installation_bin_path, python_executable + '-config'), os.path.join(SYMLINKS_DIR, 'python-config')) - os.symlink(os.path.join(installation_bin_path, pip_executable), os.path.join(SYMLINKS_DIR, 'pip')) + (SYMLINKS_DIR / 'python').symlink_to(installation_bin_path / python_executable) + (SYMLINKS_DIR / 'python-config').symlink_to(installation_bin_path / (python_executable + '-config')) + (SYMLINKS_DIR / 'pip').symlink_to(installation_bin_path / pip_executable) -def install_cpython(version: str, url: str) -> str: +def install_cpython(version: str, url: str) -> Path: installed_system_packages = subprocess.check_output(['pkgutil', '--pkgs'], universal_newlines=True).splitlines() # if this version of python isn't installed, get it from python.org and install python_package_identifier = f'org.python.Python.PythonFramework-{version}' if python_package_identifier not in installed_system_packages: # download the pkg - download(url, '/tmp/Python.pkg') + download(url, Path('/tmp/Python.pkg')) # install call(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/']) # patch open ssl if version == '3.5': open_ssl_patch_url = f'https://github.com/mayeut/patch-macos-python-openssl/releases/download/v1.0.2u/patch-macos-python-{version}-openssl-v1.0.2u.tar.gz' - download(open_ssl_patch_url, '/tmp/python-patch.tar.gz') + download(open_ssl_patch_url, Path('/tmp/python-patch.tar.gz')) call(['sudo', 'tar', '-C', f'/Library/Frameworks/Python.framework/Versions/{version}/', '-xmf', '/tmp/python-patch.tar.gz']) - installation_bin_path = f'/Library/Frameworks/Python.framework/Versions/{version}/bin' + installation_bin_path = Path(f'/Library/Frameworks/Python.framework/Versions/{version}/bin') python_executable = 'python3' if version[0] == '3' else 'python' pip_executable = 'pip3' if version[0] == '3' else 'pip' make_symlinks(installation_bin_path, python_executable, pip_executable) @@ -94,16 +95,18 @@ def install_cpython(version: str, url: str) -> str: return installation_bin_path -def install_pypy(version: str, url: str) -> str: +def install_pypy(version: str, url: str) -> Path: pypy_tar_bz2 = url.rsplit('/', 1)[-1] - assert pypy_tar_bz2.endswith(".tar.bz2") - pypy_base_filename = os.path.splitext(os.path.splitext(pypy_tar_bz2)[0])[0] - installation_path = os.path.join('/tmp', pypy_base_filename) - if not os.path.exists(installation_path): - download(url, os.path.join("/tmp", pypy_tar_bz2)) - call(['tar', '-C', '/tmp', '-xf', os.path.join("/tmp", pypy_tar_bz2)]) + extension = ".tar.bz2" + assert pypy_tar_bz2.endswith(extension) + pypy_base_filename = pypy_tar_bz2[:-len(extension)] + installation_path = Path('/tmp') / pypy_base_filename + if not installation_path.exists(): + downloaded_tar_bz2 = Path("/tmp") / pypy_tar_bz2 + download(url, downloaded_tar_bz2) + call(['tar', '-C', '/tmp', '-xf', str(downloaded_tar_bz2)]) - installation_bin_path = os.path.join(installation_path, 'bin') + installation_bin_path = installation_path / 'bin' python_executable = 'pypy3' if version[0] == '3' else 'pypy' pip_executable = 'pip3' if version[0] == '3' else 'pip' make_symlinks(installation_bin_path, python_executable, pip_executable) @@ -121,8 +124,8 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain env = os.environ.copy() env['PATH'] = os.pathsep.join([ - SYMLINKS_DIR, - installation_bin_path, + str(SYMLINKS_DIR), + str(installation_bin_path), env['PATH'], ]) @@ -144,8 +147,8 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain exit(1) # install pip & wheel - call(['python', get_pip_script] + dependency_constraint_flags, env=env, cwd="/tmp") - assert os.path.exists(os.path.join(installation_bin_path, 'pip')) + call(['python', str(get_pip_script)] + dependency_constraint_flags, env=env, cwd="/tmp") + assert (installation_bin_path / 'pip').exists() call(['which', 'pip'], env=env) call(['pip', '--version'], env=env) which_pip = subprocess.check_output(['which', 'pip'], env=env, universal_newlines=True).strip() @@ -170,9 +173,9 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain def build(options: BuildOptions) -> None: - temp_dir = tempfile.mkdtemp(prefix='cibuildwheel') - built_wheel_dir = os.path.join(temp_dir, 'built_wheel') - repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel') + temp_dir = Path(tempfile.mkdtemp(prefix='cibuildwheel')) + built_wheel_dir = temp_dir / 'built_wheel' + repaired_wheel_dir = temp_dir / 'repaired_wheel' python_configurations = get_python_configurations(options.build_selector) @@ -180,7 +183,7 @@ def build(options: BuildOptions) -> None: dependency_constraint_flags = [] if options.dependency_constraints: dependency_constraint_flags = [ - '-c', options.dependency_constraints.get_for_python_version(config.version) + '-c', str(options.dependency_constraints.get_for_python_version(config.version)) ] env = setup_python(config, dependency_constraint_flags, options.environment) @@ -191,39 +194,39 @@ def build(options: BuildOptions) -> None: call(before_build_prepared, env=env, shell=True) # build the wheel - if os.path.exists(built_wheel_dir): + if built_wheel_dir.exists(): shutil.rmtree(built_wheel_dir) - os.makedirs(built_wheel_dir) - # os.path.abspath is need. Without it pip wheel may try to fetch package from pypi.org + built_wheel_dir.mkdir(parents=True) + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org # see https://github.com/joerick/cibuildwheel/pull/369 - call(['pip', 'wheel', os.path.abspath(options.package_dir), '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env) - built_wheel = glob(os.path.join(built_wheel_dir, '*.whl'))[0] + call(['pip', 'wheel', str(options.package_dir.resolve()), '-w', str(built_wheel_dir), '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env) + built_wheel = next(built_wheel_dir.glob('*.whl')) # repair the wheel - if os.path.exists(repaired_wheel_dir): + if repaired_wheel_dir.exists(): shutil.rmtree(repaired_wheel_dir) - os.makedirs(repaired_wheel_dir) - if built_wheel.endswith('none-any.whl') or not options.repair_command: + repaired_wheel_dir.mkdir(parents=True) + if built_wheel.name.endswith('none-any.whl') or not options.repair_command: # pure Python wheel or empty repair command - shutil.move(built_wheel, repaired_wheel_dir) + shutil.move(str(built_wheel), repaired_wheel_dir) else: repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) call(repair_command_prepared, env=env, shell=True) - repaired_wheel = glob(os.path.join(repaired_wheel_dir, '*.whl'))[0] + repaired_wheel = next(repaired_wheel_dir.glob('*.whl')) if options.test_command: # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. call(['pip', 'install', 'virtualenv'] + dependency_constraint_flags, env=env) - venv_dir = tempfile.mkdtemp() + venv_dir = Path(tempfile.mkdtemp()) # Use --no-download to ensure determinism by using seed libraries # built into virtualenv - call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) + call(['python', '-m', 'virtualenv', '--no-download', str(venv_dir)], env=env) virtualenv_env = env.copy() virtualenv_env['PATH'] = os.pathsep.join([ - os.path.join(venv_dir, 'bin'), + str(venv_dir / 'bin'), virtualenv_env['PATH'], ]) @@ -235,7 +238,7 @@ def build(options: BuildOptions) -> None: call(before_test_prepared, env=virtualenv_env, shell=True) # install the wheel - call(['pip', 'install', repaired_wheel + options.test_extras], env=virtualenv_env) + call(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env) # test the wheel if options.test_requires: @@ -246,8 +249,8 @@ def build(options: BuildOptions) -> None: # and not the repo code) test_command_prepared = prepare_command( options.test_command, - project=os.path.abspath('.'), - package=os.path.abspath(options.package_dir) + project=Path('.').resolve(), + package=options.package_dir ) call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True) @@ -255,5 +258,5 @@ def build(options: BuildOptions) -> None: shutil.rmtree(venv_dir) # we're all done here; move it to output (overwrite existing) - dst = os.path.join(options.output_dir, os.path.basename(repaired_wheel)) - shutil.move(repaired_wheel, dst) + dst = options.output_dir / repaired_wheel.name + shutil.move(str(repaired_wheel), dst) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 499954b9..0b585b86 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -1,14 +1,15 @@ import os import urllib.request from fnmatch import fnmatch +from pathlib import Path from time import sleep -from typing import Dict, List, NamedTuple, Optional +from typing import Dict, List, NamedTuple, Optional, Union from .environment import ParsedEnvironment -def prepare_command(command: str, **kwargs: str) -> str: +def prepare_command(command: str, **kwargs: Union[str, os.PathLike]) -> str: ''' Preprocesses a command by expanding variables like {python}. @@ -58,11 +59,11 @@ class Unbuffered: return getattr(self.stream, attr) -def download(url: str, dest: str) -> None: - print('+ Download ' + url + ' to ' + dest) - dest_dir = os.path.dirname(dest) - if not os.path.exists(dest_dir): - os.makedirs(dest_dir) +def download(url: str, dest: Path) -> None: + print(f'+ Download {url} to {dest}') + dest_dir = dest.parent + if not dest_dir.exists(): + dest_dir.mkdir(parents=True) repeat_num = 3 for i in range(repeat_num): @@ -76,40 +77,39 @@ def download(url: str, dest: str) -> None: break try: - with open(dest, 'wb') as file: - file.write(response.read()) + dest.write_bytes(response.read()) finally: response.close() class DependencyConstraints: - def __init__(self, base_file_path: str): - assert os.path.exists(base_file_path) - self.base_file_path = os.path.abspath(base_file_path) + def __init__(self, base_file_path: Path): + assert base_file_path.exists() + self.base_file_path = base_file_path.resolve() @staticmethod def with_defaults() -> 'DependencyConstraints': return DependencyConstraints( - base_file_path=os.path.join(os.path.dirname(__file__), 'resources', 'constraints.txt') + base_file_path=Path(__file__).parent / 'resources' / 'constraints.txt' ) - def get_for_python_version(self, version: str) -> str: + def get_for_python_version(self, version: str) -> Path: version_parts = version.split('.') # try to find a version-specific dependency file e.g. if # ./constraints.txt is the base, look for ./constraints-python27.txt - base, ext = os.path.splitext(self.base_file_path) - specific = base + f'-python{version_parts[0]}{version_parts[1]}' - specific_file_path = specific + ext - if os.path.exists(specific_file_path): + specific_stem = self.base_file_path.stem + f'-python{version_parts[0]}{version_parts[1]}' + sepcific_name = specific_stem + self.base_file_path.suffix + specific_file_path = self.base_file_path.with_name(sepcific_name) + if specific_file_path.exists(): return specific_file_path else: return self.base_file_path class BuildOptions(NamedTuple): - package_dir: str - output_dir: str + package_dir: Path + output_dir: Path build_selector: BuildSelector environment: ParsedEnvironment before_build: Optional[str] @@ -123,5 +123,5 @@ class BuildOptions(NamedTuple): build_verbosity: int -resources_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'resources')) -get_pip_script = os.path.join(resources_dir, 'get-pip.py') +resources_dir = Path(__file__).resolve().parent / 'resources' +get_pip_script = resources_dir / 'get-pip.py' diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index c625b39a..97d84384 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -4,6 +4,7 @@ import subprocess import sys import tempfile from glob import glob +from pathlib import Path from zipfile import ZipFile from typing import Dict, List, Optional, NamedTuple @@ -19,7 +20,7 @@ from .util import ( ) -IS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache') +IS_RUNNING_ON_AZURE = Path('C:\\hostedtoolcache').exists() IS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows' @@ -71,36 +72,38 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi return python_configurations -def extract_zip(zip_src: str, dest: str) -> None: +def extract_zip(zip_src: Path, dest: Path) -> None: with ZipFile(zip_src) as zip: zip.extractall(dest) -def install_cpython(version: str, arch: str, nuget: str) -> str: +def install_cpython(version: str, arch: str, nuget: Path) -> Path: nuget_args = get_nuget_args(version, arch) - installation_path = os.path.join(nuget_args[-1], nuget_args[0] + '.' + version, 'tools') - shell([nuget, 'install'] + nuget_args) + installation_path = Path(nuget_args[-1]) / (nuget_args[0] + '.' + version) / 'tools' + shell([str(nuget), 'install'] + nuget_args) return installation_path -def install_pypy(version: str, arch: str, url: str) -> str: +def install_pypy(version: str, arch: str, url: str) -> Path: assert arch == '32' # Inside the PyPy zip file is a directory with the same name zip_filename = url.rsplit('/', 1)[-1] - installation_path = os.path.join('C:\\cibw', os.path.splitext(zip_filename)[0]) - if not os.path.exists(installation_path): - pypy_zip = os.path.join('C:\\cibw', zip_filename) + extension = ".zip" + assert zip_filename.endswith(extension) + installation_path = Path('C:\\cibw') / zip_filename[:-len(extension)] + if not installation_path.exists(): + pypy_zip = Path('C:\\cibw') / zip_filename download(url, pypy_zip) # Extract to the parent directory because the zip file still contains a directory - extract_zip(pypy_zip, os.path.dirname(installation_path)) + extract_zip(pypy_zip, installation_path.parent) pypy_exe = 'pypy3.exe' if version[0] == '3' else 'pypy.exe' - shell(['mklink', os.path.join(installation_path, 'python.exe'), os.path.join(installation_path, pypy_exe)]) + (installation_path / pypy_exe).symlink_to(installation_path / 'python.exe') return installation_path def setup_python(python_configuration: PythonConfiguration, dependency_constraint_flags: List[str], environment: ParsedEnvironment) -> Dict[str, str]: - nuget = 'C:\\cibw\\nuget.exe' - if not os.path.exists(nuget): + nuget = Path('C:\\cibw\\nuget.exe') + if not nuget.exists(): download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget) if python_configuration.identifier.startswith('cp'): @@ -111,15 +114,15 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain else: raise ValueError("Unknown Python implementation") - assert os.path.exists(os.path.join(installation_path, 'python.exe')) + assert (installation_path / 'python.exe').exists() # set up PATH and environment variables for run_with_env env = os.environ.copy() env['PYTHON_VERSION'] = python_configuration.version env['PYTHON_ARCH'] = python_configuration.arch env['PATH'] = os.pathsep.join([ - installation_path, - os.path.join(installation_path, 'Scripts'), + str(installation_path), + str(installation_path / 'Scripts'), env['PATH'] ]) # update env with results from CIBW_ENVIRONMENT @@ -130,16 +133,16 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain shell(['python', '--version'], env=env) shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)"'], env=env) where_python = subprocess.check_output(['where', 'python'], env=env, universal_newlines=True).splitlines()[0].strip() - if where_python != os.path.join(installation_path, 'python.exe'): + if where_python != str(installation_path / 'python.exe'): print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr) exit(1) # make sure pip is installed - if not os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe')): - shell(['python', get_pip_script] + dependency_constraint_flags, env=env, cwd="C:\\cibw") - assert os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe')) + if not (installation_path / 'Scripts' / 'pip.exe').exists(): + shell(['python', str(get_pip_script)] + dependency_constraint_flags, env=env, cwd="C:\\cibw") + assert (installation_path / 'Scripts' / 'pip.exe').exists() where_pip = subprocess.check_output(['where', 'pip'], env=env, universal_newlines=True).splitlines()[0].strip() - if where_pip.strip() != os.path.join(installation_path, 'Scripts', 'pip.exe'): + if where_pip.strip() != str(installation_path / 'Scripts' / 'pip.exe'): print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr) exit(1) @@ -152,12 +155,12 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain def build(options: BuildOptions) -> None: - temp_dir = tempfile.mkdtemp(prefix='cibuildwheel') - built_wheel_dir = os.path.join(temp_dir, 'built_wheel') - repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel') + temp_dir = Path(tempfile.mkdtemp(prefix='cibuildwheel')) + built_wheel_dir = temp_dir / 'built_wheel' + repaired_wheel_dir = temp_dir / 'repaired_wheel' # install nuget as best way to provide python - nuget = 'C:\\cibw\\nuget.exe' + nuget = Path('C:\\cibw\\nuget.exe') download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget) python_configurations = get_python_configurations(options.build_selector) @@ -165,7 +168,7 @@ def build(options: BuildOptions) -> None: dependency_constraint_flags = [] if options.dependency_constraints: dependency_constraint_flags = [ - '-c', options.dependency_constraints.get_for_python_version(config.version) + '-c', str(options.dependency_constraints.get_for_python_version(config.version)) ] # install Python @@ -177,39 +180,39 @@ def build(options: BuildOptions) -> None: shell([before_build_prepared], env=env) # build the wheel - if os.path.exists(built_wheel_dir): + if built_wheel_dir.exists(): shutil.rmtree(built_wheel_dir) - os.makedirs(built_wheel_dir) - # os.path.abspath is need. Without it pip wheel may try to fetch package from pypi.org + built_wheel_dir.mkdir(parents=True) + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org # see https://github.com/joerick/cibuildwheel/pull/369 - shell(['pip', 'wheel', os.path.abspath(options.package_dir), '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env) - built_wheel = glob(os.path.join(built_wheel_dir, '*.whl'))[0] + shell(['pip', 'wheel', str(options.package_dir.resolve()), '-w', str(built_wheel_dir), '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env) + built_wheel = next(built_wheel_dir.glob('*.whl')) # repair the wheel - if os.path.exists(repaired_wheel_dir): + if repaired_wheel_dir.exists(): shutil.rmtree(repaired_wheel_dir) - os.makedirs(repaired_wheel_dir) - if built_wheel.endswith('none-any.whl') or not options.repair_command: + repaired_wheel_dir.mkdir(parents=True) + if built_wheel.name.endswith('none-any.whl') or not options.repair_command: # pure Python wheel or empty repair command - shutil.move(built_wheel, repaired_wheel_dir) + shutil.move(str(built_wheel), repaired_wheel_dir) else: repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) shell([repair_command_prepared], env=env) - repaired_wheel = glob(os.path.join(repaired_wheel_dir, '*.whl'))[0] + repaired_wheel = next(repaired_wheel_dir.glob('*.whl')) if options.test_command: # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. shell(['pip', 'install', 'virtualenv'] + dependency_constraint_flags, env=env) - venv_dir = tempfile.mkdtemp() + venv_dir = Path(tempfile.mkdtemp()) # Use --no-download to ensure determinism by using seed libraries # built into virtualenv - shell(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) + shell(['python', '-m', 'virtualenv', '--no-download', str(venv_dir)], env=env) virtualenv_env = env.copy() virtualenv_env['PATH'] = os.pathsep.join([ - os.path.join(venv_dir, 'Scripts'), + str(venv_dir / 'Scripts'), virtualenv_env['PATH'], ]) @@ -225,7 +228,7 @@ def build(options: BuildOptions) -> None: shell([before_test_prepared], env=virtualenv_env) # install the wheel - shell(['pip', 'install', repaired_wheel + options.test_extras], env=virtualenv_env) + shell(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env) # test the wheel if options.test_requires: @@ -236,8 +239,8 @@ def build(options: BuildOptions) -> None: # and not the repo code) test_command_prepared = prepare_command( options.test_command, - project=os.path.abspath('.'), - package=os.path.abspath(options.package_dir) + project=Path('.').resolve(), + package=options.package_dir.resolve() ) shell([test_command_prepared], cwd='c:\\', env=virtualenv_env) @@ -245,7 +248,7 @@ def build(options: BuildOptions) -> None: shutil.rmtree(venv_dir) # we're all done here; move it to output (remove if already exists) - dst = os.path.join(options.output_dir, os.path.basename(repaired_wheel)) - if os.path.isfile(dst): - os.remove(dst) - shutil.move(repaired_wheel, dst) + dst = options.output_dir / repaired_wheel.name + if dst.is_file(): + dst.unlink() + shutil.move(str(repaired_wheel), dst) From 46a39b06cbd0b55c256d11af679b2ae76fd97fde Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Tue, 16 Jun 2020 14:54:50 +0200 Subject: [PATCH 004/117] Fix paths in unit tests --- unit_test/main_tests/conftest.py | 15 ++++++++------- unit_test/main_tests/main_options_test.py | 11 ++++++----- 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/unit_test/main_tests/conftest.py b/unit_test/main_tests/conftest.py index cd896915..1bfa0e50 100644 --- a/unit_test/main_tests/conftest.py +++ b/unit_test/main_tests/conftest.py @@ -1,6 +1,7 @@ import os import subprocess import sys +from pathlib import Path import pytest @@ -18,7 +19,7 @@ class ArgsInterceptor: self.kwargs = kwargs -MOCK_PACKAGE_DIR = 'some_package_dir' +MOCK_PACKAGE_DIR = Path('some_package_dir') @pytest.fixture(autouse=True) @@ -43,16 +44,16 @@ def fake_package_dir(monkeypatch): ''' Monkey-patch enough for the main() function to run ''' - real_os_path_exists = os.path.exists + real_path_exists = Path.exists - def mock_os_path_exists(path): - if path == os.path.join(MOCK_PACKAGE_DIR, 'setup.py'): + def mock_path_exists(path): + if path == MOCK_PACKAGE_DIR / 'setup.py': return True else: - return real_os_path_exists(path) + return real_path_exists(path) - monkeypatch.setattr(os.path, 'exists', mock_os_path_exists) - monkeypatch.setattr(sys, 'argv', ['cibuildwheel', MOCK_PACKAGE_DIR]) + monkeypatch.setattr(Path, 'exists', mock_path_exists) + monkeypatch.setattr(sys, 'argv', ['cibuildwheel', str(MOCK_PACKAGE_DIR)]) @pytest.fixture(params=['linux', 'macos', 'windows']) diff --git a/unit_test/main_tests/main_options_test.py b/unit_test/main_tests/main_options_test.py index 633e9cc5..14252e67 100644 --- a/unit_test/main_tests/main_options_test.py +++ b/unit_test/main_tests/main_options_test.py @@ -1,5 +1,6 @@ import sys from fnmatch import fnmatch +from pathlib import Path import pytest @@ -12,9 +13,9 @@ from cibuildwheel.util import BuildSelector def test_output_dir(platform, intercepted_build_args, monkeypatch): - OUTPUT_DIR = 'some_output_dir' + OUTPUT_DIR = Path('some_output_dir') - monkeypatch.setenv('CIBW_OUTPUT_DIR', OUTPUT_DIR) + monkeypatch.setenv('CIBW_OUTPUT_DIR', str(OUTPUT_DIR)) main() @@ -24,14 +25,14 @@ def test_output_dir(platform, intercepted_build_args, monkeypatch): def test_output_dir_default(platform, intercepted_build_args, monkeypatch): main() - assert intercepted_build_args.args[0].output_dir == 'wheelhouse' + assert intercepted_build_args.args[0].output_dir == Path('wheelhouse') @pytest.mark.parametrize('also_set_environment', [False, True]) def test_output_dir_argument(also_set_environment, platform, intercepted_build_args, monkeypatch): - OUTPUT_DIR = 'some_output_dir' + OUTPUT_DIR = Path('some_output_dir') - monkeypatch.setattr(sys, 'argv', sys.argv + ['--output-dir', OUTPUT_DIR]) + monkeypatch.setattr(sys, 'argv', sys.argv + ['--output-dir', str(OUTPUT_DIR)]) if also_set_environment: monkeypatch.setenv('CIBW_OUTPUT_DIR', 'not_this_output_dir') From c53e7824a7ddd792703a4e0c373e2460c1c45b7d Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Tue, 16 Jun 2020 15:02:09 +0200 Subject: [PATCH 005/117] Remove unused imports --- cibuildwheel/macos.py | 1 - cibuildwheel/windows.py | 1 - unit_test/main_tests/conftest.py | 1 - 3 files changed, 3 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index a4d611c4..a1f49110 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -4,7 +4,6 @@ import shutil import subprocess import sys import tempfile -from glob import glob from pathlib import Path from typing import Dict, List, Optional, NamedTuple, Union diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 97d84384..1f87bfa1 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -3,7 +3,6 @@ import shutil import subprocess import sys import tempfile -from glob import glob from pathlib import Path from zipfile import ZipFile diff --git a/unit_test/main_tests/conftest.py b/unit_test/main_tests/conftest.py index 1bfa0e50..d6cc479b 100644 --- a/unit_test/main_tests/conftest.py +++ b/unit_test/main_tests/conftest.py @@ -1,4 +1,3 @@ -import os import subprocess import sys from pathlib import Path From b3ffb04a2c8dac3deae41bdfda466214f206abdc Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Tue, 16 Jun 2020 15:18:32 +0200 Subject: [PATCH 006/117] Fix order of pathlib.Path.symlink_to arguments in windows.py --- cibuildwheel/windows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 1f87bfa1..c81566d1 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -96,7 +96,7 @@ def install_pypy(version: str, arch: str, url: str) -> Path: # Extract to the parent directory because the zip file still contains a directory extract_zip(pypy_zip, installation_path.parent) pypy_exe = 'pypy3.exe' if version[0] == '3' else 'pypy.exe' - (installation_path / pypy_exe).symlink_to(installation_path / 'python.exe') + (installation_path / 'python.exe').symlink_to(installation_path / pypy_exe) return installation_path From e58aed1731392a373523a0256545524842d969f5 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Tue, 16 Jun 2020 18:56:25 +0200 Subject: [PATCH 007/117] Fix overlooked os.path.abspath --- cibuildwheel/macos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index a1f49110..5576641f 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -249,7 +249,7 @@ def build(options: BuildOptions) -> None: test_command_prepared = prepare_command( options.test_command, project=Path('.').resolve(), - package=options.package_dir + package=options.package_dir.resolve() ) call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True) From 265708854a8e6e71642d6d4675f76149cda80886 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Wed, 17 Jun 2020 00:18:40 +0200 Subject: [PATCH 008/117] Attend to review remarks --- cibuildwheel/__main__.py | 5 +++-- cibuildwheel/linux.py | 10 +++++----- cibuildwheel/macos.py | 8 ++++---- cibuildwheel/util.py | 6 +++--- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index fd4c418e..521710a6 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -21,6 +21,7 @@ from cibuildwheel.util import ( BuildSelector, DependencyConstraints, Unbuffered, + resources_dir, ) @@ -165,7 +166,7 @@ def main() -> None: # This needs to be passed on to the docker container in linux.py os.environ['CIBUILDWHEEL'] = '1' - if not any((package_dir / name).exists() + if not any(package_dir.joinpath(name).exists() for name in ["setup.py", "setup.cfg", "pyproject.toml"]): print('cibuildwheel: Could not find setup.py, setup.cfg or pyproject.toml at root of package', file=sys.stderr) exit(2) @@ -176,7 +177,7 @@ def main() -> None: manylinux_images: Optional[Dict[str, str]] = None if platform == 'linux': - pinned_docker_images_file = Path(__file__).parent / 'resources' / 'pinned_docker_images.cfg' + pinned_docker_images_file = resources_dir / 'pinned_docker_images.cfg' all_pinned_docker_images = ConfigParser() all_pinned_docker_images.read(pinned_docker_images_file) # all_pinned_docker_images looks like a dict of dicts, e.g. diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index cdaef019..99e779c4 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -5,7 +5,7 @@ import subprocess import sys import textwrap import uuid -from pathlib import Path +from pathlib import Path, PurePath from typing import List, NamedTuple, Optional, Union @@ -105,12 +105,12 @@ def build(options: BuildOptions) -> None: ('pp', 'manylinux_x86_64', options.manylinux_images['pypy_x86_64']), ] - pwd = Path().resolve() + cwd = Path.cwd() abs_package_dir = options.package_dir.resolve() - if pwd != abs_package_dir and pwd not in abs_package_dir.parents: + if cwd != abs_package_dir and cwd not in abs_package_dir.parents: raise Exception('package_dir must be inside the working directory') - container_package_dir = Path('/project') / abs_package_dir.relative_to(pwd) + container_package_dir = PurePath('/project') / abs_package_dir.relative_to(cwd) for implementation, platform_tag, docker_image in platforms: platform_configs = [c for c in python_configurations if c.identifier.startswith(implementation) and c.identifier.endswith(platform_tag)] @@ -302,5 +302,5 @@ def troubleshoot(package_dir: Path, error: Exception) -> None: ''')) print(' Files detected:') - print('\n'.join([' ' + str(f) for f in so_files])) + print('\n'.join([f' {f}' for f in so_files])) print('') diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 5576641f..0e24f22d 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -24,7 +24,7 @@ def call(args: Union[str, List[str]], env: Optional[Dict[str, str]] = None, cwd: if shell: print(f'+ {args}') else: - print('+ ' + ' '.join(shlex.quote(str(a)) for a in args)) + print('+ ' + ' '.join(shlex.quote(a) for a in args)) return subprocess.check_call(args, env=env, cwd=cwd, shell=shell) @@ -65,9 +65,9 @@ def make_symlinks(installation_bin_path: Path, python_executable: str, pip_execu shutil.rmtree(SYMLINKS_DIR) SYMLINKS_DIR.mkdir(parents=True) - (SYMLINKS_DIR / 'python').symlink_to(installation_bin_path / python_executable) - (SYMLINKS_DIR / 'python-config').symlink_to(installation_bin_path / (python_executable + '-config')) - (SYMLINKS_DIR / 'pip').symlink_to(installation_bin_path / pip_executable) + SYMLINKS_DIR.joinpath('python').symlink_to(installation_bin_path / python_executable) + SYMLINKS_DIR.joinpath('python-config').symlink_to(installation_bin_path / (python_executable + '-config')) + SYMLINKS_DIR.joinpath('pip').symlink_to(installation_bin_path / pip_executable) def install_cpython(version: str, url: str) -> Path: diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 0b585b86..feef6185 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -90,7 +90,7 @@ class DependencyConstraints: @staticmethod def with_defaults() -> 'DependencyConstraints': return DependencyConstraints( - base_file_path=Path(__file__).parent / 'resources' / 'constraints.txt' + base_file_path=resources_dir / 'constraints.txt' ) def get_for_python_version(self, version: str) -> Path: @@ -99,8 +99,8 @@ class DependencyConstraints: # try to find a version-specific dependency file e.g. if # ./constraints.txt is the base, look for ./constraints-python27.txt specific_stem = self.base_file_path.stem + f'-python{version_parts[0]}{version_parts[1]}' - sepcific_name = specific_stem + self.base_file_path.suffix - specific_file_path = self.base_file_path.with_name(sepcific_name) + specific_name = specific_stem + self.base_file_path.suffix + specific_file_path = self.base_file_path.with_name(specific_name) if specific_file_path.exists(): return specific_file_path else: From b62b17a223dfce88d35a2301b3aeaab1ade271d1 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Wed, 17 Jun 2020 11:57:16 +0200 Subject: [PATCH 009/117] Replace shutil.move with Path.rename or Path.replace --- cibuildwheel/macos.py | 5 ++--- cibuildwheel/windows.py | 7 ++----- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 0e24f22d..bd78499d 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -207,7 +207,7 @@ def build(options: BuildOptions) -> None: repaired_wheel_dir.mkdir(parents=True) if built_wheel.name.endswith('none-any.whl') or not options.repair_command: # pure Python wheel or empty repair command - shutil.move(str(built_wheel), repaired_wheel_dir) + built_wheel.rename(repaired_wheel_dir / built_wheel.name) else: repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) call(repair_command_prepared, env=env, shell=True) @@ -257,5 +257,4 @@ def build(options: BuildOptions) -> None: shutil.rmtree(venv_dir) # we're all done here; move it to output (overwrite existing) - dst = options.output_dir / repaired_wheel.name - shutil.move(str(repaired_wheel), dst) + repaired_wheel.replace(options.output_dir / repaired_wheel.name) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index c81566d1..e81fe301 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -193,7 +193,7 @@ def build(options: BuildOptions) -> None: repaired_wheel_dir.mkdir(parents=True) if built_wheel.name.endswith('none-any.whl') or not options.repair_command: # pure Python wheel or empty repair command - shutil.move(str(built_wheel), repaired_wheel_dir) + built_wheel.rename(repaired_wheel_dir / built_wheel.name) else: repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) shell([repair_command_prepared], env=env) @@ -247,7 +247,4 @@ def build(options: BuildOptions) -> None: shutil.rmtree(venv_dir) # we're all done here; move it to output (remove if already exists) - dst = options.output_dir / repaired_wheel.name - if dst.is_file(): - dst.unlink() - shutil.move(str(repaired_wheel), dst) + repaired_wheel.replace(options.output_dir / repaired_wheel.name) From f012fa1af0a08be3a236dea23035d66f1a2c62fb Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Wed, 17 Jun 2020 12:06:39 +0200 Subject: [PATCH 010/117] Mock Path.mkdir calls in unit tests --- unit_test/main_tests/conftest.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/unit_test/main_tests/conftest.py b/unit_test/main_tests/conftest.py index d6cc479b..3f684b6c 100644 --- a/unit_test/main_tests/conftest.py +++ b/unit_test/main_tests/conftest.py @@ -31,12 +31,17 @@ def mock_protection(monkeypatch): def fail_on_call(*args, **kwargs): raise RuntimeError("This should never be called") + def ignore_call(*args, **kwargs): + pass + monkeypatch.setattr(subprocess, 'Popen', fail_on_call) monkeypatch.setattr(util, 'download', fail_on_call) monkeypatch.setattr(windows, 'build', fail_on_call) monkeypatch.setattr(linux, 'build', fail_on_call) monkeypatch.setattr(macos, 'build', fail_on_call) + monkeypatch.setattr(Path, 'mkdir', ignore_call) + @pytest.fixture(autouse=True) def fake_package_dir(monkeypatch): From acf5c248816dbfb5bc9f0aa4735825b8a411b222 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Thu, 18 Jun 2020 02:00:30 +0200 Subject: [PATCH 011/117] Use patlib.Path in tests and tools --- bin/run_tests.py | 3 +- .../mkdocs_include_markdown_plugin/plugin.py | 17 ++++------ setup.py | 8 ++--- test/test_dependency_versions.py | 7 ++-- test/test_projects/base.py | 12 ++++--- test/test_subdir_package.py | 4 +-- test/utils.py | 5 +-- unit_test/dependency_constraints_test.py | 32 ++++++------------- 8 files changed, 35 insertions(+), 53 deletions(-) diff --git a/bin/run_tests.py b/bin/run_tests.py index 93c23e88..117f8b9f 100755 --- a/bin/run_tests.py +++ b/bin/run_tests.py @@ -3,10 +3,11 @@ import os import subprocess import sys +from pathlib import Path if __name__ == '__main__': # move cwd to the project root - os.chdir(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + os.chdir(Path(__file__).resolve().parents[1]) # run the unit tests subprocess.check_call([sys.executable, '-m', 'pytest', 'unit_test']) diff --git a/docs/mkdocs_include_markdown_plugin/mkdocs_include_markdown_plugin/plugin.py b/docs/mkdocs_include_markdown_plugin/mkdocs_include_markdown_plugin/plugin.py index b6b7a723..f12ea469 100644 --- a/docs/mkdocs_include_markdown_plugin/mkdocs_include_markdown_plugin/plugin.py +++ b/docs/mkdocs_include_markdown_plugin/mkdocs_include_markdown_plugin/plugin.py @@ -1,7 +1,6 @@ import cgi -import io -import os import re +from pathlib import Path import mkdocs @@ -41,13 +40,12 @@ class ImportMarkdownPlugin(mkdocs.plugins.BasePlugin): def found_include_tag(match): filename = match.group('filename') - file_path_abs = os.path.join(os.path.dirname(page_src_path), filename) + file_path_abs = Path(page_src_path).parent / filename - if not os.path.exists(file_path_abs): + if not file_path_abs.exists(): raise ValueError('file not found', filename) - with io.open(file_path_abs, encoding='utf8') as f: - text_to_include = f.read() + text_to_include = file_path_abs.read_text(encoding='utf8') # Allow good practice of having a final newline in the file if text_to_include.endswith('\n'): @@ -60,13 +58,12 @@ class ImportMarkdownPlugin(mkdocs.plugins.BasePlugin): start = match.group('start') end = match.group('end') - file_path_abs = os.path.join(os.path.dirname(page_src_path), filename) + file_path_abs = Path(page_src_path).parent / filename - if not os.path.exists(file_path_abs): + if not file_path_abs.exists(): raise ValueError('file not found', filename) - with io.open(file_path_abs, encoding='utf8') as f: - text_to_include = f.read() + text_to_include = file_path_abs.read_text(encoding='utf8') if start: _, _, text_to_include = text_to_include.partition(start) diff --git a/setup.py b/setup.py index 3e4c90cb..d05407e5 100644 --- a/setup.py +++ b/setup.py @@ -1,15 +1,13 @@ # -*- coding: utf-8 -*- -import io -import os +from pathlib import Path try: from setuptools import setup except ImportError: from distutils.core import setup -this_directory = os.path.dirname(__file__) -with io.open(os.path.join(this_directory, 'README.md'), encoding='utf-8') as f: - long_description = f.read() +this_directory = Path(__file__).parent +long_description = this_directory.joinpath('README.md').read_text(encoding='utf-8') setup( name='cibuildwheel', diff --git a/test/test_dependency_versions.py b/test/test_dependency_versions.py index 611350f1..2f79bd2a 100644 --- a/test/test_dependency_versions.py +++ b/test/test_dependency_versions.py @@ -1,4 +1,3 @@ -import os import re import pytest import textwrap @@ -39,11 +38,9 @@ VERSION_REGEX = r'([\w-]+)==([^\s]+)' def get_versions_from_constraint_file(constraint_file): - with open(constraint_file, encoding='utf8') as f: - constraint_file_text = f.read() + constraint_file_text = constraint_file.read_text(encoding='utf8') versions = {} - for package, version in re.findall(VERSION_REGEX, constraint_file_text): versions[package] = version @@ -73,7 +70,7 @@ def test_pinned_versions(tmp_path, python_version): constraint_filename = 'constraints.txt' build_pattern = '[cp]p38-*' - constraint_file = os.path.join(cibuildwheel.util.resources_dir, constraint_filename) + constraint_file = cibuildwheel.util.resources_dir / constraint_filename constraint_versions = get_versions_from_constraint_file(constraint_file) for package in ['pip', 'setuptools', 'wheel', 'virtualenv']: diff --git a/test/test_projects/base.py b/test/test_projects/base.py index 5cac80ad..b869f4f2 100644 --- a/test/test_projects/base.py +++ b/test/test_projects/base.py @@ -1,5 +1,7 @@ -import os +from pathlib import Path + import jinja2 + from typing import Union, Dict, Any @@ -21,12 +23,12 @@ class TestProject: self.files = {} self.template_context = {} - def generate(self, path: str): + def generate(self, path: Path): for filename, content in self.files.items(): - file_path = os.path.join(path, filename) - os.makedirs(os.path.dirname(file_path), exist_ok=True) + file_path = path / filename + file_path.parent.mkdir(parents=True, exist_ok=True) - with open(file_path, 'w', encoding='utf8') as f: + with file_path.open('w', encoding='utf8') as f: if isinstance(content, jinja2.Template): content = content.render(self.template_context) diff --git a/test/test_subdir_package.py b/test/test_subdir_package.py index 98b1b8cd..14a7a58b 100644 --- a/test/test_subdir_package.py +++ b/test/test_subdir_package.py @@ -1,4 +1,4 @@ -import os +from pathlib import Path import jinja2 @@ -35,7 +35,7 @@ def test(capfd, tmp_path): project_dir = tmp_path / 'project' subdir_package_project.generate(project_dir) - package_dir = os.path.join('src', 'spam') + package_dir = Path('src', 'spam') # build the wheels actual_wheels = utils.cibuildwheel_run(project_dir, package_dir=package_dir, add_env={ 'CIBW_BEFORE_BUILD': 'python {project}/bin/before_build.py', diff --git a/test/utils.py b/test/utils.py index 15ce402a..217a616f 100644 --- a/test/utils.py +++ b/test/utils.py @@ -10,9 +10,10 @@ import shutil import subprocess import sys from contextlib import contextmanager +from pathlib import Path from tempfile import mkdtemp -IS_WINDOWS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache') +IS_WINDOWS_RUNNING_ON_AZURE = Path('C:\\hostedtoolcache').exists() IS_WINDOWS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows' @@ -66,7 +67,7 @@ def cibuildwheel_run(project_path, package_dir='.', env=None, add_env=None, outp with TemporaryDirectoryIfNone(output_dir) as _output_dir: subprocess.check_call( - [sys.executable, '-m', 'cibuildwheel', '--output-dir', str(_output_dir), package_dir], + [sys.executable, '-m', 'cibuildwheel', '--output-dir', str(_output_dir), str(package_dir)], env=env, cwd=project_path, ) diff --git a/unit_test/dependency_constraints_test.py b/unit_test/dependency_constraints_test.py index b3fb0cb3..5347011c 100644 --- a/unit_test/dependency_constraints_test.py +++ b/unit_test/dependency_constraints_test.py @@ -1,30 +1,16 @@ from cibuildwheel.util import DependencyConstraints -import os + +from pathlib import Path def test_defaults(): dependency_constraints = DependencyConstraints.with_defaults() - project_root = os.path.dirname(os.path.dirname(__file__)) - resources_dir = os.path.join(project_root, 'cibuildwheel', 'resources') + project_root = Path(__file__).parents[1] + resources_dir = project_root / 'cibuildwheel' / 'resources' - assert os.path.samefile( - dependency_constraints.base_file_path, - os.path.join(resources_dir, 'constraints.txt') - ) - assert os.path.samefile( - dependency_constraints.get_for_python_version('3.8'), - os.path.join(resources_dir, 'constraints.txt') - ) - assert os.path.samefile( - dependency_constraints.get_for_python_version('3.6'), - os.path.join(resources_dir, 'constraints-python36.txt') - ) - assert os.path.samefile( - dependency_constraints.get_for_python_version('3.5'), - os.path.join(resources_dir, 'constraints-python35.txt') - ) - assert os.path.samefile( - dependency_constraints.get_for_python_version('2.7'), - os.path.join(resources_dir, 'constraints-python27.txt') - ) + assert dependency_constraints.base_file_path.samefile(resources_dir / 'constraints.txt') + assert dependency_constraints.get_for_python_version('3.8').samefile(resources_dir / 'constraints.txt') + assert dependency_constraints.get_for_python_version('3.6').samefile(resources_dir / 'constraints-python36.txt') + assert dependency_constraints.get_for_python_version('3.5').samefile(resources_dir / 'constraints-python35.txt') + assert dependency_constraints.get_for_python_version('2.7').samefile(resources_dir / 'constraints-python27.txt') From 3b1f0dd4536cc5cef2c4337ae04392e558b5e3fb Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Sun, 17 May 2020 18:24:58 +0200 Subject: [PATCH 012/117] add pre-commit configuration --- .pre-commit-config.yaml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 .pre-commit-config.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..6a3e303e --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,22 @@ +repos: +- repo: https://gitlab.com/pycqa/flake8 + rev: 3.8.1 + hooks: + - id: flake8 + pass_filenames: true + # this seems to need to be here in addition to setup.cfg + exclude: cibuildwheel/resources/ +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: check-json + - id: check-yaml + - id: check-merge-conflict + - id: end-of-file-fixer +- repo: https://github.com/pre-commit/mirrors-mypy + rev: '' # Use the sha / tag you want to point at + hooks: + - id: mypy + args: [--strict] + files: cibuildwheel/* + exclude: test|cibuildwheel/resources From d43817b49a8bea0b7517b91f77c851432c2a5e72 Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Sun, 17 May 2020 18:26:25 +0200 Subject: [PATCH 013/117] fixed files --- .circleci/config.yml | 2 +- azure-pipelines.yml | 1 - cibuildwheel/resources/pinned_docker_images.cfg | 1 - docs/extra.js | 1 - 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index cf0e47b2..ae7228a8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,7 +9,7 @@ jobs: - run: name: Install flake8 & mypy - command: sudo python -m pip install flake8 mypy + command: sudo python -m pip install flake8==3.8.1 mypy - run: name: flake8 command: flake8 diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 4fb53b11..3eaed5f4 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -42,4 +42,3 @@ jobs: - bash: | python -m pip install -r requirements-dev.txt pytest-azurepipelines python ./bin/run_tests.py - diff --git a/cibuildwheel/resources/pinned_docker_images.cfg b/cibuildwheel/resources/pinned_docker_images.cfg index 534d350b..ea617735 100644 --- a/cibuildwheel/resources/pinned_docker_images.cfg +++ b/cibuildwheel/resources/pinned_docker_images.cfg @@ -19,4 +19,3 @@ manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2020-05-17-2f8ac3b [s390x] manylinux2014 = quay.io/pypa/manylinux2014_s390x:2020-05-17-2f8ac3b - diff --git a/docs/extra.js b/docs/extra.js index d32c3cc0..1965a84e 100644 --- a/docs/extra.js +++ b/docs/extra.js @@ -2,4 +2,3 @@ $('.rst-content pre') .prev('blockquote') .addClass('code-block-filename'); - From 1a8396726141bdb0a112b9f58f4e3def9d6a9e77 Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Sun, 17 May 2020 22:30:17 +0200 Subject: [PATCH 014/117] remove json check --- .pre-commit-config.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 6a3e303e..90b33545 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -9,7 +9,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks rev: v2.3.0 hooks: - - id: check-json - id: check-yaml - id: check-merge-conflict - id: end-of-file-fixer From 39101ae8ea053ff6cbdf1dc198f0c2ed39dabfee Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Sun, 14 Jun 2020 21:20:05 +0200 Subject: [PATCH 015/117] update mypy config --- .pre-commit-config.yaml | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 90b33545..1112c38c 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -15,7 +15,6 @@ repos: - repo: https://github.com/pre-commit/mirrors-mypy rev: '' # Use the sha / tag you want to point at hooks: - - id: mypy - args: [--strict] - files: cibuildwheel/* - exclude: test|cibuildwheel/resources + - id: mypy + files: cibuildwheel|test + exclude: cibuildwheel/resources From aa99405a9c9826275e2efb74190e85c6332d95e5 Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Fri, 19 Jun 2020 21:14:07 +0200 Subject: [PATCH 016/117] unpin flake, exclude pinned_docker_images.cfg --- .circleci/config.yml | 2 +- .pre-commit-config.yaml | 1 + cibuildwheel/resources/pinned_docker_images.cfg | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index ae7228a8..cf0e47b2 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -9,7 +9,7 @@ jobs: - run: name: Install flake8 & mypy - command: sudo python -m pip install flake8==3.8.1 mypy + command: sudo python -m pip install flake8 mypy - run: name: flake8 command: flake8 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1112c38c..15c843f9 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -12,6 +12,7 @@ repos: - id: check-yaml - id: check-merge-conflict - id: end-of-file-fixer + exclude: cibuildwheel/resources/pinned_docker_images.cfg - repo: https://github.com/pre-commit/mirrors-mypy rev: '' # Use the sha / tag you want to point at hooks: diff --git a/cibuildwheel/resources/pinned_docker_images.cfg b/cibuildwheel/resources/pinned_docker_images.cfg index ea617735..534d350b 100644 --- a/cibuildwheel/resources/pinned_docker_images.cfg +++ b/cibuildwheel/resources/pinned_docker_images.cfg @@ -19,3 +19,4 @@ manylinux2014 = quay.io/pypa/manylinux2014_ppc64le:2020-05-17-2f8ac3b [s390x] manylinux2014 = quay.io/pypa/manylinux2014_s390x:2020-05-17-2f8ac3b + From d552d29eae256dca594df79dc6b2d828cad2c6ab Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Sun, 21 Jun 2020 13:14:53 +0200 Subject: [PATCH 017/117] Revert use of Path.joinpath instead of Path.__div__ --- cibuildwheel/__main__.py | 2 +- cibuildwheel/macos.py | 6 +++--- setup.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 521710a6..137bc19e 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -166,7 +166,7 @@ def main() -> None: # This needs to be passed on to the docker container in linux.py os.environ['CIBUILDWHEEL'] = '1' - if not any(package_dir.joinpath(name).exists() + if not any((package_dir / name).exists() for name in ["setup.py", "setup.cfg", "pyproject.toml"]): print('cibuildwheel: Could not find setup.py, setup.cfg or pyproject.toml at root of package', file=sys.stderr) exit(2) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index bd78499d..2fd28d20 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -65,9 +65,9 @@ def make_symlinks(installation_bin_path: Path, python_executable: str, pip_execu shutil.rmtree(SYMLINKS_DIR) SYMLINKS_DIR.mkdir(parents=True) - SYMLINKS_DIR.joinpath('python').symlink_to(installation_bin_path / python_executable) - SYMLINKS_DIR.joinpath('python-config').symlink_to(installation_bin_path / (python_executable + '-config')) - SYMLINKS_DIR.joinpath('pip').symlink_to(installation_bin_path / pip_executable) + (SYMLINKS_DIR / 'python').symlink_to(installation_bin_path / python_executable) + (SYMLINKS_DIR / 'python-config').symlink_to(installation_bin_path / (python_executable + '-config')) + (SYMLINKS_DIR / 'pip').symlink_to(installation_bin_path / pip_executable) def install_cpython(version: str, url: str) -> Path: diff --git a/setup.py b/setup.py index d05407e5..34528b4d 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ except ImportError: from distutils.core import setup this_directory = Path(__file__).parent -long_description = this_directory.joinpath('README.md').read_text(encoding='utf-8') +long_description = (this_directory / 'README.md').read_text(encoding='utf-8') setup( name='cibuildwheel', From 2b22ffda9bd6b8433b90e3dd1183c09a5632143c Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 22 Jun 2020 11:42:33 +0100 Subject: [PATCH 018/117] Add python3.9 note to version matrix --- README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 6842b249..d3805547 100644 --- a/README.md +++ b/README.md @@ -22,11 +22,13 @@ What does it do? | CPython 3.6 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅² | | CPython 3.7 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅² | | CPython 3.8 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅² | +| CPython 3.9³ | 🛠 | 🛠 | 🛠 | 🛠 | 🛠 | 🛠 | 🛠 | | | PyPy 2.7 v7.3.0 | ✅ | | ✅ | ✅ | | | | | | PyPy 3.6 v7.3.0 | ✅ | | ✅ | ✅ | | | | | ¹ Not supported on Travis
-² Beta support until Travis CI fixes a bug +² Beta support until Travis CI fixes a bug
+³ Python 3.9 is not yet ABI stable, so you shouldn't publish wheels with it yet. But if you want to check that your wheels build on Python 3.9, try our [`python3.9` branch](https://github.com/joerick/cibuildwheel/pull/382)! - Builds manylinux, macOS and Windows wheels for CPython and PyPy using Azure Pipelines, Travis CI, AppVeyor, and CircleCI - Bundles shared library dependencies on Linux and macOS through [auditwheel](https://github.com/pypa/auditwheel) and [delocate](https://github.com/matthew-brett/delocate) From e6ac5274160279826b301a14514d3270fc575592 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Mon, 22 Jun 2020 12:08:40 +0100 Subject: [PATCH 019/117] Convert the workaround to use Path objects --- cibuildwheel/windows.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 63b22b48..f8a50015 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -154,7 +154,7 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain return env -def pep_518_cp35_workaround(package_dir: str, env: Dict[str, str]) -> None: +def pep_518_cp35_workaround(package_dir: Path, env: Dict[str, str]) -> None: """ Python 3.5 PEP 518 hack (see https://github.com/pypa/pip/issues/8392#issuecomment-639563494) Basically, nuget's Python is an embedded Python distribution, which is not supported by pip. @@ -166,9 +166,9 @@ def pep_518_cp35_workaround(package_dir: str, env: Dict[str, str]) -> None: mostly "isolated". """ - pyproject_path = os.path.join(package_dir, 'pyproject.toml') + pyproject_path = package_dir / 'pyproject.toml' - if os.path.exists(pyproject_path): + if pyproject_path.exists(): data = toml.load(pyproject_path) requirements = ( data['build-system'].get('requires', []) From 6501470357e6ccec3090f5eaafe0f834e42eab69 Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Mon, 11 May 2020 17:09:54 +0200 Subject: [PATCH 020/117] Add BEFORE_ALL options --- cibuildwheel/__main__.py | 2 ++ cibuildwheel/linux.py | 4 +++ cibuildwheel/macos.py | 4 +++ cibuildwheel/util.py | 3 +- cibuildwheel/windows.py | 4 +++ docs/options.md | 19 ++++++++++ test/14_before_all/cibuildwheel_test.py | 23 ++++++++++++ test/14_before_all/setup.py | 23 ++++++++++++ test/14_before_all/spam.c | 48 +++++++++++++++++++++++++ 9 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 test/14_before_all/cibuildwheel_test.py create mode 100644 test/14_before_all/setup.py create mode 100644 test/14_before_all/spam.c diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 137bc19e..bce3f754 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -128,6 +128,7 @@ def main() -> None: build_config, skip_config = os.environ.get('CIBW_BUILD', '*'), os.environ.get('CIBW_SKIP', '') environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform, default='') + before_all = get_option_from_environment('CIBW_BEFORE_ALL', platform=platform) before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform) repair_command = get_option_from_environment('CIBW_REPAIR_WHEEL_COMMAND', platform=platform, default=repair_command_default) dependency_versions = get_option_from_environment('CIBW_DEPENDENCY_VERSIONS', platform=platform, default='pinned') @@ -212,6 +213,7 @@ def main() -> None: test_extras=test_extras, before_test=before_test, before_build=before_build, + before_all=before_all, build_verbosity=build_verbosity, build_selector=build_selector, repair_command=repair_command, diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 99e779c4..8bbda24d 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -133,6 +133,10 @@ def build(options: BuildOptions) -> None: call(['docker', 'start', container_name]) + if options.before_all: + before_all_prepared = prepare_command(options.before_all, project='/project', package=options.package_dir) + call(['docker', 'exec', '-i', container_name] + shell_cmd, universal_newlines=True, input=before_all_prepared) + for config in platform_configs: if options.dependency_constraints: constraints_file = options.dependency_constraints.get_for_python_version(config.version) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 2fd28d20..3e67b534 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -176,6 +176,10 @@ def build(options: BuildOptions) -> None: built_wheel_dir = temp_dir / 'built_wheel' repaired_wheel_dir = temp_dir / 'repaired_wheel' + if options.before_all: + before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) + call([before_all_prepared], shell=True) + python_configurations = get_python_configurations(options.build_selector) for config in python_configurations: diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index feef6185..f7201462 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -112,6 +112,7 @@ class BuildOptions(NamedTuple): output_dir: Path build_selector: BuildSelector environment: ParsedEnvironment + before_all: str before_build: Optional[str] repair_command: str manylinux_images: Optional[Dict[str, str]] @@ -120,7 +121,7 @@ class BuildOptions(NamedTuple): before_test: Optional[str] test_requires: List[str] test_extras: str - build_verbosity: int +x build_verbosity: int resources_dir = Path(__file__).resolve().parent / 'resources' diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index e81fe301..73628c7f 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -162,6 +162,10 @@ def build(options: BuildOptions) -> None: nuget = Path('C:\\cibw\\nuget.exe') download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget) + if options.before_all: + before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) + shell([before_all_prepared]) + python_configurations = get_python_configurations(options.build_selector) for config in python_configurations: dependency_constraint_flags = [] diff --git a/docs/options.md b/docs/options.md index 18d5417b..0d61060d 100644 --- a/docs/options.md +++ b/docs/options.md @@ -190,6 +190,25 @@ CIBW_ENVIRONMENT: "BUILD_TIME=$(date) SAMPLE_TEXT=\"sample text\"" !!! note `cibuildwheel` always defines the environment variable `CIBUILDWHEEL=1`. This can be useful for [building wheels with optional extensions](faq.md#building-packages-with-optional-c-extensions). +### `CIBW_BEFORE_ALL` {: #before-all} +> Execute a shell command preparing common part for each wheel. + +Shell command to prepare common part of project (ex. build libraries which does not depend on python). +This option is added mainly for linux build, because linux wheels are build in isolated from host docker containers. + +The placeholder `{package}` can be used here; it will be replaced by the path to the package being built by `cibuildwheel`. + +Platform-specific variants also available:
+ `CIBW_BEFORE_ALL_MACOS` | `CIBW_BEFORE_ALL_WINDOWS` | `CIBW_BEFORE_ALL_LINUX` + +#### Examples +```yaml +# build third party library +CIBW_BEFORE_ALL: make -C third_party_lib + +# install system library +CIBW_BEFORE_ALL: yum install -y libffi-dev +``` ### `CIBW_BEFORE_BUILD` {: #before-build} > Execute a shell command preparing each wheel's build diff --git a/test/14_before_all/cibuildwheel_test.py b/test/14_before_all/cibuildwheel_test.py new file mode 100644 index 00000000..e5340e35 --- /dev/null +++ b/test/14_before_all/cibuildwheel_test.py @@ -0,0 +1,23 @@ +import os + +import utils + + +def test(): + project_dir = os.path.dirname(__file__) + + with open(os.path.join(project_dir, "text_info.txt"), mode='w') as ff: + print("dummy text", file=ff) + + # build the wheels + actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ + # write python version information to a temporary file, this is + # checked in setup.py + 'CIBW_BEFORE_ALL': '''python -c "open('{project}/text_info.txt', 'w').write('sample text')"''', + }) + + + # also check that we got the right wheels + os.remove(os.path.join(project_dir, "text_info.txt")) + expected_wheels = utils.expected_wheels('spam', '0.1.0') + assert set(actual_wheels) == set(expected_wheels) diff --git a/test/14_before_all/setup.py b/test/14_before_all/setup.py new file mode 100644 index 00000000..7b68808b --- /dev/null +++ b/test/14_before_all/setup.py @@ -0,0 +1,23 @@ +import os +import sys + +from setuptools import ( + Extension, + setup, +) + +# assert that the Python version as written to text_info.txt in the CIBW_BEFORE_ALL step +# is the same one as is currently running. +with open("text_info.txt") as f: + stored_text = f.read() + +print("## stored text: "+stored_text) +assert stored_text == "sample text" + + + +setup( + name="spam", + ext_modules=[Extension('spam', sources=['spam.c'])], + version="0.1.0", +) diff --git a/test/14_before_all/spam.c b/test/14_before_all/spam.c new file mode 100644 index 00000000..d1ab0f22 --- /dev/null +++ b/test/14_before_all/spam.c @@ -0,0 +1,48 @@ +#include + +static PyObject * +spam_system(PyObject *self, PyObject *args) +{ + const char *command; + int sts; + + if (!PyArg_ParseTuple(args, "s", &command)) + return NULL; + sts = system(command); + return PyLong_FromLong(sts); +} + +/* Module initialization */ + +#if PY_MAJOR_VERSION >= 3 + #define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void) + #define MOD_DEF(m, name, doc, methods, module_state_size) \ + static struct PyModuleDef moduledef = { \ + PyModuleDef_HEAD_INIT, name, doc, module_state_size, methods, }; \ + m = PyModule_Create(&moduledef); + #define MOD_RETURN(m) return m; +#else + #define MOD_INIT(name) PyMODINIT_FUNC init##name(void) + #define MOD_DEF(m, name, doc, methods, module_state_size) \ + m = Py_InitModule3(name, methods, doc); + #define MOD_RETURN(m) return; +#endif + +static PyMethodDef module_methods[] = { + {"system", (PyCFunction)spam_system, METH_VARARGS, + "Execute a shell command."}, + {NULL} /* Sentinel */ +}; + +MOD_INIT(spam) +{ + PyObject* m; + + MOD_DEF(m, + "spam", + "Example module", + module_methods, + -1) + + MOD_RETURN(m) +} From 75a0384d027f5b2c5fe16945d652a2c3dee31d58 Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Mon, 11 May 2020 17:28:47 +0200 Subject: [PATCH 021/117] fix flake8 errors --- test/14_before_all/cibuildwheel_test.py | 1 - test/14_before_all/setup.py | 6 +----- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/test/14_before_all/cibuildwheel_test.py b/test/14_before_all/cibuildwheel_test.py index e5340e35..d5be005a 100644 --- a/test/14_before_all/cibuildwheel_test.py +++ b/test/14_before_all/cibuildwheel_test.py @@ -16,7 +16,6 @@ def test(): 'CIBW_BEFORE_ALL': '''python -c "open('{project}/text_info.txt', 'w').write('sample text')"''', }) - # also check that we got the right wheels os.remove(os.path.join(project_dir, "text_info.txt")) expected_wheels = utils.expected_wheels('spam', '0.1.0') diff --git a/test/14_before_all/setup.py b/test/14_before_all/setup.py index 7b68808b..d1cf0f48 100644 --- a/test/14_before_all/setup.py +++ b/test/14_before_all/setup.py @@ -1,6 +1,3 @@ -import os -import sys - from setuptools import ( Extension, setup, @@ -11,11 +8,10 @@ from setuptools import ( with open("text_info.txt") as f: stored_text = f.read() -print("## stored text: "+stored_text) +print("## stored text: " + stored_text) assert stored_text == "sample text" - setup( name="spam", ext_modules=[Extension('spam', sources=['spam.c'])], From 3fe560cf14a13508d9e46585d85f92bdb5bc696a Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Mon, 11 May 2020 18:04:09 +0200 Subject: [PATCH 022/117] fix mypy --- cibuildwheel/util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index f7201462..012afa1a 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -121,7 +121,7 @@ class BuildOptions(NamedTuple): before_test: Optional[str] test_requires: List[str] test_extras: str -x build_verbosity: int + build_verbosity: int resources_dir = Path(__file__).resolve().parent / 'resources' From c1f155c0f2e506f1f8553a8a72ab9bbefcc82ade Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Fri, 15 May 2020 13:13:13 +0200 Subject: [PATCH 023/117] add environment evaluation --- cibuildwheel/linux.py | 16 +++++++++++++++- cibuildwheel/macos.py | 3 ++- cibuildwheel/windows.py | 3 ++- docs/options.md | 2 ++ test/14_before_all/cibuildwheel_test.py | 3 ++- test/14_before_all/setup.py | 2 +- 6 files changed, 24 insertions(+), 5 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 8bbda24d..aa672f7b 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -135,7 +135,21 @@ def build(options: BuildOptions) -> None: if options.before_all: before_all_prepared = prepare_command(options.before_all, project='/project', package=options.package_dir) - call(['docker', 'exec', '-i', container_name] + shell_cmd, universal_newlines=True, input=before_all_prepared) + task = """ + PS4=' + ' + + set -o errexit + set -o xtrace + + {environment_exports} + + {before_all_prepared} + + """.format( + environment_exports='\n'.join(options.environment.as_shell_commands()), + before_all_prepared=before_all_prepared + ) + call(['docker', 'exec', '-i', container_name] + shell_cmd, universal_newlines=True, input=task) for config in platform_configs: if options.dependency_constraints: diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 3e67b534..d9d892b9 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -177,8 +177,9 @@ def build(options: BuildOptions) -> None: repaired_wheel_dir = temp_dir / 'repaired_wheel' if options.before_all: + env = options.environment.as_dictionary(prev_environment=os.environ) before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) - call([before_all_prepared], shell=True) + call([before_all_prepared], shell=True, env=env) python_configurations = get_python_configurations(options.build_selector) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 73628c7f..bae831de 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -163,8 +163,9 @@ def build(options: BuildOptions) -> None: download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget) if options.before_all: + env = options.environment.as_dictionary(prev_environment=os.environ) before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) - shell([before_all_prepared]) + shell([before_all_prepared], env=env) python_configurations = get_python_configurations(options.build_selector) for config in python_configurations: diff --git a/docs/options.md b/docs/options.md index 0d61060d..ad2326d8 100644 --- a/docs/options.md +++ b/docs/options.md @@ -198,6 +198,8 @@ This option is added mainly for linux build, because linux wheels are build in i The placeholder `{package}` can be used here; it will be replaced by the path to the package being built by `cibuildwheel`. +For windows and macos `CIBW_BEFORE_ALL` python interpreter is same as `cibuildwheel` is run. For linux it is default interpreter for docker image. + Platform-specific variants also available:
`CIBW_BEFORE_ALL_MACOS` | `CIBW_BEFORE_ALL_WINDOWS` | `CIBW_BEFORE_ALL_LINUX` diff --git a/test/14_before_all/cibuildwheel_test.py b/test/14_before_all/cibuildwheel_test.py index d5be005a..1b7c8d03 100644 --- a/test/14_before_all/cibuildwheel_test.py +++ b/test/14_before_all/cibuildwheel_test.py @@ -13,7 +13,8 @@ def test(): actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ # write python version information to a temporary file, this is # checked in setup.py - 'CIBW_BEFORE_ALL': '''python -c "open('{project}/text_info.txt', 'w').write('sample text')"''', + 'CIBW_BEFORE_ALL': '''python -c "import os;open('{project}/text_info.txt', 'w').write('sample text '+os.environ.get('TEST_VAL', ''))"''', + 'CIBW_ENVIRONMENT': "TEST_VAL='123'" }) # also check that we got the right wheels diff --git a/test/14_before_all/setup.py b/test/14_before_all/setup.py index d1cf0f48..bdcadcaa 100644 --- a/test/14_before_all/setup.py +++ b/test/14_before_all/setup.py @@ -9,7 +9,7 @@ with open("text_info.txt") as f: stored_text = f.read() print("## stored text: " + stored_text) -assert stored_text == "sample text" +assert stored_text == "sample text 123" setup( From 4d5cf4629e2a320986c80a02d15474b01ec33f7f Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Fri, 15 May 2020 13:46:59 +0200 Subject: [PATCH 024/117] fix mypy type warning --- cibuildwheel/environment.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/environment.py b/cibuildwheel/environment.py index 086a253c..d0e14a02 100644 --- a/cibuildwheel/environment.py +++ b/cibuildwheel/environment.py @@ -1,6 +1,6 @@ import bashlex # type: ignore -from typing import Dict, List +from typing import Dict, List, MutableMapping from . import bashlex_eval @@ -61,8 +61,8 @@ class ParsedEnvironment: def __init__(self, assignments: List[EnvironmentAssignment]): self.assignments = assignments - def as_dictionary(self, prev_environment: Dict[str, str]) -> Dict[str, str]: - environment = prev_environment.copy() + def as_dictionary(self, prev_environment: MutableMapping[str, str]) -> Dict[str, str]: + environment = dict(**prev_environment) for assignment in self.assignments: value = assignment.evaluated_value(environment=environment) From 36b8f7fff252a75f0b371c4cd8caf4a71c651eaf Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Sun, 17 May 2020 17:16:23 +0200 Subject: [PATCH 025/117] changes in code from code review --- cibuildwheel/environment.py | 4 ++-- cibuildwheel/linux.py | 24 +++++++++++++----------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/cibuildwheel/environment.py b/cibuildwheel/environment.py index d0e14a02..9f7ea2c4 100644 --- a/cibuildwheel/environment.py +++ b/cibuildwheel/environment.py @@ -1,6 +1,6 @@ import bashlex # type: ignore -from typing import Dict, List, MutableMapping +from typing import Dict, List, Mapping from . import bashlex_eval @@ -61,7 +61,7 @@ class ParsedEnvironment: def __init__(self, assignments: List[EnvironmentAssignment]): self.assignments = assignments - def as_dictionary(self, prev_environment: MutableMapping[str, str]) -> Dict[str, str]: + def as_dictionary(self, prev_environment: Mapping[str, str]) -> Dict[str, str]: environment = dict(**prev_environment) for assignment in self.assignments: diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index aa672f7b..aa620f21 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -134,22 +134,24 @@ def build(options: BuildOptions) -> None: call(['docker', 'start', container_name]) if options.before_all: - before_all_prepared = prepare_command(options.before_all, project='/project', package=options.package_dir) - task = """ - PS4=' + ' + call( + ['docker', 'exec', '-i', container_name] + shell_cmd, + universal_newlines=True, + input=''' + PS4=' + ' - set -o errexit - set -o xtrace + set -o errexit + set -o xtrace - {environment_exports} + {environment_exports} - {before_all_prepared} + sh -c {before_all} - """.format( - environment_exports='\n'.join(options.environment.as_shell_commands()), - before_all_prepared=before_all_prepared + '''.format( + environment_exports='\n'.join(options.environment.as_shell_commands()), + before_all=prepare_command(options.before_all, project='/project', package=container_package_dir) + ) ) - call(['docker', 'exec', '-i', container_name] + shell_cmd, universal_newlines=True, input=task) for config in platform_configs: if options.dependency_constraints: From 219a65d5e380dd96361741f3d102eaf64d4749b2 Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Sun, 17 May 2020 17:14:50 +0200 Subject: [PATCH 026/117] Apply suggestions from code review Co-authored-by: Joe Rickerby --- docs/options.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/options.md b/docs/options.md index ad2326d8..5fbde070 100644 --- a/docs/options.md +++ b/docs/options.md @@ -191,14 +191,15 @@ CIBW_ENVIRONMENT: "BUILD_TIME=$(date) SAMPLE_TEXT=\"sample text\"" `cibuildwheel` always defines the environment variable `CIBUILDWHEEL=1`. This can be useful for [building wheels with optional extensions](faq.md#building-packages-with-optional-c-extensions). ### `CIBW_BEFORE_ALL` {: #before-all} -> Execute a shell command preparing common part for each wheel. +> Execute a shell command on the build system before any wheels are built. -Shell command to prepare common part of project (ex. build libraries which does not depend on python). -This option is added mainly for linux build, because linux wheels are build in isolated from host docker containers. +Shell command to prepare a common part of the project (e.g. build libraries which does not depend on the specific version of Python). + +This option is very useful for the Linux build, where builds take place in isolated Docker containers managed by cibuildwheel. This command will run inside the container before the wheel builds start. Note, if you're building both x86_64 and i686 wheels (the default), your build uses two different Docker images. In that case, this command will execute twice - once per build container. The placeholder `{package}` can be used here; it will be replaced by the path to the package being built by `cibuildwheel`. -For windows and macos `CIBW_BEFORE_ALL` python interpreter is same as `cibuildwheel` is run. For linux it is default interpreter for docker image. +The version of Python available inside `CIBW_BEFORE_ALL` is not controlled by cibuildwheel. On Windows and macOS, it's whatever is available on the host machine. For Linux, this comes from the Docker image, which on manylinux, is something rather old. Consider using a Python inside `/opt/python`, e.g. `/opt/python/cp38-cp38/bin/python`. Platform-specific variants also available:
`CIBW_BEFORE_ALL_MACOS` | `CIBW_BEFORE_ALL_WINDOWS` | `CIBW_BEFORE_ALL_LINUX` @@ -209,7 +210,7 @@ Platform-specific variants also available:
CIBW_BEFORE_ALL: make -C third_party_lib # install system library -CIBW_BEFORE_ALL: yum install -y libffi-dev +CIBW_BEFORE_ALL_LINUX: yum install -y libffi-dev ``` ### `CIBW_BEFORE_BUILD` {: #before-build} From 9f6c91404dc5910ee142fa35097c78f48261f9be Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Sun, 17 May 2020 17:41:53 +0200 Subject: [PATCH 027/117] fix flake8 --- cibuildwheel/linux.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index aa620f21..02cd28c8 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -135,7 +135,7 @@ def build(options: BuildOptions) -> None: if options.before_all: call( - ['docker', 'exec', '-i', container_name] + shell_cmd, + ['docker', 'exec', '-i', container_name] + shell_cmd, universal_newlines=True, input=''' PS4=' + ' @@ -147,10 +147,10 @@ def build(options: BuildOptions) -> None: sh -c {before_all} - '''.format( - environment_exports='\n'.join(options.environment.as_shell_commands()), - before_all=prepare_command(options.before_all, project='/project', package=container_package_dir) - ) + '''.format( + environment_exports='\n'.join(options.environment.as_shell_commands()), + before_all=prepare_command(options.before_all, project='/project', package=container_package_dir) + ) ) for config in platform_configs: From 7b7e24d36da160c8d17ab6ceada66f4bee248e05 Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Sun, 17 May 2020 18:45:15 +0200 Subject: [PATCH 028/117] add missed quote of before_all command --- cibuildwheel/linux.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 02cd28c8..3def204b 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -149,7 +149,7 @@ def build(options: BuildOptions) -> None: '''.format( environment_exports='\n'.join(options.environment.as_shell_commands()), - before_all=prepare_command(options.before_all, project='/project', package=container_package_dir) + before_all=shlex.quote(prepare_command(options.before_all, project='/project', package=container_package_dir)) ) ) From fc52c5e7b6d19e4e220a36b1cd960380981f68a3 Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Sun, 17 May 2020 18:53:42 +0200 Subject: [PATCH 029/117] add unit_test --- unit_test/main_tests/main_options_test.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/unit_test/main_tests/main_options_test.py b/unit_test/main_tests/main_options_test.py index 14252e67..b0422589 100644 --- a/unit_test/main_tests/main_options_test.py +++ b/unit_test/main_tests/main_options_test.py @@ -233,3 +233,18 @@ def test_build_selector_migrations(intercepted_build_args, monkeypatch, option_n assert intercepted_build_selector.build_patterns == build_selector_patterns else: assert intercepted_build_selector.skip_patterns == build_selector_patterns + + +@pytest.mark.parametrize('before_all', [None, 'test text']) +@pytest.mark.parametrize('platform_specific', [False, True]) +def test_before_all(before_all, platform_specific, platform, intercepted_build_args, monkeypatch): + if before_all is not None: + if platform_specific: + monkeypatch.setenv('CIBW_BEFORE_ALL_' + platform.upper(), before_all) + monkeypatch.setenv('CIBW_BEFORE_ALL', 'overwritten') + else: + monkeypatch.setenv('CIBW_BEFORE_ALL', before_all) + + main() + + assert intercepted_build_args.args[0].before_all == before_all From 2e15fdddfd9e44c07d573d18a7e12e914036dbde Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Mon, 18 May 2020 10:42:13 +0200 Subject: [PATCH 030/117] change to new test engine --- test/14_before_all/setup.py | 19 -------- test/14_before_all/spam.c | 48 ------------------- ...ibuildwheel_test.py => test_before_all.py} | 21 ++++++-- 3 files changed, 18 insertions(+), 70 deletions(-) delete mode 100644 test/14_before_all/setup.py delete mode 100644 test/14_before_all/spam.c rename test/{14_before_all/cibuildwheel_test.py => test_before_all.py} (54%) diff --git a/test/14_before_all/setup.py b/test/14_before_all/setup.py deleted file mode 100644 index bdcadcaa..00000000 --- a/test/14_before_all/setup.py +++ /dev/null @@ -1,19 +0,0 @@ -from setuptools import ( - Extension, - setup, -) - -# assert that the Python version as written to text_info.txt in the CIBW_BEFORE_ALL step -# is the same one as is currently running. -with open("text_info.txt") as f: - stored_text = f.read() - -print("## stored text: " + stored_text) -assert stored_text == "sample text 123" - - -setup( - name="spam", - ext_modules=[Extension('spam', sources=['spam.c'])], - version="0.1.0", -) diff --git a/test/14_before_all/spam.c b/test/14_before_all/spam.c deleted file mode 100644 index d1ab0f22..00000000 --- a/test/14_before_all/spam.c +++ /dev/null @@ -1,48 +0,0 @@ -#include - -static PyObject * -spam_system(PyObject *self, PyObject *args) -{ - const char *command; - int sts; - - if (!PyArg_ParseTuple(args, "s", &command)) - return NULL; - sts = system(command); - return PyLong_FromLong(sts); -} - -/* Module initialization */ - -#if PY_MAJOR_VERSION >= 3 - #define MOD_INIT(name) PyMODINIT_FUNC PyInit_##name(void) - #define MOD_DEF(m, name, doc, methods, module_state_size) \ - static struct PyModuleDef moduledef = { \ - PyModuleDef_HEAD_INIT, name, doc, module_state_size, methods, }; \ - m = PyModule_Create(&moduledef); - #define MOD_RETURN(m) return m; -#else - #define MOD_INIT(name) PyMODINIT_FUNC init##name(void) - #define MOD_DEF(m, name, doc, methods, module_state_size) \ - m = Py_InitModule3(name, methods, doc); - #define MOD_RETURN(m) return; -#endif - -static PyMethodDef module_methods[] = { - {"system", (PyCFunction)spam_system, METH_VARARGS, - "Execute a shell command."}, - {NULL} /* Sentinel */ -}; - -MOD_INIT(spam) -{ - PyObject* m; - - MOD_DEF(m, - "spam", - "Example module", - module_methods, - -1) - - MOD_RETURN(m) -} diff --git a/test/14_before_all/cibuildwheel_test.py b/test/test_before_all.py similarity index 54% rename from test/14_before_all/cibuildwheel_test.py rename to test/test_before_all.py index 1b7c8d03..177ec555 100644 --- a/test/14_before_all/cibuildwheel_test.py +++ b/test/test_before_all.py @@ -1,10 +1,25 @@ import os +import textwrap -import utils +from . import utils +from . import test_projects + +project_with_before_build_asserts = test_projects.new_c_project( + setup_py_add=textwrap.dedent(r''' + # assert that the Python version as written to text_info.txt in the CIBW_BEFORE_ALL step + # is the same one as is currently running. + with open("text_info.txt") as f: + stored_text = f.read() + + print("## stored text: " + stored_text) + assert stored_text == "sample text 123" + ''') +) -def test(): - project_dir = os.path.dirname(__file__) +def test(tmp_path): + project_dir = tmp_path / 'project' + project_with_before_build_asserts.generate(project_dir) with open(os.path.join(project_dir, "text_info.txt"), mode='w') as ff: print("dummy text", file=ff) From 46352c8e309973cb44dd84168b1c444f33d7815d Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 14 Jun 2020 10:34:44 +0100 Subject: [PATCH 031/117] Add a modern python to linux build before_all environment --- cibuildwheel/linux.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 3def204b..6e42f47f 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -143,6 +143,10 @@ def build(options: BuildOptions) -> None: set -o errexit set -o xtrace + # add a modern Python interpreter to PATH so it can be used by BEFORE_ALL + # commands + export PATH=/opt/python/cp38-cp38:$PATH + {environment_exports} sh -c {before_all} From 2057bba07f0b78b838d506e9a167e13875c94d8e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 14 Jun 2020 10:35:54 +0100 Subject: [PATCH 032/117] Update docs to reflect linux available python change --- docs/options.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/options.md b/docs/options.md index 5fbde070..aba121db 100644 --- a/docs/options.md +++ b/docs/options.md @@ -193,13 +193,13 @@ CIBW_ENVIRONMENT: "BUILD_TIME=$(date) SAMPLE_TEXT=\"sample text\"" ### `CIBW_BEFORE_ALL` {: #before-all} > Execute a shell command on the build system before any wheels are built. -Shell command to prepare a common part of the project (e.g. build libraries which does not depend on the specific version of Python). +Shell command to prepare a common part of the project (e.g. build or install libraries which does not depend on the specific version of Python). This option is very useful for the Linux build, where builds take place in isolated Docker containers managed by cibuildwheel. This command will run inside the container before the wheel builds start. Note, if you're building both x86_64 and i686 wheels (the default), your build uses two different Docker images. In that case, this command will execute twice - once per build container. The placeholder `{package}` can be used here; it will be replaced by the path to the package being built by `cibuildwheel`. -The version of Python available inside `CIBW_BEFORE_ALL` is not controlled by cibuildwheel. On Windows and macOS, it's whatever is available on the host machine. For Linux, this comes from the Docker image, which on manylinux, is something rather old. Consider using a Python inside `/opt/python`, e.g. `/opt/python/cp38-cp38/bin/python`. +On Windows and macOS, the version of Python available inside `CIBW_BEFORE_ALL` is whatever is available on the host machine. On Linux, a modern Python version is available on PATH. Platform-specific variants also available:
`CIBW_BEFORE_ALL_MACOS` | `CIBW_BEFORE_ALL_WINDOWS` | `CIBW_BEFORE_ALL_LINUX` From b4b46a1f066e4657e75ab4ac392f0d4076a5a704 Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Sun, 14 Jun 2020 21:35:28 +0200 Subject: [PATCH 033/117] fix mypy --- cibuildwheel/__main__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index bce3f754..4e25befa 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -128,7 +128,7 @@ def main() -> None: build_config, skip_config = os.environ.get('CIBW_BUILD', '*'), os.environ.get('CIBW_SKIP', '') environment_config = get_option_from_environment('CIBW_ENVIRONMENT', platform=platform, default='') - before_all = get_option_from_environment('CIBW_BEFORE_ALL', platform=platform) + before_all = get_option_from_environment('CIBW_BEFORE_ALL', platform=platform, default='') before_build = get_option_from_environment('CIBW_BEFORE_BUILD', platform=platform) repair_command = get_option_from_environment('CIBW_REPAIR_WHEEL_COMMAND', platform=platform, default=repair_command_default) dependency_versions = get_option_from_environment('CIBW_DEPENDENCY_VERSIONS', platform=platform, default='pinned') From 43fcb31231ee540686c5dd0e3be67d6fbb816911 Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Mon, 22 Jun 2020 13:52:56 +0200 Subject: [PATCH 034/117] fix tests --- unit_test/main_tests/main_options_test.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/unit_test/main_tests/main_options_test.py b/unit_test/main_tests/main_options_test.py index b0422589..c7e6bfdd 100644 --- a/unit_test/main_tests/main_options_test.py +++ b/unit_test/main_tests/main_options_test.py @@ -235,7 +235,7 @@ def test_build_selector_migrations(intercepted_build_args, monkeypatch, option_n assert intercepted_build_selector.skip_patterns == build_selector_patterns -@pytest.mark.parametrize('before_all', [None, 'test text']) +@pytest.mark.parametrize('before_all', ["", None, 'test text']) @pytest.mark.parametrize('platform_specific', [False, True]) def test_before_all(before_all, platform_specific, platform, intercepted_build_args, monkeypatch): if before_all is not None: @@ -247,4 +247,7 @@ def test_before_all(before_all, platform_specific, platform, intercepted_build_a main() + if before_all is None: + before_all = "" + assert intercepted_build_args.args[0].before_all == before_all From a03a26c90e6f709f1c836686c1c75e1afec73f7d Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 23 Jun 2020 09:59:22 +0100 Subject: [PATCH 035/117] Update README options table --- README.md | 11 +++++++---- docs/options.md | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index d3805547..3412159e 100644 --- a/README.md +++ b/README.md @@ -102,17 +102,20 @@ Options | | Option | Description | |---|--------|-------------| | **Build selection** | [`CIBW_PLATFORM`](https://cibuildwheel.readthedocs.io/en/stable/options/#platform) | Override the auto-detected target platform | -| | [`CIBW_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip) [`CIBW_SKIP`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip) | Choose the Python versions to build | -| **Build environment** | [`CIBW_ENVIRONMENT`](https://cibuildwheel.readthedocs.io/en/stable/options/#environment) | Set environment variables needed during the build | +| | [`CIBW_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip)
[`CIBW_SKIP`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-skip) | Choose the Python versions to build | +| **Build customization** | [`CIBW_ENVIRONMENT`](https://cibuildwheel.readthedocs.io/en/stable/options/#environment) | Set environment variables needed during the build | +| | [`CIBW_BEFORE_ALL`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-all) | Execute a shell command on the build system before any wheels are built. | | | [`CIBW_BEFORE_BUILD`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-build) | Execute a shell command preparing each wheel's build | | | [`CIBW_REPAIR_WHEEL_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#repair-wheel-command) | Execute a shell command to repair each (non-pure Python) built wheel | -| | [`CIBW_MANYLINUX_X86_64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) [`CIBW_MANYLINUX_I686_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) [`CIBW_MANYLINUX_PYPY_X86_64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) | Specify alternative manylinux docker images | +| | [`CIBW_MANYLINUX_X86_64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image)
[`CIBW_MANYLINUX_I686_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image)
[`CIBW_MANYLINUX_PYPY_X86_64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image)
[`CIBW_MANYLINUX_AARCH64_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image)
[`CIBW_MANYLINUX_PPC64LE_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image)
[`CIBW_MANYLINUX_S390X_IMAGE`](https://cibuildwheel.readthedocs.io/en/stable/options/#manylinux-image) | Specify alternative manylinux docker images | +| | [`CIBW_DEPENDENCY_VERSIONS`](https://cibuildwheel.readthedocs.io/en/stable/options/#dependency-versions) | Specify how cibuildwheel controls the versions of the tools it uses | | **Testing** | [`CIBW_TEST_COMMAND`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-command) | Execute a shell command to test each built wheel | -| | [`CIBW_BEFORE_TEST`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-test) | Execute shell command to prepare test environment | +| | [`CIBW_BEFORE_TEST`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-test) | Execute a shell command before testing each wheel | | | [`CIBW_TEST_REQUIRES`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-requires) | Install Python dependencies before running the tests | | | [`CIBW_TEST_EXTRAS`](https://cibuildwheel.readthedocs.io/en/stable/options/#test-extras) | Install your wheel for testing using extras_require | | **Other** | [`CIBW_BUILD_VERBOSITY`](https://cibuildwheel.readthedocs.io/en/stable/options/#build-verbosity) | Increase/decrease the output of pip wheel | + Working examples ---------------- diff --git a/docs/options.md b/docs/options.md index aba121db..856f0bad 100644 --- a/docs/options.md +++ b/docs/options.md @@ -625,7 +625,7 @@ optional arguments: var url = 'https://cibuildwheel.readthedocs.io/en/stable/options/#'+option.id; var namesMarkdown = $.map(optionNames, function(n) { return '[`'+n+'`]('+url+') ' - }).join(' ') + }).join('
') markdown += '| '+namesMarkdown+' ' markdown += '| '+option.description.trim()+' ' From c68d50a07387272f6f31962930842421e99648e6 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 23 Jun 2020 10:13:54 +0100 Subject: [PATCH 036/117] Lock virtualenv version --- .circleci/prepare.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.circleci/prepare.sh b/.circleci/prepare.sh index dfd3aeef..035cdde7 100644 --- a/.circleci/prepare.sh +++ b/.circleci/prepare.sh @@ -1,6 +1,9 @@ +set -o errexit +set -o xtrace + $PYTHON --version $PYTHON -m pip --version -$PYTHON -m pip install -q --user --ignore-installed --upgrade virtualenv +$PYTHON -m pip install -q --user --ignore-installed virtualenv==20.0.23 $PYTHON -m virtualenv -p $PYTHON venv venv/bin/python -m pip install -r requirements-dev.txt venv/bin/python -m pip freeze From 4302090f080b2dfc709210f41750ce66e06f530b Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 23 Jun 2020 10:22:01 +0100 Subject: [PATCH 037/117] Uninstall poetry --- .circleci/prepare.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.circleci/prepare.sh b/.circleci/prepare.sh index 035cdde7..354c2d20 100644 --- a/.circleci/prepare.sh +++ b/.circleci/prepare.sh @@ -3,7 +3,8 @@ set -o xtrace $PYTHON --version $PYTHON -m pip --version -$PYTHON -m pip install -q --user --ignore-installed virtualenv==20.0.23 +sudo $PYTHON -m pip uninstall -y poetry +$PYTHON -m pip install --user --ignore-installed virtualenv==20.0.21 $PYTHON -m virtualenv -p $PYTHON venv venv/bin/python -m pip install -r requirements-dev.txt venv/bin/python -m pip freeze From 947059bbf453079bbc571755b0f0d329fde34612 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 23 Jun 2020 10:27:23 +0100 Subject: [PATCH 038/117] Don't try to install virtualenv --- .circleci/prepare.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/.circleci/prepare.sh b/.circleci/prepare.sh index 354c2d20..7256c8c3 100644 --- a/.circleci/prepare.sh +++ b/.circleci/prepare.sh @@ -3,8 +3,6 @@ set -o xtrace $PYTHON --version $PYTHON -m pip --version -sudo $PYTHON -m pip uninstall -y poetry -$PYTHON -m pip install --user --ignore-installed virtualenv==20.0.21 $PYTHON -m virtualenv -p $PYTHON venv venv/bin/python -m pip install -r requirements-dev.txt venv/bin/python -m pip freeze From 666699a51e3e4b59f7822976308e3b001f0f83ef Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 23 Jun 2020 11:16:23 +0100 Subject: [PATCH 039/117] Configure probot-auto-merge --- .github/auto-merge.yml | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 .github/auto-merge.yml diff --git a/.github/auto-merge.yml b/.github/auto-merge.yml new file mode 100644 index 00000000..a461ed1d --- /dev/null +++ b/.github/auto-merge.yml @@ -0,0 +1,7 @@ +# config file for https://github.com/bobvanderlinden/probot-auto-merge + +# no approvals required, just the label is enough +minApprovals: {} + +requiredLabels: +- merge when green From e417791ae4a366c75b3b5fbb0ae1a2d895e06a56 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 23 Jun 2020 19:46:23 +0100 Subject: [PATCH 040/117] Remove the linux bash script, replaced with a `docker exec` approach --- cibuildwheel/bashlex_eval.py | 17 +- cibuildwheel/environment.py | 12 +- cibuildwheel/linux.py | 445 +++++++++++++++++++---------------- requirements-dev.txt | 2 + 4 files changed, 261 insertions(+), 215 deletions(-) diff --git a/cibuildwheel/bashlex_eval.py b/cibuildwheel/bashlex_eval.py index 60ce9a81..e4865d6d 100644 --- a/cibuildwheel/bashlex_eval.py +++ b/cibuildwheel/bashlex_eval.py @@ -1,17 +1,26 @@ import shlex import subprocess -from typing import Dict, NamedTuple +from typing import Dict, NamedTuple, Callable, Optional import bashlex # type: ignore +# a function that takes a shell command and the environment, and returns the result +EnvironmentExecutor = Callable[[str, Dict[str, str]], str] + + +def local_environment_executor(command: str, env: Dict[str, str]) -> str: + return subprocess.check_output(shlex.split(command), env=env, universal_newlines=True) + + class NodeExecutionContext(NamedTuple): environment: Dict[str, str] input: str + executor: EnvironmentExecutor -def evaluate(value: str, environment: Dict[str, str]) -> str: +def evaluate(value: str, environment: Dict[str, str], executor: Optional[EnvironmentExecutor] = None) -> str: if not value: # empty string evaluates to empty string # (but trips up bashlex) @@ -26,7 +35,7 @@ def evaluate(value: str, environment: Dict[str, str]) -> str: return evaluate_node( value_word_node, - context=NodeExecutionContext(environment=environment, input=value) + context=NodeExecutionContext(environment=environment, input=value, executor=executor or local_environment_executor) ) @@ -67,7 +76,7 @@ def evaluate_word_node(node: bashlex.ast.node, context: NodeExecutionContext) -> def evaluate_command_node(node: bashlex.ast.node, context: NodeExecutionContext) -> str: words = [evaluate_node(part, context=context) for part in node.parts] command = ' '.join(words) - return subprocess.check_output(shlex.split(command), env=context.environment, universal_newlines=True) + return context.executor(command, context.environment) def evaluate_parameter_node(node: bashlex.ast.node, context: NodeExecutionContext) -> str: diff --git a/cibuildwheel/environment.py b/cibuildwheel/environment.py index 9f7ea2c4..5ecd8c68 100644 --- a/cibuildwheel/environment.py +++ b/cibuildwheel/environment.py @@ -1,6 +1,6 @@ import bashlex # type: ignore -from typing import Dict, List, Mapping +from typing import Dict, List, Mapping, Optional from . import bashlex_eval @@ -46,9 +46,9 @@ class EnvironmentAssignment: self.name = name self.value = value - def evaluated_value(self, environment: Dict[str, str]) -> str: + def evaluated_value(self, environment: Dict[str, str], executor: Optional[bashlex_eval.EnvironmentExecutor] = None) -> str: '''Returns the value of this assignment, as evaluated in the environment''' - return bashlex_eval.evaluate(self.value, environment=environment) + return bashlex_eval.evaluate(self.value, environment=environment, executor=executor) def as_shell_assignment(self) -> str: return f'export {self.name}={self.value}' @@ -61,11 +61,13 @@ class ParsedEnvironment: def __init__(self, assignments: List[EnvironmentAssignment]): self.assignments = assignments - def as_dictionary(self, prev_environment: Mapping[str, str]) -> Dict[str, str]: + def as_dictionary(self, + prev_environment: Mapping[str, str], + executor: Optional[bashlex_eval.EnvironmentExecutor] = None) -> Dict[str, str]: environment = dict(**prev_environment) for assignment in self.assignments: - value = assignment.evaluated_value(environment=environment) + value = assignment.evaluated_value(environment=environment, executor=executor) environment[assignment.name] = value return environment diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 6e42f47f..f66cc5e1 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -1,27 +1,16 @@ -import os +import json import platform import shlex import subprocess import sys import textwrap import uuid +from os import PathLike from pathlib import Path, PurePath +from typing import Dict, List, NamedTuple, Optional, Union, Sequence -from typing import List, NamedTuple, Optional, Union - -from .util import ( - BuildOptions, - BuildSelector, - get_build_verbosity_extra_flags, - prepare_command, -) - - -def call(args: List[str], input: Optional[Union[str, bytes]] = None, universal_newlines: bool = False) -> None: - print('+ ' + ' '.join(shlex.quote(a) for a in args)) - subprocess.run( - args, input=input, universal_newlines=universal_newlines, check=True - ) +from .util import (BuildOptions, BuildSelector, + get_build_verbosity_extra_flags, prepare_command) def matches_platform(identifier: str) -> bool: @@ -48,37 +37,41 @@ def matches_platform(identifier: str) -> bool: class PythonConfiguration(NamedTuple): version: str identifier: str - path: str + path_str: str + + @property + def path(self): + return PurePath(self.path_str) def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfiguration]: python_configurations = [ - PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path='/opt/python/cp27-cp27m'), - PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path='/opt/python/cp27-cp27mu'), - PythonConfiguration(version='3.5', identifier='cp35-manylinux_x86_64', path='/opt/python/cp35-cp35m'), - PythonConfiguration(version='3.6', identifier='cp36-manylinux_x86_64', path='/opt/python/cp36-cp36m'), - PythonConfiguration(version='3.7', identifier='cp37-manylinux_x86_64', path='/opt/python/cp37-cp37m'), - PythonConfiguration(version='3.8', identifier='cp38-manylinux_x86_64', path='/opt/python/cp38-cp38'), - PythonConfiguration(version='2.7', identifier='cp27-manylinux_i686', path='/opt/python/cp27-cp27m'), - PythonConfiguration(version='2.7', identifier='cp27-manylinux_i686', path='/opt/python/cp27-cp27mu'), - PythonConfiguration(version='3.5', identifier='cp35-manylinux_i686', path='/opt/python/cp35-cp35m'), - PythonConfiguration(version='3.6', identifier='cp36-manylinux_i686', path='/opt/python/cp36-cp36m'), - PythonConfiguration(version='3.7', identifier='cp37-manylinux_i686', path='/opt/python/cp37-cp37m'), - PythonConfiguration(version='3.8', identifier='cp38-manylinux_i686', path='/opt/python/cp38-cp38'), - PythonConfiguration(version='2.7', identifier='pp27-manylinux_x86_64', path='/opt/python/pp27-pypy_73'), - PythonConfiguration(version='3.6', identifier='pp36-manylinux_x86_64', path='/opt/python/pp36-pypy36_pp73'), - PythonConfiguration(version='3.5', identifier='cp35-manylinux_aarch64', path='/opt/python/cp35-cp35m'), - PythonConfiguration(version='3.6', identifier='cp36-manylinux_aarch64', path='/opt/python/cp36-cp36m'), - PythonConfiguration(version='3.7', identifier='cp37-manylinux_aarch64', path='/opt/python/cp37-cp37m'), - PythonConfiguration(version='3.8', identifier='cp38-manylinux_aarch64', path='/opt/python/cp38-cp38'), - PythonConfiguration(version='3.5', identifier='cp35-manylinux_ppc64le', path='/opt/python/cp35-cp35m'), - PythonConfiguration(version='3.6', identifier='cp36-manylinux_ppc64le', path='/opt/python/cp36-cp36m'), - PythonConfiguration(version='3.7', identifier='cp37-manylinux_ppc64le', path='/opt/python/cp37-cp37m'), - PythonConfiguration(version='3.8', identifier='cp38-manylinux_ppc64le', path='/opt/python/cp38-cp38'), - PythonConfiguration(version='3.5', identifier='cp35-manylinux_s390x', path='/opt/python/cp35-cp35m'), - PythonConfiguration(version='3.6', identifier='cp36-manylinux_s390x', path='/opt/python/cp36-cp36m'), - PythonConfiguration(version='3.7', identifier='cp37-manylinux_s390x', path='/opt/python/cp37-cp37m'), - PythonConfiguration(version='3.8', identifier='cp38-manylinux_s390x', path='/opt/python/cp38-cp38'), + PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path_str='/opt/python/cp27-cp27m'), + PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path_str='/opt/python/cp27-cp27mu'), + PythonConfiguration(version='3.5', identifier='cp35-manylinux_x86_64', path_str='/opt/python/cp35-cp35m'), + PythonConfiguration(version='3.6', identifier='cp36-manylinux_x86_64', path_str='/opt/python/cp36-cp36m'), + PythonConfiguration(version='3.7', identifier='cp37-manylinux_x86_64', path_str='/opt/python/cp37-cp37m'), + PythonConfiguration(version='3.8', identifier='cp38-manylinux_x86_64', path_str='/opt/python/cp38-cp38'), + PythonConfiguration(version='2.7', identifier='cp27-manylinux_i686', path_str='/opt/python/cp27-cp27m'), + PythonConfiguration(version='2.7', identifier='cp27-manylinux_i686', path_str='/opt/python/cp27-cp27mu'), + PythonConfiguration(version='3.5', identifier='cp35-manylinux_i686', path_str='/opt/python/cp35-cp35m'), + PythonConfiguration(version='3.6', identifier='cp36-manylinux_i686', path_str='/opt/python/cp36-cp36m'), + PythonConfiguration(version='3.7', identifier='cp37-manylinux_i686', path_str='/opt/python/cp37-cp37m'), + PythonConfiguration(version='3.8', identifier='cp38-manylinux_i686', path_str='/opt/python/cp38-cp38'), + PythonConfiguration(version='2.7', identifier='pp27-manylinux_x86_64', path_str='/opt/python/pp27-pypy_73'), + PythonConfiguration(version='3.6', identifier='pp36-manylinux_x86_64', path_str='/opt/python/pp36-pypy36_pp73'), + PythonConfiguration(version='3.5', identifier='cp35-manylinux_aarch64', path_str='/opt/python/cp35-cp35m'), + PythonConfiguration(version='3.6', identifier='cp36-manylinux_aarch64', path_str='/opt/python/cp36-cp36m'), + PythonConfiguration(version='3.7', identifier='cp37-manylinux_aarch64', path_str='/opt/python/cp37-cp37m'), + PythonConfiguration(version='3.8', identifier='cp38-manylinux_aarch64', path_str='/opt/python/cp38-cp38'), + PythonConfiguration(version='3.5', identifier='cp35-manylinux_ppc64le', path_str='/opt/python/cp35-cp35m'), + PythonConfiguration(version='3.6', identifier='cp36-manylinux_ppc64le', path_str='/opt/python/cp36-cp36m'), + PythonConfiguration(version='3.7', identifier='cp37-manylinux_ppc64le', path_str='/opt/python/cp37-cp37m'), + PythonConfiguration(version='3.8', identifier='cp38-manylinux_ppc64le', path_str='/opt/python/cp38-cp38'), + PythonConfiguration(version='3.5', identifier='cp35-manylinux_s390x', path_str='/opt/python/cp35-cp35m'), + PythonConfiguration(version='3.6', identifier='cp36-manylinux_s390x', path_str='/opt/python/cp36-cp36m'), + PythonConfiguration(version='3.7', identifier='cp37-manylinux_s390x', path_str='/opt/python/cp37-cp37m'), + PythonConfiguration(version='3.8', identifier='cp38-manylinux_s390x', path_str='/opt/python/cp38-cp38'), ] # skip builds as required return [c for c in python_configurations if matches_platform(c.identifier) and build_selector(c.identifier)] @@ -111,201 +104,133 @@ def build(options: BuildOptions) -> None: raise Exception('package_dir must be inside the working directory') container_package_dir = PurePath('/project') / abs_package_dir.relative_to(cwd) + container_output_dir = PurePath('/output') for implementation, platform_tag, docker_image in platforms: platform_configs = [c for c in python_configurations if c.identifier.startswith(implementation) and c.identifier.endswith(platform_tag)] if not platform_configs: continue - shell_cmd = ['linux32', '/bin/bash'] if platform_tag.endswith("i686") else ['/bin/bash'] - - container_name = f'cibuildwheel-{uuid.uuid4()}' - call(['docker', 'create', - '--env', 'CIBUILDWHEEL', - '--name', container_name, - '-i', - '-v', '/:/host', # ignored on CircleCI - docker_image, - '/bin/bash']) - try: - call(['docker', 'cp', '.', container_name + ':/project']) + with DockerContainer(docker_image, simulate_32_bit=platform_tag.endswith('i686')) as docker: + docker.copy_into(Path.cwd(), Path('/project')) - call(['docker', 'start', container_name]) + if options.before_all: + env = docker.get_environment() + env['PATH'] = f'/opt/python/cp38-cp38:{env["PATH"]}' + env = options.environment.as_dictionary(env, executor=docker.environment_executor) - if options.before_all: - call( - ['docker', 'exec', '-i', container_name] + shell_cmd, - universal_newlines=True, - input=''' - PS4=' + ' + before_all_prepared = prepare_command(options.before_all, project='/project', package=container_package_dir) + docker.call(['sh', '-c', before_all_prepared], env=env) - set -o errexit - set -o xtrace + for config in platform_configs: + dependency_constraint_flags: List[Union[str, PathLike]] = [] - # add a modern Python interpreter to PATH so it can be used by BEFORE_ALL - # commands - export PATH=/opt/python/cp38-cp38:$PATH + if options.dependency_constraints: + constraints_file = options.dependency_constraints.get_for_python_version(config.version) + container_constraints_file = PurePath('/constraints.txt') - {environment_exports} + docker.copy_into(constraints_file, container_constraints_file) + dependency_constraint_flags = ['-c', container_constraints_file] - sh -c {before_all} + env = docker.get_environment() - '''.format( - environment_exports='\n'.join(options.environment.as_shell_commands()), - before_all=shlex.quote(prepare_command(options.before_all, project='/project', package=container_package_dir)) + # put this config's python top of the list + python_bin = config.path / 'bin' + env['PATH'] = f'{str(python_bin)}:{env["PATH"]}' + + env = options.environment.as_dictionary(env, executor=docker.environment_executor) + + # check config python and pip are still on PATH + which_python = docker.call(['which', 'python'], env=env, capture_output=True).strip() + if PurePath(which_python) != python_bin / 'python': + print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr) + exit(1) + + which_pip = docker.call(['which', 'pip'], env=env, capture_output=True).strip() + if PurePath(which_pip) != python_bin / 'pip': + print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr) + exit(1) + + if options.before_build: + before_build_prepared = prepare_command(options.before_build, project='/project', package=container_package_dir) + docker.call(['sh', '-c', before_build_prepared], env=env) + + temp_dir = PurePath('/tmp/cibuildwheel') + built_wheel_dir = temp_dir / 'built_wheel' + docker.call(['rm', '-rf', built_wheel_dir]) + docker.call(['mkdir', '-p', built_wheel_dir]) + + docker.call( + [ + 'pip', 'wheel', + container_package_dir, + '-w', built_wheel_dir, + '--no-deps', + *get_build_verbosity_extra_flags(options.build_verbosity) + ], + env=env, ) - ) - for config in platform_configs: - if options.dependency_constraints: - constraints_file = options.dependency_constraints.get_for_python_version(config.version) + built_wheel = docker.glob(built_wheel_dir / '*.whl')[0] - # `docker cp` causes 'no space left on device' error when - # a container is running and the host filesystem is - # mounted. https://github.com/moby/moby/issues/38995 - # Use `docker exec` instead. - with open(constraints_file, 'rb') as f: - call( - ['docker', 'exec', '-i', container_name, 'sh', '-c', 'cat > /constraints.txt'], - input=f.read(), - ) + repaired_wheel_dir = temp_dir / 'repaired_wheel' + docker.call(['rm', '-rf', repaired_wheel_dir]) + docker.call(['mkdir', '-p', repaired_wheel_dir]) - call( - ['docker', 'exec', '-i', container_name] + shell_cmd, - universal_newlines=True, - input=''' - # give xtrace output an extra level of indent inside docker - PS4=' + ' + if built_wheel.name.endswith('none-any.whl') or not options.repair_command: + docker.call(['mv', built_wheel, repaired_wheel_dir]) + else: + repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) + docker.call(['sh', '-c', repair_command_prepared], env=env) - set -o errexit - set -o xtrace - mkdir -p /output - cd /project + repaired_wheels = docker.glob(repaired_wheel_dir / '*.whl') - PYBIN="{config_python_bin}" + if options.test_command: + # set up a virtual environment to install and test from, to make sure + # there are no dependencies that were pulled in at build time. + docker.call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) + venv_dir = PurePath(docker.call(['mktemp', '-d'], capture_output=True).strip()) / 'venv' - export PATH="$PYBIN:$PATH" - {environment_exports} + docker.call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) - # check the active python and pip are in PYBIN - if [ "$(which pip)" != "$PYBIN/pip" ]; then - echo "cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it." - exit 1 - fi - if [ "$(which python)" != "$PYBIN/python" ]; then - echo "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it." - exit 1 - fi + virtualenv_env = env.copy() + virtualenv_env['PATH'] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}" - if [ ! -z {before_build} ]; then - sh -c {before_build} - fi + if options.before_test: + before_test_prepared = prepare_command(options.before_test, project='/project', package=container_package_dir) + docker.call(['sh', '-c', before_test_prepared], env=virtualenv_env) - # Build the wheel - rm -rf /tmp/built_wheel - mkdir /tmp/built_wheel - pip wheel {package_dir} -w /tmp/built_wheel --no-deps {build_verbosity_flag} - built_wheel=(/tmp/built_wheel/*.whl) + # Install the wheel we just built + # Note: If auditwheel produced two wheels, it's because the earlier produced wheel + # conforms to multiple manylinux standards. These multiple versions of the wheel are + # functionally the same, differing only in name, wheel metadata, and possibly include + # different external shared libraries. so it doesn't matter which one we run the tests on. + # Let's just pick the first one. + wheel_to_test = repaired_wheels[0] + docker.call(['pip', 'install', str(wheel_to_test) + options.test_extras], env=virtualenv_env) - # repair the wheel - rm -rf /tmp/repaired_wheels - mkdir /tmp/repaired_wheels - # NOTE: 'built_wheel' here is a bash array of glob matches; "$built_wheel" returns - # the first element - if [[ "$built_wheel" == *none-any.whl ]] || [ -z {repair_command} ]; then - # pure Python wheel or empty repair command - mv "$built_wheel" /tmp/repaired_wheels - else - sh -c {repair_command} repair_command "$built_wheel" - fi - repaired_wheels=(/tmp/repaired_wheels/*.whl) + # Install any requirements to run the tests + if options.test_requires: + docker.call(['pip', 'install', *options.test_requires], env=virtualenv_env) - if [ ! -z {test_command} ]; then - # Set up a virtual environment to install and test from, to make sure - # there are no dependencies that were pulled in at build time. - pip install {dependency_install_flags} virtualenv - venv_dir=`mktemp -d`/venv - python -m virtualenv --no-download "$venv_dir" + # Run the tests from a different directory + test_command_prepared = prepare_command(options.test_command, project='/project', package=container_package_dir) + docker.call(['sh', '-c', test_command_prepared], cwd='/root', env=virtualenv_env) - # run the tests in a subshell to keep that `activate` - # script from polluting the env - ( - source "$venv_dir/bin/activate" + # clean up test environment + docker.call(['rm', '-rf', venv_dir]) - echo "Running tests using `which python`" + # move repaired wheels to output + docker.call(['mkdir', '-p', container_output_dir]) + docker.call(['mv', *repaired_wheels, container_output_dir]) - if [ ! -z {before_test} ]; then - sh -c {before_test} - fi - - # Install the wheel we just built - # Note: If auditwheel produced two wheels, it's because the earlier produced wheel - # conforms to multiple manylinux standards. These multiple versions of the wheel are - # functionally the same, differing only in name, wheel metadata, and possibly include - # different external shared libraries. so it doesn't matter which one we run the tests on. - # Let's just pick the first one. - pip install "${{repaired_wheels[0]}}"{test_extras} - - # Install any requirements to run the tests - if [ ! -z "{test_requires}" ]; then - pip install {test_requires} - fi - - # Run the tests from a different directory - pushd $HOME - sh -c {test_command} - popd - ) - # exit if tests failed (needed for older bash versions) - if [ $? -ne 0 ]; then - exit 1; - fi - - # clean up - rm -rf "$venv_dir" - fi - - # we're all done here; move it to output - mv "${{repaired_wheels[@]}}" /output - for repaired_wheel in "${{repaired_wheels[@]}}"; do - chown {uid}:{gid} "/output/$(basename "$repaired_wheel")" - done - '''.format( - config_python_bin=config.path + '/bin', - package_dir=container_package_dir, - test_requires=' '.join(options.test_requires), - test_extras=options.test_extras, - test_command=shlex.quote( - prepare_command(options.test_command, project='/project', package=container_package_dir) if options.test_command else '' - ), - before_build=shlex.quote( - prepare_command(options.before_build, project='/project', package=container_package_dir) if options.before_build else '' - ), - build_verbosity_flag=' '.join(get_build_verbosity_extra_flags(options.build_verbosity)), - repair_command=shlex.quote( - prepare_command(options.repair_command, wheel='"$1"', dest_dir='/tmp/repaired_wheels') if options.repair_command else '' - ), - environment_exports='\n'.join(options.environment.as_shell_commands()), - uid=os.getuid(), - gid=os.getgid(), - before_test=shlex.quote( - prepare_command(options.before_test, project='/project', package=container_package_dir) if options.before_test else '' - ), - dependency_install_flags='-c /constraints.txt' if options.dependency_constraints else '', - ) - ) - - # copy the output back into the host - call(['docker', 'cp', - container_name + ':/output/.', - str(options.output_dir.resolve())]) + # copy the output back into the host + docker.copy_out(container_output_dir, options.output_dir) except subprocess.CalledProcessError as error: + print(f'Command {error.cmd} failed with code {error.returncode}. {error.stdout}') troubleshoot(options.package_dir, error) exit(1) - finally: - # Still gets executed, even when 'exit(1)' gets called - call(['docker', 'rm', '--force', '-v', container_name]) def troubleshoot(package_dir: Path, error: Exception) -> None: @@ -328,3 +253,111 @@ def troubleshoot(package_dir: Path, error: Exception) -> None: print(' Files detected:') print('\n'.join([f' {f}' for f in so_files])) print('') + + +class DockerContainer: + UTILITY_PYTHON = '/opt/python/cp38-cp38/bin/python' + + def __init__(self, docker_image, simulate_32_bit=False): + self.docker_image = docker_image + self.simulate_32_bit = simulate_32_bit + + def __enter__(self) -> 'DockerContainer': + self.container_name = f'cibuildwheel-{uuid.uuid4()}' + subprocess.run( + [ + 'docker', 'create', + '--env', 'CIBUILDWHEEL', + '--name', self.container_name, + '-i', + '-v', '/:/host', # ignored on CircleCI + self.docker_image, + '/bin/bash' + ], + check=True, + ) + subprocess.run(['docker', 'start', self.container_name], check=True) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + subprocess.run(['docker', 'rm', '--force', '-v', self.container_name]) + self.container_name = None + + def copy_into(self, from_path: Path, to_path: PurePath) -> None: + # `docker cp` causes 'no space left on device' error when + # a container is running and the host filesystem is + # mounted. https://github.com/moby/moby/issues/38995 + # Use `docker exec` instead. + if from_path.is_dir(): + self.call(['mkdir', '-p', to_path]) + subprocess.run( + f'tar cf - . | docker exec -i {self.container_name} tar -xC {to_path} -f -', + shell=True, + check=True, + cwd=from_path) + else: + subprocess.run( + f'cat {from_path} | docker exec -i {self.container_name} sh -c "cat > {to_path}"', + shell=True, + check=True) + + def copy_out(self, from_path: PurePath, to_path: Path) -> None: + # note: we assume from_path is a dir + to_path.mkdir(parents=True, exist_ok=True) + + subprocess.run( + f'docker exec -i -w {from_path} {self.container_name} tar cf - . | tar -xf -', + shell=True, + check=True, + cwd=to_path + ) + + def glob(self, pattern: PurePath) -> List[PurePath]: + path_strs = json.loads(self.call([ + self.UTILITY_PYTHON, + '-c', + f'import sys, json, glob; json.dump(glob.glob({str(pattern)!r}), sys.stdout)' + ], capture_output=True)) + + return [PurePath(p) for p in path_strs] + + def call(self, args: Sequence[Union[str, PathLike]], env: Dict[str, str] = {}, + capture_output=False, cwd: Optional[Union[str, PathLike]] = None) -> str: + # ensure paths are converted to strs + args = [str(a) for a in args] + + if self.simulate_32_bit: + args = ['linux32', *args] + + options = ['-i'] + + for key, value in env.items(): + options += ['--env', f'{key}={value}'] + + if cwd: + options += ['-w', str(cwd)] + + print('+ ' + ' '.join(shlex.quote(str(a)) for a in args)) + + process = subprocess.run( + ['docker', 'exec', *options, self.container_name, *args], + check=True, + universal_newlines=True, + stdout=subprocess.PIPE if capture_output else None + ) + + if capture_output: + return process.stdout + else: + return '' + + def get_environment(self) -> Dict[str, str]: + return json.loads(self.call([ + self.UTILITY_PYTHON, + '-c', + 'import sys, json, os; json.dump(os.environ.copy(), sys.stdout)' + ], capture_output=True)) + + def environment_executor(self, command: str, environment: Dict[str, str]) -> str: + # used as an EnvironmentExecutor to evaluate commands and capture output + return self.call(shlex.split(command), env=environment) diff --git a/requirements-dev.txt b/requirements-dev.txt index 44043506..e6129684 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -6,3 +6,5 @@ pymdown-extensions pip-tools requests click +mypy +typing-extensions From 50a263df21b4a2e9b99fe61e026caee42609bd4a Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Tue, 23 Jun 2020 20:28:54 +0100 Subject: [PATCH 041/117] Remove str() calls in call/shell functions using destructuring syntax --- cibuildwheel/macos.py | 49 +++++++++++++++++++++---------------- cibuildwheel/windows.py | 54 +++++++++++++++++++++++------------------ 2 files changed, 58 insertions(+), 45 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index d9d892b9..0c268f4c 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -4,27 +4,22 @@ import shutil import subprocess import sys import tempfile +from os import PathLike from pathlib import Path - -from typing import Dict, List, Optional, NamedTuple, Union +from typing import Dict, List, NamedTuple, Optional, Sequence, Union from .environment import ParsedEnvironment -from .util import ( - BuildOptions, - BuildSelector, - download, - get_build_verbosity_extra_flags, - get_pip_script, - prepare_command, -) +from .util import (BuildOptions, BuildSelector, download, + get_build_verbosity_extra_flags, get_pip_script, + prepare_command) -def call(args: Union[str, List[str]], env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None, shell: bool = False) -> int: +def call(args: Union[str, Sequence[Union[str, PathLike]]], env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None, shell: bool = False) -> int: # print the command executing for the logs if shell: print(f'+ {args}') else: - print('+ ' + ' '.join(shlex.quote(a) for a in args)) + print('+ ' + ' '.join(shlex.quote(str(a)) for a in args)) return subprocess.check_call(args, env=env, cwd=cwd, shell=shell) @@ -103,7 +98,7 @@ def install_pypy(version: str, url: str) -> Path: if not installation_path.exists(): downloaded_tar_bz2 = Path("/tmp") / pypy_tar_bz2 download(url, downloaded_tar_bz2) - call(['tar', '-C', '/tmp', '-xf', str(downloaded_tar_bz2)]) + call(['tar', '-C', '/tmp', '-xf', downloaded_tar_bz2]) installation_bin_path = installation_path / 'bin' python_executable = 'pypy3' if version[0] == '3' else 'pypy' @@ -113,7 +108,9 @@ def install_pypy(version: str, url: str) -> Path: return installation_bin_path -def setup_python(python_configuration: PythonConfiguration, dependency_constraint_flags: List[str], environment: ParsedEnvironment) -> Dict[str, str]: +def setup_python(python_configuration: PythonConfiguration, + dependency_constraint_flags: Sequence[Union[str, PathLike]], + environment: ParsedEnvironment) -> Dict[str, str]: if python_configuration.identifier.startswith('cp'): installation_bin_path = install_cpython(python_configuration.version, python_configuration.url) elif python_configuration.identifier.startswith('pp'): @@ -146,7 +143,7 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain exit(1) # install pip & wheel - call(['python', str(get_pip_script)] + dependency_constraint_flags, env=env, cwd="/tmp") + call(['python', get_pip_script, *dependency_constraint_flags], env=env, cwd="/tmp") assert (installation_bin_path / 'pip').exists() call(['which', 'pip'], env=env) call(['pip', '--version'], env=env) @@ -154,7 +151,7 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain if which_pip != '/tmp/cibw_bin/pip': print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr) exit(1) - call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate'] + dependency_constraint_flags, env=env) + call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate', *dependency_constraint_flags], env=env) # Set MACOSX_DEPLOYMENT_TARGET to 10.9, if the user didn't set it. # CPython 3.5 defaults to 10.6, and pypy defaults to 10.7, causing @@ -184,10 +181,10 @@ def build(options: BuildOptions) -> None: python_configurations = get_python_configurations(options.build_selector) for config in python_configurations: - dependency_constraint_flags = [] + dependency_constraint_flags: Sequence[Union[str, PathLike]] = [] if options.dependency_constraints: dependency_constraint_flags = [ - '-c', str(options.dependency_constraints.get_for_python_version(config.version)) + '-c', options.dependency_constraints.get_for_python_version(config.version) ] env = setup_python(config, dependency_constraint_flags, options.environment) @@ -201,9 +198,19 @@ def build(options: BuildOptions) -> None: if built_wheel_dir.exists(): shutil.rmtree(built_wheel_dir) built_wheel_dir.mkdir(parents=True) + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org # see https://github.com/joerick/cibuildwheel/pull/369 - call(['pip', 'wheel', str(options.package_dir.resolve()), '-w', str(built_wheel_dir), '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env) + call( + [ + 'pip', 'wheel', + options.package_dir.resolve(), + '-w', built_wheel_dir, + '--no-deps', + *get_build_verbosity_extra_flags(options.build_verbosity) + ], + env=env, + ) built_wheel = next(built_wheel_dir.glob('*.whl')) # repair the wheel @@ -221,12 +228,12 @@ def build(options: BuildOptions) -> None: if options.test_command: # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. - call(['pip', 'install', 'virtualenv'] + dependency_constraint_flags, env=env) + call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) venv_dir = Path(tempfile.mkdtemp()) # Use --no-download to ensure determinism by using seed libraries # built into virtualenv - call(['python', '-m', 'virtualenv', '--no-download', str(venv_dir)], env=env) + call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) virtualenv_env = env.copy() virtualenv_env['PATH'] = os.pathsep.join([ diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index bae831de..9f67c6c4 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -3,29 +3,25 @@ import shutil import subprocess import sys import tempfile +from os import PathLike from pathlib import Path +from typing import Dict, List, NamedTuple, Optional, Sequence, Union from zipfile import ZipFile -from typing import Dict, List, Optional, NamedTuple - from .environment import ParsedEnvironment -from .util import ( - BuildOptions, - BuildSelector, - download, - get_build_verbosity_extra_flags, - get_pip_script, - prepare_command, -) - +from .util import (BuildOptions, BuildSelector, download, + get_build_verbosity_extra_flags, get_pip_script, + prepare_command) IS_RUNNING_ON_AZURE = Path('C:\\hostedtoolcache').exists() IS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows' -def shell(args: List[str], env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None) -> int: - print('+ ' + ' '.join(args)) - return subprocess.check_call(' '.join(args), env=env, cwd=cwd, shell=True) +def shell(args: Sequence[Union[str, PathLike]], env: Optional[Dict[str, str]] = None, + cwd: Optional[str] = None) -> int: + command = ' '.join(str(a) for a in args) + print(f'+ {command}') + return subprocess.check_call(command, env=env, cwd=cwd, shell=True) def get_nuget_args(version: str, arch: str) -> List[str]: @@ -79,7 +75,7 @@ def extract_zip(zip_src: Path, dest: Path) -> None: def install_cpython(version: str, arch: str, nuget: Path) -> Path: nuget_args = get_nuget_args(version, arch) installation_path = Path(nuget_args[-1]) / (nuget_args[0] + '.' + version) / 'tools' - shell([str(nuget), 'install'] + nuget_args) + shell([nuget, 'install', *nuget_args]) return installation_path @@ -100,7 +96,7 @@ def install_pypy(version: str, arch: str, url: str) -> Path: return installation_path -def setup_python(python_configuration: PythonConfiguration, dependency_constraint_flags: List[str], environment: ParsedEnvironment) -> Dict[str, str]: +def setup_python(python_configuration: PythonConfiguration, dependency_constraint_flags: Sequence[Union[str, PathLike]], environment: ParsedEnvironment) -> Dict[str, str]: nuget = Path('C:\\cibw\\nuget.exe') if not nuget.exists(): download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget) @@ -138,7 +134,7 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain # make sure pip is installed if not (installation_path / 'Scripts' / 'pip.exe').exists(): - shell(['python', str(get_pip_script)] + dependency_constraint_flags, env=env, cwd="C:\\cibw") + shell(['python', get_pip_script, *dependency_constraint_flags], env=env, cwd="C:\\cibw") assert (installation_path / 'Scripts' / 'pip.exe').exists() where_pip = subprocess.check_output(['where', 'pip'], env=env, universal_newlines=True).splitlines()[0].strip() if where_pip.strip() != str(installation_path / 'Scripts' / 'pip.exe'): @@ -146,9 +142,9 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain exit(1) # prepare the Python environment - shell(['python', '-m', 'pip', 'install', '--upgrade', 'pip'] + dependency_constraint_flags, env=env) + shell(['python', '-m', 'pip', 'install', '--upgrade', 'pip', *dependency_constraint_flags], env=env) shell(['pip', '--version'], env=env) - shell(['pip', 'install', '--upgrade', 'setuptools', 'wheel'] + dependency_constraint_flags, env=env) + shell(['pip', 'install', '--upgrade', 'setuptools', 'wheel', *dependency_constraint_flags], env=env) return env @@ -169,10 +165,10 @@ def build(options: BuildOptions) -> None: python_configurations = get_python_configurations(options.build_selector) for config in python_configurations: - dependency_constraint_flags = [] + dependency_constraint_flags: Sequence[Union[str, PathLike]] = [] if options.dependency_constraints: dependency_constraint_flags = [ - '-c', str(options.dependency_constraints.get_for_python_version(config.version)) + '-c', options.dependency_constraints.get_for_python_version(config.version) ] # install Python @@ -189,7 +185,17 @@ def build(options: BuildOptions) -> None: built_wheel_dir.mkdir(parents=True) # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org # see https://github.com/joerick/cibuildwheel/pull/369 - shell(['pip', 'wheel', str(options.package_dir.resolve()), '-w', str(built_wheel_dir), '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env) + shell( + [ + 'pip', 'wheel', + options.package_dir.resolve(), + '-w', built_wheel_dir, + '--no-deps', + *get_build_verbosity_extra_flags(options.build_verbosity) + ], + env=env, + ) + built_wheel = next(built_wheel_dir.glob('*.whl')) # repair the wheel @@ -207,12 +213,12 @@ def build(options: BuildOptions) -> None: if options.test_command: # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. - shell(['pip', 'install', 'virtualenv'] + dependency_constraint_flags, env=env) + shell(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) venv_dir = Path(tempfile.mkdtemp()) # Use --no-download to ensure determinism by using seed libraries # built into virtualenv - shell(['python', '-m', 'virtualenv', '--no-download', str(venv_dir)], env=env) + shell(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) virtualenv_env = env.copy() virtualenv_env['PATH'] = os.pathsep.join([ From 41891277d33c728059fb89ac2ce0c2b32d6d459b Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 24 Jun 2020 09:54:55 +0100 Subject: [PATCH 042/117] Bump version: v1.5.0 --- README.md | 40 +++++++++++++++++++------- cibuildwheel/__init__.py | 2 +- examples/appveyor-minimal.yml | 2 +- examples/azure-pipelines-minimal.yml | 6 ++-- examples/circleci-minimal.yml | 4 +-- examples/github-deploy.yml | 2 +- examples/github-minimal.yml | 2 +- examples/travis-ci-deploy.yml | 2 +- examples/travis-ci-minimal.yml | 2 +- examples/travis-ci-test-and-deploy.yml | 6 ++-- setup.py | 2 +- 11 files changed, 44 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 3412159e..31cc943c 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ env: # Note: TWINE_PASSWORD is set to a PyPI API token in Travis settings install: - - python3 -m pip install cibuildwheel==1.4.2 + - python3 -m pip install cibuildwheel==1.5.0 script: # build the wheels, put them into './wheelhouse' @@ -152,6 +152,24 @@ This is similar to static linking, so it might have some licence implications. C Changelog ========= +### 1.5.0 + +_24 June 2020_ + +- 🌟 Add [`CIBW_BEFORE_ALL`](https://cibuildwheel.readthedocs.io/en/stable/options/#before-all) + option, which lets you run a command on the build machine before any wheels + are built. This is especially useful when building on Linux, to `make` + something external to Python, or to `yum install` a dependency. (#342) +- ✨ Added support for projects using pyproject.toml instead of setup.py + (#360, #358) +- ✨ Added workaround to allow Python 3.5 on Windows to pull dependencies from + pyproject.toml. (#358) +- 📚 Improved Github Actions examples and docs (#354, #362) +- 🐛 Ensure pip wheel uses the specified package, and doesn't build a wheel + from PyPI (#369) +- 🛠 Internal changes: using pathlib.Path, precommit hooks, testing + improvements. + ### 1.4.2 _25 May 2020_ @@ -174,7 +192,7 @@ _4 May 2020_ _2 May 2020_ -- ✨ Deterministic builds. cibuildwheel now locks the versions of the tools it +- 🌟 Deterministic builds. cibuildwheel now locks the versions of the tools it uses. This means that pinning your version of cibuildwheel pins the versions of pip, setuptools, manylinux etc. that are used under the hood. This should make things more reliable. But note that we don't control the entire build @@ -204,7 +222,7 @@ _2 May 2020_ _12 March 2020_ -- ✨ Add support for building on Github Actions! Check out the +- 🌟 Add support for building on Github Actions! Check out the [docs](https://cibuildwheel.readthedocs.io/en/stable/setup/#github-actions) for information on how to set it up. (#194) - ✨ Add the `CIBW_BEFORE_TEST` option, which lets you run a command to @@ -214,9 +232,9 @@ _12 March 2020_ _8 March 2020_ -- ✨ Add support for building PyPy wheels, across Manylinux, macOS, and +- 🌟 Add support for building PyPy wheels, across Manylinux, macOS, and Windows. (#185) -- ✨ Added the ability to build ARM64 (aarch64), ppc64le, and s390x wheels, +- 🌟 Added the ability to build ARM64 (aarch64), ppc64le, and s390x wheels, using manylinux2014 and Travis CI. (#273) - ✨ You can now build macOS wheels on Appveyor. (#230) - 🛠 Changed default macOS minimum target to 10.9, from 10.6. This allows the @@ -237,7 +255,7 @@ _8 March 2020_ _7 December 2019_ -- ✨ Add support for building manylinux2014 wheels. To use, set +- 🌟 Add support for building manylinux2014 wheels. To use, set `CIBW_MANYLINUX_X86_64_IMAGE` and CIBW_MANYLINUX_I686_IMAGE to `manylinux2014`. - ✨ Add support for [Linux on Appveyor](https://www.appveyor.com/blog/2018/03/06/appveyor-for-linux/) (#204, #207) @@ -250,8 +268,8 @@ _7 December 2019_ _10 November 2019_ -- ✨ Add support for building Python 3.8 wheels! (#180) -- ✨ Add support for building manylinux2010 wheels. cibuildwheel will now +- 🌟 Add support for building Python 3.8 wheels! (#180) +- 🌟 Add support for building manylinux2010 wheels. cibuildwheel will now build using the manylinux2010 images by default. If your project is still manylinux1 compatible, you should get both manylinux1 and manylinux2010 wheels - you can upload both to PyPI. If you always require manylinux1 wheels, you can @@ -287,7 +305,7 @@ _28 May 2019_ _26 May 2019_ -- ✨ Add support for building on Azure pipelines! This lets you build all +- 🌟 Add support for building on Azure pipelines! This lets you build all Linux, Mac and Windows wheels on one service, so it promises to be the easiest to set up! Check out the quickstart in the docs, or [cibuildwheel-azure-example](https://github.com/joerick/cibuildwheel-azure-example) @@ -317,8 +335,8 @@ _3 February 2019_ _23 September 2018_ -- ✨ Add `CIBW_BUILD` option, for specifying which specific builds to perform (#101) -- ✨ Add support for building Mac and Linux on CircleCI (#91, #97) +- 🌟 Add `CIBW_BUILD` option, for specifying which specific builds to perform (#101) +- 🌟 Add support for building Mac and Linux on CircleCI (#91, #97) - 🛠 Improved support for building universal wheels (#95) - 🛠 Ensure log output is unbuffered and therefore in the correct order (#92) - 🛠 Improved error reporting for errors that occur inside a package's setup.py (#88) diff --git a/cibuildwheel/__init__.py b/cibuildwheel/__init__.py index 98d186be..77f1c8e6 100644 --- a/cibuildwheel/__init__.py +++ b/cibuildwheel/__init__.py @@ -1 +1 @@ -__version__ = '1.4.2' +__version__ = '1.5.0' diff --git a/examples/appveyor-minimal.yml b/examples/appveyor-minimal.yml index 849fe274..47bfde21 100644 --- a/examples/appveyor-minimal.yml +++ b/examples/appveyor-minimal.yml @@ -12,7 +12,7 @@ stack: python 3.7 init: - cmd: set PATH=C:\Python37;C:\Python37\Scripts;%PATH% -install: python -m pip install cibuildwheel==1.4.2 +install: python -m pip install cibuildwheel==1.5.0 build_script: python -m cibuildwheel --output-dir wheelhouse diff --git a/examples/azure-pipelines-minimal.yml b/examples/azure-pipelines-minimal.yml index 3266e888..c6597d13 100644 --- a/examples/azure-pipelines-minimal.yml +++ b/examples/azure-pipelines-minimal.yml @@ -5,7 +5,7 @@ jobs: - task: UsePythonVersion@0 - bash: | python3 -m pip install --upgrade pip - pip3 install cibuildwheel==1.4.2 + pip3 install cibuildwheel==1.5.0 cibuildwheel --output-dir wheelhouse . - task: PublishBuildArtifacts@1 inputs: {pathtoPublish: 'wheelhouse'} @@ -16,7 +16,7 @@ jobs: - task: UsePythonVersion@0 - bash: | python3 -m pip install --upgrade pip - pip3 install cibuildwheel==1.4.2 + pip3 install cibuildwheel==1.5.0 cibuildwheel --output-dir wheelhouse . - task: PublishBuildArtifacts@1 inputs: {pathtoPublish: 'wheelhouse'} @@ -29,7 +29,7 @@ jobs: displayName: Install Visual C++ for Python 2.7 - bash: | python -m pip install --upgrade pip - pip install cibuildwheel==1.4.2 + pip install cibuildwheel==1.5.0 cibuildwheel --output-dir wheelhouse . - task: PublishBuildArtifacts@1 inputs: {pathtoPublish: 'wheelhouse'} diff --git a/examples/circleci-minimal.yml b/examples/circleci-minimal.yml index 747b3f8e..fb11ff3b 100644 --- a/examples/circleci-minimal.yml +++ b/examples/circleci-minimal.yml @@ -11,7 +11,7 @@ jobs: - run: name: Build the Linux wheels. command: | - pip3 install --user cibuildwheel==1.4.2 + pip3 install --user cibuildwheel==1.5.0 cibuildwheel --output-dir wheelhouse - store_artifacts: path: wheelhouse/ @@ -25,7 +25,7 @@ jobs: - run: name: Build the OS X wheels. command: | - pip3 install --user cibuildwheel==1.4.2 + pip3 install --user cibuildwheel==1.5.0 cibuildwheel --output-dir wheelhouse - store_artifacts: path: wheelhouse/ diff --git a/examples/github-deploy.yml b/examples/github-deploy.yml index 746c5847..d86eccde 100644 --- a/examples/github-deploy.yml +++ b/examples/github-deploy.yml @@ -28,7 +28,7 @@ jobs: - name: Install cibuildwheel run: | - python -m pip install cibuildwheel==1.4.2 + python -m pip install cibuildwheel==1.5.0 - name: Install Visual C++ for Python 2.7 if: runner.os == 'Windows' diff --git a/examples/github-minimal.yml b/examples/github-minimal.yml index 8501cb84..30c20f60 100644 --- a/examples/github-minimal.yml +++ b/examples/github-minimal.yml @@ -20,7 +20,7 @@ jobs: - name: Install cibuildwheel run: | - python -m pip install cibuildwheel==1.4.2 + python -m pip install cibuildwheel==1.5.0 - name: Install Visual C++ for Python 2.7 if: runner.os == 'Windows' diff --git a/examples/travis-ci-deploy.yml b/examples/travis-ci-deploy.yml index 5e82149f..c9c085f5 100644 --- a/examples/travis-ci-deploy.yml +++ b/examples/travis-ci-deploy.yml @@ -25,7 +25,7 @@ env: # Note: TWINE_PASSWORD is set to a PyPI API token in Travis settings install: - - python3 -m pip install cibuildwheel==1.4.2 + - python3 -m pip install cibuildwheel==1.5.0 script: # build the wheels, put them into './wheelhouse' diff --git a/examples/travis-ci-minimal.yml b/examples/travis-ci-minimal.yml index b03a0da5..2be04ce2 100644 --- a/examples/travis-ci-minimal.yml +++ b/examples/travis-ci-minimal.yml @@ -26,7 +26,7 @@ jobs: - ln -s /c/Python38/python.exe /c/Python38/python3.exe install: - - python3 -m pip install cibuildwheel==1.4.2 + - python3 -m pip install cibuildwheel==1.5.0 script: # build the wheels, put them into './wheelhouse' diff --git a/examples/travis-ci-test-and-deploy.yml b/examples/travis-ci-test-and-deploy.yml index b9d920f6..f29ea81b 100644 --- a/examples/travis-ci-test-and-deploy.yml +++ b/examples/travis-ci-test-and-deploy.yml @@ -55,7 +55,7 @@ jobs: - stage: deploy name: Build and deploy Linux wheels services: docker - install: python3 -m pip install cibuildwheel==1.4.2 + install: python3 -m pip install cibuildwheel==1.5.0 script: python3 -m cibuildwheel --output-dir wheelhouse after_success: | python3 -m pip install twine @@ -65,7 +65,7 @@ jobs: name: Build and deploy macOS wheels os: osx language: shell - install: python3 -m pip install cibuildwheel==1.4.2 + install: python3 -m pip install cibuildwheel==1.5.0 script: python3 -m cibuildwheel --output-dir wheelhouse after_success: | python3 -m pip install twine @@ -75,7 +75,7 @@ jobs: name: Build and deploy Windows wheels os: windows language: shell - install: python3 -m pip install cibuildwheel==1.4.2 + install: python3 -m pip install cibuildwheel==1.5.0 script: python3 -m cibuildwheel --output-dir wheelhouse after_success: | python3 -m pip install twine diff --git a/setup.py b/setup.py index 542d29f7..4396b928 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ long_description = (this_directory / 'README.md').read_text(encoding='utf-8') setup( name='cibuildwheel', - version='1.4.2', + version='1.5.0', install_requires=['bashlex!=0.13', 'toml'], description="Build Python wheels on CI with minimal configuration.", long_description=long_description, From 53c00a0af686399ae12ecb9f0fed01245fadb93f Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 24 Jun 2020 16:39:20 +0100 Subject: [PATCH 043/117] Use a long-running remote bash shell, rather than `docker exec` --- cibuildwheel/linux.py | 102 +++++++++++++++++++++++++++++++----------- setup.cfg | 2 +- 2 files changed, 78 insertions(+), 26 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index f66cc5e1..14c63adb 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -1,3 +1,4 @@ +import io import json import platform import shlex @@ -7,7 +8,8 @@ import textwrap import uuid from os import PathLike from pathlib import Path, PurePath -from typing import Dict, List, NamedTuple, Optional, Union, Sequence +from typing import (IO, Dict, List, NamedTuple, Optional, Sequence, TextIO, + Union) from .util import (BuildOptions, BuildSelector, get_build_verbosity_extra_flags, prepare_command) @@ -258,12 +260,17 @@ def troubleshoot(package_dir: Path, error: Exception) -> None: class DockerContainer: UTILITY_PYTHON = '/opt/python/cp38-cp38/bin/python' - def __init__(self, docker_image, simulate_32_bit=False): + process: subprocess.Popen + bash_stdin: IO[str] + bash_stdout: IO[str] + + def __init__(self, docker_image: str, simulate_32_bit=False): self.docker_image = docker_image self.simulate_32_bit = simulate_32_bit def __enter__(self) -> 'DockerContainer': self.container_name = f'cibuildwheel-{uuid.uuid4()}' + shell_args = ['linux32', '/bin/bash'] if self.simulate_32_bit else ['/bin/bash'] subprocess.run( [ 'docker', 'create', @@ -272,14 +279,34 @@ class DockerContainer: '-i', '-v', '/:/host', # ignored on CircleCI self.docker_image, - '/bin/bash' + *shell_args ], check=True, ) - subprocess.run(['docker', 'start', self.container_name], check=True) + process = subprocess.Popen( + [ + 'docker', 'start', + '--attach', '--interactive', + self.container_name, + ], + encoding='utf8', + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + # make the input buffer large enough to carry a lot of environment + # variables. We choose 256kB. + bufsize=262144, + ) + self.process = process + assert process.stdin and process.stdout + self.bash_stdin = process.stdin + self.bash_stdout = process.stdout return self def __exit__(self, exc_type, exc_val, exc_tb): + self.bash_stdin.close() + self.process.terminate() + self.process.wait() + subprocess.run(['docker', 'rm', '--force', '-v', self.container_name]) self.container_name = None @@ -323,33 +350,58 @@ class DockerContainer: def call(self, args: Sequence[Union[str, PathLike]], env: Dict[str, str] = {}, capture_output=False, cwd: Optional[Union[str, PathLike]] = None) -> str: - # ensure paths are converted to strs - args = [str(a) for a in args] + env_exports = '\n'.join(f'export {k}={v}' for k, v in env.items()) + chdir = f'cd {cwd}' if cwd else '' + command = ' '.join(shlex.quote(str(a)) for a in args) + end_of_message = str(uuid.uuid4()) - if self.simulate_32_bit: - args = ['linux32', *args] + # log the command we're executing + print(f' + {command}') - options = ['-i'] - - for key, value in env.items(): - options += ['--env', f'{key}={value}'] - - if cwd: - options += ['-w', str(cwd)] - - print('+ ' + ' '.join(shlex.quote(str(a)) for a in args)) - - process = subprocess.run( - ['docker', 'exec', *options, self.container_name, *args], - check=True, - universal_newlines=True, - stdout=subprocess.PIPE if capture_output else None + # Write a command to the remote shell. First we write the + # environment variables, exported inside the subshell. We change the + # cwd, if that's required. Then, the command is written. Finally, the + # remote shell is told to write a footer - this will show up in the + # output so we know when to stop reading, and will include the + # returncode of `command`. + self.bash_stdin.write(f'''( + {env_exports} + {chdir} + {command} + printf "%04d%s\n" $? {end_of_message} ) + ''') + self.bash_stdin.flush() if capture_output: - return process.stdout + output_io: TextIO = io.StringIO() else: - return '' + output_io = sys.stdout + + while True: + line = self.bash_stdout.readline() + + if line.endswith(end_of_message+'\n'): + footer_offset = ( + len(line) + - 1 # newline character + - len(end_of_message) # delimiter + - 4 # 4 returncode decimals + ) + returncode_str = line[footer_offset:footer_offset+4] + returncode = int(returncode_str) + # add the last line to output, without the footer + output_io.write(line[0:footer_offset]) + break + else: + output_io.write(line) + + output = output_io.getvalue() if isinstance(output_io, io.StringIO) else None + + if returncode != 0: + raise subprocess.CalledProcessError(returncode, args, output) + + return output if output else '' def get_environment(self) -> Dict[str, str]: return json.loads(self.call([ diff --git a/setup.cfg b/setup.cfg index d54a42bb..e98764a2 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [flake8] -ignore = E501,W503,E741 +ignore = E501,W503,E741,E226 application-import-names = cibuildwheel exclude = cibuildwheel/resources/, From 31339ba30e8e73e8d0956e13f62399d627f066b1 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 24 Jun 2020 16:43:33 +0100 Subject: [PATCH 044/117] Extract DockerContainer into its own file --- cibuildwheel/docker_container.py | 177 +++++++++++++++++++++++++++++++ cibuildwheel/linux.py | 166 +---------------------------- 2 files changed, 179 insertions(+), 164 deletions(-) create mode 100644 cibuildwheel/docker_container.py diff --git a/cibuildwheel/docker_container.py b/cibuildwheel/docker_container.py new file mode 100644 index 00000000..9ba7509d --- /dev/null +++ b/cibuildwheel/docker_container.py @@ -0,0 +1,177 @@ +import io +import json +import shlex +import subprocess +import sys +import uuid +from os import PathLike +from pathlib import Path, PurePath +from typing import IO, Dict, List, Optional, Sequence, TextIO, Union + + +class DockerContainer: + ''' + An object that represents a running Docker container. + + Intended for use as a context manager e.g. + `with DockerContainer('ubuntu') as docker:` + + A bash shell is running in the remote container. When `call()` is invoked, + the command is relayed to the remote shell, and the results are streamed + back to cibuildwheel. + ''' + UTILITY_PYTHON = '/opt/python/cp38-cp38/bin/python' + + process: subprocess.Popen + bash_stdin: IO[str] + bash_stdout: IO[str] + + def __init__(self, docker_image: str, simulate_32_bit=False): + self.docker_image = docker_image + self.simulate_32_bit = simulate_32_bit + + def __enter__(self) -> 'DockerContainer': + self.container_name = f'cibuildwheel-{uuid.uuid4()}' + shell_args = ['linux32', '/bin/bash'] if self.simulate_32_bit else ['/bin/bash'] + subprocess.run( + [ + 'docker', 'create', + '--env', 'CIBUILDWHEEL', + '--name', self.container_name, + '-i', + '-v', '/:/host', # ignored on CircleCI + self.docker_image, + *shell_args + ], + check=True, + ) + process = subprocess.Popen( + [ + 'docker', 'start', + '--attach', '--interactive', + self.container_name, + ], + encoding='utf8', + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + # make the input buffer large enough to carry a lot of environment + # variables. We choose 256kB. + bufsize=262144, + ) + self.process = process + assert process.stdin and process.stdout + self.bash_stdin = process.stdin + self.bash_stdout = process.stdout + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.bash_stdin.close() + self.process.terminate() + self.process.wait() + + subprocess.run(['docker', 'rm', '--force', '-v', self.container_name]) + self.container_name = None + + def copy_into(self, from_path: Path, to_path: PurePath) -> None: + # `docker cp` causes 'no space left on device' error when + # a container is running and the host filesystem is + # mounted. https://github.com/moby/moby/issues/38995 + # Use `docker exec` instead. + if from_path.is_dir(): + self.call(['mkdir', '-p', to_path]) + subprocess.run( + f'tar cf - . | docker exec -i {self.container_name} tar -xC {to_path} -f -', + shell=True, + check=True, + cwd=from_path) + else: + subprocess.run( + f'cat {from_path} | docker exec -i {self.container_name} sh -c "cat > {to_path}"', + shell=True, + check=True) + + def copy_out(self, from_path: PurePath, to_path: Path) -> None: + # note: we assume from_path is a dir + to_path.mkdir(parents=True, exist_ok=True) + + subprocess.run( + f'docker exec -i -w {from_path} {self.container_name} tar cf - . | tar -xf -', + shell=True, + check=True, + cwd=to_path + ) + + def glob(self, pattern: PurePath) -> List[PurePath]: + path_strs = json.loads(self.call([ + self.UTILITY_PYTHON, + '-c', + f'import sys, json, glob; json.dump(glob.glob({str(pattern)!r}), sys.stdout)' + ], capture_output=True)) + + return [PurePath(p) for p in path_strs] + + def call(self, args: Sequence[Union[str, PathLike]], env: Dict[str, str] = {}, + capture_output=False, cwd: Optional[Union[str, PathLike]] = None) -> str: + env_exports = '\n'.join(f'export {k}={v}' for k, v in env.items()) + chdir = f'cd {cwd}' if cwd else '' + command = ' '.join(shlex.quote(str(a)) for a in args) + end_of_message = str(uuid.uuid4()) + + # log the command we're executing + print(f' + {command}') + + # Write a command to the remote shell. First we write the + # environment variables, exported inside the subshell. We change the + # cwd, if that's required. Then, the command is written. Finally, the + # remote shell is told to write a footer - this will show up in the + # output so we know when to stop reading, and will include the + # returncode of `command`. + self.bash_stdin.write(f'''( + {env_exports} + {chdir} + {command} + printf "%04d%s\n" $? {end_of_message} + ) + ''') + self.bash_stdin.flush() + + if capture_output: + output_io: TextIO = io.StringIO() + else: + output_io = sys.stdout + + while True: + line = self.bash_stdout.readline() + + if line.endswith(end_of_message+'\n'): + footer_offset = ( + len(line) + - 1 # newline character + - len(end_of_message) # delimiter + - 4 # 4 returncode decimals + ) + returncode_str = line[footer_offset:footer_offset+4] + returncode = int(returncode_str) + # add the last line to output, without the footer + output_io.write(line[0:footer_offset]) + break + else: + output_io.write(line) + + output = output_io.getvalue() if isinstance(output_io, io.StringIO) else None + + if returncode != 0: + raise subprocess.CalledProcessError(returncode, args, output) + + return output if output else '' + + def get_environment(self) -> Dict[str, str]: + return json.loads(self.call([ + self.UTILITY_PYTHON, + '-c', + 'import sys, json, os; json.dump(os.environ.copy(), sys.stdout)' + ], capture_output=True)) + + def environment_executor(self, command: str, environment: Dict[str, str]) -> str: + # used as an EnvironmentExecutor to evaluate commands and capture output + return self.call(shlex.split(command), env=environment) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 14c63adb..30481008 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -1,16 +1,12 @@ -import io -import json import platform -import shlex import subprocess import sys import textwrap -import uuid from os import PathLike from pathlib import Path, PurePath -from typing import (IO, Dict, List, NamedTuple, Optional, Sequence, TextIO, - Union) +from typing import List, NamedTuple, Union +from .docker_container import DockerContainer from .util import (BuildOptions, BuildSelector, get_build_verbosity_extra_flags, prepare_command) @@ -255,161 +251,3 @@ def troubleshoot(package_dir: Path, error: Exception) -> None: print(' Files detected:') print('\n'.join([f' {f}' for f in so_files])) print('') - - -class DockerContainer: - UTILITY_PYTHON = '/opt/python/cp38-cp38/bin/python' - - process: subprocess.Popen - bash_stdin: IO[str] - bash_stdout: IO[str] - - def __init__(self, docker_image: str, simulate_32_bit=False): - self.docker_image = docker_image - self.simulate_32_bit = simulate_32_bit - - def __enter__(self) -> 'DockerContainer': - self.container_name = f'cibuildwheel-{uuid.uuid4()}' - shell_args = ['linux32', '/bin/bash'] if self.simulate_32_bit else ['/bin/bash'] - subprocess.run( - [ - 'docker', 'create', - '--env', 'CIBUILDWHEEL', - '--name', self.container_name, - '-i', - '-v', '/:/host', # ignored on CircleCI - self.docker_image, - *shell_args - ], - check=True, - ) - process = subprocess.Popen( - [ - 'docker', 'start', - '--attach', '--interactive', - self.container_name, - ], - encoding='utf8', - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - # make the input buffer large enough to carry a lot of environment - # variables. We choose 256kB. - bufsize=262144, - ) - self.process = process - assert process.stdin and process.stdout - self.bash_stdin = process.stdin - self.bash_stdout = process.stdout - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - self.bash_stdin.close() - self.process.terminate() - self.process.wait() - - subprocess.run(['docker', 'rm', '--force', '-v', self.container_name]) - self.container_name = None - - def copy_into(self, from_path: Path, to_path: PurePath) -> None: - # `docker cp` causes 'no space left on device' error when - # a container is running and the host filesystem is - # mounted. https://github.com/moby/moby/issues/38995 - # Use `docker exec` instead. - if from_path.is_dir(): - self.call(['mkdir', '-p', to_path]) - subprocess.run( - f'tar cf - . | docker exec -i {self.container_name} tar -xC {to_path} -f -', - shell=True, - check=True, - cwd=from_path) - else: - subprocess.run( - f'cat {from_path} | docker exec -i {self.container_name} sh -c "cat > {to_path}"', - shell=True, - check=True) - - def copy_out(self, from_path: PurePath, to_path: Path) -> None: - # note: we assume from_path is a dir - to_path.mkdir(parents=True, exist_ok=True) - - subprocess.run( - f'docker exec -i -w {from_path} {self.container_name} tar cf - . | tar -xf -', - shell=True, - check=True, - cwd=to_path - ) - - def glob(self, pattern: PurePath) -> List[PurePath]: - path_strs = json.loads(self.call([ - self.UTILITY_PYTHON, - '-c', - f'import sys, json, glob; json.dump(glob.glob({str(pattern)!r}), sys.stdout)' - ], capture_output=True)) - - return [PurePath(p) for p in path_strs] - - def call(self, args: Sequence[Union[str, PathLike]], env: Dict[str, str] = {}, - capture_output=False, cwd: Optional[Union[str, PathLike]] = None) -> str: - env_exports = '\n'.join(f'export {k}={v}' for k, v in env.items()) - chdir = f'cd {cwd}' if cwd else '' - command = ' '.join(shlex.quote(str(a)) for a in args) - end_of_message = str(uuid.uuid4()) - - # log the command we're executing - print(f' + {command}') - - # Write a command to the remote shell. First we write the - # environment variables, exported inside the subshell. We change the - # cwd, if that's required. Then, the command is written. Finally, the - # remote shell is told to write a footer - this will show up in the - # output so we know when to stop reading, and will include the - # returncode of `command`. - self.bash_stdin.write(f'''( - {env_exports} - {chdir} - {command} - printf "%04d%s\n" $? {end_of_message} - ) - ''') - self.bash_stdin.flush() - - if capture_output: - output_io: TextIO = io.StringIO() - else: - output_io = sys.stdout - - while True: - line = self.bash_stdout.readline() - - if line.endswith(end_of_message+'\n'): - footer_offset = ( - len(line) - - 1 # newline character - - len(end_of_message) # delimiter - - 4 # 4 returncode decimals - ) - returncode_str = line[footer_offset:footer_offset+4] - returncode = int(returncode_str) - # add the last line to output, without the footer - output_io.write(line[0:footer_offset]) - break - else: - output_io.write(line) - - output = output_io.getvalue() if isinstance(output_io, io.StringIO) else None - - if returncode != 0: - raise subprocess.CalledProcessError(returncode, args, output) - - return output if output else '' - - def get_environment(self) -> Dict[str, str]: - return json.loads(self.call([ - self.UTILITY_PYTHON, - '-c', - 'import sys, json, os; json.dump(os.environ.copy(), sys.stdout)' - ], capture_output=True)) - - def environment_executor(self, command: str, environment: Dict[str, str]) -> str: - # used as an EnvironmentExecutor to evaluate commands and capture output - return self.call(shlex.split(command), env=environment) From a052c71aec1fcf17ee8da22166ed6258877fb94d Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 24 Jun 2020 16:50:08 +0100 Subject: [PATCH 045/117] Remove hard-coded shell info to use the docker ENTRYPOINT instead This lets us use the 'linux32 bash' that's specified by the i686 docker images, without specifying it explicitly. --- cibuildwheel/docker_container.py | 7 ++----- cibuildwheel/linux.py | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/cibuildwheel/docker_container.py b/cibuildwheel/docker_container.py index 9ba7509d..66eee1e0 100644 --- a/cibuildwheel/docker_container.py +++ b/cibuildwheel/docker_container.py @@ -26,13 +26,11 @@ class DockerContainer: bash_stdin: IO[str] bash_stdout: IO[str] - def __init__(self, docker_image: str, simulate_32_bit=False): + def __init__(self, docker_image: str): self.docker_image = docker_image - self.simulate_32_bit = simulate_32_bit def __enter__(self) -> 'DockerContainer': self.container_name = f'cibuildwheel-{uuid.uuid4()}' - shell_args = ['linux32', '/bin/bash'] if self.simulate_32_bit else ['/bin/bash'] subprocess.run( [ 'docker', 'create', @@ -40,8 +38,7 @@ class DockerContainer: '--name', self.container_name, '-i', '-v', '/:/host', # ignored on CircleCI - self.docker_image, - *shell_args + self.docker_image ], check=True, ) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 30481008..529e1e5b 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -110,7 +110,7 @@ def build(options: BuildOptions) -> None: continue try: - with DockerContainer(docker_image, simulate_32_bit=platform_tag.endswith('i686')) as docker: + with DockerContainer(docker_image) as docker: docker.copy_into(Path.cwd(), Path('/project')) if options.before_all: From 850d109582e80a821c89b1e91ccf5fadc48e4853 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 24 Jun 2020 17:12:06 +0100 Subject: [PATCH 046/117] Don't use `docker exec -w` because CircleCI doesn't support it --- cibuildwheel/docker_container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/docker_container.py b/cibuildwheel/docker_container.py index 66eee1e0..a3609890 100644 --- a/cibuildwheel/docker_container.py +++ b/cibuildwheel/docker_container.py @@ -92,7 +92,7 @@ class DockerContainer: to_path.mkdir(parents=True, exist_ok=True) subprocess.run( - f'docker exec -i -w {from_path} {self.container_name} tar cf - . | tar -xf -', + f'docker exec -i {self.container_name} tar -cC {from_path} -f - . | tar -xf -', shell=True, check=True, cwd=to_path From a2f4d2aa474aff176240e73d91e63ac8b4e3bd63 Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Wed, 24 Jun 2020 22:29:15 +0000 Subject: [PATCH 047/117] Switch back to shutil.move from Path.rename --- cibuildwheel/macos.py | 4 ++-- cibuildwheel/windows.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index d9d892b9..7369bc82 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -212,7 +212,7 @@ def build(options: BuildOptions) -> None: repaired_wheel_dir.mkdir(parents=True) if built_wheel.name.endswith('none-any.whl') or not options.repair_command: # pure Python wheel or empty repair command - built_wheel.rename(repaired_wheel_dir / built_wheel.name) + shutil.move(str(built_wheel), str(repaired_wheel_dir / built_wheel.name)) else: repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) call(repair_command_prepared, env=env, shell=True) @@ -262,4 +262,4 @@ def build(options: BuildOptions) -> None: shutil.rmtree(venv_dir) # we're all done here; move it to output (overwrite existing) - repaired_wheel.replace(options.output_dir / repaired_wheel.name) + shutil.move(str(repaired_wheel), str(options.output_dir / repaired_wheel.name)) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 31cbccb4..9a6e2b79 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -229,7 +229,7 @@ def build(options: BuildOptions) -> None: repaired_wheel_dir.mkdir(parents=True) if built_wheel.name.endswith('none-any.whl') or not options.repair_command: # pure Python wheel or empty repair command - built_wheel.rename(repaired_wheel_dir / built_wheel.name) + shutil.move(str(built_wheel), str(repaired_wheel_dir / built_wheel.name)) else: repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) shell([repair_command_prepared], env=env) @@ -283,4 +283,4 @@ def build(options: BuildOptions) -> None: shutil.rmtree(venv_dir) # we're all done here; move it to output (remove if already exists) - repaired_wheel.replace(options.output_dir / repaired_wheel.name) + shutil.move(str(repaired_wheel), str(options.output_dir / repaired_wheel.name)) From 4dd5c0e11c60ddd75409145ca2e7be46aeeb4afc Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Wed, 24 Jun 2020 23:04:15 +0000 Subject: [PATCH 048/117] use only dir name as target of shutil.move --- cibuildwheel/macos.py | 4 ++-- cibuildwheel/windows.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 7369bc82..df71f6e4 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -212,7 +212,7 @@ def build(options: BuildOptions) -> None: repaired_wheel_dir.mkdir(parents=True) if built_wheel.name.endswith('none-any.whl') or not options.repair_command: # pure Python wheel or empty repair command - shutil.move(str(built_wheel), str(repaired_wheel_dir / built_wheel.name)) + shutil.move(str(built_wheel), repaired_wheel_dir) else: repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) call(repair_command_prepared, env=env, shell=True) @@ -262,4 +262,4 @@ def build(options: BuildOptions) -> None: shutil.rmtree(venv_dir) # we're all done here; move it to output (overwrite existing) - shutil.move(str(repaired_wheel), str(options.output_dir / repaired_wheel.name)) + shutil.move(str(repaired_wheel), options.output_dir) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 9a6e2b79..92b8005e 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -229,7 +229,7 @@ def build(options: BuildOptions) -> None: repaired_wheel_dir.mkdir(parents=True) if built_wheel.name.endswith('none-any.whl') or not options.repair_command: # pure Python wheel or empty repair command - shutil.move(str(built_wheel), str(repaired_wheel_dir / built_wheel.name)) + shutil.move(str(built_wheel), repaired_wheel_dir) else: repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) shell([repair_command_prepared], env=env) @@ -283,4 +283,4 @@ def build(options: BuildOptions) -> None: shutil.rmtree(venv_dir) # we're all done here; move it to output (remove if already exists) - shutil.move(str(repaired_wheel), str(options.output_dir / repaired_wheel.name)) + shutil.move(str(repaired_wheel), options.output_dir) From dd4fc69e7c25b9b964ef9424e265d82d1f5867bc Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 25 Jun 2020 10:39:26 +0100 Subject: [PATCH 049/117] Revert "Remove hard-coded shell info to use the docker ENTRYPOINT instead" This reverts commit a052c71aec1fcf17ee8da22166ed6258877fb94d. --- cibuildwheel/docker_container.py | 7 +++++-- cibuildwheel/linux.py | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/docker_container.py b/cibuildwheel/docker_container.py index a3609890..cef31d57 100644 --- a/cibuildwheel/docker_container.py +++ b/cibuildwheel/docker_container.py @@ -26,11 +26,13 @@ class DockerContainer: bash_stdin: IO[str] bash_stdout: IO[str] - def __init__(self, docker_image: str): + def __init__(self, docker_image: str, simulate_32_bit=False): self.docker_image = docker_image + self.simulate_32_bit = simulate_32_bit def __enter__(self) -> 'DockerContainer': self.container_name = f'cibuildwheel-{uuid.uuid4()}' + shell_args = ['linux32', '/bin/bash'] if self.simulate_32_bit else ['/bin/bash'] subprocess.run( [ 'docker', 'create', @@ -38,7 +40,8 @@ class DockerContainer: '--name', self.container_name, '-i', '-v', '/:/host', # ignored on CircleCI - self.docker_image + self.docker_image, + *shell_args ], check=True, ) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 529e1e5b..30481008 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -110,7 +110,7 @@ def build(options: BuildOptions) -> None: continue try: - with DockerContainer(docker_image) as docker: + with DockerContainer(docker_image, simulate_32_bit=platform_tag.endswith('i686')) as docker: docker.copy_into(Path.cwd(), Path('/project')) if options.before_all: From 321335e756422765cffb0c9ab2caad14b1fd9f31 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 25 Jun 2020 10:53:30 +0100 Subject: [PATCH 050/117] Bump version: v1.5.1 --- README.md | 9 ++++++++- cibuildwheel/__init__.py | 2 +- examples/appveyor-minimal.yml | 2 +- examples/azure-pipelines-minimal.yml | 6 +++--- examples/circleci-minimal.yml | 4 ++-- examples/github-deploy.yml | 2 +- examples/github-minimal.yml | 2 +- examples/travis-ci-deploy.yml | 2 +- examples/travis-ci-minimal.yml | 2 +- examples/travis-ci-test-and-deploy.yml | 6 +++--- setup.py | 2 +- 11 files changed, 23 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 31cc943c..c238d787 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ env: # Note: TWINE_PASSWORD is set to a PyPI API token in Travis settings install: - - python3 -m pip install cibuildwheel==1.5.0 + - python3 -m pip install cibuildwheel==1.5.1 script: # build the wheels, put them into './wheelhouse' @@ -152,6 +152,13 @@ This is similar to static linking, so it might have some licence implications. C Changelog ========= +### 1.5.1 + +_25 June 2020_ + +- 🐛 Fix "OSError: [WinError 17] The system cannot move the file to a different + disk drive" on Github Actions (#388, #389) + ### 1.5.0 _24 June 2020_ diff --git a/cibuildwheel/__init__.py b/cibuildwheel/__init__.py index 77f1c8e6..51ed7c48 100644 --- a/cibuildwheel/__init__.py +++ b/cibuildwheel/__init__.py @@ -1 +1 @@ -__version__ = '1.5.0' +__version__ = '1.5.1' diff --git a/examples/appveyor-minimal.yml b/examples/appveyor-minimal.yml index 47bfde21..3227e749 100644 --- a/examples/appveyor-minimal.yml +++ b/examples/appveyor-minimal.yml @@ -12,7 +12,7 @@ stack: python 3.7 init: - cmd: set PATH=C:\Python37;C:\Python37\Scripts;%PATH% -install: python -m pip install cibuildwheel==1.5.0 +install: python -m pip install cibuildwheel==1.5.1 build_script: python -m cibuildwheel --output-dir wheelhouse diff --git a/examples/azure-pipelines-minimal.yml b/examples/azure-pipelines-minimal.yml index c6597d13..45bc8d56 100644 --- a/examples/azure-pipelines-minimal.yml +++ b/examples/azure-pipelines-minimal.yml @@ -5,7 +5,7 @@ jobs: - task: UsePythonVersion@0 - bash: | python3 -m pip install --upgrade pip - pip3 install cibuildwheel==1.5.0 + pip3 install cibuildwheel==1.5.1 cibuildwheel --output-dir wheelhouse . - task: PublishBuildArtifacts@1 inputs: {pathtoPublish: 'wheelhouse'} @@ -16,7 +16,7 @@ jobs: - task: UsePythonVersion@0 - bash: | python3 -m pip install --upgrade pip - pip3 install cibuildwheel==1.5.0 + pip3 install cibuildwheel==1.5.1 cibuildwheel --output-dir wheelhouse . - task: PublishBuildArtifacts@1 inputs: {pathtoPublish: 'wheelhouse'} @@ -29,7 +29,7 @@ jobs: displayName: Install Visual C++ for Python 2.7 - bash: | python -m pip install --upgrade pip - pip install cibuildwheel==1.5.0 + pip install cibuildwheel==1.5.1 cibuildwheel --output-dir wheelhouse . - task: PublishBuildArtifacts@1 inputs: {pathtoPublish: 'wheelhouse'} diff --git a/examples/circleci-minimal.yml b/examples/circleci-minimal.yml index fb11ff3b..90afe669 100644 --- a/examples/circleci-minimal.yml +++ b/examples/circleci-minimal.yml @@ -11,7 +11,7 @@ jobs: - run: name: Build the Linux wheels. command: | - pip3 install --user cibuildwheel==1.5.0 + pip3 install --user cibuildwheel==1.5.1 cibuildwheel --output-dir wheelhouse - store_artifacts: path: wheelhouse/ @@ -25,7 +25,7 @@ jobs: - run: name: Build the OS X wheels. command: | - pip3 install --user cibuildwheel==1.5.0 + pip3 install --user cibuildwheel==1.5.1 cibuildwheel --output-dir wheelhouse - store_artifacts: path: wheelhouse/ diff --git a/examples/github-deploy.yml b/examples/github-deploy.yml index d86eccde..49e67f23 100644 --- a/examples/github-deploy.yml +++ b/examples/github-deploy.yml @@ -28,7 +28,7 @@ jobs: - name: Install cibuildwheel run: | - python -m pip install cibuildwheel==1.5.0 + python -m pip install cibuildwheel==1.5.1 - name: Install Visual C++ for Python 2.7 if: runner.os == 'Windows' diff --git a/examples/github-minimal.yml b/examples/github-minimal.yml index 30c20f60..ecbdc396 100644 --- a/examples/github-minimal.yml +++ b/examples/github-minimal.yml @@ -20,7 +20,7 @@ jobs: - name: Install cibuildwheel run: | - python -m pip install cibuildwheel==1.5.0 + python -m pip install cibuildwheel==1.5.1 - name: Install Visual C++ for Python 2.7 if: runner.os == 'Windows' diff --git a/examples/travis-ci-deploy.yml b/examples/travis-ci-deploy.yml index c9c085f5..db1b1317 100644 --- a/examples/travis-ci-deploy.yml +++ b/examples/travis-ci-deploy.yml @@ -25,7 +25,7 @@ env: # Note: TWINE_PASSWORD is set to a PyPI API token in Travis settings install: - - python3 -m pip install cibuildwheel==1.5.0 + - python3 -m pip install cibuildwheel==1.5.1 script: # build the wheels, put them into './wheelhouse' diff --git a/examples/travis-ci-minimal.yml b/examples/travis-ci-minimal.yml index 2be04ce2..4fc3673e 100644 --- a/examples/travis-ci-minimal.yml +++ b/examples/travis-ci-minimal.yml @@ -26,7 +26,7 @@ jobs: - ln -s /c/Python38/python.exe /c/Python38/python3.exe install: - - python3 -m pip install cibuildwheel==1.5.0 + - python3 -m pip install cibuildwheel==1.5.1 script: # build the wheels, put them into './wheelhouse' diff --git a/examples/travis-ci-test-and-deploy.yml b/examples/travis-ci-test-and-deploy.yml index f29ea81b..95771ba0 100644 --- a/examples/travis-ci-test-and-deploy.yml +++ b/examples/travis-ci-test-and-deploy.yml @@ -55,7 +55,7 @@ jobs: - stage: deploy name: Build and deploy Linux wheels services: docker - install: python3 -m pip install cibuildwheel==1.5.0 + install: python3 -m pip install cibuildwheel==1.5.1 script: python3 -m cibuildwheel --output-dir wheelhouse after_success: | python3 -m pip install twine @@ -65,7 +65,7 @@ jobs: name: Build and deploy macOS wheels os: osx language: shell - install: python3 -m pip install cibuildwheel==1.5.0 + install: python3 -m pip install cibuildwheel==1.5.1 script: python3 -m cibuildwheel --output-dir wheelhouse after_success: | python3 -m pip install twine @@ -75,7 +75,7 @@ jobs: name: Build and deploy Windows wheels os: windows language: shell - install: python3 -m pip install cibuildwheel==1.5.0 + install: python3 -m pip install cibuildwheel==1.5.1 script: python3 -m cibuildwheel --output-dir wheelhouse after_success: | python3 -m pip install twine diff --git a/setup.py b/setup.py index 4396b928..01088a55 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ long_description = (this_directory / 'README.md').read_text(encoding='utf-8') setup( name='cibuildwheel', - version='1.5.0', + version='1.5.1', install_requires=['bashlex!=0.13', 'toml'], description="Build Python wheels on CI with minimal configuration.", long_description=long_description, From a92959f19132a87605b3940c695b85556bfba6a0 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 25 Jun 2020 10:57:58 +0100 Subject: [PATCH 051/117] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c238d787..ee1f5a42 100644 --- a/README.md +++ b/README.md @@ -94,7 +94,7 @@ after_success: fi ``` -For more information, including how to build on Appveyor, Azure, CircleCI, check out the [documentation](https://cibuildwheel.readthedocs.org) and also check out [the examples](https://github.com/joerick/cibuildwheel/tree/master/examples). +For more information, including how to build on GitHub Actions, Appveyor, Azure Pipelines, or CircleCI, check out the [documentation](https://cibuildwheel.readthedocs.org) and the [examples](https://github.com/joerick/cibuildwheel/tree/master/examples). Options ------- From fefe128a15c6a278f0d69e3afd8b11808aceab83 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 25 Jun 2020 11:21:10 +0100 Subject: [PATCH 052/117] Fix issues with spaces in environment variables --- cibuildwheel/docker_container.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/cibuildwheel/docker_container.py b/cibuildwheel/docker_container.py index cef31d57..73529fff 100644 --- a/cibuildwheel/docker_container.py +++ b/cibuildwheel/docker_container.py @@ -112,24 +112,25 @@ class DockerContainer: def call(self, args: Sequence[Union[str, PathLike]], env: Dict[str, str] = {}, capture_output=False, cwd: Optional[Union[str, PathLike]] = None) -> str: - env_exports = '\n'.join(f'export {k}={v}' for k, v in env.items()) chdir = f'cd {cwd}' if cwd else '' + env_assignments = ' '.join(f'{shlex.quote(k)}={shlex.quote(v)}' + for k, v in env.items()) command = ' '.join(shlex.quote(str(a)) for a in args) end_of_message = str(uuid.uuid4()) # log the command we're executing print(f' + {command}') - # Write a command to the remote shell. First we write the - # environment variables, exported inside the subshell. We change the - # cwd, if that's required. Then, the command is written. Finally, the - # remote shell is told to write a footer - this will show up in the - # output so we know when to stop reading, and will include the - # returncode of `command`. + # Write a command to the remote shell. First we change the + # cwd, if that's required. Then, we use the `env` utility to run + # `command` inside the specified environment. We use `env` because it + # can cope with spaces and strange characters in the name or value. + # Finally, the remote shell is told to write a footer - this will show + # up in the output so we know when to stop reading, and will include + # the returncode of `command`. self.bash_stdin.write(f'''( - {env_exports} {chdir} - {command} + env {env_assignments} {command} printf "%04d%s\n" $? {end_of_message} ) ''') From 1d88c27530c76e423c32cf1342d040bb88551fe7 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 25 Jun 2020 12:41:21 +0100 Subject: [PATCH 053/117] Neaten function call whitespace --- cibuildwheel/macos.py | 18 ++++++++---------- cibuildwheel/windows.py | 17 +++++++---------- 2 files changed, 15 insertions(+), 20 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 0c268f4c..99e089df 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -201,16 +201,14 @@ def build(options: BuildOptions) -> None: # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org # see https://github.com/joerick/cibuildwheel/pull/369 - call( - [ - 'pip', 'wheel', - options.package_dir.resolve(), - '-w', built_wheel_dir, - '--no-deps', - *get_build_verbosity_extra_flags(options.build_verbosity) - ], - env=env, - ) + call([ + 'pip', 'wheel', + options.package_dir.resolve(), + '-w', built_wheel_dir, + '--no-deps', + *get_build_verbosity_extra_flags(options.build_verbosity) + ], env=env) + built_wheel = next(built_wheel_dir.glob('*.whl')) # repair the wheel diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 9f67c6c4..f5dc686c 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -185,16 +185,13 @@ def build(options: BuildOptions) -> None: built_wheel_dir.mkdir(parents=True) # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org # see https://github.com/joerick/cibuildwheel/pull/369 - shell( - [ - 'pip', 'wheel', - options.package_dir.resolve(), - '-w', built_wheel_dir, - '--no-deps', - *get_build_verbosity_extra_flags(options.build_verbosity) - ], - env=env, - ) + shell([ + 'pip', 'wheel', + options.package_dir.resolve(), + '-w', built_wheel_dir, + '--no-deps', + *get_build_verbosity_extra_flags(options.build_verbosity) + ], env=env) built_wheel = next(built_wheel_dir.glob('*.whl')) From 68d8f89fb3bc01240eae889f1ba3458c1db4018f Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 26 Jun 2020 10:59:47 +0100 Subject: [PATCH 054/117] Style tweaks --- cibuildwheel/docker_container.py | 9 ++++----- cibuildwheel/linux.py | 19 ++++++++----------- 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/cibuildwheel/docker_container.py b/cibuildwheel/docker_container.py index 73529fff..3855c733 100644 --- a/cibuildwheel/docker_container.py +++ b/cibuildwheel/docker_container.py @@ -45,7 +45,7 @@ class DockerContainer: ], check=True, ) - process = subprocess.Popen( + self.process = subprocess.Popen( [ 'docker', 'start', '--attach', '--interactive', @@ -58,10 +58,9 @@ class DockerContainer: # variables. We choose 256kB. bufsize=262144, ) - self.process = process - assert process.stdin and process.stdout - self.bash_stdin = process.stdin - self.bash_stdout = process.stdout + assert self.process.stdin and self.process.stdout + self.bash_stdin = self.process.stdin + self.bash_stdout = self.process.stdout return self def __exit__(self, exc_type, exc_val, exc_tb): diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 30481008..60de4743 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -135,7 +135,7 @@ def build(options: BuildOptions) -> None: # put this config's python top of the list python_bin = config.path / 'bin' - env['PATH'] = f'{str(python_bin)}:{env["PATH"]}' + env['PATH'] = f'{python_bin}:{env["PATH"]}' env = options.environment.as_dictionary(env, executor=docker.environment_executor) @@ -159,16 +159,13 @@ def build(options: BuildOptions) -> None: docker.call(['rm', '-rf', built_wheel_dir]) docker.call(['mkdir', '-p', built_wheel_dir]) - docker.call( - [ - 'pip', 'wheel', - container_package_dir, - '-w', built_wheel_dir, - '--no-deps', - *get_build_verbosity_extra_flags(options.build_verbosity) - ], - env=env, - ) + docker.call([ + 'pip', 'wheel', + container_package_dir, + '-w', built_wheel_dir, + '--no-deps', + *get_build_verbosity_extra_flags(options.build_verbosity) + ], env=env) built_wheel = docker.glob(built_wheel_dir / '*.whl')[0] From 1d9333689f40caadac92c92f227737207562982c Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 26 Jun 2020 12:55:38 +0100 Subject: [PATCH 055/117] Explicit buffer size isn't necessary --- cibuildwheel/docker_container.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/cibuildwheel/docker_container.py b/cibuildwheel/docker_container.py index 3855c733..fd0166e4 100644 --- a/cibuildwheel/docker_container.py +++ b/cibuildwheel/docker_container.py @@ -54,8 +54,6 @@ class DockerContainer: encoding='utf8', stdin=subprocess.PIPE, stdout=subprocess.PIPE, - # make the input buffer large enough to carry a lot of environment - # variables. We choose 256kB. bufsize=262144, ) assert self.process.stdin and self.process.stdout From df69dda44ad21a629eb5dd02491f84fe956bcc4f Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 26 Jun 2020 12:59:26 +0100 Subject: [PATCH 056/117] Fix broken PATH test - it was erroring for a different reason --- cibuildwheel/bashlex_eval.py | 33 ++++++++++++++++++++++++++++++--- test/test_environment.py | 7 +++++-- unit_test/environment_test.py | 8 ++++++++ 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/cibuildwheel/bashlex_eval.py b/cibuildwheel/bashlex_eval.py index e4865d6d..c055a0a6 100644 --- a/cibuildwheel/bashlex_eval.py +++ b/cibuildwheel/bashlex_eval.py @@ -1,7 +1,7 @@ import shlex import subprocess -from typing import Dict, NamedTuple, Callable, Optional +from typing import Dict, NamedTuple, Callable, Optional, List, Sequence import bashlex # type: ignore @@ -47,7 +47,7 @@ def evaluate_node(node: bashlex.ast.node, context: NodeExecutionContext) -> str: elif node.kind == 'parameter': return evaluate_parameter_node(node, context=context) else: - raise ValueError(f'Unsupported bash construct: "{node.word}"') + raise ValueError(f'Unsupported bash construct: "{node.kind}"') def evaluate_word_node(node: bashlex.ast.node, context: NodeExecutionContext) -> str: @@ -74,7 +74,34 @@ def evaluate_word_node(node: bashlex.ast.node, context: NodeExecutionContext) -> def evaluate_command_node(node: bashlex.ast.node, context: NodeExecutionContext) -> str: - words = [evaluate_node(part, context=context) for part in node.parts] + if any(n.kind == 'operator' for n in node.parts): + return evaluate_nodes_as_compound_command(node.parts, context=context) + else: + return evaluate_nodes_as_simple_command(node.parts, context=context) + + +def evaluate_nodes_as_compound_command(nodes: Sequence[bashlex.ast.node], context: NodeExecutionContext) -> str: + # bashlex doesn't support any operators besides ';' inside command + # substitutions, so we only need to handle that case. We do so assuming + # that `set -o errexit` is on, because it's easier to code! + + result = '' + for node in nodes: + if node.kind == 'command': + result += evaluate_command_node(node, context=context) + elif node.kind == 'operator': + if node.op == ';': + pass + else: + raise ValueError(f'Unsupported bash operator: "{node.op}"') + else: + raise ValueError(f'Unsupported bash node in compound command: "{node.kind}"') + + return result + + +def evaluate_nodes_as_simple_command(nodes: List[bashlex.ast.node], context: NodeExecutionContext): + words = [evaluate_node(part, context=context) for part in nodes] command = ' '.join(words) return context.executor(command, context.environment) diff --git a/test/test_environment.py b/test/test_environment.py index 470a151f..7e5f8d10 100644 --- a/test/test_environment.py +++ b/test/test_environment.py @@ -44,7 +44,7 @@ def test(tmp_path): assert set(actual_wheels) == set(expected_wheels) -def test_overridden_path(tmp_path): +def test_overridden_path(tmp_path, capfd): project_dir = tmp_path / 'project' output_dir = tmp_path / 'output' @@ -55,7 +55,10 @@ def test_overridden_path(tmp_path): # mess up PATH, somehow with pytest.raises(subprocess.CalledProcessError): utils.cibuildwheel_run(project_dir, output_dir=output_dir, add_env={ - 'CIBW_ENVIRONMENT': '''SOMETHING="$(mkdir new_path && touch new_path/python)" PATH="$(realpath new_path):$PATH"''', + 'CIBW_ENVIRONMENT': '''SOMETHING="$(mkdir -p /new_path ; touch /new_path/python ; chmod +x /new_path/python)" PATH="/new_path:$PATH"''', 'CIBW_ENVIRONMENT_WINDOWS': '''SOMETHING="$(mkdir new_path && type nul > new_path/python.exe)" PATH="$CD\\new_path;$PATH"''', }) + assert len(os.listdir(output_dir)) == 0 + captured = capfd.readouterr() + assert "python available on PATH doesn't match our installed instance" in captured.err diff --git a/unit_test/environment_test.py b/unit_test/environment_test.py index b8b961a8..850233e3 100644 --- a/unit_test/environment_test.py +++ b/unit_test/environment_test.py @@ -94,3 +94,11 @@ def test_no_vars_pass_through(): environment_dict = environment_recipe.as_dictionary(prev_environment={'CIBUILDWHEEL': 'awesome'}) assert environment_dict == {'CIBUILDWHEEL': 'awesome'} + + +def test_operators_inside_eval(): + environment_recipe = parse_environment('SOMETHING="$(echo a ; echo b ; echo c)"') + + environment_dict = environment_recipe.as_dictionary({}) + + assert environment_dict == {'SOMETHING': 'a\nb\nc'} From 2edb33662b46f6012785b57f3f249587412cd865 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 26 Jun 2020 20:22:49 +0100 Subject: [PATCH 057/117] Import organisation --- cibuildwheel/bashlex_eval.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cibuildwheel/bashlex_eval.py b/cibuildwheel/bashlex_eval.py index c055a0a6..f4f0641d 100644 --- a/cibuildwheel/bashlex_eval.py +++ b/cibuildwheel/bashlex_eval.py @@ -1,11 +1,9 @@ import shlex import subprocess - -from typing import Dict, NamedTuple, Callable, Optional, List, Sequence +from typing import Callable, Dict, List, NamedTuple, Optional, Sequence import bashlex # type: ignore - # a function that takes a shell command and the environment, and returns the result EnvironmentExecutor = Callable[[str, Dict[str, str]], str] From 762dc807cd1ffdfcffd32e0b6493e5744668ae57 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 26 Jun 2020 21:36:52 +0100 Subject: [PATCH 058/117] Convert DockerContainer to use binary PIPEs, to handle arbitrary data --- cibuildwheel/docker_container.py | 43 +++++++++++++++++--------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/cibuildwheel/docker_container.py b/cibuildwheel/docker_container.py index fd0166e4..3131c782 100644 --- a/cibuildwheel/docker_container.py +++ b/cibuildwheel/docker_container.py @@ -1,3 +1,4 @@ +import codecs import io import json import shlex @@ -6,7 +7,7 @@ import sys import uuid from os import PathLike from pathlib import Path, PurePath -from typing import IO, Dict, List, Optional, Sequence, TextIO, Union +from typing import IO, Any, Dict, List, Optional, Sequence, TextIO, Union, cast class DockerContainer: @@ -23,21 +24,21 @@ class DockerContainer: UTILITY_PYTHON = '/opt/python/cp38-cp38/bin/python' process: subprocess.Popen - bash_stdin: IO[str] - bash_stdout: IO[str] + bash_stdin: IO[bytes] + bash_stdout: IO[bytes] def __init__(self, docker_image: str, simulate_32_bit=False): self.docker_image = docker_image self.simulate_32_bit = simulate_32_bit def __enter__(self) -> 'DockerContainer': - self.container_name = f'cibuildwheel-{uuid.uuid4()}' + self.name = f'cibuildwheel-{uuid.uuid4()}' shell_args = ['linux32', '/bin/bash'] if self.simulate_32_bit else ['/bin/bash'] subprocess.run( [ 'docker', 'create', '--env', 'CIBUILDWHEEL', - '--name', self.container_name, + '--name', self.name, '-i', '-v', '/:/host', # ignored on CircleCI self.docker_image, @@ -49,13 +50,12 @@ class DockerContainer: [ 'docker', 'start', '--attach', '--interactive', - self.container_name, + self.name, ], - encoding='utf8', stdin=subprocess.PIPE, stdout=subprocess.PIPE, - bufsize=262144, ) + assert self.process.stdin and self.process.stdout self.bash_stdin = self.process.stdin self.bash_stdout = self.process.stdout @@ -66,8 +66,8 @@ class DockerContainer: self.process.terminate() self.process.wait() - subprocess.run(['docker', 'rm', '--force', '-v', self.container_name]) - self.container_name = None + subprocess.run(['docker', 'rm', '--force', '-v', self.name]) + self.name = None def copy_into(self, from_path: Path, to_path: PurePath) -> None: # `docker cp` causes 'no space left on device' error when @@ -77,13 +77,13 @@ class DockerContainer: if from_path.is_dir(): self.call(['mkdir', '-p', to_path]) subprocess.run( - f'tar cf - . | docker exec -i {self.container_name} tar -xC {to_path} -f -', + f'tar cf - . | docker exec -i {self.name} tar -xC {to_path} -f -', shell=True, check=True, cwd=from_path) else: subprocess.run( - f'cat {from_path} | docker exec -i {self.container_name} sh -c "cat > {to_path}"', + f'cat {from_path} | docker exec -i {self.name} sh -c "cat > {to_path}"', shell=True, check=True) @@ -92,7 +92,7 @@ class DockerContainer: to_path.mkdir(parents=True, exist_ok=True) subprocess.run( - f'docker exec -i {self.container_name} tar -cC {from_path} -f - . | tar -xf -', + f'docker exec -i {self.name} tar -cC {from_path} -f - . | tar -xf -', shell=True, check=True, cwd=to_path @@ -125,23 +125,23 @@ class DockerContainer: # Finally, the remote shell is told to write a footer - this will show # up in the output so we know when to stop reading, and will include # the returncode of `command`. - self.bash_stdin.write(f'''( + self.bash_stdin.write(bytes(f'''( {chdir} env {env_assignments} {command} printf "%04d%s\n" $? {end_of_message} ) - ''') + ''', encoding='utf8', errors='surrogateescape')) self.bash_stdin.flush() if capture_output: - output_io: TextIO = io.StringIO() + output_io: IO[bytes] = io.BytesIO() else: - output_io = sys.stdout + output_io = sys.stdout.buffer while True: line = self.bash_stdout.readline() - if line.endswith(end_of_message+'\n'): + if line.endswith(b'%s\n' % (bytes(end_of_message, encoding='utf8'))): footer_offset = ( len(line) - 1 # newline character @@ -156,12 +156,15 @@ class DockerContainer: else: output_io.write(line) - output = output_io.getvalue() if isinstance(output_io, io.StringIO) else None + if isinstance(output_io, io.BytesIO): + output = str(output_io.getvalue(), encoding='utf8', errors='surrogateescape') + else: + output = '' if returncode != 0: raise subprocess.CalledProcessError(returncode, args, output) - return output if output else '' + return output def get_environment(self) -> Dict[str, str]: return json.loads(self.call([ From c956f118f6400f0d1f21c94557050efc41af72e5 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 26 Jun 2020 21:37:02 +0100 Subject: [PATCH 059/117] Add unit tests for DockerContainer --- bin/run_tests.py | 2 +- unit_test/conftest.py | 21 ++++++++ unit_test/docker_container_test.py | 85 ++++++++++++++++++++++++++++++ 3 files changed, 107 insertions(+), 1 deletion(-) create mode 100644 unit_test/conftest.py create mode 100644 unit_test/docker_container_test.py diff --git a/bin/run_tests.py b/bin/run_tests.py index 117f8b9f..1a1895cf 100755 --- a/bin/run_tests.py +++ b/bin/run_tests.py @@ -10,7 +10,7 @@ if __name__ == '__main__': os.chdir(Path(__file__).resolve().parents[1]) # run the unit tests - subprocess.check_call([sys.executable, '-m', 'pytest', 'unit_test']) + subprocess.check_call([sys.executable, '-m', 'pytest', 'unit_test', '--runslow']) # run the integration tests subprocess.check_call([sys.executable, '-m', 'pytest', '-x', '--durations', '0', 'test']) diff --git a/unit_test/conftest.py b/unit_test/conftest.py new file mode 100644 index 00000000..e446d0a1 --- /dev/null +++ b/unit_test/conftest.py @@ -0,0 +1,21 @@ +import pytest + + +def pytest_addoption(parser): + parser.addoption( + "--runslow", action="store_true", default=False, help="run slow tests" + ) + + +def pytest_configure(config): + config.addinivalue_line("markers", "slow: mark test as slow to run") + + +def pytest_collection_modifyitems(config, items): + if config.getoption("--runslow"): + # --runslow given in cli: do not skip slow tests + return + skip_slow = pytest.mark.skip(reason="need --runslow option to run") + for item in items: + if "slow" in item.keywords: + item.add_marker(skip_slow) diff --git a/unit_test/docker_container_test.py b/unit_test/docker_container_test.py new file mode 100644 index 00000000..e8ddc7d1 --- /dev/null +++ b/unit_test/docker_container_test.py @@ -0,0 +1,85 @@ +import pytest +from cibuildwheel.docker_container import DockerContainer +import subprocess +import time +import textwrap + +DEFAULT_IMAGE = 'centos:6' + +@pytest.mark.slow +def test_simple(): + with DockerContainer(DEFAULT_IMAGE) as container: + assert container.call(['echo', 'hello'], capture_output=True) == 'hello\n' + +@pytest.mark.slow +def test_no_lf(): + with DockerContainer(DEFAULT_IMAGE) as container: + assert container.call(['printf', 'hello'], capture_output=True) == 'hello' + +@pytest.mark.slow +def test_environment(): + with DockerContainer(DEFAULT_IMAGE) as container: + assert container.call(['sh', '-c', 'echo $TEST_VAR'], env={'TEST_VAR': '1'}, capture_output=True) == '1\n' + +@pytest.mark.slow +def test_container_removed(): + start_time = time.time() + with DockerContainer(DEFAULT_IMAGE) as container: + container.call(['true']) + docker_containers_listing = subprocess.run('docker container ls', shell=True, check=True, capture_output=True, universal_newlines=True).stdout + assert container.name in docker_containers_listing + old_container_name = container.name + + docker_containers_listing = subprocess.run('docker container ls', shell=True, check=True, capture_output=True, universal_newlines=True).stdout + assert old_container_name not in docker_containers_listing + +@pytest.mark.slow +def test_large_environment(): + # max environment variable size is 128kB + long_env_var_length = 127*1024 + large_environment = { + 'a': '0'*long_env_var_length, + 'b': '0'*long_env_var_length, + 'c': '0'*long_env_var_length, + 'd': '0'*long_env_var_length, + } + + with DockerContainer(DEFAULT_IMAGE) as container: + # check the length of d + assert container.call(['sh', '-c', 'echo ${#d}'], env=large_environment, capture_output=True) == f'{long_env_var_length}\n' + +@pytest.mark.slow +def test_binary_output(): + with DockerContainer(DEFAULT_IMAGE) as container: + # the centos image only has python 2.6, so the below embedded snippets + # are in python2 + + # check that we can pass though arbitrary binary data without erroring + container.call(['/usr/bin/python2', '-c', textwrap.dedent(''' + import sys + sys.stdout.write(''.join(chr(n) for n in range(0, 256))) + ''')]) + + # check that we can capture arbitrary binary data + output = container.call(['/usr/bin/python2', '-c', textwrap.dedent(''' + import sys + sys.stdout.write(''.join(chr(n % 256) for n in range(0, 512))) + ''')], capture_output=True) + + data = bytes(output, encoding='utf8', errors='surrogateescape') + + for i in range(0, 512): + assert data[i] == i % 256 + + # check that environment variables can carry binary data, except null characters + # (https://www.gnu.org/software/libc/manual/html_node/Environment-Variables.html) + binary_data = bytes(n for n in range(1, 256)) + binary_data_string = str(binary_data, encoding='utf8', errors='surrogateescape') + output = container.call( + ['python2', '-c', 'import os, sys; sys.stdout.write(os.environ["TEST_VAR"])'], + env={'TEST_VAR': binary_data_string}, + capture_output=True, + ) + assert output == binary_data_string + + From ad871f886a550ca7cd2ff60afc07fd7efef627bd Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 26 Jun 2020 21:41:00 +0100 Subject: [PATCH 060/117] Fix style --- cibuildwheel/docker_container.py | 3 +-- unit_test/docker_container_test.py | 16 ++++++++++------ 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/cibuildwheel/docker_container.py b/cibuildwheel/docker_container.py index 3131c782..837c06c1 100644 --- a/cibuildwheel/docker_container.py +++ b/cibuildwheel/docker_container.py @@ -1,4 +1,3 @@ -import codecs import io import json import shlex @@ -7,7 +6,7 @@ import sys import uuid from os import PathLike from pathlib import Path, PurePath -from typing import IO, Any, Dict, List, Optional, Sequence, TextIO, Union, cast +from typing import IO, Dict, List, Optional, Sequence, Union class DockerContainer: diff --git a/unit_test/docker_container_test.py b/unit_test/docker_container_test.py index e8ddc7d1..d66155ef 100644 --- a/unit_test/docker_container_test.py +++ b/unit_test/docker_container_test.py @@ -1,29 +1,33 @@ -import pytest -from cibuildwheel.docker_container import DockerContainer import subprocess -import time import textwrap +import pytest + +from cibuildwheel.docker_container import DockerContainer + DEFAULT_IMAGE = 'centos:6' + @pytest.mark.slow def test_simple(): with DockerContainer(DEFAULT_IMAGE) as container: assert container.call(['echo', 'hello'], capture_output=True) == 'hello\n' + @pytest.mark.slow def test_no_lf(): with DockerContainer(DEFAULT_IMAGE) as container: assert container.call(['printf', 'hello'], capture_output=True) == 'hello' + @pytest.mark.slow def test_environment(): with DockerContainer(DEFAULT_IMAGE) as container: assert container.call(['sh', '-c', 'echo $TEST_VAR'], env={'TEST_VAR': '1'}, capture_output=True) == '1\n' + @pytest.mark.slow def test_container_removed(): - start_time = time.time() with DockerContainer(DEFAULT_IMAGE) as container: container.call(['true']) docker_containers_listing = subprocess.run('docker container ls', shell=True, check=True, capture_output=True, universal_newlines=True).stdout @@ -33,6 +37,7 @@ def test_container_removed(): docker_containers_listing = subprocess.run('docker container ls', shell=True, check=True, capture_output=True, universal_newlines=True).stdout assert old_container_name not in docker_containers_listing + @pytest.mark.slow def test_large_environment(): # max environment variable size is 128kB @@ -48,6 +53,7 @@ def test_large_environment(): # check the length of d assert container.call(['sh', '-c', 'echo ${#d}'], env=large_environment, capture_output=True) == f'{long_env_var_length}\n' + @pytest.mark.slow def test_binary_output(): with DockerContainer(DEFAULT_IMAGE) as container: @@ -81,5 +87,3 @@ def test_binary_output(): capture_output=True, ) assert output == binary_data_string - - From 91c3e90017db468e12d04497666202120f0d44c2 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 26 Jun 2020 21:47:42 +0100 Subject: [PATCH 061/117] Fix Python 3.6 compatibility --- unit_test/docker_container_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/unit_test/docker_container_test.py b/unit_test/docker_container_test.py index d66155ef..7e124d38 100644 --- a/unit_test/docker_container_test.py +++ b/unit_test/docker_container_test.py @@ -29,12 +29,13 @@ def test_environment(): @pytest.mark.slow def test_container_removed(): with DockerContainer(DEFAULT_IMAGE) as container: + # call a command to ensure it has started container.call(['true']) - docker_containers_listing = subprocess.run('docker container ls', shell=True, check=True, capture_output=True, universal_newlines=True).stdout + docker_containers_listing = subprocess.run('docker container ls', shell=True, check=True, stdout=subprocess.PIPE, universal_newlines=True).stdout assert container.name in docker_containers_listing old_container_name = container.name - docker_containers_listing = subprocess.run('docker container ls', shell=True, check=True, capture_output=True, universal_newlines=True).stdout + docker_containers_listing = subprocess.run('docker container ls', shell=True, check=True, stdout=subprocess.PIPE, universal_newlines=True).stdout assert old_container_name not in docker_containers_listing From faf6a7637583ff7c383dbff0a2863e155013777f Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 28 Jun 2020 10:53:03 +0100 Subject: [PATCH 062/117] Change 'slow' tests to 'docker' tests, only run them on Linux --- bin/run_tests.py | 6 +++++- unit_test/conftest.py | 14 +++++++------- unit_test/docker_container_test.py | 12 ++++++------ 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/bin/run_tests.py b/bin/run_tests.py index 1a1895cf..17f06aa5 100755 --- a/bin/run_tests.py +++ b/bin/run_tests.py @@ -10,7 +10,11 @@ if __name__ == '__main__': os.chdir(Path(__file__).resolve().parents[1]) # run the unit tests - subprocess.check_call([sys.executable, '-m', 'pytest', 'unit_test', '--runslow']) + unit_test_args = [sys.executable, '-m', 'pytest', 'unit_test'] + # run the docker unit tests only on Linux + if sys.platform.startswith('linux'): + unit_test_args += ['--run-docker'] + subprocess.check_call(unit_test_args) # run the integration tests subprocess.check_call([sys.executable, '-m', 'pytest', '-x', '--durations', '0', 'test']) diff --git a/unit_test/conftest.py b/unit_test/conftest.py index e446d0a1..a54658a9 100644 --- a/unit_test/conftest.py +++ b/unit_test/conftest.py @@ -3,19 +3,19 @@ import pytest def pytest_addoption(parser): parser.addoption( - "--runslow", action="store_true", default=False, help="run slow tests" + "--run-docker", action="store_true", default=False, help="run docker tests" ) def pytest_configure(config): - config.addinivalue_line("markers", "slow: mark test as slow to run") + config.addinivalue_line("markers", "docker: mark test requiring docker to run") def pytest_collection_modifyitems(config, items): - if config.getoption("--runslow"): - # --runslow given in cli: do not skip slow tests + if config.getoption("--run-docker"): + # --run-docker given in cli: do not skip docker tests return - skip_slow = pytest.mark.skip(reason="need --runslow option to run") + skip_docker = pytest.mark.skip(reason="need --run-docker option to run") for item in items: - if "slow" in item.keywords: - item.add_marker(skip_slow) + if "docker" in item.keywords: + item.add_marker(skip_docker) diff --git a/unit_test/docker_container_test.py b/unit_test/docker_container_test.py index 7e124d38..cba0fb50 100644 --- a/unit_test/docker_container_test.py +++ b/unit_test/docker_container_test.py @@ -8,25 +8,25 @@ from cibuildwheel.docker_container import DockerContainer DEFAULT_IMAGE = 'centos:6' -@pytest.mark.slow +@pytest.mark.docker def test_simple(): with DockerContainer(DEFAULT_IMAGE) as container: assert container.call(['echo', 'hello'], capture_output=True) == 'hello\n' -@pytest.mark.slow +@pytest.mark.docker def test_no_lf(): with DockerContainer(DEFAULT_IMAGE) as container: assert container.call(['printf', 'hello'], capture_output=True) == 'hello' -@pytest.mark.slow +@pytest.mark.docker def test_environment(): with DockerContainer(DEFAULT_IMAGE) as container: assert container.call(['sh', '-c', 'echo $TEST_VAR'], env={'TEST_VAR': '1'}, capture_output=True) == '1\n' -@pytest.mark.slow +@pytest.mark.docker def test_container_removed(): with DockerContainer(DEFAULT_IMAGE) as container: # call a command to ensure it has started @@ -39,7 +39,7 @@ def test_container_removed(): assert old_container_name not in docker_containers_listing -@pytest.mark.slow +@pytest.mark.docker def test_large_environment(): # max environment variable size is 128kB long_env_var_length = 127*1024 @@ -55,7 +55,7 @@ def test_large_environment(): assert container.call(['sh', '-c', 'echo ${#d}'], env=large_environment, capture_output=True) == f'{long_env_var_length}\n' -@pytest.mark.slow +@pytest.mark.docker def test_binary_output(): with DockerContainer(DEFAULT_IMAGE) as container: # the centos image only has python 2.6, so the below embedded snippets From 27d97daf13b6aa04301640d225845344aad17fac Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 28 Jun 2020 11:33:22 +0100 Subject: [PATCH 063/117] Don't use redirection syntax in Windows environment. Fix absolute path use for Mac --- test/test_environment.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/test_environment.py b/test/test_environment.py index 7e5f8d10..124867e1 100644 --- a/test/test_environment.py +++ b/test/test_environment.py @@ -55,8 +55,8 @@ def test_overridden_path(tmp_path, capfd): # mess up PATH, somehow with pytest.raises(subprocess.CalledProcessError): utils.cibuildwheel_run(project_dir, output_dir=output_dir, add_env={ - 'CIBW_ENVIRONMENT': '''SOMETHING="$(mkdir -p /new_path ; touch /new_path/python ; chmod +x /new_path/python)" PATH="/new_path:$PATH"''', - 'CIBW_ENVIRONMENT_WINDOWS': '''SOMETHING="$(mkdir new_path && type nul > new_path/python.exe)" PATH="$CD\\new_path;$PATH"''', + 'CIBW_ENVIRONMENT': '''SOMETHING="$(mkdir -p new_path ; touch new_path/python ; chmod +x new_path/python)" PATH="$(pwd)/new_path:$PATH"''', + 'CIBW_ENVIRONMENT_WINDOWS': '''SOMETHING="$(mkdir new_path ; fsutil file createnew new_path/python.exe 0)" PATH="$CD\\new_path;$PATH"''', }) assert len(os.listdir(output_dir)) == 0 From e31c24d3bb4db9dde2be2309e11cf5d5ee07d8b4 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 28 Jun 2020 21:50:26 +0100 Subject: [PATCH 064/117] Fix overridden PATH test, by ensuring bash command substitution rules --- cibuildwheel/bashlex_eval.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/bashlex_eval.py b/cibuildwheel/bashlex_eval.py index f4f0641d..b5e478c8 100644 --- a/cibuildwheel/bashlex_eval.py +++ b/cibuildwheel/bashlex_eval.py @@ -41,7 +41,9 @@ def evaluate_node(node: bashlex.ast.node, context: NodeExecutionContext) -> str: if node.kind == 'word': return evaluate_word_node(node, context=context) elif node.kind == 'commandsubstitution': - return evaluate_command_node(node.command, context=context) + node_result = evaluate_command_node(node.command, context=context) + # bash removes training newlines in command substitution + return node_result.rstrip() elif node.kind == 'parameter': return evaluate_parameter_node(node, context=context) else: From b12a560e87863244c7297a1c2d95a703662b062d Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 1 Jul 2020 13:22:24 +0100 Subject: [PATCH 065/117] Add docker test images for other architectures --- unit_test/docker_container_test.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/unit_test/docker_container_test.py b/unit_test/docker_container_test.py index cba0fb50..6318d8a6 100644 --- a/unit_test/docker_container_test.py +++ b/unit_test/docker_container_test.py @@ -1,3 +1,4 @@ +import platform import subprocess import textwrap @@ -5,7 +6,15 @@ import pytest from cibuildwheel.docker_container import DockerContainer -DEFAULT_IMAGE = 'centos:6' +pm = platform.machine() +if pm == "x86_64": + DEFAULT_IMAGE = 'centos:7' +elif pm == "aarch64": + DEFAULT_IMAGE = 'arm64v8/centos:7' +elif pm == "ppc64le": + DEFAULT_IMAGE = 'ppc64le/centos:7' +elif pm == "s390x": + DEFAULT_IMAGE = 's390x/clefos:7' @pytest.mark.docker From ad51e796c0ea3aeca6cdfdb9c692a128d59084b1 Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Sat, 4 Jul 2020 12:01:38 +0200 Subject: [PATCH 066/117] Update links in documentation fix #391 --- docs/deliver-to-pypi.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deliver-to-pypi.md b/docs/deliver-to-pypi.md index 35d36062..648f1bdd 100644 --- a/docs/deliver-to-pypi.md +++ b/docs/deliver-to-pypi.md @@ -33,4 +33,4 @@ Obviously, manual steps are for chumps, so we can automate this a little by usin If you don't need much control over the release of a package, you can set up cibuildwheel to deliver the wheels straight to PyPI. This doesn't require anycloud storage to work - you just need to bump the version and tag it. -[`examples/travis-deploy.yml`](https://github.com/joerick/cibuildwheel/blob/master/examples/travis-deploy.yml) and [`examples/github-deploy.yml`](https://github.com/joerick/cibuildwheel/blob/master/examples/travis-deploy.yml) are example configurations that automatocially upload wheels to PyPI. Also check out [this example repo](https://github.com/joerick/cibuildwheel-autopypi-example) for more detailed instructions on how to set this up. +[`examples/travis-ci-deploy.yml`](https://github.com/joerick/cibuildwheel/blob/master/examples/travis-ci-deploy.yml) and [`examples/github-deploy.yml`](https://github.com/joerick/cibuildwheel/blob/master/examples/github-deploy.yml) are example configurations that automatocially upload wheels to PyPI. Also check out [this example repo](https://github.com/joerick/cibuildwheel-autopypi-example) for more detailed instructions on how to set this up. From 141b3e031e36c94d98d54efad14234222be3e8cf Mon Sep 17 00:00:00 2001 From: mayeut Date: Sat, 4 Jul 2020 18:11:06 +0200 Subject: [PATCH 067/117] Update CPython 3.7 from 3.7.7 to 3.7.8 Changelog: https://docs.python.org/release/3.7.8/whatsnew/changelog.html#changelog --- cibuildwheel/macos.py | 2 +- cibuildwheel/windows.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index b0734a6c..ce7bd974 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -36,7 +36,7 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi PythonConfiguration(version='2.7', identifier='cp27-macosx_x86_64', url='https://www.python.org/ftp/python/2.7.18/python-2.7.18-macosx10.9.pkg'), PythonConfiguration(version='3.5', identifier='cp35-macosx_x86_64', url='https://www.python.org/ftp/python/3.5.4/python-3.5.4-macosx10.6.pkg'), PythonConfiguration(version='3.6', identifier='cp36-macosx_x86_64', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg'), - PythonConfiguration(version='3.7', identifier='cp37-macosx_x86_64', url='https://www.python.org/ftp/python/3.7.7/python-3.7.7-macosx10.9.pkg'), + PythonConfiguration(version='3.7', identifier='cp37-macosx_x86_64', url='https://www.python.org/ftp/python/3.7.8/python-3.7.8-macosx10.9.pkg'), PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.3/python-3.8.3-macosx10.9.pkg'), # PyPy PythonConfiguration(version='2.7', identifier='pp27-macosx_x86_64', url='https://downloads.python.org/pypy/pypy2.7-v7.3.1-osx64.tar.bz2'), diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index f27fca8f..7a694797 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -48,8 +48,8 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi PythonConfiguration(version='3.5.4', arch='64', identifier='cp35-win_amd64', url=None), PythonConfiguration(version='3.6.8', arch='32', identifier='cp36-win32', url=None), PythonConfiguration(version='3.6.8', arch='64', identifier='cp36-win_amd64', url=None), - PythonConfiguration(version='3.7.7', arch='32', identifier='cp37-win32', url=None), - PythonConfiguration(version='3.7.7', arch='64', identifier='cp37-win_amd64', url=None), + PythonConfiguration(version='3.7.8', arch='32', identifier='cp37-win32', url=None), + PythonConfiguration(version='3.7.8', arch='64', identifier='cp37-win_amd64', url=None), PythonConfiguration(version='3.8.3', arch='32', identifier='cp38-win32', url=None), PythonConfiguration(version='3.8.3', arch='64', identifier='cp38-win_amd64', url=None), # PyPy From cc913ee7cf0c9cd91f9968a8ad10bc28fb3c2549 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 5 Jul 2020 17:27:29 +0100 Subject: [PATCH 068/117] Use a different method for the Windows test --- test/test_environment.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/test/test_environment.py b/test/test_environment.py index 124867e1..03bf3bd2 100644 --- a/test/test_environment.py +++ b/test/test_environment.py @@ -54,10 +54,19 @@ def test_overridden_path(tmp_path, capfd): # mess up PATH, somehow with pytest.raises(subprocess.CalledProcessError): - utils.cibuildwheel_run(project_dir, output_dir=output_dir, add_env={ - 'CIBW_ENVIRONMENT': '''SOMETHING="$(mkdir -p new_path ; touch new_path/python ; chmod +x new_path/python)" PATH="$(pwd)/new_path:$PATH"''', - 'CIBW_ENVIRONMENT_WINDOWS': '''SOMETHING="$(mkdir new_path ; fsutil file createnew new_path/python.exe 0)" PATH="$CD\\new_path;$PATH"''', - }) + if utils.platform == 'linux': + utils.cibuildwheel_run(project_dir, output_dir=output_dir, add_env={ + 'CIBW_BEFORE_ALL': 'mkdir new_path && touch new_path/python && chmod +x new_path/python', + 'CIBW_ENVIRONMENT': '''PATH="$(pwd)/new_path:$PATH"''', + }) + else: + new_path = tmp_path / 'another_bin' + new_path.mkdir() + (new_path / 'python').touch(mode=0o777) + + utils.cibuildwheel_run(project_dir, output_dir=output_dir, add_env={ + 'CIBW_ENVIRONMENT': f'''PATH="{new_path}{os.pathsep}$PATH"''', + }) assert len(os.listdir(output_dir)) == 0 captured = capfd.readouterr() From 833a0692079aa18ee781403d3ab94122a1ef8fe1 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 5 Jul 2020 17:39:40 +0100 Subject: [PATCH 069/117] Pass PATH during command substitution test --- unit_test/environment_test.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/unit_test/environment_test.py b/unit_test/environment_test.py index 850233e3..4e10e07c 100644 --- a/unit_test/environment_test.py +++ b/unit_test/environment_test.py @@ -97,8 +97,9 @@ def test_no_vars_pass_through(): def test_operators_inside_eval(): - environment_recipe = parse_environment('SOMETHING="$(echo a ; echo b ; echo c)"') + environment_recipe = parse_environment('SOMETHING="$(echo a; echo b; echo c)"') - environment_dict = environment_recipe.as_dictionary({}) + # pass the existing process env so PATH is available + environment_dict = environment_recipe.as_dictionary(os.environ.copy()) - assert environment_dict == {'SOMETHING': 'a\nb\nc'} + assert environment_dict.get('SOMETHING') == 'a\nb\nc' From 7a27ae941ce8e4e1113cc9e7392b807d7ea4efdf Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sun, 5 Jul 2020 23:09:02 -0400 Subject: [PATCH 070/117] fix: Support environment markers in PEP 518 workaround --- cibuildwheel/windows.py | 4 +++- test/test_pep518.py | 9 +++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 7a694797..1996d061 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -8,6 +8,7 @@ from pathlib import Path from typing import Dict, List, NamedTuple, Optional, Sequence, Union from zipfile import ZipFile import toml +import shlex from .environment import ParsedEnvironment from .util import (BuildOptions, BuildSelector, download, @@ -172,7 +173,8 @@ def pep_518_cp35_workaround(package_dir: Path, env: Dict[str, str]) -> None: else [] ) if requirements: - shell(['pip', 'install'] + requirements, env=env) + escaped_requirements = [shlex.quote(s) for s in requirements] + shell(['pip', 'install'] + escaped_requirements, env=env) def build(options: BuildOptions) -> None: diff --git a/test/test_pep518.py b/test/test_pep518.py index 7d5f5081..2054616b 100644 --- a/test/test_pep518.py +++ b/test/test_pep518.py @@ -6,8 +6,12 @@ basic_project = test_projects.new_c_project( setup_py_add=textwrap.dedent( """ # Will fail if PEP 518 does work + import sys import requests - assert requests.__version__ == "2.23.0", "Requests found but wrong version ({0})".format(requests.__version__) + if sys.version_info < (3, 6, 0): + assert requests.__version__ == "2.22.0", "Requests found but wrong version ({0})".format(requests.__version__) + else: + assert requests.__version__ == "2.23.0", "Requests found but wrong version ({0})".format(requests.__version__) # Just making sure environment is still set import os @@ -24,7 +28,8 @@ basic_project.files[ requires = [ "setuptools>=42", "wheel", - "requests==2.23.0" + "requests==2.22.0; python_version<'3.6'", + "requests==2.23.0; python_version>='3.6'" ] build-backend = "setuptools.build_meta" From c36be04ee8285fd25c759fc0dad47109cf673c16 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Sun, 5 Jul 2020 23:51:28 -0400 Subject: [PATCH 071/117] fix: Use repr --- cibuildwheel/windows.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 1996d061..dfbfc368 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -8,7 +8,6 @@ from pathlib import Path from typing import Dict, List, NamedTuple, Optional, Sequence, Union from zipfile import ZipFile import toml -import shlex from .environment import ParsedEnvironment from .util import (BuildOptions, BuildSelector, download, @@ -173,7 +172,7 @@ def pep_518_cp35_workaround(package_dir: Path, env: Dict[str, str]) -> None: else [] ) if requirements: - escaped_requirements = [shlex.quote(s) for s in requirements] + escaped_requirements = [f'{s!r}' for s in requirements] shell(['pip', 'install'] + escaped_requirements, env=env) From fc24230cb4269c264cbfadaba2692eee9aca5ccd Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 6 Jul 2020 01:31:47 -0400 Subject: [PATCH 072/117] Try original --- cibuildwheel/windows.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index dfbfc368..7a694797 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -172,8 +172,7 @@ def pep_518_cp35_workaround(package_dir: Path, env: Dict[str, str]) -> None: else [] ) if requirements: - escaped_requirements = [f'{s!r}' for s in requirements] - shell(['pip', 'install'] + escaped_requirements, env=env) + shell(['pip', 'install'] + requirements, env=env) def build(options: BuildOptions) -> None: From c68be58e7845b6918726877735f842897f3b4d19 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 6 Jul 2020 08:55:36 -0400 Subject: [PATCH 073/117] fix: Try simple quoting --- cibuildwheel/windows.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 7a694797..e7dab1c4 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -172,7 +172,8 @@ def pep_518_cp35_workaround(package_dir: Path, env: Dict[str, str]) -> None: else [] ) if requirements: - shell(['pip', 'install'] + requirements, env=env) + escaped_requirements = [f'"{s}"' for s in requirements] + shell(['pip', 'install'] + escaped_requirements, env=env) def build(options: BuildOptions) -> None: From 84661945763ca50c7eb383282cbdccf10a2c7506 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 6 Jul 2020 10:42:51 -0400 Subject: [PATCH 074/117] fix: Try removing shell --- cibuildwheel/windows.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index e7dab1c4..c94515e7 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -22,7 +22,7 @@ def shell(args: Sequence[Union[str, PathLike]], env: Optional[Dict[str, str]] = cwd: Optional[str] = None) -> int: command = ' '.join(str(a) for a in args) print(f'+ {command}') - return subprocess.check_call(command, env=env, cwd=cwd, shell=True) + return subprocess.check_call(list(str(a) for a in args), env=env, cwd=cwd) def get_nuget_args(version: str, arch: str) -> List[str]: @@ -172,8 +172,7 @@ def pep_518_cp35_workaround(package_dir: Path, env: Dict[str, str]) -> None: else [] ) if requirements: - escaped_requirements = [f'"{s}"' for s in requirements] - shell(['pip', 'install'] + escaped_requirements, env=env) + shell(['pip', 'install'] + requirements, env=env) def build(options: BuildOptions) -> None: From 317397b39adfc67e1e3d15d3faaade46f0925127 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 6 Jul 2020 13:18:50 -0400 Subject: [PATCH 075/117] fix: Try resolving --- cibuildwheel/windows.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index c94515e7..ad844844 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -18,11 +18,18 @@ IS_RUNNING_ON_AZURE = Path('C:\\hostedtoolcache').exists() IS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows' +def normalize(x: [Union[str, PathLike]], cwd: Optional[PathLike]) -> str: + if hasattr(x, 'resolve'): + return str((cwd / x if cwd is not None else x).resolve()) + else: + return x + + def shell(args: Sequence[Union[str, PathLike]], env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None) -> int: - command = ' '.join(str(a) for a in args) + command = ' '.join(normalize(a, cwd) for a in args) print(f'+ {command}') - return subprocess.check_call(list(str(a) for a in args), env=env, cwd=cwd) + return subprocess.check_call(list(normalize(a, cwd) for a in args), env=env, cwd=cwd) def get_nuget_args(version: str, arch: str) -> List[str]: From 53ba9bd0d08253eb8c804db6cfd947b4696a9e48 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Mon, 6 Jul 2020 18:11:29 -0400 Subject: [PATCH 076/117] Try adding PATH to env --- cibuildwheel/windows.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index ad844844..6c19e4dd 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -1,3 +1,4 @@ +import copy import os import shutil import subprocess @@ -29,6 +30,9 @@ def shell(args: Sequence[Union[str, PathLike]], env: Optional[Dict[str, str]] = cwd: Optional[str] = None) -> int: command = ' '.join(normalize(a, cwd) for a in args) print(f'+ {command}') + if env is not None: + env = copy.copy(env) + env["PATH"] = os.environ["PATH"] return subprocess.check_call(list(normalize(a, cwd) for a in args), env=env, cwd=cwd) From 47fe71efb4b7f154c7385275f6541c3c91c4b76e Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 7 Jul 2020 23:09:40 -0400 Subject: [PATCH 077/117] Try just converting to list --- cibuildwheel/windows.py | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 6c19e4dd..4bc5a973 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -1,4 +1,3 @@ -import copy import os import shutil import subprocess @@ -19,21 +18,11 @@ IS_RUNNING_ON_AZURE = Path('C:\\hostedtoolcache').exists() IS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows' -def normalize(x: [Union[str, PathLike]], cwd: Optional[PathLike]) -> str: - if hasattr(x, 'resolve'): - return str((cwd / x if cwd is not None else x).resolve()) - else: - return x - - def shell(args: Sequence[Union[str, PathLike]], env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None) -> int: - command = ' '.join(normalize(a, cwd) for a in args) + command = ' '.join(str(a) for a in args) print(f'+ {command}') - if env is not None: - env = copy.copy(env) - env["PATH"] = os.environ["PATH"] - return subprocess.check_call(list(normalize(a, cwd) for a in args), env=env, cwd=cwd) + return subprocess.check_call([str(s) for a in args], env=env, cwd=cwd, shell=True) def get_nuget_args(version: str, arch: str) -> List[str]: From ac2b39179065eef7da7143b890381c9585433e74 Mon Sep 17 00:00:00 2001 From: Henry Schreiner Date: Tue, 7 Jul 2020 23:33:03 -0400 Subject: [PATCH 078/117] Typo --- cibuildwheel/windows.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 4bc5a973..e1d416eb 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -22,7 +22,7 @@ def shell(args: Sequence[Union[str, PathLike]], env: Optional[Dict[str, str]] = cwd: Optional[str] = None) -> int: command = ' '.join(str(a) for a in args) print(f'+ {command}') - return subprocess.check_call([str(s) for a in args], env=env, cwd=cwd, shell=True) + return subprocess.check_call([str(a) for a in args], env=env, cwd=cwd, shell=True) def get_nuget_args(version: str, arch: str) -> List[str]: From c521dca40bd24a6a17e0de01c7694eadbc3b3d3e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 8 Jul 2020 18:17:55 +0100 Subject: [PATCH 079/117] My own unique attempt on the shell/call debacle --- cibuildwheel/windows.py | 52 +++++++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 23 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index e1d416eb..1baf66f5 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -18,11 +18,17 @@ IS_RUNNING_ON_AZURE = Path('C:\\hostedtoolcache').exists() IS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows' -def shell(args: Sequence[Union[str, PathLike]], env: Optional[Dict[str, str]] = None, - cwd: Optional[str] = None) -> int: - command = ' '.join(str(a) for a in args) +def call(args: Sequence[Union[str, PathLike]], env: Optional[Dict[str, str]] = None, + cwd: Optional[str] = None) -> None: + print('+ ' + ' '.join(str(a) for a in args)) + # we use shell=True here, even though we don't need a shell due to a bug + # https://bugs.python.org/issue8557 + subprocess.check_call([str(a) for a in args], env=env, cwd=cwd, shell=True) + + +def shell(command: str, env: Optional[Dict[str, str]] = None, cwd: Optional[str] = None) -> None: print(f'+ {command}') - return subprocess.check_call([str(a) for a in args], env=env, cwd=cwd, shell=True) + subprocess.check_call(command, env=env, cwd=cwd, shell=True) def get_nuget_args(version: str, arch: str) -> List[str]: @@ -76,7 +82,7 @@ def extract_zip(zip_src: Path, dest: Path) -> None: def install_cpython(version: str, arch: str, nuget: Path) -> Path: nuget_args = get_nuget_args(version, arch) installation_path = Path(nuget_args[-1]) / (nuget_args[0] + '.' + version) / 'tools' - shell([nuget, 'install', *nuget_args]) + call([nuget, 'install', *nuget_args]) return installation_path @@ -125,9 +131,9 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain env = environment.as_dictionary(prev_environment=env) # for the logs - check we're running the right version of python - shell(['where', 'python'], env=env) - shell(['python', '--version'], env=env) - shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)"'], env=env) + call(['where', 'python'], env=env) + call(['python', '--version'], env=env) + call(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)"'], env=env) where_python = subprocess.check_output(['where', 'python'], env=env, universal_newlines=True).splitlines()[0].strip() if where_python != str(installation_path / 'python.exe'): print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr) @@ -135,7 +141,7 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain # make sure pip is installed if not (installation_path / 'Scripts' / 'pip.exe').exists(): - shell(['python', get_pip_script, *dependency_constraint_flags], env=env, cwd="C:\\cibw") + call(['python', get_pip_script, *dependency_constraint_flags], env=env, cwd="C:\\cibw") assert (installation_path / 'Scripts' / 'pip.exe').exists() where_pip = subprocess.check_output(['where', 'pip'], env=env, universal_newlines=True).splitlines()[0].strip() if where_pip.strip() != str(installation_path / 'Scripts' / 'pip.exe'): @@ -143,9 +149,9 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain exit(1) # prepare the Python environment - shell(['python', '-m', 'pip', 'install', '--upgrade', 'pip', *dependency_constraint_flags], env=env) - shell(['pip', '--version'], env=env) - shell(['pip', 'install', '--upgrade', 'setuptools', 'wheel', *dependency_constraint_flags], env=env) + call(['python', '-m', 'pip', 'install', '--upgrade', 'pip', *dependency_constraint_flags], env=env) + call(['pip', '--version'], env=env) + call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', *dependency_constraint_flags], env=env) return env @@ -187,7 +193,7 @@ def build(options: BuildOptions) -> None: if options.before_all: env = options.environment.as_dictionary(prev_environment=os.environ) before_all_prepared = prepare_command(options.before_all, project='.', package=options.package_dir) - shell([before_all_prepared], env=env) + shell(before_all_prepared, env=env) python_configurations = get_python_configurations(options.build_selector) for config in python_configurations: @@ -203,7 +209,7 @@ def build(options: BuildOptions) -> None: # run the before_build command if options.before_build: before_build_prepared = prepare_command(options.before_build, project='.', package=options.package_dir) - shell([before_build_prepared], env=env) + shell(before_build_prepared, env=env) # activate the PEP 518 patch if on Windows Python 3.5 # (will only have an effect if PEP 517 builds are used): @@ -216,7 +222,7 @@ def build(options: BuildOptions) -> None: built_wheel_dir.mkdir(parents=True) # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org # see https://github.com/joerick/cibuildwheel/pull/369 - shell([ + call([ 'pip', 'wheel', options.package_dir.resolve(), '-w', built_wheel_dir, @@ -235,18 +241,18 @@ def build(options: BuildOptions) -> None: shutil.move(str(built_wheel), repaired_wheel_dir) else: repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) - shell([repair_command_prepared], env=env) + shell(repair_command_prepared, env=env) repaired_wheel = next(repaired_wheel_dir.glob('*.whl')) if options.test_command: # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. - shell(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) + call(['pip', 'install', 'virtualenv', *dependency_constraint_flags], env=env) venv_dir = Path(tempfile.mkdtemp()) # Use --no-download to ensure determinism by using seed libraries # built into virtualenv - shell(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) + call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) virtualenv_env = env.copy() virtualenv_env['PATH'] = os.pathsep.join([ @@ -255,7 +261,7 @@ def build(options: BuildOptions) -> None: ]) # check that we are using the Python from the virtual environment - shell(['which', 'python'], env=virtualenv_env) + call(['which', 'python'], env=virtualenv_env) if options.before_test: before_test_prepared = prepare_command( @@ -263,14 +269,14 @@ def build(options: BuildOptions) -> None: project='.', package=options.package_dir ) - shell([before_test_prepared], env=virtualenv_env) + shell(before_test_prepared, env=virtualenv_env) # install the wheel - shell(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env) + call(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env) # test the wheel if options.test_requires: - shell(['pip', 'install'] + options.test_requires, env=virtualenv_env) + call(['pip', 'install'] + options.test_requires, env=virtualenv_env) # run the tests from c:\, with an absolute path in the command # (this ensures that Python runs the tests against the installed wheel @@ -280,7 +286,7 @@ def build(options: BuildOptions) -> None: project=Path('.').resolve(), package=options.package_dir.resolve() ) - shell([test_command_prepared], cwd='c:\\', env=virtualenv_env) + shell(test_command_prepared, cwd='c:\\', env=virtualenv_env) # clean up shutil.rmtree(venv_dir) From 048ddc4af36e3231091d372bfc0b26d2c90f30be Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 8 Jul 2020 20:12:50 +0100 Subject: [PATCH 080/117] Update test/test_pep518.py Co-authored-by: Henry Schreiner --- test/test_pep518.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_pep518.py b/test/test_pep518.py index 2054616b..9257f7db 100644 --- a/test/test_pep518.py +++ b/test/test_pep518.py @@ -26,7 +26,7 @@ basic_project.files[ ] = """ [build-system] requires = [ - "setuptools>=42", + "setuptools >= 42", "wheel", "requests==2.22.0; python_version<'3.6'", "requests==2.23.0; python_version>='3.6'" From 9e35d65de438d0b42e9218a9b7a7ca4633388eb0 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 8 Jul 2020 22:56:33 +0100 Subject: [PATCH 081/117] Add file-based unit tests --- cibuildwheel/docker_container.py | 4 +++ unit_test/docker_container_test.py | 43 ++++++++++++++++++++++++------ 2 files changed, 39 insertions(+), 8 deletions(-) diff --git a/cibuildwheel/docker_container.py b/cibuildwheel/docker_container.py index 837c06c1..8a119136 100644 --- a/cibuildwheel/docker_container.py +++ b/cibuildwheel/docker_container.py @@ -58,6 +58,10 @@ class DockerContainer: assert self.process.stdin and self.process.stdout self.bash_stdin = self.process.stdin self.bash_stdout = self.process.stdout + + # run a noop command to block until the container is responding + self.call(['/bin/true']) + return self def __exit__(self, exc_type, exc_val, exc_tb): diff --git a/unit_test/docker_container_test.py b/unit_test/docker_container_test.py index 6318d8a6..689ff19b 100644 --- a/unit_test/docker_container_test.py +++ b/unit_test/docker_container_test.py @@ -1,20 +1,24 @@ +from pathlib import Path, PurePath import platform import subprocess import textwrap +from uuid import uuid4 import pytest from cibuildwheel.docker_container import DockerContainer +# for these tests we use manylinux2014 images, because they're available on +# multi architectures and include python3.8 pm = platform.machine() if pm == "x86_64": - DEFAULT_IMAGE = 'centos:7' + DEFAULT_IMAGE = 'quay.io/pypa/manylinux2014_x86_64:2020-05-17-2f8ac3b' elif pm == "aarch64": - DEFAULT_IMAGE = 'arm64v8/centos:7' + DEFAULT_IMAGE = 'quay.io/pypa/manylinux2014_aarch64:2020-05-17-2f8ac3b' elif pm == "ppc64le": - DEFAULT_IMAGE = 'ppc64le/centos:7' + DEFAULT_IMAGE = 'quay.io/pypa/manylinux2014_ppc64le:2020-05-17-2f8ac3b' elif pm == "s390x": - DEFAULT_IMAGE = 's390x/clefos:7' + DEFAULT_IMAGE = 'quay.io/pypa/manylinux2014_s390x:2020-05-17-2f8ac3b' @pytest.mark.docker @@ -38,8 +42,6 @@ def test_environment(): @pytest.mark.docker def test_container_removed(): with DockerContainer(DEFAULT_IMAGE) as container: - # call a command to ensure it has started - container.call(['true']) docker_containers_listing = subprocess.run('docker container ls', shell=True, check=True, stdout=subprocess.PIPE, universal_newlines=True).stdout assert container.name in docker_containers_listing old_container_name = container.name @@ -67,8 +69,7 @@ def test_large_environment(): @pytest.mark.docker def test_binary_output(): with DockerContainer(DEFAULT_IMAGE) as container: - # the centos image only has python 2.6, so the below embedded snippets - # are in python2 + # note: the below embedded snippets are in python2 # check that we can pass though arbitrary binary data without erroring container.call(['/usr/bin/python2', '-c', textwrap.dedent(''' @@ -97,3 +98,29 @@ def test_binary_output(): capture_output=True, ) assert output == binary_data_string + + +@pytest.mark.docker +def test_file_operations(tmp_path: Path): + with DockerContainer(DEFAULT_IMAGE) as container: + # test copying a file in + test_binary_data = uuid4().bytes + uuid4().bytes + uuid4().bytes + uuid4().bytes + + original_test_file = tmp_path / 'test.dat' + original_test_file.write_bytes(test_binary_data) + + dst_file = PurePath('/tmp/test.dat') + + container.copy_into(original_test_file, dst_file) + + output = container.call(['cat', dst_file], capture_output=True) + assert test_binary_data == bytes(output, encoding='utf8', errors='surrogateescape') + + # test copying a dir in + test_dir = tmp_path / 'test_dir' + new_test_file = tmp_path / 'test-new.dat' + container.copy_out(dst_file, new_test_file) + + assert original_test_file.read_bytes() == new_test_file.read_bytes() + + assert container.glob(PurePath('/tmp/*.dat')) == [dst_file] From 8ddc6d57a25c959308d27e48be69f1f84187c8bf Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 8 Jul 2020 22:59:40 +0100 Subject: [PATCH 082/117] Bump version: v1.5.2 --- README.md | 10 +++++++++- cibuildwheel/__init__.py | 2 +- examples/appveyor-minimal.yml | 2 +- examples/azure-pipelines-minimal.yml | 6 +++--- examples/circleci-minimal.yml | 4 ++-- examples/github-deploy.yml | 2 +- examples/github-minimal.yml | 2 +- examples/travis-ci-deploy.yml | 2 +- examples/travis-ci-minimal.yml | 2 +- examples/travis-ci-test-and-deploy.yml | 6 +++--- setup.py | 2 +- 11 files changed, 24 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index ee1f5a42..14607d78 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ env: # Note: TWINE_PASSWORD is set to a PyPI API token in Travis settings install: - - python3 -m pip install cibuildwheel==1.5.1 + - python3 -m pip install cibuildwheel==1.5.2 script: # build the wheels, put them into './wheelhouse' @@ -152,6 +152,14 @@ This is similar to static linking, so it might have some licence implications. C Changelog ========= +### 1.5.2 + +_8 July 2020_ + +- 🐛 Fix an issue on Windows where pyproject.toml would cause an error when + some requirements formats were used. (#401) +- 🛠 Update CPython 3.7 to 3.7.8 (#394) + ### 1.5.1 _25 June 2020_ diff --git a/cibuildwheel/__init__.py b/cibuildwheel/__init__.py index 51ed7c48..c3b38415 100644 --- a/cibuildwheel/__init__.py +++ b/cibuildwheel/__init__.py @@ -1 +1 @@ -__version__ = '1.5.1' +__version__ = '1.5.2' diff --git a/examples/appveyor-minimal.yml b/examples/appveyor-minimal.yml index 3227e749..311332aa 100644 --- a/examples/appveyor-minimal.yml +++ b/examples/appveyor-minimal.yml @@ -12,7 +12,7 @@ stack: python 3.7 init: - cmd: set PATH=C:\Python37;C:\Python37\Scripts;%PATH% -install: python -m pip install cibuildwheel==1.5.1 +install: python -m pip install cibuildwheel==1.5.2 build_script: python -m cibuildwheel --output-dir wheelhouse diff --git a/examples/azure-pipelines-minimal.yml b/examples/azure-pipelines-minimal.yml index 45bc8d56..d7ae6c96 100644 --- a/examples/azure-pipelines-minimal.yml +++ b/examples/azure-pipelines-minimal.yml @@ -5,7 +5,7 @@ jobs: - task: UsePythonVersion@0 - bash: | python3 -m pip install --upgrade pip - pip3 install cibuildwheel==1.5.1 + pip3 install cibuildwheel==1.5.2 cibuildwheel --output-dir wheelhouse . - task: PublishBuildArtifacts@1 inputs: {pathtoPublish: 'wheelhouse'} @@ -16,7 +16,7 @@ jobs: - task: UsePythonVersion@0 - bash: | python3 -m pip install --upgrade pip - pip3 install cibuildwheel==1.5.1 + pip3 install cibuildwheel==1.5.2 cibuildwheel --output-dir wheelhouse . - task: PublishBuildArtifacts@1 inputs: {pathtoPublish: 'wheelhouse'} @@ -29,7 +29,7 @@ jobs: displayName: Install Visual C++ for Python 2.7 - bash: | python -m pip install --upgrade pip - pip install cibuildwheel==1.5.1 + pip install cibuildwheel==1.5.2 cibuildwheel --output-dir wheelhouse . - task: PublishBuildArtifacts@1 inputs: {pathtoPublish: 'wheelhouse'} diff --git a/examples/circleci-minimal.yml b/examples/circleci-minimal.yml index 90afe669..272efba8 100644 --- a/examples/circleci-minimal.yml +++ b/examples/circleci-minimal.yml @@ -11,7 +11,7 @@ jobs: - run: name: Build the Linux wheels. command: | - pip3 install --user cibuildwheel==1.5.1 + pip3 install --user cibuildwheel==1.5.2 cibuildwheel --output-dir wheelhouse - store_artifacts: path: wheelhouse/ @@ -25,7 +25,7 @@ jobs: - run: name: Build the OS X wheels. command: | - pip3 install --user cibuildwheel==1.5.1 + pip3 install --user cibuildwheel==1.5.2 cibuildwheel --output-dir wheelhouse - store_artifacts: path: wheelhouse/ diff --git a/examples/github-deploy.yml b/examples/github-deploy.yml index 49e67f23..93aeaf27 100644 --- a/examples/github-deploy.yml +++ b/examples/github-deploy.yml @@ -28,7 +28,7 @@ jobs: - name: Install cibuildwheel run: | - python -m pip install cibuildwheel==1.5.1 + python -m pip install cibuildwheel==1.5.2 - name: Install Visual C++ for Python 2.7 if: runner.os == 'Windows' diff --git a/examples/github-minimal.yml b/examples/github-minimal.yml index ecbdc396..1095775c 100644 --- a/examples/github-minimal.yml +++ b/examples/github-minimal.yml @@ -20,7 +20,7 @@ jobs: - name: Install cibuildwheel run: | - python -m pip install cibuildwheel==1.5.1 + python -m pip install cibuildwheel==1.5.2 - name: Install Visual C++ for Python 2.7 if: runner.os == 'Windows' diff --git a/examples/travis-ci-deploy.yml b/examples/travis-ci-deploy.yml index db1b1317..ec3a3d74 100644 --- a/examples/travis-ci-deploy.yml +++ b/examples/travis-ci-deploy.yml @@ -25,7 +25,7 @@ env: # Note: TWINE_PASSWORD is set to a PyPI API token in Travis settings install: - - python3 -m pip install cibuildwheel==1.5.1 + - python3 -m pip install cibuildwheel==1.5.2 script: # build the wheels, put them into './wheelhouse' diff --git a/examples/travis-ci-minimal.yml b/examples/travis-ci-minimal.yml index 4fc3673e..5a533441 100644 --- a/examples/travis-ci-minimal.yml +++ b/examples/travis-ci-minimal.yml @@ -26,7 +26,7 @@ jobs: - ln -s /c/Python38/python.exe /c/Python38/python3.exe install: - - python3 -m pip install cibuildwheel==1.5.1 + - python3 -m pip install cibuildwheel==1.5.2 script: # build the wheels, put them into './wheelhouse' diff --git a/examples/travis-ci-test-and-deploy.yml b/examples/travis-ci-test-and-deploy.yml index 95771ba0..56c0d11d 100644 --- a/examples/travis-ci-test-and-deploy.yml +++ b/examples/travis-ci-test-and-deploy.yml @@ -55,7 +55,7 @@ jobs: - stage: deploy name: Build and deploy Linux wheels services: docker - install: python3 -m pip install cibuildwheel==1.5.1 + install: python3 -m pip install cibuildwheel==1.5.2 script: python3 -m cibuildwheel --output-dir wheelhouse after_success: | python3 -m pip install twine @@ -65,7 +65,7 @@ jobs: name: Build and deploy macOS wheels os: osx language: shell - install: python3 -m pip install cibuildwheel==1.5.1 + install: python3 -m pip install cibuildwheel==1.5.2 script: python3 -m cibuildwheel --output-dir wheelhouse after_success: | python3 -m pip install twine @@ -75,7 +75,7 @@ jobs: name: Build and deploy Windows wheels os: windows language: shell - install: python3 -m pip install cibuildwheel==1.5.1 + install: python3 -m pip install cibuildwheel==1.5.2 script: python3 -m cibuildwheel --output-dir wheelhouse after_success: | python3 -m pip install twine diff --git a/setup.py b/setup.py index 01088a55..a9510c4f 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ long_description = (this_directory / 'README.md').read_text(encoding='utf-8') setup( name='cibuildwheel', - version='1.5.1', + version='1.5.2', install_requires=['bashlex!=0.13', 'toml'], description="Build Python wheels on CI with minimal configuration.", long_description=long_description, From 4c0af9f152df7e53c71a4203631ffa06f4c73e01 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 10 Jul 2020 12:46:32 +0100 Subject: [PATCH 083/117] Finish file/dir tests --- cibuildwheel/docker_container.py | 10 +++++---- unit_test/docker_container_test.py | 34 ++++++++++++++++++++++++------ 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/cibuildwheel/docker_container.py b/cibuildwheel/docker_container.py index 8a119136..f5d6e965 100644 --- a/cibuildwheel/docker_container.py +++ b/cibuildwheel/docker_container.py @@ -6,7 +6,7 @@ import sys import uuid from os import PathLike from pathlib import Path, PurePath -from typing import IO, Dict, List, Optional, Sequence, Union +from typing import Any, IO, Dict, List, Optional, Sequence, Union class DockerContainer: @@ -77,16 +77,17 @@ class DockerContainer: # a container is running and the host filesystem is # mounted. https://github.com/moby/moby/issues/38995 # Use `docker exec` instead. + quote = lambda p: shlex.quote(str(p)) if from_path.is_dir(): self.call(['mkdir', '-p', to_path]) subprocess.run( - f'tar cf - . | docker exec -i {self.name} tar -xC {to_path} -f -', + f'tar cf - . | docker exec -i {self.name} tar -xC {quote(to_path)} -f -', shell=True, check=True, cwd=from_path) else: subprocess.run( - f'cat {from_path} | docker exec -i {self.name} sh -c "cat > {to_path}"', + f'cat {quote(from_path)} | docker exec -i {self.name} sh -c "cat > {quote(to_path)}"', shell=True, check=True) @@ -94,8 +95,9 @@ class DockerContainer: # note: we assume from_path is a dir to_path.mkdir(parents=True, exist_ok=True) + quote = lambda p: shlex.quote(str(p)) subprocess.run( - f'docker exec -i {self.name} tar -cC {from_path} -f - . | tar -xf -', + f'docker exec -i {self.name} tar -cC {quote(from_path)} -f - . | tar -xf -', shell=True, check=True, cwd=to_path diff --git a/unit_test/docker_container_test.py b/unit_test/docker_container_test.py index 689ff19b..c3b3c951 100644 --- a/unit_test/docker_container_test.py +++ b/unit_test/docker_container_test.py @@ -1,5 +1,7 @@ from pathlib import Path, PurePath import platform +import random +import shutil import subprocess import textwrap from uuid import uuid4 @@ -104,8 +106,7 @@ def test_binary_output(): def test_file_operations(tmp_path: Path): with DockerContainer(DEFAULT_IMAGE) as container: # test copying a file in - test_binary_data = uuid4().bytes + uuid4().bytes + uuid4().bytes + uuid4().bytes - + test_binary_data = bytes(random.randrange(256) for _ in range(1000)) original_test_file = tmp_path / 'test.dat' original_test_file.write_bytes(test_binary_data) @@ -116,11 +117,32 @@ def test_file_operations(tmp_path: Path): output = container.call(['cat', dst_file], capture_output=True) assert test_binary_data == bytes(output, encoding='utf8', errors='surrogateescape') +@pytest.mark.docker +def test_dir_operations(tmp_path: Path): + with DockerContainer(DEFAULT_IMAGE) as container: + test_binary_data = bytes(random.randrange(256) for _ in range(1000)) + original_test_file = tmp_path / 'test.dat' + original_test_file.write_bytes(test_binary_data) + # test copying a dir in test_dir = tmp_path / 'test_dir' - new_test_file = tmp_path / 'test-new.dat' - container.copy_out(dst_file, new_test_file) + test_dir.mkdir() + test_file = test_dir / 'test.dat' + shutil.copyfile(original_test_file, test_file) - assert original_test_file.read_bytes() == new_test_file.read_bytes() + dst_dir = PurePath('/tmp/test_dir') + dst_file = dst_dir / 'test.dat' + container.copy_into(test_dir, dst_dir) + + output = container.call(['cat', dst_file], capture_output=True) + assert test_binary_data == bytes(output, encoding='utf8', errors='surrogateescape') + + # test glob + assert container.glob(dst_dir / '*.dat') == [dst_file] + + # test copy dir out + new_test_dir = tmp_path / 'test_dir_new' + container.copy_out(dst_dir, new_test_dir) + + assert test_binary_data == (new_test_dir / 'test.dat').read_bytes() - assert container.glob(PurePath('/tmp/*.dat')) == [dst_file] From 44e31988805c30b1e22f43552094940dd2e30a86 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 10 Jul 2020 12:57:05 +0100 Subject: [PATCH 084/117] Convert glob to take a Path and a str pattern --- cibuildwheel/docker_container.py | 7 +++++-- cibuildwheel/linux.py | 4 ++-- unit_test/docker_container_test.py | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/cibuildwheel/docker_container.py b/cibuildwheel/docker_container.py index f5d6e965..6bd5cdf3 100644 --- a/cibuildwheel/docker_container.py +++ b/cibuildwheel/docker_container.py @@ -1,5 +1,6 @@ import io import json +import os import shlex import subprocess import sys @@ -103,11 +104,13 @@ class DockerContainer: cwd=to_path ) - def glob(self, pattern: PurePath) -> List[PurePath]: + def glob(self, path: PurePath, pattern: str) -> List[PurePath]: + glob_pattern = os.path.join(str(path), pattern) + path_strs = json.loads(self.call([ self.UTILITY_PYTHON, '-c', - f'import sys, json, glob; json.dump(glob.glob({str(pattern)!r}), sys.stdout)' + f'import sys, json, glob; json.dump(glob.glob({glob_pattern!r}), sys.stdout)' ], capture_output=True)) return [PurePath(p) for p in path_strs] diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 60de4743..e4607f0c 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -167,7 +167,7 @@ def build(options: BuildOptions) -> None: *get_build_verbosity_extra_flags(options.build_verbosity) ], env=env) - built_wheel = docker.glob(built_wheel_dir / '*.whl')[0] + built_wheel = docker.glob(built_wheel_dir, '*.whl')[0] repaired_wheel_dir = temp_dir / 'repaired_wheel' docker.call(['rm', '-rf', repaired_wheel_dir]) @@ -179,7 +179,7 @@ def build(options: BuildOptions) -> None: repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) docker.call(['sh', '-c', repair_command_prepared], env=env) - repaired_wheels = docker.glob(repaired_wheel_dir / '*.whl') + repaired_wheels = docker.glob(repaired_wheel_dir, '*.whl') if options.test_command: # set up a virtual environment to install and test from, to make sure diff --git a/unit_test/docker_container_test.py b/unit_test/docker_container_test.py index c3b3c951..6fe9f957 100644 --- a/unit_test/docker_container_test.py +++ b/unit_test/docker_container_test.py @@ -138,7 +138,7 @@ def test_dir_operations(tmp_path: Path): assert test_binary_data == bytes(output, encoding='utf8', errors='surrogateescape') # test glob - assert container.glob(dst_dir / '*.dat') == [dst_file] + assert container.glob(dst_dir, '*.dat') == [dst_file] # test copy dir out new_test_dir = tmp_path / 'test_dir_new' From 5842a7592727e0b4f58adb563629f2bace786976 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 10 Jul 2020 13:04:50 +0100 Subject: [PATCH 085/117] Flake8 fixes --- cibuildwheel/docker_container.py | 15 +++++++++------ unit_test/docker_container_test.py | 5 ++--- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/cibuildwheel/docker_container.py b/cibuildwheel/docker_container.py index 6bd5cdf3..41b2f42f 100644 --- a/cibuildwheel/docker_container.py +++ b/cibuildwheel/docker_container.py @@ -7,7 +7,7 @@ import sys import uuid from os import PathLike from pathlib import Path, PurePath -from typing import Any, IO, Dict, List, Optional, Sequence, Union +from typing import IO, Dict, List, Optional, Sequence, Union class DockerContainer: @@ -78,17 +78,17 @@ class DockerContainer: # a container is running and the host filesystem is # mounted. https://github.com/moby/moby/issues/38995 # Use `docker exec` instead. - quote = lambda p: shlex.quote(str(p)) + if from_path.is_dir(): self.call(['mkdir', '-p', to_path]) subprocess.run( - f'tar cf - . | docker exec -i {self.name} tar -xC {quote(to_path)} -f -', + f'tar cf - . | docker exec -i {self.name} tar -xC {shell_quote(to_path)} -f -', shell=True, check=True, cwd=from_path) else: subprocess.run( - f'cat {quote(from_path)} | docker exec -i {self.name} sh -c "cat > {quote(to_path)}"', + f'cat {shell_quote(from_path)} | docker exec -i {self.name} sh -c "cat > {shell_quote(to_path)}"', shell=True, check=True) @@ -96,9 +96,8 @@ class DockerContainer: # note: we assume from_path is a dir to_path.mkdir(parents=True, exist_ok=True) - quote = lambda p: shlex.quote(str(p)) subprocess.run( - f'docker exec -i {self.name} tar -cC {quote(from_path)} -f - . | tar -xf -', + f'docker exec -i {self.name} tar -cC {shell_quote(from_path)} -f - . | tar -xf -', shell=True, check=True, cwd=to_path @@ -184,3 +183,7 @@ class DockerContainer: def environment_executor(self, command: str, environment: Dict[str, str]) -> str: # used as an EnvironmentExecutor to evaluate commands and capture output return self.call(shlex.split(command), env=environment) + + +def shell_quote(path: PurePath) -> str: + return shlex.quote(str(path)) diff --git a/unit_test/docker_container_test.py b/unit_test/docker_container_test.py index 6fe9f957..fed06174 100644 --- a/unit_test/docker_container_test.py +++ b/unit_test/docker_container_test.py @@ -1,10 +1,9 @@ -from pathlib import Path, PurePath import platform import random import shutil import subprocess import textwrap -from uuid import uuid4 +from pathlib import Path, PurePath import pytest @@ -117,6 +116,7 @@ def test_file_operations(tmp_path: Path): output = container.call(['cat', dst_file], capture_output=True) assert test_binary_data == bytes(output, encoding='utf8', errors='surrogateescape') + @pytest.mark.docker def test_dir_operations(tmp_path: Path): with DockerContainer(DEFAULT_IMAGE) as container: @@ -145,4 +145,3 @@ def test_dir_operations(tmp_path: Path): container.copy_out(dst_dir, new_test_dir) assert test_binary_data == (new_test_dir / 'test.dat').read_bytes() - From e098e1450d59f022129324f43446c29b3f5e9ff9 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 12 Jul 2020 12:35:30 +0100 Subject: [PATCH 086/117] Fix test project generation script --- test/test_projects/__main__.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/test/test_projects/__main__.py b/test/test_projects/__main__.py index af5b14dc..06fe696f 100644 --- a/test/test_projects/__main__.py +++ b/test/test_projects/__main__.py @@ -1,8 +1,9 @@ -from argparse import ArgumentParser import importlib -import tempfile -import sys import subprocess +import sys +import tempfile +from argparse import ArgumentParser +from pathlib import Path def main(): @@ -19,7 +20,7 @@ def main(): project = getattr(importlib.import_module(module), name) - project_dir = tempfile.mkdtemp() + project_dir = Path(tempfile.mkdtemp()) project.generate(project_dir) print('Project generated at', project_dir) From 9f71900637d41b8a82c27501ee13a1ad049d1a7e Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 12 Jul 2020 12:39:33 +0100 Subject: [PATCH 087/117] add debugging code --- bin/run_tests.py | 3 ++- test/test_environment.py | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/bin/run_tests.py b/bin/run_tests.py index 17f06aa5..96813ae6 100755 --- a/bin/run_tests.py +++ b/bin/run_tests.py @@ -17,4 +17,5 @@ if __name__ == '__main__': subprocess.check_call(unit_test_args) # run the integration tests - subprocess.check_call([sys.executable, '-m', 'pytest', '-x', '--durations', '0', 'test']) + # TODO: REMOVE 'overridden' + subprocess.check_call([sys.executable, '-m', 'pytest', '-x', '--durations', '0', 'test', '-k', 'overridden']) diff --git a/test/test_environment.py b/test/test_environment.py index 03bf3bd2..7f1c641c 100644 --- a/test/test_environment.py +++ b/test/test_environment.py @@ -70,4 +70,7 @@ def test_overridden_path(tmp_path, capfd): assert len(os.listdir(output_dir)) == 0 captured = capfd.readouterr() + # TODO: REMOVE THIS PRINT + print('captured.out', captured.out) + print('captured.err', captured.err) assert "python available on PATH doesn't match our installed instance" in captured.err From dc446d3cac4d8ca49cb2b010ce686aa346145c79 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 12 Jul 2020 12:45:59 +0100 Subject: [PATCH 088/117] moar debug --- cibuildwheel/bashlex_eval.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cibuildwheel/bashlex_eval.py b/cibuildwheel/bashlex_eval.py index b5e478c8..4cc9c053 100644 --- a/cibuildwheel/bashlex_eval.py +++ b/cibuildwheel/bashlex_eval.py @@ -69,6 +69,8 @@ def evaluate_word_node(node: bashlex.ast.node, context: NodeExecutionContext) -> # remove the None letters and concat value = ''.join(letters) + print('node value:', value) + # apply bash-like quotes/whitespace treatment return ' '.join(word.strip() for word in shlex.split(value)) From 9fb748bfb0411dbdfbdd7d204eeb232ac3172e23 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 12 Jul 2020 21:45:46 +0100 Subject: [PATCH 089/117] Add unit test for this failure mode --- unit_test/environment_test.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/unit_test/environment_test.py b/unit_test/environment_test.py index 4e10e07c..5a0cf613 100644 --- a/unit_test/environment_test.py +++ b/unit_test/environment_test.py @@ -103,3 +103,14 @@ def test_operators_inside_eval(): environment_dict = environment_recipe.as_dictionary(os.environ.copy()) assert environment_dict.get('SOMETHING') == 'a\nb\nc' + + +def test_substitution_with_backslash(): + environment_recipe = parse_environment('PATH2="somewhere_else;$PATH1"') + + # pass the existing process env so PATH is available + environment_dict = environment_recipe.as_dictionary(prev_environment={ + 'PATH1': 'c:\\folder\\' + }) + + assert environment_dict.get('PATH2') == 'somewhere_else;c:\\folder\\' From 08eddeff8bc0e2ca7699a0c1af71b15f12615132 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 15 Jul 2020 21:07:33 +0100 Subject: [PATCH 090/117] Fix test and add another more awkward example --- cibuildwheel/bashlex_eval.py | 26 +++++++++++--------------- unit_test/environment_test.py | 9 +++++++++ 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/cibuildwheel/bashlex_eval.py b/cibuildwheel/bashlex_eval.py index 4cc9c053..122b50b3 100644 --- a/cibuildwheel/bashlex_eval.py +++ b/cibuildwheel/bashlex_eval.py @@ -51,28 +51,24 @@ def evaluate_node(node: bashlex.ast.node, context: NodeExecutionContext) -> str: def evaluate_word_node(node: bashlex.ast.node, context: NodeExecutionContext) -> str: - word_start = node.pos[0] - word_end = node.pos[1] - word_string = context.input[word_start:word_end] - letters = list(word_string) + value = node.word for part in node.parts: - part_start = part.pos[0] - word_start - part_end = part.pos[1] - word_start + part_string = context.input[part.pos[0]:part.pos[1]] + part_value = evaluate_node(part, context=context) - # Set all the characters in the part to None - for i in range(part_start, part_end): - letters[i] = '' + if part_string not in value: + raise RuntimeError( + 'bash parse failed. part "{}" not found in "{}". Word was "{}". Full input was "{}"'.format( + part_string, value, node.word, context.input, + ) + ) - letters[part_start] = evaluate_node(part, context=context) - - # remove the None letters and concat - value = ''.join(letters) + value = value.replace(part_string, part_value, 1) print('node value:', value) - # apply bash-like quotes/whitespace treatment - return ' '.join(word.strip() for word in shlex.split(value)) + return value def evaluate_command_node(node: bashlex.ast.node, context: NodeExecutionContext) -> str: diff --git a/unit_test/environment_test.py b/unit_test/environment_test.py index 5a0cf613..0827cc0d 100644 --- a/unit_test/environment_test.py +++ b/unit_test/environment_test.py @@ -114,3 +114,12 @@ def test_substitution_with_backslash(): }) assert environment_dict.get('PATH2') == 'somewhere_else;c:\\folder\\' + + +def test_awkwardly_quoted_variable(): + environment_recipe = parse_environment('VAR2=something"like this""$VAR1"$VAR1$(echo "theres more")"$(echo "and more!")"') + + # pass the existing process env so PATH is available + environment_dict = environment_recipe.as_dictionary({'VAR1': 'but wait'}) + + assert environment_dict.get('VAR2') == 'somethinglike thisbut waitbut waittheres moreand more!' From 69fe085ba3b88c220558f6e5a2347f2151fa52c0 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 15 Jul 2020 21:25:29 +0100 Subject: [PATCH 091/117] more back-slash hilarity --- test/test_environment.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/test/test_environment.py b/test/test_environment.py index 7f1c641c..3bcb6e3a 100644 --- a/test/test_environment.py +++ b/test/test_environment.py @@ -65,7 +65,9 @@ def test_overridden_path(tmp_path, capfd): (new_path / 'python').touch(mode=0o777) utils.cibuildwheel_run(project_dir, output_dir=output_dir, add_env={ - 'CIBW_ENVIRONMENT': f'''PATH="{new_path}{os.pathsep}$PATH"''', + # use single-quotes for new_path, because on windows, paths with backslashes + # are interpreted by bash as escape sequences! + 'CIBW_ENVIRONMENT': f'''PATH='{new_path}'{os.pathsep}"$PATH"''', }) assert len(os.listdir(output_dir)) == 0 From 5df97a4121312ab5f7af98c119767ca1c12bad65 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 15 Jul 2020 21:34:30 +0100 Subject: [PATCH 092/117] Correct quoting of semicolon --- test/test_environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_environment.py b/test/test_environment.py index 3bcb6e3a..04d4d3d9 100644 --- a/test/test_environment.py +++ b/test/test_environment.py @@ -67,7 +67,7 @@ def test_overridden_path(tmp_path, capfd): utils.cibuildwheel_run(project_dir, output_dir=output_dir, add_env={ # use single-quotes for new_path, because on windows, paths with backslashes # are interpreted by bash as escape sequences! - 'CIBW_ENVIRONMENT': f'''PATH='{new_path}'{os.pathsep}"$PATH"''', + 'CIBW_ENVIRONMENT': f'''PATH='{new_path}{os.pathsep}'$PATH''', }) assert len(os.listdir(output_dir)) == 0 From 56cd6c965d4835d12530b2be97171e1c7063e1dc Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 15 Jul 2020 22:02:20 +0100 Subject: [PATCH 093/117] Avoid backslash-in-env-var entirely --- test/test_environment.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/test/test_environment.py b/test/test_environment.py index 04d4d3d9..6ff4dbd1 100644 --- a/test/test_environment.py +++ b/test/test_environment.py @@ -65,9 +65,8 @@ def test_overridden_path(tmp_path, capfd): (new_path / 'python').touch(mode=0o777) utils.cibuildwheel_run(project_dir, output_dir=output_dir, add_env={ - # use single-quotes for new_path, because on windows, paths with backslashes - # are interpreted by bash as escape sequences! - 'CIBW_ENVIRONMENT': f'''PATH='{new_path}{os.pathsep}'$PATH''', + 'NEW_PATH': new_path, + 'CIBW_ENVIRONMENT': f'''PATH="$NEW_PATH{os.pathsep}$PATH''', }) assert len(os.listdir(output_dir)) == 0 From b90c4035b381c9bdb09f12203c2470b8cbca1c24 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Thu, 16 Jul 2020 07:56:15 +0100 Subject: [PATCH 094/117] Update test_environment.py --- test/test_environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_environment.py b/test/test_environment.py index 6ff4dbd1..7a608836 100644 --- a/test/test_environment.py +++ b/test/test_environment.py @@ -65,7 +65,7 @@ def test_overridden_path(tmp_path, capfd): (new_path / 'python').touch(mode=0o777) utils.cibuildwheel_run(project_dir, output_dir=output_dir, add_env={ - 'NEW_PATH': new_path, + 'NEW_PATH': str(new_path), 'CIBW_ENVIRONMENT': f'''PATH="$NEW_PATH{os.pathsep}$PATH''', }) From af2a1719946f1723d19af61447e29893405f76dc Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 17 Jul 2020 09:09:02 +0100 Subject: [PATCH 095/117] Fix quotes --- test/test_environment.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_environment.py b/test/test_environment.py index 7a608836..c9b66d0c 100644 --- a/test/test_environment.py +++ b/test/test_environment.py @@ -66,7 +66,7 @@ def test_overridden_path(tmp_path, capfd): utils.cibuildwheel_run(project_dir, output_dir=output_dir, add_env={ 'NEW_PATH': str(new_path), - 'CIBW_ENVIRONMENT': f'''PATH="$NEW_PATH{os.pathsep}$PATH''', + 'CIBW_ENVIRONMENT': f'''PATH="$NEW_PATH{os.pathsep}$PATH"''', }) assert len(os.listdir(output_dir)) == 0 From 684745485091cb7922132e0d82fba380ae53f8b8 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 17 Jul 2020 09:31:47 +0100 Subject: [PATCH 096/117] Remove debug code --- bin/run_tests.py | 1 - cibuildwheel/bashlex_eval.py | 2 -- test/test_environment.py | 3 --- 3 files changed, 6 deletions(-) diff --git a/bin/run_tests.py b/bin/run_tests.py index 96813ae6..316f9a54 100755 --- a/bin/run_tests.py +++ b/bin/run_tests.py @@ -17,5 +17,4 @@ if __name__ == '__main__': subprocess.check_call(unit_test_args) # run the integration tests - # TODO: REMOVE 'overridden' subprocess.check_call([sys.executable, '-m', 'pytest', '-x', '--durations', '0', 'test', '-k', 'overridden']) diff --git a/cibuildwheel/bashlex_eval.py b/cibuildwheel/bashlex_eval.py index 122b50b3..57791b2c 100644 --- a/cibuildwheel/bashlex_eval.py +++ b/cibuildwheel/bashlex_eval.py @@ -66,8 +66,6 @@ def evaluate_word_node(node: bashlex.ast.node, context: NodeExecutionContext) -> value = value.replace(part_string, part_value, 1) - print('node value:', value) - return value diff --git a/test/test_environment.py b/test/test_environment.py index c9b66d0c..dd2de0ea 100644 --- a/test/test_environment.py +++ b/test/test_environment.py @@ -71,7 +71,4 @@ def test_overridden_path(tmp_path, capfd): assert len(os.listdir(output_dir)) == 0 captured = capfd.readouterr() - # TODO: REMOVE THIS PRINT - print('captured.out', captured.out) - print('captured.err', captured.err) assert "python available on PATH doesn't match our installed instance" in captured.err From 7d983cdf8c720a86490744718bcc342d95b1fcba Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Fri, 17 Jul 2020 09:39:22 +0100 Subject: [PATCH 097/117] Reenable all tests --- bin/run_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/run_tests.py b/bin/run_tests.py index 316f9a54..17f06aa5 100755 --- a/bin/run_tests.py +++ b/bin/run_tests.py @@ -17,4 +17,4 @@ if __name__ == '__main__': subprocess.check_call(unit_test_args) # run the integration tests - subprocess.check_call([sys.executable, '-m', 'pytest', '-x', '--durations', '0', 'test', '-k', 'overridden']) + subprocess.check_call([sys.executable, '-m', 'pytest', '-x', '--durations', '0', 'test']) From 5368b3bc645a8e3d2a6c7118c7b8e2592e488b09 Mon Sep 17 00:00:00 2001 From: mayeut Date: Sat, 18 Jul 2020 11:05:23 +0200 Subject: [PATCH 098/117] Update CPython 3.8 from 3.8.3 to 3.8.4 Changelog: https://docs.python.org/release/3.8.4/whatsnew/changelog.html#changelog --- cibuildwheel/macos.py | 2 +- cibuildwheel/windows.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index ce7bd974..48ec210b 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -37,7 +37,7 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi PythonConfiguration(version='3.5', identifier='cp35-macosx_x86_64', url='https://www.python.org/ftp/python/3.5.4/python-3.5.4-macosx10.6.pkg'), PythonConfiguration(version='3.6', identifier='cp36-macosx_x86_64', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg'), PythonConfiguration(version='3.7', identifier='cp37-macosx_x86_64', url='https://www.python.org/ftp/python/3.7.8/python-3.7.8-macosx10.9.pkg'), - PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.3/python-3.8.3-macosx10.9.pkg'), + PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.4/python-3.8.4-macosx10.9.pkg'), # PyPy PythonConfiguration(version='2.7', identifier='pp27-macosx_x86_64', url='https://downloads.python.org/pypy/pypy2.7-v7.3.1-osx64.tar.bz2'), PythonConfiguration(version='3.6', identifier='pp36-macosx_x86_64', url='https://downloads.python.org/pypy/pypy3.6-v7.3.1-osx64.tar.bz2'), diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 1baf66f5..c958b9b6 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -56,8 +56,8 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi PythonConfiguration(version='3.6.8', arch='64', identifier='cp36-win_amd64', url=None), PythonConfiguration(version='3.7.8', arch='32', identifier='cp37-win32', url=None), PythonConfiguration(version='3.7.8', arch='64', identifier='cp37-win_amd64', url=None), - PythonConfiguration(version='3.8.3', arch='32', identifier='cp38-win32', url=None), - PythonConfiguration(version='3.8.3', arch='64', identifier='cp38-win_amd64', url=None), + PythonConfiguration(version='3.8.4', arch='32', identifier='cp38-win32', url=None), + PythonConfiguration(version='3.8.4', arch='64', identifier='cp38-win_amd64', url=None), # PyPy PythonConfiguration(version='2.7', arch='32', identifier='pp27-win32', url='https://downloads.python.org/pypy/pypy2.7-v7.3.1-win32.zip'), PythonConfiguration(version='3.6', arch='32', identifier='pp36-win32', url='https://downloads.python.org/pypy/pypy3.6-v7.3.1-win32.zip'), From c884530a1da933814eaaaa9fa2db605e8ecaf116 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 19 Jul 2020 11:59:56 +0100 Subject: [PATCH 099/117] Bump version: v1.5.3 --- README.md | 9 ++++++++- cibuildwheel/__init__.py | 2 +- examples/appveyor-minimal.yml | 2 +- examples/azure-pipelines-minimal.yml | 6 +++--- examples/circleci-minimal.yml | 4 ++-- examples/github-deploy.yml | 2 +- examples/github-minimal.yml | 2 +- examples/travis-ci-deploy.yml | 2 +- examples/travis-ci-minimal.yml | 2 +- examples/travis-ci-test-and-deploy.yml | 6 +++--- setup.py | 2 +- 11 files changed, 23 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 14607d78..c445ac3a 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ env: # Note: TWINE_PASSWORD is set to a PyPI API token in Travis settings install: - - python3 -m pip install cibuildwheel==1.5.2 + - python3 -m pip install cibuildwheel==1.5.3 script: # build the wheels, put them into './wheelhouse' @@ -152,6 +152,13 @@ This is similar to static linking, so it might have some licence implications. C Changelog ========= +### 1.5.3 + +_19 July 2020_ + +- 🛠 Update CPython 3.8 to 3.8.3 (#405) +- 🛠 Internal refactoring of Linux build, to move control flow into Python (#386) + ### 1.5.2 _8 July 2020_ diff --git a/cibuildwheel/__init__.py b/cibuildwheel/__init__.py index c3b38415..0bb84ff2 100644 --- a/cibuildwheel/__init__.py +++ b/cibuildwheel/__init__.py @@ -1 +1 @@ -__version__ = '1.5.2' +__version__ = '1.5.3' diff --git a/examples/appveyor-minimal.yml b/examples/appveyor-minimal.yml index 311332aa..ee178280 100644 --- a/examples/appveyor-minimal.yml +++ b/examples/appveyor-minimal.yml @@ -12,7 +12,7 @@ stack: python 3.7 init: - cmd: set PATH=C:\Python37;C:\Python37\Scripts;%PATH% -install: python -m pip install cibuildwheel==1.5.2 +install: python -m pip install cibuildwheel==1.5.3 build_script: python -m cibuildwheel --output-dir wheelhouse diff --git a/examples/azure-pipelines-minimal.yml b/examples/azure-pipelines-minimal.yml index d7ae6c96..6c0b3c97 100644 --- a/examples/azure-pipelines-minimal.yml +++ b/examples/azure-pipelines-minimal.yml @@ -5,7 +5,7 @@ jobs: - task: UsePythonVersion@0 - bash: | python3 -m pip install --upgrade pip - pip3 install cibuildwheel==1.5.2 + pip3 install cibuildwheel==1.5.3 cibuildwheel --output-dir wheelhouse . - task: PublishBuildArtifacts@1 inputs: {pathtoPublish: 'wheelhouse'} @@ -16,7 +16,7 @@ jobs: - task: UsePythonVersion@0 - bash: | python3 -m pip install --upgrade pip - pip3 install cibuildwheel==1.5.2 + pip3 install cibuildwheel==1.5.3 cibuildwheel --output-dir wheelhouse . - task: PublishBuildArtifacts@1 inputs: {pathtoPublish: 'wheelhouse'} @@ -29,7 +29,7 @@ jobs: displayName: Install Visual C++ for Python 2.7 - bash: | python -m pip install --upgrade pip - pip install cibuildwheel==1.5.2 + pip install cibuildwheel==1.5.3 cibuildwheel --output-dir wheelhouse . - task: PublishBuildArtifacts@1 inputs: {pathtoPublish: 'wheelhouse'} diff --git a/examples/circleci-minimal.yml b/examples/circleci-minimal.yml index 272efba8..7dc7fd5f 100644 --- a/examples/circleci-minimal.yml +++ b/examples/circleci-minimal.yml @@ -11,7 +11,7 @@ jobs: - run: name: Build the Linux wheels. command: | - pip3 install --user cibuildwheel==1.5.2 + pip3 install --user cibuildwheel==1.5.3 cibuildwheel --output-dir wheelhouse - store_artifacts: path: wheelhouse/ @@ -25,7 +25,7 @@ jobs: - run: name: Build the OS X wheels. command: | - pip3 install --user cibuildwheel==1.5.2 + pip3 install --user cibuildwheel==1.5.3 cibuildwheel --output-dir wheelhouse - store_artifacts: path: wheelhouse/ diff --git a/examples/github-deploy.yml b/examples/github-deploy.yml index 93aeaf27..192c762f 100644 --- a/examples/github-deploy.yml +++ b/examples/github-deploy.yml @@ -28,7 +28,7 @@ jobs: - name: Install cibuildwheel run: | - python -m pip install cibuildwheel==1.5.2 + python -m pip install cibuildwheel==1.5.3 - name: Install Visual C++ for Python 2.7 if: runner.os == 'Windows' diff --git a/examples/github-minimal.yml b/examples/github-minimal.yml index 1095775c..8eb83814 100644 --- a/examples/github-minimal.yml +++ b/examples/github-minimal.yml @@ -20,7 +20,7 @@ jobs: - name: Install cibuildwheel run: | - python -m pip install cibuildwheel==1.5.2 + python -m pip install cibuildwheel==1.5.3 - name: Install Visual C++ for Python 2.7 if: runner.os == 'Windows' diff --git a/examples/travis-ci-deploy.yml b/examples/travis-ci-deploy.yml index ec3a3d74..8a17ac62 100644 --- a/examples/travis-ci-deploy.yml +++ b/examples/travis-ci-deploy.yml @@ -25,7 +25,7 @@ env: # Note: TWINE_PASSWORD is set to a PyPI API token in Travis settings install: - - python3 -m pip install cibuildwheel==1.5.2 + - python3 -m pip install cibuildwheel==1.5.3 script: # build the wheels, put them into './wheelhouse' diff --git a/examples/travis-ci-minimal.yml b/examples/travis-ci-minimal.yml index 5a533441..dd0665b8 100644 --- a/examples/travis-ci-minimal.yml +++ b/examples/travis-ci-minimal.yml @@ -26,7 +26,7 @@ jobs: - ln -s /c/Python38/python.exe /c/Python38/python3.exe install: - - python3 -m pip install cibuildwheel==1.5.2 + - python3 -m pip install cibuildwheel==1.5.3 script: # build the wheels, put them into './wheelhouse' diff --git a/examples/travis-ci-test-and-deploy.yml b/examples/travis-ci-test-and-deploy.yml index 56c0d11d..e3edd0a9 100644 --- a/examples/travis-ci-test-and-deploy.yml +++ b/examples/travis-ci-test-and-deploy.yml @@ -55,7 +55,7 @@ jobs: - stage: deploy name: Build and deploy Linux wheels services: docker - install: python3 -m pip install cibuildwheel==1.5.2 + install: python3 -m pip install cibuildwheel==1.5.3 script: python3 -m cibuildwheel --output-dir wheelhouse after_success: | python3 -m pip install twine @@ -65,7 +65,7 @@ jobs: name: Build and deploy macOS wheels os: osx language: shell - install: python3 -m pip install cibuildwheel==1.5.2 + install: python3 -m pip install cibuildwheel==1.5.3 script: python3 -m cibuildwheel --output-dir wheelhouse after_success: | python3 -m pip install twine @@ -75,7 +75,7 @@ jobs: name: Build and deploy Windows wheels os: windows language: shell - install: python3 -m pip install cibuildwheel==1.5.2 + install: python3 -m pip install cibuildwheel==1.5.3 script: python3 -m cibuildwheel --output-dir wheelhouse after_success: | python3 -m pip install twine diff --git a/setup.py b/setup.py index a9510c4f..fddd0923 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ long_description = (this_directory / 'README.md').read_text(encoding='utf-8') setup( name='cibuildwheel', - version='1.5.2', + version='1.5.3', install_requires=['bashlex!=0.13', 'toml'], description="Build Python wheels on CI with minimal configuration.", long_description=long_description, From fddfaba0a4c68f6dcd191ab46a3d2f0c334964c9 Mon Sep 17 00:00:00 2001 From: Samuel St-Jean Date: Sun, 19 Jul 2020 10:32:27 -0400 Subject: [PATCH 100/117] fix typo --- docs/deliver-to-pypi.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/deliver-to-pypi.md b/docs/deliver-to-pypi.md index 648f1bdd..5c1fb2ee 100644 --- a/docs/deliver-to-pypi.md +++ b/docs/deliver-to-pypi.md @@ -33,4 +33,4 @@ Obviously, manual steps are for chumps, so we can automate this a little by usin If you don't need much control over the release of a package, you can set up cibuildwheel to deliver the wheels straight to PyPI. This doesn't require anycloud storage to work - you just need to bump the version and tag it. -[`examples/travis-ci-deploy.yml`](https://github.com/joerick/cibuildwheel/blob/master/examples/travis-ci-deploy.yml) and [`examples/github-deploy.yml`](https://github.com/joerick/cibuildwheel/blob/master/examples/github-deploy.yml) are example configurations that automatocially upload wheels to PyPI. Also check out [this example repo](https://github.com/joerick/cibuildwheel-autopypi-example) for more detailed instructions on how to set this up. +[`examples/travis-ci-deploy.yml`](https://github.com/joerick/cibuildwheel/blob/master/examples/travis-ci-deploy.yml) and [`examples/github-deploy.yml`](https://github.com/joerick/cibuildwheel/blob/master/examples/github-deploy.yml) are example configurations that automatically upload wheels to PyPI. Also check out [this example repo](https://github.com/joerick/cibuildwheel-autopypi-example) for more detailed instructions on how to set this up. From 44fd8b5c2622f7f53c9ca32d3144257a1eb8ab9c Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Sun, 19 Jul 2020 17:19:48 +0200 Subject: [PATCH 101/117] Fix test to demonstrate failure --- unit_test/environment_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/unit_test/environment_test.py b/unit_test/environment_test.py index 0827cc0d..ab690a84 100644 --- a/unit_test/environment_test.py +++ b/unit_test/environment_test.py @@ -40,7 +40,7 @@ def test_inheritance(): def test_shell_eval(): - environment_recipe = parse_environment('VAR="$(echo "a test" string)"') + environment_recipe = parse_environment('VAR="$(echo "a test" string)"') env_copy = os.environ.copy() env_copy.pop('VAR', None) @@ -50,8 +50,8 @@ def test_shell_eval(): ) environment_cmds = environment_recipe.as_shell_commands() - assert environment_dict['VAR'] == 'a test string' - assert environment_cmds == ['export VAR="$(echo "a test" string)"'] + assert environment_dict['VAR'] == 'a test string' + assert environment_cmds == ['export VAR="$(echo "a test" string)"'] def test_shell_eval_and_env(): From 3a2d7a21b114c14ee93d29bbc7b16affec251ad5 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Sun, 19 Jul 2020 17:32:07 +0200 Subject: [PATCH 102/117] Fix quotes-in-commandsubstitution problem with well-placed shlex.quote --- cibuildwheel/bashlex_eval.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/bashlex_eval.py b/cibuildwheel/bashlex_eval.py index 57791b2c..e09752f2 100644 --- a/cibuildwheel/bashlex_eval.py +++ b/cibuildwheel/bashlex_eval.py @@ -97,7 +97,7 @@ def evaluate_nodes_as_compound_command(nodes: Sequence[bashlex.ast.node], contex def evaluate_nodes_as_simple_command(nodes: List[bashlex.ast.node], context: NodeExecutionContext): - words = [evaluate_node(part, context=context) for part in nodes] + words = [shlex.quote(evaluate_node(part, context=context)) for part in nodes] command = ' '.join(words) return context.executor(command, context.environment) From 28af49a0069916a1a23e21363fead380ab0ff292 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Sun, 19 Jul 2020 17:37:25 +0200 Subject: [PATCH 103/117] Simplify by removing shlex.quote and shlex.split when using EnvironmentExecutor --- cibuildwheel/bashlex_eval.py | 10 ++++------ cibuildwheel/docker_container.py | 4 ++-- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/cibuildwheel/bashlex_eval.py b/cibuildwheel/bashlex_eval.py index e09752f2..8461d4c8 100644 --- a/cibuildwheel/bashlex_eval.py +++ b/cibuildwheel/bashlex_eval.py @@ -1,15 +1,14 @@ -import shlex import subprocess from typing import Callable, Dict, List, NamedTuple, Optional, Sequence import bashlex # type: ignore # a function that takes a shell command and the environment, and returns the result -EnvironmentExecutor = Callable[[str, Dict[str, str]], str] +EnvironmentExecutor = Callable[[List[str], Dict[str, str]], str] -def local_environment_executor(command: str, env: Dict[str, str]) -> str: - return subprocess.check_output(shlex.split(command), env=env, universal_newlines=True) +def local_environment_executor(command: List[str], env: Dict[str, str]) -> str: + return subprocess.check_output(command, env=env, universal_newlines=True) class NodeExecutionContext(NamedTuple): @@ -97,8 +96,7 @@ def evaluate_nodes_as_compound_command(nodes: Sequence[bashlex.ast.node], contex def evaluate_nodes_as_simple_command(nodes: List[bashlex.ast.node], context: NodeExecutionContext): - words = [shlex.quote(evaluate_node(part, context=context)) for part in nodes] - command = ' '.join(words) + command = [evaluate_node(part, context=context) for part in nodes] return context.executor(command, context.environment) diff --git a/cibuildwheel/docker_container.py b/cibuildwheel/docker_container.py index 41b2f42f..c3b226aa 100644 --- a/cibuildwheel/docker_container.py +++ b/cibuildwheel/docker_container.py @@ -180,9 +180,9 @@ class DockerContainer: 'import sys, json, os; json.dump(os.environ.copy(), sys.stdout)' ], capture_output=True)) - def environment_executor(self, command: str, environment: Dict[str, str]) -> str: + def environment_executor(self, command: List[str], environment: Dict[str, str]) -> str: # used as an EnvironmentExecutor to evaluate commands and capture output - return self.call(shlex.split(command), env=environment) + return self.call(command, env=environment) def shell_quote(path: PurePath) -> str: From c5a98377843b37127ff4a069fa84b2562949b9df Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 19 Jul 2020 19:46:40 +0100 Subject: [PATCH 104/117] Update cibuildwheel/bashlex_eval.py --- cibuildwheel/bashlex_eval.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/bashlex_eval.py b/cibuildwheel/bashlex_eval.py index 8461d4c8..2c61bc54 100644 --- a/cibuildwheel/bashlex_eval.py +++ b/cibuildwheel/bashlex_eval.py @@ -3,7 +3,7 @@ from typing import Callable, Dict, List, NamedTuple, Optional, Sequence import bashlex # type: ignore -# a function that takes a shell command and the environment, and returns the result +# a function that takes a command and the environment, and returns the result EnvironmentExecutor = Callable[[List[str], Dict[str, str]], str] From c0cb43800b6740677e373889705b61b17b879a8d Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sun, 19 Jul 2020 22:40:33 +0100 Subject: [PATCH 105/117] Bump version: v1.5.4 --- README.md | 9 ++++++++- cibuildwheel/__init__.py | 2 +- examples/appveyor-minimal.yml | 2 +- examples/azure-pipelines-minimal.yml | 6 +++--- examples/circleci-minimal.yml | 4 ++-- examples/github-deploy.yml | 2 +- examples/github-minimal.yml | 2 +- examples/travis-ci-deploy.yml | 2 +- examples/travis-ci-minimal.yml | 2 +- examples/travis-ci-test-and-deploy.yml | 6 +++--- setup.py | 2 +- 11 files changed, 23 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index c445ac3a..491fd831 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ env: # Note: TWINE_PASSWORD is set to a PyPI API token in Travis settings install: - - python3 -m pip install cibuildwheel==1.5.3 + - python3 -m pip install cibuildwheel==1.5.4 script: # build the wheels, put them into './wheelhouse' @@ -152,6 +152,13 @@ This is similar to static linking, so it might have some licence implications. C Changelog ========= +### 1.5.4 + +_19 June 2020_ + +- 🐛 Fix a bug that would cause command substitutions in CIBW_ENVIRONMENT + variables to not interpret quotes in commands correctly (#406, #408) + ### 1.5.3 _19 July 2020_ diff --git a/cibuildwheel/__init__.py b/cibuildwheel/__init__.py index 0bb84ff2..c24ed73b 100644 --- a/cibuildwheel/__init__.py +++ b/cibuildwheel/__init__.py @@ -1 +1 @@ -__version__ = '1.5.3' +__version__ = '1.5.4' diff --git a/examples/appveyor-minimal.yml b/examples/appveyor-minimal.yml index ee178280..959c4267 100644 --- a/examples/appveyor-minimal.yml +++ b/examples/appveyor-minimal.yml @@ -12,7 +12,7 @@ stack: python 3.7 init: - cmd: set PATH=C:\Python37;C:\Python37\Scripts;%PATH% -install: python -m pip install cibuildwheel==1.5.3 +install: python -m pip install cibuildwheel==1.5.4 build_script: python -m cibuildwheel --output-dir wheelhouse diff --git a/examples/azure-pipelines-minimal.yml b/examples/azure-pipelines-minimal.yml index 6c0b3c97..213c101f 100644 --- a/examples/azure-pipelines-minimal.yml +++ b/examples/azure-pipelines-minimal.yml @@ -5,7 +5,7 @@ jobs: - task: UsePythonVersion@0 - bash: | python3 -m pip install --upgrade pip - pip3 install cibuildwheel==1.5.3 + pip3 install cibuildwheel==1.5.4 cibuildwheel --output-dir wheelhouse . - task: PublishBuildArtifacts@1 inputs: {pathtoPublish: 'wheelhouse'} @@ -16,7 +16,7 @@ jobs: - task: UsePythonVersion@0 - bash: | python3 -m pip install --upgrade pip - pip3 install cibuildwheel==1.5.3 + pip3 install cibuildwheel==1.5.4 cibuildwheel --output-dir wheelhouse . - task: PublishBuildArtifacts@1 inputs: {pathtoPublish: 'wheelhouse'} @@ -29,7 +29,7 @@ jobs: displayName: Install Visual C++ for Python 2.7 - bash: | python -m pip install --upgrade pip - pip install cibuildwheel==1.5.3 + pip install cibuildwheel==1.5.4 cibuildwheel --output-dir wheelhouse . - task: PublishBuildArtifacts@1 inputs: {pathtoPublish: 'wheelhouse'} diff --git a/examples/circleci-minimal.yml b/examples/circleci-minimal.yml index 7dc7fd5f..f890e3d5 100644 --- a/examples/circleci-minimal.yml +++ b/examples/circleci-minimal.yml @@ -11,7 +11,7 @@ jobs: - run: name: Build the Linux wheels. command: | - pip3 install --user cibuildwheel==1.5.3 + pip3 install --user cibuildwheel==1.5.4 cibuildwheel --output-dir wheelhouse - store_artifacts: path: wheelhouse/ @@ -25,7 +25,7 @@ jobs: - run: name: Build the OS X wheels. command: | - pip3 install --user cibuildwheel==1.5.3 + pip3 install --user cibuildwheel==1.5.4 cibuildwheel --output-dir wheelhouse - store_artifacts: path: wheelhouse/ diff --git a/examples/github-deploy.yml b/examples/github-deploy.yml index 192c762f..0db47913 100644 --- a/examples/github-deploy.yml +++ b/examples/github-deploy.yml @@ -28,7 +28,7 @@ jobs: - name: Install cibuildwheel run: | - python -m pip install cibuildwheel==1.5.3 + python -m pip install cibuildwheel==1.5.4 - name: Install Visual C++ for Python 2.7 if: runner.os == 'Windows' diff --git a/examples/github-minimal.yml b/examples/github-minimal.yml index 8eb83814..5bfddc1f 100644 --- a/examples/github-minimal.yml +++ b/examples/github-minimal.yml @@ -20,7 +20,7 @@ jobs: - name: Install cibuildwheel run: | - python -m pip install cibuildwheel==1.5.3 + python -m pip install cibuildwheel==1.5.4 - name: Install Visual C++ for Python 2.7 if: runner.os == 'Windows' diff --git a/examples/travis-ci-deploy.yml b/examples/travis-ci-deploy.yml index 8a17ac62..e1862508 100644 --- a/examples/travis-ci-deploy.yml +++ b/examples/travis-ci-deploy.yml @@ -25,7 +25,7 @@ env: # Note: TWINE_PASSWORD is set to a PyPI API token in Travis settings install: - - python3 -m pip install cibuildwheel==1.5.3 + - python3 -m pip install cibuildwheel==1.5.4 script: # build the wheels, put them into './wheelhouse' diff --git a/examples/travis-ci-minimal.yml b/examples/travis-ci-minimal.yml index dd0665b8..0b73aff3 100644 --- a/examples/travis-ci-minimal.yml +++ b/examples/travis-ci-minimal.yml @@ -26,7 +26,7 @@ jobs: - ln -s /c/Python38/python.exe /c/Python38/python3.exe install: - - python3 -m pip install cibuildwheel==1.5.3 + - python3 -m pip install cibuildwheel==1.5.4 script: # build the wheels, put them into './wheelhouse' diff --git a/examples/travis-ci-test-and-deploy.yml b/examples/travis-ci-test-and-deploy.yml index e3edd0a9..193bce04 100644 --- a/examples/travis-ci-test-and-deploy.yml +++ b/examples/travis-ci-test-and-deploy.yml @@ -55,7 +55,7 @@ jobs: - stage: deploy name: Build and deploy Linux wheels services: docker - install: python3 -m pip install cibuildwheel==1.5.3 + install: python3 -m pip install cibuildwheel==1.5.4 script: python3 -m cibuildwheel --output-dir wheelhouse after_success: | python3 -m pip install twine @@ -65,7 +65,7 @@ jobs: name: Build and deploy macOS wheels os: osx language: shell - install: python3 -m pip install cibuildwheel==1.5.3 + install: python3 -m pip install cibuildwheel==1.5.4 script: python3 -m cibuildwheel --output-dir wheelhouse after_success: | python3 -m pip install twine @@ -75,7 +75,7 @@ jobs: name: Build and deploy Windows wheels os: windows language: shell - install: python3 -m pip install cibuildwheel==1.5.3 + install: python3 -m pip install cibuildwheel==1.5.4 script: python3 -m cibuildwheel --output-dir wheelhouse after_success: | python3 -m pip install twine diff --git a/setup.py b/setup.py index fddd0923..64b0b39f 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ long_description = (this_directory / 'README.md').read_text(encoding='utf-8') setup( name='cibuildwheel', - version='1.5.3', + version='1.5.4', install_requires=['bashlex!=0.13', 'toml'], description="Build Python wheels on CI with minimal configuration.", long_description=long_description, From e28e43e7f98dbe8d7f3cd0ef26b5137270092eef Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Mon, 20 Jul 2020 02:02:50 +0200 Subject: [PATCH 106/117] Fix test_environment's checks on a command substitution in an environment --- test/test_environment.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/test_environment.py b/test/test_environment.py index dd2de0ea..7de1c0bf 100644 --- a/test/test_environment.py +++ b/test/test_environment.py @@ -13,12 +13,15 @@ project_with_environment_asserts = test_projects.new_c_project( # explode if environment isn't correct, as set in CIBW_ENVIRONMENT CIBW_TEST_VAR = os.environ.get("CIBW_TEST_VAR") CIBW_TEST_VAR_2 = os.environ.get("CIBW_TEST_VAR_2") + CIBW_TEST_VAR_3 = os.environ.get("CIBW_TEST_VAR_3") PATH = os.environ.get("PATH") if CIBW_TEST_VAR != "a b c": raise Exception('CIBW_TEST_VAR should equal "a b c". It was "%s"' % CIBW_TEST_VAR) if CIBW_TEST_VAR_2 != "1": raise Exception('CIBW_TEST_VAR_2 should equal "1". It was "%s"' % CIBW_TEST_VAR_2) + if CIBW_TEST_VAR_3 != "test string 3": + raise Exception('CIBW_TEST_VAR_3 should equal "test string 3". It was "%s"' % CIBW_TEST_VAR_3) if "/opt/cibw_test_path" not in PATH: raise Exception('PATH should contain "/opt/cibw_test_path". It was "%s"' % PATH) if "$PATH" in PATH: From 3142806f4b02411995f2857992fa24f14d7e5d3e Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Mon, 20 Jul 2020 02:23:07 +0200 Subject: [PATCH 107/117] Capture output in DockerContainer.environment_executor to fix command substitutions in CIBW_ENVIRONMENT on Linux --- cibuildwheel/docker_container.py | 2 +- unit_test/docker_container_test.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/docker_container.py b/cibuildwheel/docker_container.py index c3b226aa..ecf9c52f 100644 --- a/cibuildwheel/docker_container.py +++ b/cibuildwheel/docker_container.py @@ -182,7 +182,7 @@ class DockerContainer: def environment_executor(self, command: List[str], environment: Dict[str, str]) -> str: # used as an EnvironmentExecutor to evaluate commands and capture output - return self.call(command, env=environment) + return self.call(command, env=environment, capture_output=True) def shell_quote(path: PurePath) -> str: diff --git a/unit_test/docker_container_test.py b/unit_test/docker_container_test.py index fed06174..a3b6f06d 100644 --- a/unit_test/docker_container_test.py +++ b/unit_test/docker_container_test.py @@ -8,6 +8,7 @@ from pathlib import Path, PurePath import pytest from cibuildwheel.docker_container import DockerContainer +from cibuildwheel.environment import EnvironmentAssignment # for these tests we use manylinux2014 images, because they're available on # multi architectures and include python3.8 @@ -145,3 +146,10 @@ def test_dir_operations(tmp_path: Path): container.copy_out(dst_dir, new_test_dir) assert test_binary_data == (new_test_dir / 'test.dat').read_bytes() + + +@pytest.mark.docker +def test_environment_executor(): + with DockerContainer(DEFAULT_IMAGE) as container: + assignment = EnvironmentAssignment("TEST=$(echo 42)") + assert assignment.evaluated_value({}, container.environment_executor) == "42" From 2b7da3fcbebe31b22f3bf0c2922482f50bc1c454 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Mon, 20 Jul 2020 00:46:36 +0200 Subject: [PATCH 108/117] Test whether CIBW_BEFORE_BUILD is executed from the right directory and confirm it is not --- test/test_before_build.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/test/test_before_build.py b/test/test_before_build.py index 11c752f0..437a0ae8 100644 --- a/test/test_before_build.py +++ b/test/test_before_build.py @@ -24,6 +24,12 @@ project_with_before_build_asserts = test_projects.new_c_project( print('sys.executable', sys.executable) # windows/mac are case insensitive assert os.path.realpath(stored_executable).lower() == os.path.realpath(sys.executable).lower() + + if sys.platform == 'linux': + cwd_file = '/tmp/cwd.txt' + with open(cwd_file) as f: + stored_cwd = f.read() + assert stored_cwd == '/project' ''') ) @@ -32,12 +38,18 @@ def test(tmp_path): project_dir = tmp_path / 'project' project_with_before_build_asserts.generate(project_dir) + before_build = textwrap.dedent(''' + python -c "import sys; open('{output_dir}pythonversion.txt', 'w').write(sys.version)" && + python -c "import sys; open('{output_dir}pythonexecutable.txt', 'w').write(sys.executable)" && + python -c "import os; open('{output_dir}cwd.txt', 'w').write(os.getcwd())" + ''') + # build the wheels actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ # write python version information to a temporary file, this is # checked in setup.py - 'CIBW_BEFORE_BUILD': '''python -c "import sys; open('/tmp/pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('/tmp/pythonexecutable.txt', 'w').write(sys.executable)"''', - 'CIBW_BEFORE_BUILD_WINDOWS': '''python -c "import sys; open('c:\\pythonversion.txt', 'w').write(sys.version)" && python -c "import sys; open('c:\\pythonexecutable.txt', 'w').write(sys.executable)"''', + 'CIBW_BEFORE_BUILD': before_build.format(output_dir="/tmp/"), + 'CIBW_BEFORE_BUILD_WINDOWS': before_build.format(output_dir="c:\\"), }) # also check that we got the right wheels From 29b0cdb08a1da8cd35c8ef68b9a424be48d66160 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Mon, 20 Jul 2020 01:04:26 +0200 Subject: [PATCH 109/117] Add cwd argument to DockerContainer --- cibuildwheel/docker_container.py | 5 ++++- unit_test/docker_container_test.py | 7 +++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/docker_container.py b/cibuildwheel/docker_container.py index ecf9c52f..8c3a608b 100644 --- a/cibuildwheel/docker_container.py +++ b/cibuildwheel/docker_container.py @@ -27,12 +27,14 @@ class DockerContainer: bash_stdin: IO[bytes] bash_stdout: IO[bytes] - def __init__(self, docker_image: str, simulate_32_bit=False): + def __init__(self, docker_image: str, simulate_32_bit: bool = False, cwd: Optional[Union[str, PathLike]] = None): self.docker_image = docker_image self.simulate_32_bit = simulate_32_bit + self.cwd = cwd def __enter__(self) -> 'DockerContainer': self.name = f'cibuildwheel-{uuid.uuid4()}' + cwd_args = ['-w', str(self.cwd)] if self.cwd else [] shell_args = ['linux32', '/bin/bash'] if self.simulate_32_bit else ['/bin/bash'] subprocess.run( [ @@ -41,6 +43,7 @@ class DockerContainer: '--name', self.name, '-i', '-v', '/:/host', # ignored on CircleCI + *cwd_args, self.docker_image, *shell_args ], diff --git a/unit_test/docker_container_test.py b/unit_test/docker_container_test.py index a3b6f06d..22771ed2 100644 --- a/unit_test/docker_container_test.py +++ b/unit_test/docker_container_test.py @@ -41,6 +41,13 @@ def test_environment(): assert container.call(['sh', '-c', 'echo $TEST_VAR'], env={'TEST_VAR': '1'}, capture_output=True) == '1\n' +@pytest.mark.docker +def test_cwd(): + with DockerContainer(DEFAULT_IMAGE, cwd='/cibuildwheel/working_directory') as container: + assert container.call(['pwd'], capture_output=True) == '/cibuildwheel/working_directory\n' + assert container.call(['pwd'], capture_output=True, cwd='/opt') == '/opt\n' + + @pytest.mark.docker def test_container_removed(): with DockerContainer(DEFAULT_IMAGE) as container: From 2610f3220a75909565e416d3b21e39f078c94c14 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Mon, 20 Jul 2020 01:14:33 +0200 Subject: [PATCH 110/117] Run Linux docker images with /project as cwd --- cibuildwheel/linux.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index e4607f0c..344b944a 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -110,8 +110,8 @@ def build(options: BuildOptions) -> None: continue try: - with DockerContainer(docker_image, simulate_32_bit=platform_tag.endswith('i686')) as docker: - docker.copy_into(Path.cwd(), Path('/project')) + with DockerContainer(docker_image, simulate_32_bit=platform_tag.endswith('i686'), cwd='/project') as docker: + docker.copy_into(Path.cwd(), PurePath('/project')) if options.before_all: env = docker.get_environment() From f81c9474d6142de2259afe2079cc7d30097e7a2e Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Mon, 20 Jul 2020 01:42:57 +0200 Subject: [PATCH 111/117] Fix test_before_build test on Windows --- test/test_before_build.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/test/test_before_build.py b/test/test_before_build.py index 437a0ae8..46eca871 100644 --- a/test/test_before_build.py +++ b/test/test_before_build.py @@ -38,18 +38,16 @@ def test(tmp_path): project_dir = tmp_path / 'project' project_with_before_build_asserts.generate(project_dir) - before_build = textwrap.dedent(''' - python -c "import sys; open('{output_dir}pythonversion.txt', 'w').write(sys.version)" && - python -c "import sys; open('{output_dir}pythonexecutable.txt', 'w').write(sys.executable)" && - python -c "import os; open('{output_dir}cwd.txt', 'w').write(os.getcwd())" - ''') + before_build = ('''python -c "import sys; open('{output_dir}pythonversion.txt', 'w').write(sys.version)" && ''' + '''python -c "import sys; open('{output_dir}pythonexecutable.txt', 'w').write(sys.executable)" && ''' + '''python -c "import os; open('{output_dir}cwd.txt', 'w').write(os.getcwd())"''') # build the wheels actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ # write python version information to a temporary file, this is # checked in setup.py - 'CIBW_BEFORE_BUILD': before_build.format(output_dir="/tmp/"), - 'CIBW_BEFORE_BUILD_WINDOWS': before_build.format(output_dir="c:\\"), + 'CIBW_BEFORE_BUILD': before_build.format(output_dir='/tmp/'), + 'CIBW_BEFORE_BUILD_WINDOWS': before_build.format(output_dir=r'c:\\'), }) # also check that we got the right wheels From c91d669a90a73791bbca0a5e6bad4cca8f5c07c8 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Tue, 21 Jul 2020 18:41:51 +0200 Subject: [PATCH 112/117] Extend tests and add tests for CIBW_BEFORE_ALL --- test/test_before_all.py | 31 ++++++++++++++++++++++++++++--- test/test_before_build.py | 35 +++++++++++++++++++++++++++-------- 2 files changed, 55 insertions(+), 11 deletions(-) diff --git a/test/test_before_all.py b/test/test_before_all.py index 177ec555..73d36ff6 100644 --- a/test/test_before_all.py +++ b/test/test_before_all.py @@ -1,4 +1,5 @@ -import os +import pytest +import subprocess import textwrap from . import utils @@ -21,7 +22,7 @@ def test(tmp_path): project_dir = tmp_path / 'project' project_with_before_build_asserts.generate(project_dir) - with open(os.path.join(project_dir, "text_info.txt"), mode='w') as ff: + with (project_dir / 'text_info.txt').open(mode='w') as ff: print("dummy text", file=ff) # build the wheels @@ -33,6 +34,30 @@ def test(tmp_path): }) # also check that we got the right wheels - os.remove(os.path.join(project_dir, "text_info.txt")) + (project_dir / 'text_info.txt').unlink() + expected_wheels = utils.expected_wheels('spam', '0.1.0') + assert set(actual_wheels) == set(expected_wheels) + + +def test_failing_command(tmp_path): + project_dir = tmp_path / 'project' + test_projects.new_c_project().generate(project_dir) + + with pytest.raises(subprocess.CalledProcessError): + utils.cibuildwheel_run(project_dir, add_env={ + 'CIBW_BEFORE_ALL': 'false', + 'CIBW_BEFORE_ALL_WINDOWS': 'exit /b 1', + }) + + +def test_cwd(tmp_path): + project_dir = tmp_path / 'project' + test_projects.new_c_project().generate(project_dir) + + actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ + 'CIBW_BEFORE_ALL': f'''python -c "import os; assert os.getcwd() == {str(project_dir)!r}"''', + 'CIBW_BEFORE_ALL_LINUX': '''python -c "import os; assert os.getcwd() == '/project'"''', + }) + expected_wheels = utils.expected_wheels('spam', '0.1.0') assert set(actual_wheels) == set(expected_wheels) diff --git a/test/test_before_build.py b/test/test_before_build.py index 46eca871..cb92280b 100644 --- a/test/test_before_build.py +++ b/test/test_before_build.py @@ -1,3 +1,5 @@ +import pytest +import subprocess import textwrap from . import utils @@ -24,12 +26,6 @@ project_with_before_build_asserts = test_projects.new_c_project( print('sys.executable', sys.executable) # windows/mac are case insensitive assert os.path.realpath(stored_executable).lower() == os.path.realpath(sys.executable).lower() - - if sys.platform == 'linux': - cwd_file = '/tmp/cwd.txt' - with open(cwd_file) as f: - stored_cwd = f.read() - assert stored_cwd == '/project' ''') ) @@ -39,8 +35,7 @@ def test(tmp_path): project_with_before_build_asserts.generate(project_dir) before_build = ('''python -c "import sys; open('{output_dir}pythonversion.txt', 'w').write(sys.version)" && ''' - '''python -c "import sys; open('{output_dir}pythonexecutable.txt', 'w').write(sys.executable)" && ''' - '''python -c "import os; open('{output_dir}cwd.txt', 'w').write(os.getcwd())"''') + '''python -c "import sys; open('{output_dir}pythonexecutable.txt', 'w').write(sys.executable)"''') # build the wheels actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ @@ -53,3 +48,27 @@ def test(tmp_path): # also check that we got the right wheels expected_wheels = utils.expected_wheels('spam', '0.1.0') assert set(actual_wheels) == set(expected_wheels) + + +def test_failing_command(tmp_path): + project_dir = tmp_path / 'project' + test_projects.new_c_project().generate(project_dir) + + with pytest.raises(subprocess.CalledProcessError): + utils.cibuildwheel_run(project_dir, add_env={ + 'CIBW_BEFORE_BUILD': 'false', + 'CIBW_BEFORE_BUILD_WINDOWS': 'exit /b 1', + }) + + +def test_cwd(tmp_path): + project_dir = tmp_path / 'project' + test_projects.new_c_project().generate(project_dir) + + actual_wheels = utils.cibuildwheel_run(project_dir, add_env={ + 'CIBW_BEFORE_BUILD': f'''python -c "import os; assert os.getcwd() == {str(project_dir)!r}"''', + 'CIBW_BEFORE_BUILD_LINUX': '''python -c "import os; assert os.getcwd() == '/project'"''', + }) + + expected_wheels = utils.expected_wheels('spam', '0.1.0') + assert set(actual_wheels) == set(expected_wheels) From 9e40094cfc274da96884e2348a35fb22f318542b Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Tue, 21 Jul 2020 23:13:55 +0200 Subject: [PATCH 113/117] Make timeout 2 hours on GitHub Actions and Azure Pipelines because of slow Windows jobs --- .github/workflows/test.yml | 1 + azure-pipelines.yml | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b2e28e08..41c02da4 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -17,6 +17,7 @@ jobs: matrix: os: [ubuntu-18.04, windows-latest, macos-latest] python_version: ['3.7'] + timeout-minutes: 180 steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3eaed5f4..74dc215d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -21,6 +21,7 @@ jobs: - job: windows_36 pool: {vmImage: 'vs2017-win2016'} + timeoutInMinutes: 180 steps: - task: UsePythonVersion@0 inputs: @@ -33,6 +34,7 @@ jobs: - job: windows_38 pool: {vmImage: 'vs2017-win2016'} + timeoutInMinutes: 180 steps: - task: UsePythonVersion@0 inputs: From ae6d65e62e9831411accddd6437ac847924265f7 Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Wed, 22 Jul 2020 15:27:51 +0200 Subject: [PATCH 114/117] use variable instead on '/project' literal --- cibuildwheel/linux.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 344b944a..7d17b046 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -101,7 +101,8 @@ def build(options: BuildOptions) -> None: if cwd != abs_package_dir and cwd not in abs_package_dir.parents: raise Exception('package_dir must be inside the working directory') - container_package_dir = PurePath('/project') / abs_package_dir.relative_to(cwd) + project_path = '/project' + container_package_dir = PurePath(project_path) / abs_package_dir.relative_to(cwd) container_output_dir = PurePath('/output') for implementation, platform_tag, docker_image in platforms: @@ -110,15 +111,15 @@ def build(options: BuildOptions) -> None: continue try: - with DockerContainer(docker_image, simulate_32_bit=platform_tag.endswith('i686'), cwd='/project') as docker: - docker.copy_into(Path.cwd(), PurePath('/project')) + with DockerContainer(docker_image, simulate_32_bit=platform_tag.endswith('i686'), cwd=project_path) as docker: + docker.copy_into(Path.cwd(), PurePath(project_path)) if options.before_all: env = docker.get_environment() env['PATH'] = f'/opt/python/cp38-cp38:{env["PATH"]}' env = options.environment.as_dictionary(env, executor=docker.environment_executor) - before_all_prepared = prepare_command(options.before_all, project='/project', package=container_package_dir) + before_all_prepared = prepare_command(options.before_all, project=project_path, package=container_package_dir) docker.call(['sh', '-c', before_all_prepared], env=env) for config in platform_configs: @@ -151,7 +152,7 @@ def build(options: BuildOptions) -> None: exit(1) if options.before_build: - before_build_prepared = prepare_command(options.before_build, project='/project', package=container_package_dir) + before_build_prepared = prepare_command(options.before_build, project=project_path, package=container_package_dir) docker.call(['sh', '-c', before_build_prepared], env=env) temp_dir = PurePath('/tmp/cibuildwheel') @@ -193,7 +194,7 @@ def build(options: BuildOptions) -> None: virtualenv_env['PATH'] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}" if options.before_test: - before_test_prepared = prepare_command(options.before_test, project='/project', package=container_package_dir) + before_test_prepared = prepare_command(options.before_test, project=project_path, package=container_package_dir) docker.call(['sh', '-c', before_test_prepared], env=virtualenv_env) # Install the wheel we just built @@ -210,7 +211,7 @@ def build(options: BuildOptions) -> None: docker.call(['pip', 'install', *options.test_requires], env=virtualenv_env) # Run the tests from a different directory - test_command_prepared = prepare_command(options.test_command, project='/project', package=container_package_dir) + test_command_prepared = prepare_command(options.test_command, project=project_path, package=container_package_dir) docker.call(['sh', '-c', test_command_prepared], cwd='/root', env=virtualenv_env) # clean up test environment From a15ccb6ef0984b1a73ec6d8787d7e1d490574066 Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Wed, 22 Jul 2020 17:26:25 +0200 Subject: [PATCH 115/117] change project_path to PurePath --- cibuildwheel/linux.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 7d17b046..d0d41ef8 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -101,8 +101,8 @@ def build(options: BuildOptions) -> None: if cwd != abs_package_dir and cwd not in abs_package_dir.parents: raise Exception('package_dir must be inside the working directory') - project_path = '/project' - container_package_dir = PurePath(project_path) / abs_package_dir.relative_to(cwd) + project_path = PurePath('/project') + container_package_dir = project_path / abs_package_dir.relative_to(cwd) container_output_dir = PurePath('/output') for implementation, platform_tag, docker_image in platforms: @@ -112,7 +112,7 @@ def build(options: BuildOptions) -> None: try: with DockerContainer(docker_image, simulate_32_bit=platform_tag.endswith('i686'), cwd=project_path) as docker: - docker.copy_into(Path.cwd(), PurePath(project_path)) + docker.copy_into(Path.cwd(), project_path) if options.before_all: env = docker.get_environment() From 5c4767899dc4418b509a2d7ccfde07d33ab29e28 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 22 Jul 2020 18:57:47 +0100 Subject: [PATCH 116/117] Bump version: v1.5.5 --- README.md | 11 ++++++++++- cibuildwheel/__init__.py | 2 +- examples/appveyor-minimal.yml | 2 +- examples/azure-pipelines-minimal.yml | 6 +++--- examples/circleci-minimal.yml | 4 ++-- examples/github-deploy.yml | 2 +- examples/github-minimal.yml | 2 +- examples/travis-ci-deploy.yml | 2 +- examples/travis-ci-minimal.yml | 2 +- examples/travis-ci-test-and-deploy.yml | 6 +++--- setup.py | 2 +- 11 files changed, 25 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index 491fd831..e149eb99 100644 --- a/README.md +++ b/README.md @@ -79,7 +79,7 @@ env: # Note: TWINE_PASSWORD is set to a PyPI API token in Travis settings install: - - python3 -m pip install cibuildwheel==1.5.4 + - python3 -m pip install cibuildwheel==1.5.5 script: # build the wheels, put them into './wheelhouse' @@ -152,6 +152,15 @@ This is similar to static linking, so it might have some licence implications. C Changelog ========= +### 1.5.5 + +_22 July 2020_ + +- 🐛 Fix a bug that would cause command substitutions in CIBW_ENVIRONMENT to + produce no output on Linux (#411) +- 🐛 Fix regression (introduced in 1.5.3) which caused BEFORE_BUILD and + BEFORE_ALL to be executed in the wrong directory (#410) + ### 1.5.4 _19 June 2020_ diff --git a/cibuildwheel/__init__.py b/cibuildwheel/__init__.py index c24ed73b..63a58789 100644 --- a/cibuildwheel/__init__.py +++ b/cibuildwheel/__init__.py @@ -1 +1 @@ -__version__ = '1.5.4' +__version__ = '1.5.5' diff --git a/examples/appveyor-minimal.yml b/examples/appveyor-minimal.yml index 959c4267..0a632024 100644 --- a/examples/appveyor-minimal.yml +++ b/examples/appveyor-minimal.yml @@ -12,7 +12,7 @@ stack: python 3.7 init: - cmd: set PATH=C:\Python37;C:\Python37\Scripts;%PATH% -install: python -m pip install cibuildwheel==1.5.4 +install: python -m pip install cibuildwheel==1.5.5 build_script: python -m cibuildwheel --output-dir wheelhouse diff --git a/examples/azure-pipelines-minimal.yml b/examples/azure-pipelines-minimal.yml index 213c101f..055eb36a 100644 --- a/examples/azure-pipelines-minimal.yml +++ b/examples/azure-pipelines-minimal.yml @@ -5,7 +5,7 @@ jobs: - task: UsePythonVersion@0 - bash: | python3 -m pip install --upgrade pip - pip3 install cibuildwheel==1.5.4 + pip3 install cibuildwheel==1.5.5 cibuildwheel --output-dir wheelhouse . - task: PublishBuildArtifacts@1 inputs: {pathtoPublish: 'wheelhouse'} @@ -16,7 +16,7 @@ jobs: - task: UsePythonVersion@0 - bash: | python3 -m pip install --upgrade pip - pip3 install cibuildwheel==1.5.4 + pip3 install cibuildwheel==1.5.5 cibuildwheel --output-dir wheelhouse . - task: PublishBuildArtifacts@1 inputs: {pathtoPublish: 'wheelhouse'} @@ -29,7 +29,7 @@ jobs: displayName: Install Visual C++ for Python 2.7 - bash: | python -m pip install --upgrade pip - pip install cibuildwheel==1.5.4 + pip install cibuildwheel==1.5.5 cibuildwheel --output-dir wheelhouse . - task: PublishBuildArtifacts@1 inputs: {pathtoPublish: 'wheelhouse'} diff --git a/examples/circleci-minimal.yml b/examples/circleci-minimal.yml index f890e3d5..a7e71c5a 100644 --- a/examples/circleci-minimal.yml +++ b/examples/circleci-minimal.yml @@ -11,7 +11,7 @@ jobs: - run: name: Build the Linux wheels. command: | - pip3 install --user cibuildwheel==1.5.4 + pip3 install --user cibuildwheel==1.5.5 cibuildwheel --output-dir wheelhouse - store_artifacts: path: wheelhouse/ @@ -25,7 +25,7 @@ jobs: - run: name: Build the OS X wheels. command: | - pip3 install --user cibuildwheel==1.5.4 + pip3 install --user cibuildwheel==1.5.5 cibuildwheel --output-dir wheelhouse - store_artifacts: path: wheelhouse/ diff --git a/examples/github-deploy.yml b/examples/github-deploy.yml index 0db47913..d0aef52f 100644 --- a/examples/github-deploy.yml +++ b/examples/github-deploy.yml @@ -28,7 +28,7 @@ jobs: - name: Install cibuildwheel run: | - python -m pip install cibuildwheel==1.5.4 + python -m pip install cibuildwheel==1.5.5 - name: Install Visual C++ for Python 2.7 if: runner.os == 'Windows' diff --git a/examples/github-minimal.yml b/examples/github-minimal.yml index 5bfddc1f..f88b262f 100644 --- a/examples/github-minimal.yml +++ b/examples/github-minimal.yml @@ -20,7 +20,7 @@ jobs: - name: Install cibuildwheel run: | - python -m pip install cibuildwheel==1.5.4 + python -m pip install cibuildwheel==1.5.5 - name: Install Visual C++ for Python 2.7 if: runner.os == 'Windows' diff --git a/examples/travis-ci-deploy.yml b/examples/travis-ci-deploy.yml index e1862508..08dc997f 100644 --- a/examples/travis-ci-deploy.yml +++ b/examples/travis-ci-deploy.yml @@ -25,7 +25,7 @@ env: # Note: TWINE_PASSWORD is set to a PyPI API token in Travis settings install: - - python3 -m pip install cibuildwheel==1.5.4 + - python3 -m pip install cibuildwheel==1.5.5 script: # build the wheels, put them into './wheelhouse' diff --git a/examples/travis-ci-minimal.yml b/examples/travis-ci-minimal.yml index 0b73aff3..05fd5240 100644 --- a/examples/travis-ci-minimal.yml +++ b/examples/travis-ci-minimal.yml @@ -26,7 +26,7 @@ jobs: - ln -s /c/Python38/python.exe /c/Python38/python3.exe install: - - python3 -m pip install cibuildwheel==1.5.4 + - python3 -m pip install cibuildwheel==1.5.5 script: # build the wheels, put them into './wheelhouse' diff --git a/examples/travis-ci-test-and-deploy.yml b/examples/travis-ci-test-and-deploy.yml index 193bce04..6b9abd51 100644 --- a/examples/travis-ci-test-and-deploy.yml +++ b/examples/travis-ci-test-and-deploy.yml @@ -55,7 +55,7 @@ jobs: - stage: deploy name: Build and deploy Linux wheels services: docker - install: python3 -m pip install cibuildwheel==1.5.4 + install: python3 -m pip install cibuildwheel==1.5.5 script: python3 -m cibuildwheel --output-dir wheelhouse after_success: | python3 -m pip install twine @@ -65,7 +65,7 @@ jobs: name: Build and deploy macOS wheels os: osx language: shell - install: python3 -m pip install cibuildwheel==1.5.4 + install: python3 -m pip install cibuildwheel==1.5.5 script: python3 -m cibuildwheel --output-dir wheelhouse after_success: | python3 -m pip install twine @@ -75,7 +75,7 @@ jobs: name: Build and deploy Windows wheels os: windows language: shell - install: python3 -m pip install cibuildwheel==1.5.4 + install: python3 -m pip install cibuildwheel==1.5.5 script: python3 -m cibuildwheel --output-dir wheelhouse after_success: | python3 -m pip install twine diff --git a/setup.py b/setup.py index 64b0b39f..513c085e 100644 --- a/setup.py +++ b/setup.py @@ -11,7 +11,7 @@ long_description = (this_directory / 'README.md').read_text(encoding='utf-8') setup( name='cibuildwheel', - version='1.5.4', + version='1.5.5', install_requires=['bashlex!=0.13', 'toml'], description="Build Python wheels on CI with minimal configuration.", long_description=long_description, From cfec077f05a33a35547b85acb26bdfaa98152915 Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Thu, 23 Jul 2020 20:24:19 +0200 Subject: [PATCH 117/117] change project_path to container_project_path --- cibuildwheel/linux.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index d0d41ef8..a3dcb036 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -101,8 +101,8 @@ def build(options: BuildOptions) -> None: if cwd != abs_package_dir and cwd not in abs_package_dir.parents: raise Exception('package_dir must be inside the working directory') - project_path = PurePath('/project') - container_package_dir = project_path / abs_package_dir.relative_to(cwd) + container_project_path = PurePath('/project') + container_package_dir = container_project_path / abs_package_dir.relative_to(cwd) container_output_dir = PurePath('/output') for implementation, platform_tag, docker_image in platforms: @@ -111,15 +111,15 @@ def build(options: BuildOptions) -> None: continue try: - with DockerContainer(docker_image, simulate_32_bit=platform_tag.endswith('i686'), cwd=project_path) as docker: - docker.copy_into(Path.cwd(), project_path) + with DockerContainer(docker_image, simulate_32_bit=platform_tag.endswith('i686'), cwd=container_project_path) as docker: + docker.copy_into(Path.cwd(), container_project_path) if options.before_all: env = docker.get_environment() env['PATH'] = f'/opt/python/cp38-cp38:{env["PATH"]}' env = options.environment.as_dictionary(env, executor=docker.environment_executor) - before_all_prepared = prepare_command(options.before_all, project=project_path, package=container_package_dir) + before_all_prepared = prepare_command(options.before_all, project=container_project_path, package=container_package_dir) docker.call(['sh', '-c', before_all_prepared], env=env) for config in platform_configs: @@ -152,7 +152,7 @@ def build(options: BuildOptions) -> None: exit(1) if options.before_build: - before_build_prepared = prepare_command(options.before_build, project=project_path, package=container_package_dir) + before_build_prepared = prepare_command(options.before_build, project=container_project_path, package=container_package_dir) docker.call(['sh', '-c', before_build_prepared], env=env) temp_dir = PurePath('/tmp/cibuildwheel') @@ -194,7 +194,7 @@ def build(options: BuildOptions) -> None: virtualenv_env['PATH'] = f"{venv_dir / 'bin'}:{virtualenv_env['PATH']}" if options.before_test: - before_test_prepared = prepare_command(options.before_test, project=project_path, package=container_package_dir) + before_test_prepared = prepare_command(options.before_test, project=container_project_path, package=container_package_dir) docker.call(['sh', '-c', before_test_prepared], env=virtualenv_env) # Install the wheel we just built @@ -211,7 +211,7 @@ def build(options: BuildOptions) -> None: docker.call(['pip', 'install', *options.test_requires], env=virtualenv_env) # Run the tests from a different directory - test_command_prepared = prepare_command(options.test_command, project=project_path, package=container_package_dir) + test_command_prepared = prepare_command(options.test_command, project=container_project_path, package=container_package_dir) docker.call(['sh', '-c', test_command_prepared], cwd='/root', env=virtualenv_env) # clean up test environment