diff --git a/bin/run_tests.py b/bin/run_tests.py index 93c23e88..117f8b9f 100755 --- a/bin/run_tests.py +++ b/bin/run_tests.py @@ -3,10 +3,11 @@ import os import subprocess import sys +from pathlib import Path if __name__ == '__main__': # move cwd to the project root - os.chdir(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + os.chdir(Path(__file__).resolve().parents[1]) # run the unit tests subprocess.check_call([sys.executable, '-m', 'pytest', 'unit_test']) diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py index 40241ac1..137bc19e 100644 --- a/cibuildwheel/__main__.py +++ b/cibuildwheel/__main__.py @@ -4,6 +4,7 @@ import sys import textwrap import traceback from configparser import ConfigParser +from pathlib import Path from typing import Any, Dict, List, Optional, overload @@ -20,6 +21,7 @@ from cibuildwheel.util import ( BuildSelector, DependencyConstraints, Unbuffered, + resources_dir, ) @@ -114,8 +116,8 @@ def main() -> None: file=sys.stderr) exit(2) - package_dir = args.package_dir - output_dir = args.output_dir + package_dir = Path(args.package_dir) + output_dir = Path(args.output_dir) if platform == 'linux': repair_command_default = 'auditwheel repair -w {dest_dir} {wheel}' @@ -149,7 +151,8 @@ def main() -> None: elif dependency_versions == 'latest': dependency_constraints = None else: - dependency_constraints = DependencyConstraints(dependency_versions) + dependency_versions_path = Path(dependency_versions) + dependency_constraints = DependencyConstraints(dependency_versions_path) if test_extras: test_extras = f'[{test_extras}]' @@ -163,7 +166,7 @@ def main() -> None: # This needs to be passed on to the docker container in linux.py os.environ['CIBUILDWHEEL'] = '1' - if not any(os.path.exists(os.path.join(package_dir, name)) + if not any((package_dir / name).exists() for name in ["setup.py", "setup.cfg", "pyproject.toml"]): print('cibuildwheel: Could not find setup.py, setup.cfg or pyproject.toml at root of package', file=sys.stderr) exit(2) @@ -174,9 +177,7 @@ def main() -> None: manylinux_images: Optional[Dict[str, str]] = None if platform == 'linux': - pinned_docker_images_file = os.path.join( - os.path.dirname(__file__), 'resources', 'pinned_docker_images.cfg' - ) + pinned_docker_images_file = resources_dir / 'pinned_docker_images.cfg' all_pinned_docker_images = ConfigParser() all_pinned_docker_images.read(pinned_docker_images_file) # all_pinned_docker_images looks like a dict of dicts, e.g. @@ -224,8 +225,8 @@ def main() -> None: print_preamble(platform, build_options) - if not os.path.exists(output_dir): - os.makedirs(output_dir) + if not output_dir.exists(): + output_dir.mkdir(parents=True) if platform == 'linux': cibuildwheel.linux.build(build_options) diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index 0058c2db..99e779c4 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -5,6 +5,7 @@ import subprocess import sys import textwrap import uuid +from pathlib import Path, PurePath from typing import List, NamedTuple, Optional, Union @@ -104,10 +105,12 @@ def build(options: BuildOptions) -> None: ('pp', 'manylinux_x86_64', options.manylinux_images['pypy_x86_64']), ] - if not os.path.realpath(options.package_dir).startswith(os.path.realpath('.')): + cwd = Path.cwd() + abs_package_dir = options.package_dir.resolve() + if cwd != abs_package_dir and cwd not in abs_package_dir.parents: raise Exception('package_dir must be inside the working directory') - container_package_dir = os.path.join('/project', os.path.relpath(options.package_dir, '.')) + container_package_dir = PurePath('/project') / abs_package_dir.relative_to(cwd) for implementation, platform_tag, docker_image in platforms: platform_configs = [c for c in python_configurations if c.identifier.startswith(implementation) and c.identifier.endswith(platform_tag)] @@ -272,7 +275,7 @@ def build(options: BuildOptions) -> None: # copy the output back into the host call(['docker', 'cp', container_name + ':/output/.', - os.path.abspath(options.output_dir)]) + str(options.output_dir.resolve())]) except subprocess.CalledProcessError as error: troubleshoot(options.package_dir, error) exit(1) @@ -281,16 +284,11 @@ def build(options: BuildOptions) -> None: call(['docker', 'rm', '--force', '-v', container_name]) -def troubleshoot(package_dir: str, error: Exception) -> None: +def troubleshoot(package_dir: Path, error: Exception) -> None: if (isinstance(error, subprocess.CalledProcessError) and 'exec' in error.cmd): # the bash script failed print('Checking for common errors...') - so_files = [] - for root, dirs, files in os.walk(package_dir): - for name in files: - _, ext = os.path.splitext(name) - if ext == '.so': - so_files.append(os.path.join(root, name)) + so_files = list(package_dir.glob('**/*.so')) if so_files: print(textwrap.dedent(''' @@ -304,5 +302,5 @@ def troubleshoot(package_dir: str, error: Exception) -> None: ''')) print(' Files detected:') - print('\n'.join([' ' + f for f in so_files])) + print('\n'.join([f' {f}' for f in so_files])) print('') diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 69a06c04..2fd28d20 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -4,7 +4,7 @@ import shutil import subprocess import sys import tempfile -from glob import glob +from pathlib import Path from typing import Dict, List, Optional, NamedTuple, Union @@ -52,41 +52,41 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi return [c for c in python_configurations if build_selector(c.identifier)] -SYMLINKS_DIR = '/tmp/cibw_bin' +SYMLINKS_DIR = Path('/tmp/cibw_bin') -def make_symlinks(installation_bin_path: str, python_executable: str, pip_executable: str) -> None: - assert os.path.exists(os.path.join(installation_bin_path, python_executable)) +def make_symlinks(installation_bin_path: Path, python_executable: str, pip_executable: str) -> None: + assert (installation_bin_path / python_executable).exists() # Python bin folders on Mac don't symlink `python3` to `python`, and neither # does PyPy for `pypy` or `pypy3`, so we do that so `python` and `pip` always # point to the active configuration. - if os.path.exists(SYMLINKS_DIR): + if SYMLINKS_DIR.exists(): shutil.rmtree(SYMLINKS_DIR) - os.makedirs(SYMLINKS_DIR) + SYMLINKS_DIR.mkdir(parents=True) - os.symlink(os.path.join(installation_bin_path, python_executable), os.path.join(SYMLINKS_DIR, 'python')) - os.symlink(os.path.join(installation_bin_path, python_executable + '-config'), os.path.join(SYMLINKS_DIR, 'python-config')) - os.symlink(os.path.join(installation_bin_path, pip_executable), os.path.join(SYMLINKS_DIR, 'pip')) + (SYMLINKS_DIR / 'python').symlink_to(installation_bin_path / python_executable) + (SYMLINKS_DIR / 'python-config').symlink_to(installation_bin_path / (python_executable + '-config')) + (SYMLINKS_DIR / 'pip').symlink_to(installation_bin_path / pip_executable) -def install_cpython(version: str, url: str) -> str: +def install_cpython(version: str, url: str) -> Path: installed_system_packages = subprocess.check_output(['pkgutil', '--pkgs'], universal_newlines=True).splitlines() # if this version of python isn't installed, get it from python.org and install python_package_identifier = f'org.python.Python.PythonFramework-{version}' if python_package_identifier not in installed_system_packages: # download the pkg - download(url, '/tmp/Python.pkg') + download(url, Path('/tmp/Python.pkg')) # install call(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/']) # patch open ssl if version == '3.5': open_ssl_patch_url = f'https://github.com/mayeut/patch-macos-python-openssl/releases/download/v1.0.2u/patch-macos-python-{version}-openssl-v1.0.2u.tar.gz' - download(open_ssl_patch_url, '/tmp/python-patch.tar.gz') + download(open_ssl_patch_url, Path('/tmp/python-patch.tar.gz')) call(['sudo', 'tar', '-C', f'/Library/Frameworks/Python.framework/Versions/{version}/', '-xmf', '/tmp/python-patch.tar.gz']) - installation_bin_path = f'/Library/Frameworks/Python.framework/Versions/{version}/bin' + installation_bin_path = Path(f'/Library/Frameworks/Python.framework/Versions/{version}/bin') python_executable = 'python3' if version[0] == '3' else 'python' pip_executable = 'pip3' if version[0] == '3' else 'pip' make_symlinks(installation_bin_path, python_executable, pip_executable) @@ -94,16 +94,18 @@ def install_cpython(version: str, url: str) -> str: return installation_bin_path -def install_pypy(version: str, url: str) -> str: +def install_pypy(version: str, url: str) -> Path: pypy_tar_bz2 = url.rsplit('/', 1)[-1] - assert pypy_tar_bz2.endswith(".tar.bz2") - pypy_base_filename = os.path.splitext(os.path.splitext(pypy_tar_bz2)[0])[0] - installation_path = os.path.join('/tmp', pypy_base_filename) - if not os.path.exists(installation_path): - download(url, os.path.join("/tmp", pypy_tar_bz2)) - call(['tar', '-C', '/tmp', '-xf', os.path.join("/tmp", pypy_tar_bz2)]) + extension = ".tar.bz2" + assert pypy_tar_bz2.endswith(extension) + pypy_base_filename = pypy_tar_bz2[:-len(extension)] + installation_path = Path('/tmp') / pypy_base_filename + if not installation_path.exists(): + downloaded_tar_bz2 = Path("/tmp") / pypy_tar_bz2 + download(url, downloaded_tar_bz2) + call(['tar', '-C', '/tmp', '-xf', str(downloaded_tar_bz2)]) - installation_bin_path = os.path.join(installation_path, 'bin') + installation_bin_path = installation_path / 'bin' python_executable = 'pypy3' if version[0] == '3' else 'pypy' pip_executable = 'pip3' if version[0] == '3' else 'pip' make_symlinks(installation_bin_path, python_executable, pip_executable) @@ -121,8 +123,8 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain env = os.environ.copy() env['PATH'] = os.pathsep.join([ - SYMLINKS_DIR, - installation_bin_path, + str(SYMLINKS_DIR), + str(installation_bin_path), env['PATH'], ]) @@ -144,8 +146,8 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain exit(1) # install pip & wheel - call(['python', get_pip_script] + dependency_constraint_flags, env=env, cwd="/tmp") - assert os.path.exists(os.path.join(installation_bin_path, 'pip')) + call(['python', str(get_pip_script)] + dependency_constraint_flags, env=env, cwd="/tmp") + assert (installation_bin_path / 'pip').exists() call(['which', 'pip'], env=env) call(['pip', '--version'], env=env) which_pip = subprocess.check_output(['which', 'pip'], env=env, universal_newlines=True).strip() @@ -170,9 +172,9 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain def build(options: BuildOptions) -> None: - temp_dir = tempfile.mkdtemp(prefix='cibuildwheel') - built_wheel_dir = os.path.join(temp_dir, 'built_wheel') - repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel') + temp_dir = Path(tempfile.mkdtemp(prefix='cibuildwheel')) + built_wheel_dir = temp_dir / 'built_wheel' + repaired_wheel_dir = temp_dir / 'repaired_wheel' python_configurations = get_python_configurations(options.build_selector) @@ -180,7 +182,7 @@ def build(options: BuildOptions) -> None: dependency_constraint_flags = [] if options.dependency_constraints: dependency_constraint_flags = [ - '-c', options.dependency_constraints.get_for_python_version(config.version) + '-c', str(options.dependency_constraints.get_for_python_version(config.version)) ] env = setup_python(config, dependency_constraint_flags, options.environment) @@ -191,39 +193,39 @@ def build(options: BuildOptions) -> None: call(before_build_prepared, env=env, shell=True) # build the wheel - if os.path.exists(built_wheel_dir): + if built_wheel_dir.exists(): shutil.rmtree(built_wheel_dir) - os.makedirs(built_wheel_dir) - # os.path.abspath is need. Without it pip wheel may try to fetch package from pypi.org + built_wheel_dir.mkdir(parents=True) + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org # see https://github.com/joerick/cibuildwheel/pull/369 - call(['pip', 'wheel', os.path.abspath(options.package_dir), '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env) - built_wheel = glob(os.path.join(built_wheel_dir, '*.whl'))[0] + call(['pip', 'wheel', str(options.package_dir.resolve()), '-w', str(built_wheel_dir), '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env) + built_wheel = next(built_wheel_dir.glob('*.whl')) # repair the wheel - if os.path.exists(repaired_wheel_dir): + if repaired_wheel_dir.exists(): shutil.rmtree(repaired_wheel_dir) - os.makedirs(repaired_wheel_dir) - if built_wheel.endswith('none-any.whl') or not options.repair_command: + repaired_wheel_dir.mkdir(parents=True) + if built_wheel.name.endswith('none-any.whl') or not options.repair_command: # pure Python wheel or empty repair command - shutil.move(built_wheel, repaired_wheel_dir) + built_wheel.rename(repaired_wheel_dir / built_wheel.name) else: repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) call(repair_command_prepared, env=env, shell=True) - repaired_wheel = glob(os.path.join(repaired_wheel_dir, '*.whl'))[0] + repaired_wheel = next(repaired_wheel_dir.glob('*.whl')) if options.test_command: # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. call(['pip', 'install', 'virtualenv'] + dependency_constraint_flags, env=env) - venv_dir = tempfile.mkdtemp() + venv_dir = Path(tempfile.mkdtemp()) # Use --no-download to ensure determinism by using seed libraries # built into virtualenv - call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) + call(['python', '-m', 'virtualenv', '--no-download', str(venv_dir)], env=env) virtualenv_env = env.copy() virtualenv_env['PATH'] = os.pathsep.join([ - os.path.join(venv_dir, 'bin'), + str(venv_dir / 'bin'), virtualenv_env['PATH'], ]) @@ -235,7 +237,7 @@ def build(options: BuildOptions) -> None: call(before_test_prepared, env=virtualenv_env, shell=True) # install the wheel - call(['pip', 'install', repaired_wheel + options.test_extras], env=virtualenv_env) + call(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env) # test the wheel if options.test_requires: @@ -246,8 +248,8 @@ def build(options: BuildOptions) -> None: # and not the repo code) test_command_prepared = prepare_command( options.test_command, - project=os.path.abspath('.'), - package=os.path.abspath(options.package_dir) + project=Path('.').resolve(), + package=options.package_dir.resolve() ) call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True) @@ -255,5 +257,4 @@ def build(options: BuildOptions) -> None: shutil.rmtree(venv_dir) # we're all done here; move it to output (overwrite existing) - dst = os.path.join(options.output_dir, os.path.basename(repaired_wheel)) - shutil.move(repaired_wheel, dst) + repaired_wheel.replace(options.output_dir / repaired_wheel.name) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 499954b9..feef6185 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -1,14 +1,15 @@ import os import urllib.request from fnmatch import fnmatch +from pathlib import Path from time import sleep -from typing import Dict, List, NamedTuple, Optional +from typing import Dict, List, NamedTuple, Optional, Union from .environment import ParsedEnvironment -def prepare_command(command: str, **kwargs: str) -> str: +def prepare_command(command: str, **kwargs: Union[str, os.PathLike]) -> str: ''' Preprocesses a command by expanding variables like {python}. @@ -58,11 +59,11 @@ class Unbuffered: return getattr(self.stream, attr) -def download(url: str, dest: str) -> None: - print('+ Download ' + url + ' to ' + dest) - dest_dir = os.path.dirname(dest) - if not os.path.exists(dest_dir): - os.makedirs(dest_dir) +def download(url: str, dest: Path) -> None: + print(f'+ Download {url} to {dest}') + dest_dir = dest.parent + if not dest_dir.exists(): + dest_dir.mkdir(parents=True) repeat_num = 3 for i in range(repeat_num): @@ -76,40 +77,39 @@ def download(url: str, dest: str) -> None: break try: - with open(dest, 'wb') as file: - file.write(response.read()) + dest.write_bytes(response.read()) finally: response.close() class DependencyConstraints: - def __init__(self, base_file_path: str): - assert os.path.exists(base_file_path) - self.base_file_path = os.path.abspath(base_file_path) + def __init__(self, base_file_path: Path): + assert base_file_path.exists() + self.base_file_path = base_file_path.resolve() @staticmethod def with_defaults() -> 'DependencyConstraints': return DependencyConstraints( - base_file_path=os.path.join(os.path.dirname(__file__), 'resources', 'constraints.txt') + base_file_path=resources_dir / 'constraints.txt' ) - def get_for_python_version(self, version: str) -> str: + def get_for_python_version(self, version: str) -> Path: version_parts = version.split('.') # try to find a version-specific dependency file e.g. if # ./constraints.txt is the base, look for ./constraints-python27.txt - base, ext = os.path.splitext(self.base_file_path) - specific = base + f'-python{version_parts[0]}{version_parts[1]}' - specific_file_path = specific + ext - if os.path.exists(specific_file_path): + specific_stem = self.base_file_path.stem + f'-python{version_parts[0]}{version_parts[1]}' + specific_name = specific_stem + self.base_file_path.suffix + specific_file_path = self.base_file_path.with_name(specific_name) + if specific_file_path.exists(): return specific_file_path else: return self.base_file_path class BuildOptions(NamedTuple): - package_dir: str - output_dir: str + package_dir: Path + output_dir: Path build_selector: BuildSelector environment: ParsedEnvironment before_build: Optional[str] @@ -123,5 +123,5 @@ class BuildOptions(NamedTuple): build_verbosity: int -resources_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'resources')) -get_pip_script = os.path.join(resources_dir, 'get-pip.py') +resources_dir = Path(__file__).resolve().parent / 'resources' +get_pip_script = resources_dir / 'get-pip.py' diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index c625b39a..e81fe301 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -3,7 +3,7 @@ import shutil import subprocess import sys import tempfile -from glob import glob +from pathlib import Path from zipfile import ZipFile from typing import Dict, List, Optional, NamedTuple @@ -19,7 +19,7 @@ from .util import ( ) -IS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache') +IS_RUNNING_ON_AZURE = Path('C:\\hostedtoolcache').exists() IS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows' @@ -71,36 +71,38 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi return python_configurations -def extract_zip(zip_src: str, dest: str) -> None: +def extract_zip(zip_src: Path, dest: Path) -> None: with ZipFile(zip_src) as zip: zip.extractall(dest) -def install_cpython(version: str, arch: str, nuget: str) -> str: +def install_cpython(version: str, arch: str, nuget: Path) -> Path: nuget_args = get_nuget_args(version, arch) - installation_path = os.path.join(nuget_args[-1], nuget_args[0] + '.' + version, 'tools') - shell([nuget, 'install'] + nuget_args) + installation_path = Path(nuget_args[-1]) / (nuget_args[0] + '.' + version) / 'tools' + shell([str(nuget), 'install'] + nuget_args) return installation_path -def install_pypy(version: str, arch: str, url: str) -> str: +def install_pypy(version: str, arch: str, url: str) -> Path: assert arch == '32' # Inside the PyPy zip file is a directory with the same name zip_filename = url.rsplit('/', 1)[-1] - installation_path = os.path.join('C:\\cibw', os.path.splitext(zip_filename)[0]) - if not os.path.exists(installation_path): - pypy_zip = os.path.join('C:\\cibw', zip_filename) + extension = ".zip" + assert zip_filename.endswith(extension) + installation_path = Path('C:\\cibw') / zip_filename[:-len(extension)] + if not installation_path.exists(): + pypy_zip = Path('C:\\cibw') / zip_filename download(url, pypy_zip) # Extract to the parent directory because the zip file still contains a directory - extract_zip(pypy_zip, os.path.dirname(installation_path)) + extract_zip(pypy_zip, installation_path.parent) pypy_exe = 'pypy3.exe' if version[0] == '3' else 'pypy.exe' - shell(['mklink', os.path.join(installation_path, 'python.exe'), os.path.join(installation_path, pypy_exe)]) + (installation_path / 'python.exe').symlink_to(installation_path / pypy_exe) return installation_path def setup_python(python_configuration: PythonConfiguration, dependency_constraint_flags: List[str], environment: ParsedEnvironment) -> Dict[str, str]: - nuget = 'C:\\cibw\\nuget.exe' - if not os.path.exists(nuget): + nuget = Path('C:\\cibw\\nuget.exe') + if not nuget.exists(): download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget) if python_configuration.identifier.startswith('cp'): @@ -111,15 +113,15 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain else: raise ValueError("Unknown Python implementation") - assert os.path.exists(os.path.join(installation_path, 'python.exe')) + assert (installation_path / 'python.exe').exists() # set up PATH and environment variables for run_with_env env = os.environ.copy() env['PYTHON_VERSION'] = python_configuration.version env['PYTHON_ARCH'] = python_configuration.arch env['PATH'] = os.pathsep.join([ - installation_path, - os.path.join(installation_path, 'Scripts'), + str(installation_path), + str(installation_path / 'Scripts'), env['PATH'] ]) # update env with results from CIBW_ENVIRONMENT @@ -130,16 +132,16 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain shell(['python', '--version'], env=env) shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)"'], env=env) where_python = subprocess.check_output(['where', 'python'], env=env, universal_newlines=True).splitlines()[0].strip() - if where_python != os.path.join(installation_path, 'python.exe'): + if where_python != str(installation_path / 'python.exe'): print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr) exit(1) # make sure pip is installed - if not os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe')): - shell(['python', get_pip_script] + dependency_constraint_flags, env=env, cwd="C:\\cibw") - assert os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe')) + if not (installation_path / 'Scripts' / 'pip.exe').exists(): + shell(['python', str(get_pip_script)] + dependency_constraint_flags, env=env, cwd="C:\\cibw") + assert (installation_path / 'Scripts' / 'pip.exe').exists() where_pip = subprocess.check_output(['where', 'pip'], env=env, universal_newlines=True).splitlines()[0].strip() - if where_pip.strip() != os.path.join(installation_path, 'Scripts', 'pip.exe'): + if where_pip.strip() != str(installation_path / 'Scripts' / 'pip.exe'): print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr) exit(1) @@ -152,12 +154,12 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain def build(options: BuildOptions) -> None: - temp_dir = tempfile.mkdtemp(prefix='cibuildwheel') - built_wheel_dir = os.path.join(temp_dir, 'built_wheel') - repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel') + temp_dir = Path(tempfile.mkdtemp(prefix='cibuildwheel')) + built_wheel_dir = temp_dir / 'built_wheel' + repaired_wheel_dir = temp_dir / 'repaired_wheel' # install nuget as best way to provide python - nuget = 'C:\\cibw\\nuget.exe' + nuget = Path('C:\\cibw\\nuget.exe') download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget) python_configurations = get_python_configurations(options.build_selector) @@ -165,7 +167,7 @@ def build(options: BuildOptions) -> None: dependency_constraint_flags = [] if options.dependency_constraints: dependency_constraint_flags = [ - '-c', options.dependency_constraints.get_for_python_version(config.version) + '-c', str(options.dependency_constraints.get_for_python_version(config.version)) ] # install Python @@ -177,39 +179,39 @@ def build(options: BuildOptions) -> None: shell([before_build_prepared], env=env) # build the wheel - if os.path.exists(built_wheel_dir): + if built_wheel_dir.exists(): shutil.rmtree(built_wheel_dir) - os.makedirs(built_wheel_dir) - # os.path.abspath is need. Without it pip wheel may try to fetch package from pypi.org + built_wheel_dir.mkdir(parents=True) + # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org # see https://github.com/joerick/cibuildwheel/pull/369 - shell(['pip', 'wheel', os.path.abspath(options.package_dir), '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env) - built_wheel = glob(os.path.join(built_wheel_dir, '*.whl'))[0] + shell(['pip', 'wheel', str(options.package_dir.resolve()), '-w', str(built_wheel_dir), '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env) + built_wheel = next(built_wheel_dir.glob('*.whl')) # repair the wheel - if os.path.exists(repaired_wheel_dir): + if repaired_wheel_dir.exists(): shutil.rmtree(repaired_wheel_dir) - os.makedirs(repaired_wheel_dir) - if built_wheel.endswith('none-any.whl') or not options.repair_command: + repaired_wheel_dir.mkdir(parents=True) + if built_wheel.name.endswith('none-any.whl') or not options.repair_command: # pure Python wheel or empty repair command - shutil.move(built_wheel, repaired_wheel_dir) + built_wheel.rename(repaired_wheel_dir / built_wheel.name) else: repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) shell([repair_command_prepared], env=env) - repaired_wheel = glob(os.path.join(repaired_wheel_dir, '*.whl'))[0] + repaired_wheel = next(repaired_wheel_dir.glob('*.whl')) if options.test_command: # set up a virtual environment to install and test from, to make sure # there are no dependencies that were pulled in at build time. shell(['pip', 'install', 'virtualenv'] + dependency_constraint_flags, env=env) - venv_dir = tempfile.mkdtemp() + venv_dir = Path(tempfile.mkdtemp()) # Use --no-download to ensure determinism by using seed libraries # built into virtualenv - shell(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) + shell(['python', '-m', 'virtualenv', '--no-download', str(venv_dir)], env=env) virtualenv_env = env.copy() virtualenv_env['PATH'] = os.pathsep.join([ - os.path.join(venv_dir, 'Scripts'), + str(venv_dir / 'Scripts'), virtualenv_env['PATH'], ]) @@ -225,7 +227,7 @@ def build(options: BuildOptions) -> None: shell([before_test_prepared], env=virtualenv_env) # install the wheel - shell(['pip', 'install', repaired_wheel + options.test_extras], env=virtualenv_env) + shell(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env) # test the wheel if options.test_requires: @@ -236,8 +238,8 @@ def build(options: BuildOptions) -> None: # and not the repo code) test_command_prepared = prepare_command( options.test_command, - project=os.path.abspath('.'), - package=os.path.abspath(options.package_dir) + project=Path('.').resolve(), + package=options.package_dir.resolve() ) shell([test_command_prepared], cwd='c:\\', env=virtualenv_env) @@ -245,7 +247,4 @@ def build(options: BuildOptions) -> None: shutil.rmtree(venv_dir) # we're all done here; move it to output (remove if already exists) - dst = os.path.join(options.output_dir, os.path.basename(repaired_wheel)) - if os.path.isfile(dst): - os.remove(dst) - shutil.move(repaired_wheel, dst) + repaired_wheel.replace(options.output_dir / repaired_wheel.name) diff --git a/docs/mkdocs_include_markdown_plugin/mkdocs_include_markdown_plugin/plugin.py b/docs/mkdocs_include_markdown_plugin/mkdocs_include_markdown_plugin/plugin.py index b6b7a723..f12ea469 100644 --- a/docs/mkdocs_include_markdown_plugin/mkdocs_include_markdown_plugin/plugin.py +++ b/docs/mkdocs_include_markdown_plugin/mkdocs_include_markdown_plugin/plugin.py @@ -1,7 +1,6 @@ import cgi -import io -import os import re +from pathlib import Path import mkdocs @@ -41,13 +40,12 @@ class ImportMarkdownPlugin(mkdocs.plugins.BasePlugin): def found_include_tag(match): filename = match.group('filename') - file_path_abs = os.path.join(os.path.dirname(page_src_path), filename) + file_path_abs = Path(page_src_path).parent / filename - if not os.path.exists(file_path_abs): + if not file_path_abs.exists(): raise ValueError('file not found', filename) - with io.open(file_path_abs, encoding='utf8') as f: - text_to_include = f.read() + text_to_include = file_path_abs.read_text(encoding='utf8') # Allow good practice of having a final newline in the file if text_to_include.endswith('\n'): @@ -60,13 +58,12 @@ class ImportMarkdownPlugin(mkdocs.plugins.BasePlugin): start = match.group('start') end = match.group('end') - file_path_abs = os.path.join(os.path.dirname(page_src_path), filename) + file_path_abs = Path(page_src_path).parent / filename - if not os.path.exists(file_path_abs): + if not file_path_abs.exists(): raise ValueError('file not found', filename) - with io.open(file_path_abs, encoding='utf8') as f: - text_to_include = f.read() + text_to_include = file_path_abs.read_text(encoding='utf8') if start: _, _, text_to_include = text_to_include.partition(start) diff --git a/setup.py b/setup.py index 3e4c90cb..34528b4d 100644 --- a/setup.py +++ b/setup.py @@ -1,15 +1,13 @@ # -*- coding: utf-8 -*- -import io -import os +from pathlib import Path try: from setuptools import setup except ImportError: from distutils.core import setup -this_directory = os.path.dirname(__file__) -with io.open(os.path.join(this_directory, 'README.md'), encoding='utf-8') as f: - long_description = f.read() +this_directory = Path(__file__).parent +long_description = (this_directory / 'README.md').read_text(encoding='utf-8') setup( name='cibuildwheel', diff --git a/test/test_dependency_versions.py b/test/test_dependency_versions.py index 611350f1..2f79bd2a 100644 --- a/test/test_dependency_versions.py +++ b/test/test_dependency_versions.py @@ -1,4 +1,3 @@ -import os import re import pytest import textwrap @@ -39,11 +38,9 @@ VERSION_REGEX = r'([\w-]+)==([^\s]+)' def get_versions_from_constraint_file(constraint_file): - with open(constraint_file, encoding='utf8') as f: - constraint_file_text = f.read() + constraint_file_text = constraint_file.read_text(encoding='utf8') versions = {} - for package, version in re.findall(VERSION_REGEX, constraint_file_text): versions[package] = version @@ -73,7 +70,7 @@ def test_pinned_versions(tmp_path, python_version): constraint_filename = 'constraints.txt' build_pattern = '[cp]p38-*' - constraint_file = os.path.join(cibuildwheel.util.resources_dir, constraint_filename) + constraint_file = cibuildwheel.util.resources_dir / constraint_filename constraint_versions = get_versions_from_constraint_file(constraint_file) for package in ['pip', 'setuptools', 'wheel', 'virtualenv']: diff --git a/test/test_projects/base.py b/test/test_projects/base.py index b38310cc..a5710fdf 100644 --- a/test/test_projects/base.py +++ b/test/test_projects/base.py @@ -1,5 +1,7 @@ -import os +from pathlib import Path + import jinja2 + from typing import Union, Dict, Any @@ -23,12 +25,12 @@ class TestProject: self.files = {} self.template_context = {} - def generate(self, path: str): + def generate(self, path: Path): for filename, content in self.files.items(): - file_path = os.path.join(path, filename) - os.makedirs(os.path.dirname(file_path), exist_ok=True) + file_path = path / filename + file_path.parent.mkdir(parents=True, exist_ok=True) - with open(file_path, 'w', encoding='utf8') as f: + with file_path.open('w', encoding='utf8') as f: if isinstance(content, jinja2.Template): content = content.render(self.template_context) diff --git a/test/test_subdir_package.py b/test/test_subdir_package.py index 98b1b8cd..14a7a58b 100644 --- a/test/test_subdir_package.py +++ b/test/test_subdir_package.py @@ -1,4 +1,4 @@ -import os +from pathlib import Path import jinja2 @@ -35,7 +35,7 @@ def test(capfd, tmp_path): project_dir = tmp_path / 'project' subdir_package_project.generate(project_dir) - package_dir = os.path.join('src', 'spam') + package_dir = Path('src', 'spam') # build the wheels actual_wheels = utils.cibuildwheel_run(project_dir, package_dir=package_dir, add_env={ 'CIBW_BEFORE_BUILD': 'python {project}/bin/before_build.py', diff --git a/test/utils.py b/test/utils.py index 15ce402a..217a616f 100644 --- a/test/utils.py +++ b/test/utils.py @@ -10,9 +10,10 @@ import shutil import subprocess import sys from contextlib import contextmanager +from pathlib import Path from tempfile import mkdtemp -IS_WINDOWS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache') +IS_WINDOWS_RUNNING_ON_AZURE = Path('C:\\hostedtoolcache').exists() IS_WINDOWS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows' @@ -66,7 +67,7 @@ def cibuildwheel_run(project_path, package_dir='.', env=None, add_env=None, outp with TemporaryDirectoryIfNone(output_dir) as _output_dir: subprocess.check_call( - [sys.executable, '-m', 'cibuildwheel', '--output-dir', str(_output_dir), package_dir], + [sys.executable, '-m', 'cibuildwheel', '--output-dir', str(_output_dir), str(package_dir)], env=env, cwd=project_path, ) diff --git a/unit_test/dependency_constraints_test.py b/unit_test/dependency_constraints_test.py index b3fb0cb3..5347011c 100644 --- a/unit_test/dependency_constraints_test.py +++ b/unit_test/dependency_constraints_test.py @@ -1,30 +1,16 @@ from cibuildwheel.util import DependencyConstraints -import os + +from pathlib import Path def test_defaults(): dependency_constraints = DependencyConstraints.with_defaults() - project_root = os.path.dirname(os.path.dirname(__file__)) - resources_dir = os.path.join(project_root, 'cibuildwheel', 'resources') + project_root = Path(__file__).parents[1] + resources_dir = project_root / 'cibuildwheel' / 'resources' - assert os.path.samefile( - dependency_constraints.base_file_path, - os.path.join(resources_dir, 'constraints.txt') - ) - assert os.path.samefile( - dependency_constraints.get_for_python_version('3.8'), - os.path.join(resources_dir, 'constraints.txt') - ) - assert os.path.samefile( - dependency_constraints.get_for_python_version('3.6'), - os.path.join(resources_dir, 'constraints-python36.txt') - ) - assert os.path.samefile( - dependency_constraints.get_for_python_version('3.5'), - os.path.join(resources_dir, 'constraints-python35.txt') - ) - assert os.path.samefile( - dependency_constraints.get_for_python_version('2.7'), - os.path.join(resources_dir, 'constraints-python27.txt') - ) + assert dependency_constraints.base_file_path.samefile(resources_dir / 'constraints.txt') + assert dependency_constraints.get_for_python_version('3.8').samefile(resources_dir / 'constraints.txt') + assert dependency_constraints.get_for_python_version('3.6').samefile(resources_dir / 'constraints-python36.txt') + assert dependency_constraints.get_for_python_version('3.5').samefile(resources_dir / 'constraints-python35.txt') + assert dependency_constraints.get_for_python_version('2.7').samefile(resources_dir / 'constraints-python27.txt') diff --git a/unit_test/main_tests/conftest.py b/unit_test/main_tests/conftest.py index cd896915..3f684b6c 100644 --- a/unit_test/main_tests/conftest.py +++ b/unit_test/main_tests/conftest.py @@ -1,6 +1,6 @@ -import os import subprocess import sys +from pathlib import Path import pytest @@ -18,7 +18,7 @@ class ArgsInterceptor: self.kwargs = kwargs -MOCK_PACKAGE_DIR = 'some_package_dir' +MOCK_PACKAGE_DIR = Path('some_package_dir') @pytest.fixture(autouse=True) @@ -31,28 +31,33 @@ def mock_protection(monkeypatch): def fail_on_call(*args, **kwargs): raise RuntimeError("This should never be called") + def ignore_call(*args, **kwargs): + pass + monkeypatch.setattr(subprocess, 'Popen', fail_on_call) monkeypatch.setattr(util, 'download', fail_on_call) monkeypatch.setattr(windows, 'build', fail_on_call) monkeypatch.setattr(linux, 'build', fail_on_call) monkeypatch.setattr(macos, 'build', fail_on_call) + monkeypatch.setattr(Path, 'mkdir', ignore_call) + @pytest.fixture(autouse=True) def fake_package_dir(monkeypatch): ''' Monkey-patch enough for the main() function to run ''' - real_os_path_exists = os.path.exists + real_path_exists = Path.exists - def mock_os_path_exists(path): - if path == os.path.join(MOCK_PACKAGE_DIR, 'setup.py'): + def mock_path_exists(path): + if path == MOCK_PACKAGE_DIR / 'setup.py': return True else: - return real_os_path_exists(path) + return real_path_exists(path) - monkeypatch.setattr(os.path, 'exists', mock_os_path_exists) - monkeypatch.setattr(sys, 'argv', ['cibuildwheel', MOCK_PACKAGE_DIR]) + monkeypatch.setattr(Path, 'exists', mock_path_exists) + monkeypatch.setattr(sys, 'argv', ['cibuildwheel', str(MOCK_PACKAGE_DIR)]) @pytest.fixture(params=['linux', 'macos', 'windows']) diff --git a/unit_test/main_tests/main_options_test.py b/unit_test/main_tests/main_options_test.py index 633e9cc5..14252e67 100644 --- a/unit_test/main_tests/main_options_test.py +++ b/unit_test/main_tests/main_options_test.py @@ -1,5 +1,6 @@ import sys from fnmatch import fnmatch +from pathlib import Path import pytest @@ -12,9 +13,9 @@ from cibuildwheel.util import BuildSelector def test_output_dir(platform, intercepted_build_args, monkeypatch): - OUTPUT_DIR = 'some_output_dir' + OUTPUT_DIR = Path('some_output_dir') - monkeypatch.setenv('CIBW_OUTPUT_DIR', OUTPUT_DIR) + monkeypatch.setenv('CIBW_OUTPUT_DIR', str(OUTPUT_DIR)) main() @@ -24,14 +25,14 @@ def test_output_dir(platform, intercepted_build_args, monkeypatch): def test_output_dir_default(platform, intercepted_build_args, monkeypatch): main() - assert intercepted_build_args.args[0].output_dir == 'wheelhouse' + assert intercepted_build_args.args[0].output_dir == Path('wheelhouse') @pytest.mark.parametrize('also_set_environment', [False, True]) def test_output_dir_argument(also_set_environment, platform, intercepted_build_args, monkeypatch): - OUTPUT_DIR = 'some_output_dir' + OUTPUT_DIR = Path('some_output_dir') - monkeypatch.setattr(sys, 'argv', sys.argv + ['--output-dir', OUTPUT_DIR]) + monkeypatch.setattr(sys, 'argv', sys.argv + ['--output-dir', str(OUTPUT_DIR)]) if also_set_environment: monkeypatch.setenv('CIBW_OUTPUT_DIR', 'not_this_output_dir')