diff --git a/.travis.yml b/.travis.yml
index 855b565c..c30a4ab6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -35,6 +35,8 @@ jobs:
language: shell
before_install:
- choco install python3 --version 3.6.8 --no-progress -y
+ # Update root certificates to fix SSL error; see http://www.chawn.com/RootCerts.htm
+ - powershell "md C:\temp\certs; CertUtil -generateSSTFromWU C:\temp\certs\RootStore.sst; Get-ChildItem -Path C:\\temp\certs\Rootstore.sst | Import-Certificate -CertStoreLocation Cert:\\LocalMachine\\Root\\ | out-null"
env:
- PYTHON=C:\\Python36\\python
diff --git a/README.md b/README.md
index 6842b249..d3805547 100644
--- a/README.md
+++ b/README.md
@@ -22,11 +22,13 @@ What does it do?
| CPython 3.6 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅² |
| CPython 3.7 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅² |
| CPython 3.8 | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅ | ✅² |
+| CPython 3.9³ | 🛠 | 🛠 | 🛠 | 🛠 | 🛠 | 🛠 | 🛠 | |
| PyPy 2.7 v7.3.0 | ✅ | | ✅ | ✅ | | | | |
| PyPy 3.6 v7.3.0 | ✅ | | ✅ | ✅ | | | | |
¹ Not supported on Travis
-² Beta support until Travis CI fixes a bug
+² Beta support until Travis CI fixes a bug
+³ Python 3.9 is not yet ABI stable, so you shouldn't publish wheels with it yet. But if you want to check that your wheels build on Python 3.9, try our [`python3.9` branch](https://github.com/joerick/cibuildwheel/pull/382)!
- Builds manylinux, macOS and Windows wheels for CPython and PyPy using Azure Pipelines, Travis CI, AppVeyor, and CircleCI
- Bundles shared library dependencies on Linux and macOS through [auditwheel](https://github.com/pypa/auditwheel) and [delocate](https://github.com/matthew-brett/delocate)
diff --git a/appveyor.yml b/appveyor.yml
index 4bf0a18f..401eeb11 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -10,6 +10,11 @@ environment:
- APPVEYOR_BUILD_WORKER_IMAGE: macos-mojave
APPVEYOR_JOB_NAME: "python37-x64-macos-mojave"
+# while pypy libffi dependency is failing the build, allow failures on macos
+matrix:
+ allow_failures:
+ - APPVEYOR_BUILD_WORKER_IMAGE: macos-mojave
+
stack: python 3.7
build: off
diff --git a/bin/run_tests.py b/bin/run_tests.py
index 93c23e88..117f8b9f 100755
--- a/bin/run_tests.py
+++ b/bin/run_tests.py
@@ -3,10 +3,11 @@
import os
import subprocess
import sys
+from pathlib import Path
if __name__ == '__main__':
# move cwd to the project root
- os.chdir(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+ os.chdir(Path(__file__).resolve().parents[1])
# run the unit tests
subprocess.check_call([sys.executable, '-m', 'pytest', 'unit_test'])
diff --git a/cibuildwheel/__main__.py b/cibuildwheel/__main__.py
index 40241ac1..137bc19e 100644
--- a/cibuildwheel/__main__.py
+++ b/cibuildwheel/__main__.py
@@ -4,6 +4,7 @@ import sys
import textwrap
import traceback
from configparser import ConfigParser
+from pathlib import Path
from typing import Any, Dict, List, Optional, overload
@@ -20,6 +21,7 @@ from cibuildwheel.util import (
BuildSelector,
DependencyConstraints,
Unbuffered,
+ resources_dir,
)
@@ -114,8 +116,8 @@ def main() -> None:
file=sys.stderr)
exit(2)
- package_dir = args.package_dir
- output_dir = args.output_dir
+ package_dir = Path(args.package_dir)
+ output_dir = Path(args.output_dir)
if platform == 'linux':
repair_command_default = 'auditwheel repair -w {dest_dir} {wheel}'
@@ -149,7 +151,8 @@ def main() -> None:
elif dependency_versions == 'latest':
dependency_constraints = None
else:
- dependency_constraints = DependencyConstraints(dependency_versions)
+ dependency_versions_path = Path(dependency_versions)
+ dependency_constraints = DependencyConstraints(dependency_versions_path)
if test_extras:
test_extras = f'[{test_extras}]'
@@ -163,7 +166,7 @@ def main() -> None:
# This needs to be passed on to the docker container in linux.py
os.environ['CIBUILDWHEEL'] = '1'
- if not any(os.path.exists(os.path.join(package_dir, name))
+ if not any((package_dir / name).exists()
for name in ["setup.py", "setup.cfg", "pyproject.toml"]):
print('cibuildwheel: Could not find setup.py, setup.cfg or pyproject.toml at root of package', file=sys.stderr)
exit(2)
@@ -174,9 +177,7 @@ def main() -> None:
manylinux_images: Optional[Dict[str, str]] = None
if platform == 'linux':
- pinned_docker_images_file = os.path.join(
- os.path.dirname(__file__), 'resources', 'pinned_docker_images.cfg'
- )
+ pinned_docker_images_file = resources_dir / 'pinned_docker_images.cfg'
all_pinned_docker_images = ConfigParser()
all_pinned_docker_images.read(pinned_docker_images_file)
# all_pinned_docker_images looks like a dict of dicts, e.g.
@@ -224,8 +225,8 @@ def main() -> None:
print_preamble(platform, build_options)
- if not os.path.exists(output_dir):
- os.makedirs(output_dir)
+ if not output_dir.exists():
+ output_dir.mkdir(parents=True)
if platform == 'linux':
cibuildwheel.linux.build(build_options)
diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py
index 3a660ef1..99e779c4 100644
--- a/cibuildwheel/linux.py
+++ b/cibuildwheel/linux.py
@@ -5,6 +5,7 @@ import subprocess
import sys
import textwrap
import uuid
+from pathlib import Path, PurePath
from typing import List, NamedTuple, Optional, Union
@@ -104,10 +105,12 @@ def build(options: BuildOptions) -> None:
('pp', 'manylinux_x86_64', options.manylinux_images['pypy_x86_64']),
]
- if not os.path.realpath(options.package_dir).startswith(os.path.realpath('.')):
+ cwd = Path.cwd()
+ abs_package_dir = options.package_dir.resolve()
+ if cwd != abs_package_dir and cwd not in abs_package_dir.parents:
raise Exception('package_dir must be inside the working directory')
- container_package_dir = os.path.join('/project', os.path.relpath(options.package_dir, '.'))
+ container_package_dir = PurePath('/project') / abs_package_dir.relative_to(cwd)
for implementation, platform_tag, docker_image in platforms:
platform_configs = [c for c in python_configurations if c.identifier.startswith(implementation) and c.identifier.endswith(platform_tag)]
@@ -200,7 +203,6 @@ def build(options: BuildOptions) -> None:
pip install {dependency_install_flags} virtualenv
venv_dir=`mktemp -d`/venv
python -m virtualenv --no-download "$venv_dir"
- export __CIBW_VIRTUALENV_PATH__=$venv_dir
# run the tests in a subshell to keep that `activate`
# script from polluting the env
@@ -273,7 +275,7 @@ def build(options: BuildOptions) -> None:
# copy the output back into the host
call(['docker', 'cp',
container_name + ':/output/.',
- os.path.abspath(options.output_dir)])
+ str(options.output_dir.resolve())])
except subprocess.CalledProcessError as error:
troubleshoot(options.package_dir, error)
exit(1)
@@ -282,16 +284,11 @@ def build(options: BuildOptions) -> None:
call(['docker', 'rm', '--force', '-v', container_name])
-def troubleshoot(package_dir: str, error: Exception) -> None:
+def troubleshoot(package_dir: Path, error: Exception) -> None:
if (isinstance(error, subprocess.CalledProcessError) and 'exec' in error.cmd):
# the bash script failed
print('Checking for common errors...')
- so_files = []
- for root, dirs, files in os.walk(package_dir):
- for name in files:
- _, ext = os.path.splitext(name)
- if ext == '.so':
- so_files.append(os.path.join(root, name))
+ so_files = list(package_dir.glob('**/*.so'))
if so_files:
print(textwrap.dedent('''
@@ -305,5 +302,5 @@ def troubleshoot(package_dir: str, error: Exception) -> None:
'''))
print(' Files detected:')
- print('\n'.join([' ' + f for f in so_files]))
+ print('\n'.join([f' {f}' for f in so_files]))
print('')
diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py
index 745c1557..2fd28d20 100644
--- a/cibuildwheel/macos.py
+++ b/cibuildwheel/macos.py
@@ -4,7 +4,7 @@ import shutil
import subprocess
import sys
import tempfile
-from glob import glob
+from pathlib import Path
from typing import Dict, List, Optional, NamedTuple, Union
@@ -52,41 +52,41 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi
return [c for c in python_configurations if build_selector(c.identifier)]
-SYMLINKS_DIR = '/tmp/cibw_bin'
+SYMLINKS_DIR = Path('/tmp/cibw_bin')
-def make_symlinks(installation_bin_path: str, python_executable: str, pip_executable: str) -> None:
- assert os.path.exists(os.path.join(installation_bin_path, python_executable))
+def make_symlinks(installation_bin_path: Path, python_executable: str, pip_executable: str) -> None:
+ assert (installation_bin_path / python_executable).exists()
# Python bin folders on Mac don't symlink `python3` to `python`, and neither
# does PyPy for `pypy` or `pypy3`, so we do that so `python` and `pip` always
# point to the active configuration.
- if os.path.exists(SYMLINKS_DIR):
+ if SYMLINKS_DIR.exists():
shutil.rmtree(SYMLINKS_DIR)
- os.makedirs(SYMLINKS_DIR)
+ SYMLINKS_DIR.mkdir(parents=True)
- os.symlink(os.path.join(installation_bin_path, python_executable), os.path.join(SYMLINKS_DIR, 'python'))
- os.symlink(os.path.join(installation_bin_path, python_executable + '-config'), os.path.join(SYMLINKS_DIR, 'python-config'))
- os.symlink(os.path.join(installation_bin_path, pip_executable), os.path.join(SYMLINKS_DIR, 'pip'))
+ (SYMLINKS_DIR / 'python').symlink_to(installation_bin_path / python_executable)
+ (SYMLINKS_DIR / 'python-config').symlink_to(installation_bin_path / (python_executable + '-config'))
+ (SYMLINKS_DIR / 'pip').symlink_to(installation_bin_path / pip_executable)
-def install_cpython(version: str, url: str) -> str:
+def install_cpython(version: str, url: str) -> Path:
installed_system_packages = subprocess.check_output(['pkgutil', '--pkgs'], universal_newlines=True).splitlines()
# if this version of python isn't installed, get it from python.org and install
python_package_identifier = f'org.python.Python.PythonFramework-{version}'
if python_package_identifier not in installed_system_packages:
# download the pkg
- download(url, '/tmp/Python.pkg')
+ download(url, Path('/tmp/Python.pkg'))
# install
call(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/'])
# patch open ssl
if version == '3.5':
open_ssl_patch_url = f'https://github.com/mayeut/patch-macos-python-openssl/releases/download/v1.0.2u/patch-macos-python-{version}-openssl-v1.0.2u.tar.gz'
- download(open_ssl_patch_url, '/tmp/python-patch.tar.gz')
+ download(open_ssl_patch_url, Path('/tmp/python-patch.tar.gz'))
call(['sudo', 'tar', '-C', f'/Library/Frameworks/Python.framework/Versions/{version}/', '-xmf', '/tmp/python-patch.tar.gz'])
- installation_bin_path = f'/Library/Frameworks/Python.framework/Versions/{version}/bin'
+ installation_bin_path = Path(f'/Library/Frameworks/Python.framework/Versions/{version}/bin')
python_executable = 'python3' if version[0] == '3' else 'python'
pip_executable = 'pip3' if version[0] == '3' else 'pip'
make_symlinks(installation_bin_path, python_executable, pip_executable)
@@ -94,16 +94,18 @@ def install_cpython(version: str, url: str) -> str:
return installation_bin_path
-def install_pypy(version: str, url: str) -> str:
+def install_pypy(version: str, url: str) -> Path:
pypy_tar_bz2 = url.rsplit('/', 1)[-1]
- assert pypy_tar_bz2.endswith(".tar.bz2")
- pypy_base_filename = os.path.splitext(os.path.splitext(pypy_tar_bz2)[0])[0]
- installation_path = os.path.join('/tmp', pypy_base_filename)
- if not os.path.exists(installation_path):
- download(url, os.path.join("/tmp", pypy_tar_bz2))
- call(['tar', '-C', '/tmp', '-xf', os.path.join("/tmp", pypy_tar_bz2)])
+ extension = ".tar.bz2"
+ assert pypy_tar_bz2.endswith(extension)
+ pypy_base_filename = pypy_tar_bz2[:-len(extension)]
+ installation_path = Path('/tmp') / pypy_base_filename
+ if not installation_path.exists():
+ downloaded_tar_bz2 = Path("/tmp") / pypy_tar_bz2
+ download(url, downloaded_tar_bz2)
+ call(['tar', '-C', '/tmp', '-xf', str(downloaded_tar_bz2)])
- installation_bin_path = os.path.join(installation_path, 'bin')
+ installation_bin_path = installation_path / 'bin'
python_executable = 'pypy3' if version[0] == '3' else 'pypy'
pip_executable = 'pip3' if version[0] == '3' else 'pip'
make_symlinks(installation_bin_path, python_executable, pip_executable)
@@ -121,8 +123,8 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain
env = os.environ.copy()
env['PATH'] = os.pathsep.join([
- SYMLINKS_DIR,
- installation_bin_path,
+ str(SYMLINKS_DIR),
+ str(installation_bin_path),
env['PATH'],
])
@@ -144,8 +146,8 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain
exit(1)
# install pip & wheel
- call(['python', get_pip_script] + dependency_constraint_flags, env=env, cwd="/tmp")
- assert os.path.exists(os.path.join(installation_bin_path, 'pip'))
+ call(['python', str(get_pip_script)] + dependency_constraint_flags, env=env, cwd="/tmp")
+ assert (installation_bin_path / 'pip').exists()
call(['which', 'pip'], env=env)
call(['pip', '--version'], env=env)
which_pip = subprocess.check_output(['which', 'pip'], env=env, universal_newlines=True).strip()
@@ -170,9 +172,9 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain
def build(options: BuildOptions) -> None:
- temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
- built_wheel_dir = os.path.join(temp_dir, 'built_wheel')
- repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel')
+ temp_dir = Path(tempfile.mkdtemp(prefix='cibuildwheel'))
+ built_wheel_dir = temp_dir / 'built_wheel'
+ repaired_wheel_dir = temp_dir / 'repaired_wheel'
python_configurations = get_python_configurations(options.build_selector)
@@ -180,7 +182,7 @@ def build(options: BuildOptions) -> None:
dependency_constraint_flags = []
if options.dependency_constraints:
dependency_constraint_flags = [
- '-c', options.dependency_constraints.get_for_python_version(config.version)
+ '-c', str(options.dependency_constraints.get_for_python_version(config.version))
]
env = setup_python(config, dependency_constraint_flags, options.environment)
@@ -191,40 +193,41 @@ def build(options: BuildOptions) -> None:
call(before_build_prepared, env=env, shell=True)
# build the wheel
- if os.path.exists(built_wheel_dir):
+ if built_wheel_dir.exists():
shutil.rmtree(built_wheel_dir)
- os.makedirs(built_wheel_dir)
- call(['pip', 'wheel', options.package_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env)
- built_wheel = glob(os.path.join(built_wheel_dir, '*.whl'))[0]
+ built_wheel_dir.mkdir(parents=True)
+ # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
+ # see https://github.com/joerick/cibuildwheel/pull/369
+ call(['pip', 'wheel', str(options.package_dir.resolve()), '-w', str(built_wheel_dir), '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env)
+ built_wheel = next(built_wheel_dir.glob('*.whl'))
# repair the wheel
- if os.path.exists(repaired_wheel_dir):
+ if repaired_wheel_dir.exists():
shutil.rmtree(repaired_wheel_dir)
- os.makedirs(repaired_wheel_dir)
- if built_wheel.endswith('none-any.whl') or not options.repair_command:
+ repaired_wheel_dir.mkdir(parents=True)
+ if built_wheel.name.endswith('none-any.whl') or not options.repair_command:
# pure Python wheel or empty repair command
- shutil.move(built_wheel, repaired_wheel_dir)
+ built_wheel.rename(repaired_wheel_dir / built_wheel.name)
else:
repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir)
call(repair_command_prepared, env=env, shell=True)
- repaired_wheel = glob(os.path.join(repaired_wheel_dir, '*.whl'))[0]
+ repaired_wheel = next(repaired_wheel_dir.glob('*.whl'))
if options.test_command:
# set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
call(['pip', 'install', 'virtualenv'] + dependency_constraint_flags, env=env)
- venv_dir = tempfile.mkdtemp()
+ venv_dir = Path(tempfile.mkdtemp())
# Use --no-download to ensure determinism by using seed libraries
# built into virtualenv
- call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env)
+ call(['python', '-m', 'virtualenv', '--no-download', str(venv_dir)], env=env)
virtualenv_env = env.copy()
virtualenv_env['PATH'] = os.pathsep.join([
- os.path.join(venv_dir, 'bin'),
+ str(venv_dir / 'bin'),
virtualenv_env['PATH'],
])
- virtualenv_env["__CIBW_VIRTUALENV_PATH__"] = venv_dir
# check that we are using the Python from the virtual environment
call(['which', 'python'], env=virtualenv_env)
@@ -234,7 +237,7 @@ def build(options: BuildOptions) -> None:
call(before_test_prepared, env=virtualenv_env, shell=True)
# install the wheel
- call(['pip', 'install', repaired_wheel + options.test_extras], env=virtualenv_env)
+ call(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env)
# test the wheel
if options.test_requires:
@@ -245,8 +248,8 @@ def build(options: BuildOptions) -> None:
# and not the repo code)
test_command_prepared = prepare_command(
options.test_command,
- project=os.path.abspath('.'),
- package=os.path.abspath(options.package_dir)
+ project=Path('.').resolve(),
+ package=options.package_dir.resolve()
)
call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True)
@@ -254,5 +257,4 @@ def build(options: BuildOptions) -> None:
shutil.rmtree(venv_dir)
# we're all done here; move it to output (overwrite existing)
- dst = os.path.join(options.output_dir, os.path.basename(repaired_wheel))
- shutil.move(repaired_wheel, dst)
+ repaired_wheel.replace(options.output_dir / repaired_wheel.name)
diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py
index 499954b9..feef6185 100644
--- a/cibuildwheel/util.py
+++ b/cibuildwheel/util.py
@@ -1,14 +1,15 @@
import os
import urllib.request
from fnmatch import fnmatch
+from pathlib import Path
from time import sleep
-from typing import Dict, List, NamedTuple, Optional
+from typing import Dict, List, NamedTuple, Optional, Union
from .environment import ParsedEnvironment
-def prepare_command(command: str, **kwargs: str) -> str:
+def prepare_command(command: str, **kwargs: Union[str, os.PathLike]) -> str:
'''
Preprocesses a command by expanding variables like {python}.
@@ -58,11 +59,11 @@ class Unbuffered:
return getattr(self.stream, attr)
-def download(url: str, dest: str) -> None:
- print('+ Download ' + url + ' to ' + dest)
- dest_dir = os.path.dirname(dest)
- if not os.path.exists(dest_dir):
- os.makedirs(dest_dir)
+def download(url: str, dest: Path) -> None:
+ print(f'+ Download {url} to {dest}')
+ dest_dir = dest.parent
+ if not dest_dir.exists():
+ dest_dir.mkdir(parents=True)
repeat_num = 3
for i in range(repeat_num):
@@ -76,40 +77,39 @@ def download(url: str, dest: str) -> None:
break
try:
- with open(dest, 'wb') as file:
- file.write(response.read())
+ dest.write_bytes(response.read())
finally:
response.close()
class DependencyConstraints:
- def __init__(self, base_file_path: str):
- assert os.path.exists(base_file_path)
- self.base_file_path = os.path.abspath(base_file_path)
+ def __init__(self, base_file_path: Path):
+ assert base_file_path.exists()
+ self.base_file_path = base_file_path.resolve()
@staticmethod
def with_defaults() -> 'DependencyConstraints':
return DependencyConstraints(
- base_file_path=os.path.join(os.path.dirname(__file__), 'resources', 'constraints.txt')
+ base_file_path=resources_dir / 'constraints.txt'
)
- def get_for_python_version(self, version: str) -> str:
+ def get_for_python_version(self, version: str) -> Path:
version_parts = version.split('.')
# try to find a version-specific dependency file e.g. if
# ./constraints.txt is the base, look for ./constraints-python27.txt
- base, ext = os.path.splitext(self.base_file_path)
- specific = base + f'-python{version_parts[0]}{version_parts[1]}'
- specific_file_path = specific + ext
- if os.path.exists(specific_file_path):
+ specific_stem = self.base_file_path.stem + f'-python{version_parts[0]}{version_parts[1]}'
+ specific_name = specific_stem + self.base_file_path.suffix
+ specific_file_path = self.base_file_path.with_name(specific_name)
+ if specific_file_path.exists():
return specific_file_path
else:
return self.base_file_path
class BuildOptions(NamedTuple):
- package_dir: str
- output_dir: str
+ package_dir: Path
+ output_dir: Path
build_selector: BuildSelector
environment: ParsedEnvironment
before_build: Optional[str]
@@ -123,5 +123,5 @@ class BuildOptions(NamedTuple):
build_verbosity: int
-resources_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'resources'))
-get_pip_script = os.path.join(resources_dir, 'get-pip.py')
+resources_dir = Path(__file__).resolve().parent / 'resources'
+get_pip_script = resources_dir / 'get-pip.py'
diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py
index 6a62b63d..63b22b48 100644
--- a/cibuildwheel/windows.py
+++ b/cibuildwheel/windows.py
@@ -3,7 +3,7 @@ import shutil
import subprocess
import sys
import tempfile
-from glob import glob
+from pathlib import Path
from zipfile import ZipFile
import toml
@@ -20,7 +20,7 @@ from .util import (
)
-IS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache')
+IS_RUNNING_ON_AZURE = Path('C:\\hostedtoolcache').exists()
IS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows'
@@ -72,36 +72,38 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi
return python_configurations
-def extract_zip(zip_src: str, dest: str) -> None:
+def extract_zip(zip_src: Path, dest: Path) -> None:
with ZipFile(zip_src) as zip:
zip.extractall(dest)
-def install_cpython(version: str, arch: str, nuget: str) -> str:
+def install_cpython(version: str, arch: str, nuget: Path) -> Path:
nuget_args = get_nuget_args(version, arch)
- installation_path = os.path.join(nuget_args[-1], nuget_args[0] + '.' + version, 'tools')
- shell([nuget, 'install'] + nuget_args)
+ installation_path = Path(nuget_args[-1]) / (nuget_args[0] + '.' + version) / 'tools'
+ shell([str(nuget), 'install'] + nuget_args)
return installation_path
-def install_pypy(version: str, arch: str, url: str) -> str:
+def install_pypy(version: str, arch: str, url: str) -> Path:
assert arch == '32'
# Inside the PyPy zip file is a directory with the same name
zip_filename = url.rsplit('/', 1)[-1]
- installation_path = os.path.join('C:\\cibw', os.path.splitext(zip_filename)[0])
- if not os.path.exists(installation_path):
- pypy_zip = os.path.join('C:\\cibw', zip_filename)
+ extension = ".zip"
+ assert zip_filename.endswith(extension)
+ installation_path = Path('C:\\cibw') / zip_filename[:-len(extension)]
+ if not installation_path.exists():
+ pypy_zip = Path('C:\\cibw') / zip_filename
download(url, pypy_zip)
# Extract to the parent directory because the zip file still contains a directory
- extract_zip(pypy_zip, os.path.dirname(installation_path))
+ extract_zip(pypy_zip, installation_path.parent)
pypy_exe = 'pypy3.exe' if version[0] == '3' else 'pypy.exe'
- shell(['mklink', os.path.join(installation_path, 'python.exe'), os.path.join(installation_path, pypy_exe)])
+ (installation_path / 'python.exe').symlink_to(installation_path / pypy_exe)
return installation_path
def setup_python(python_configuration: PythonConfiguration, dependency_constraint_flags: List[str], environment: ParsedEnvironment) -> Dict[str, str]:
- nuget = 'C:\\cibw\\nuget.exe'
- if not os.path.exists(nuget):
+ nuget = Path('C:\\cibw\\nuget.exe')
+ if not nuget.exists():
download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget)
if python_configuration.identifier.startswith('cp'):
@@ -112,15 +114,15 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain
else:
raise ValueError("Unknown Python implementation")
- assert os.path.exists(os.path.join(installation_path, 'python.exe'))
+ assert (installation_path / 'python.exe').exists()
# set up PATH and environment variables for run_with_env
env = os.environ.copy()
env['PYTHON_VERSION'] = python_configuration.version
env['PYTHON_ARCH'] = python_configuration.arch
env['PATH'] = os.pathsep.join([
- installation_path,
- os.path.join(installation_path, 'Scripts'),
+ str(installation_path),
+ str(installation_path / 'Scripts'),
env['PATH']
])
# update env with results from CIBW_ENVIRONMENT
@@ -131,16 +133,16 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain
shell(['python', '--version'], env=env)
shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)"'], env=env)
where_python = subprocess.check_output(['where', 'python'], env=env, universal_newlines=True).splitlines()[0].strip()
- if where_python != os.path.join(installation_path, 'python.exe'):
+ if where_python != str(installation_path / 'python.exe'):
print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr)
exit(1)
# make sure pip is installed
- if not os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe')):
- shell(['python', get_pip_script] + dependency_constraint_flags, env=env, cwd="C:\\cibw")
- assert os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe'))
+ if not (installation_path / 'Scripts' / 'pip.exe').exists():
+ shell(['python', str(get_pip_script)] + dependency_constraint_flags, env=env, cwd="C:\\cibw")
+ assert (installation_path / 'Scripts' / 'pip.exe').exists()
where_pip = subprocess.check_output(['where', 'pip'], env=env, universal_newlines=True).splitlines()[0].strip()
- if where_pip.strip() != os.path.join(installation_path, 'Scripts', 'pip.exe'):
+ if where_pip.strip() != str(installation_path / 'Scripts' / 'pip.exe'):
print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr)
exit(1)
@@ -178,12 +180,12 @@ def pep_518_cp35_workaround(package_dir: str, env: Dict[str, str]) -> None:
def build(options: BuildOptions) -> None:
- temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
- built_wheel_dir = os.path.join(temp_dir, 'built_wheel')
- repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel')
+ temp_dir = Path(tempfile.mkdtemp(prefix='cibuildwheel'))
+ built_wheel_dir = temp_dir / 'built_wheel'
+ repaired_wheel_dir = temp_dir / 'repaired_wheel'
# install nuget as best way to provide python
- nuget = 'C:\\cibw\\nuget.exe'
+ nuget = Path('C:\\cibw\\nuget.exe')
download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget)
python_configurations = get_python_configurations(options.build_selector)
@@ -191,7 +193,7 @@ def build(options: BuildOptions) -> None:
dependency_constraint_flags = []
if options.dependency_constraints:
dependency_constraint_flags = [
- '-c', options.dependency_constraints.get_for_python_version(config.version)
+ '-c', str(options.dependency_constraints.get_for_python_version(config.version))
]
# install Python
@@ -208,42 +210,41 @@ def build(options: BuildOptions) -> None:
pep_518_cp35_workaround(options.package_dir, env)
# build the wheel
- if os.path.exists(built_wheel_dir):
+ if built_wheel_dir.exists():
shutil.rmtree(built_wheel_dir)
- os.makedirs(built_wheel_dir)
- shell(['pip', 'wheel', options.package_dir, '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env)
- built_wheel = glob(os.path.join(built_wheel_dir, '*.whl'))[0]
+ built_wheel_dir.mkdir(parents=True)
+ # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
+ # see https://github.com/joerick/cibuildwheel/pull/369
+ shell(['pip', 'wheel', str(options.package_dir.resolve()), '-w', str(built_wheel_dir), '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env)
+ built_wheel = next(built_wheel_dir.glob('*.whl'))
# repair the wheel
- if os.path.exists(repaired_wheel_dir):
+ if repaired_wheel_dir.exists():
shutil.rmtree(repaired_wheel_dir)
- os.makedirs(repaired_wheel_dir)
- if built_wheel.endswith('none-any.whl') or not options.repair_command:
+ repaired_wheel_dir.mkdir(parents=True)
+ if built_wheel.name.endswith('none-any.whl') or not options.repair_command:
# pure Python wheel or empty repair command
- shutil.move(built_wheel, repaired_wheel_dir)
+ built_wheel.rename(repaired_wheel_dir / built_wheel.name)
else:
repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir)
shell([repair_command_prepared], env=env)
- repaired_wheel = glob(os.path.join(repaired_wheel_dir, '*.whl'))[0]
+ repaired_wheel = next(repaired_wheel_dir.glob('*.whl'))
if options.test_command:
# set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time.
shell(['pip', 'install', 'virtualenv'] + dependency_constraint_flags, env=env)
- venv_dir = tempfile.mkdtemp()
+ venv_dir = Path(tempfile.mkdtemp())
# Use --no-download to ensure determinism by using seed libraries
# built into virtualenv
- shell(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env)
+ shell(['python', '-m', 'virtualenv', '--no-download', str(venv_dir)], env=env)
virtualenv_env = env.copy()
-
- venv_script_path = os.path.join(venv_dir, 'Scripts')
virtualenv_env['PATH'] = os.pathsep.join([
- venv_script_path,
+ str(venv_dir / 'Scripts'),
virtualenv_env['PATH'],
])
- virtualenv_env["__CIBW_VIRTUALENV_PATH__"] = venv_dir
# check that we are using the Python from the virtual environment
shell(['which', 'python'], env=virtualenv_env)
@@ -257,7 +258,7 @@ def build(options: BuildOptions) -> None:
shell([before_test_prepared], env=virtualenv_env)
# install the wheel
- shell(['pip', 'install', repaired_wheel + options.test_extras], env=virtualenv_env)
+ shell(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env)
# test the wheel
if options.test_requires:
@@ -268,8 +269,8 @@ def build(options: BuildOptions) -> None:
# and not the repo code)
test_command_prepared = prepare_command(
options.test_command,
- project=os.path.abspath('.'),
- package=os.path.abspath(options.package_dir)
+ project=Path('.').resolve(),
+ package=options.package_dir.resolve()
)
shell([test_command_prepared], cwd='c:\\', env=virtualenv_env)
@@ -277,7 +278,4 @@ def build(options: BuildOptions) -> None:
shutil.rmtree(venv_dir)
# we're all done here; move it to output (remove if already exists)
- dst = os.path.join(options.output_dir, os.path.basename(repaired_wheel))
- if os.path.isfile(dst):
- os.remove(dst)
- shutil.move(repaired_wheel, dst)
+ repaired_wheel.replace(options.output_dir / repaired_wheel.name)
diff --git a/docs/mkdocs_include_markdown_plugin/mkdocs_include_markdown_plugin/plugin.py b/docs/mkdocs_include_markdown_plugin/mkdocs_include_markdown_plugin/plugin.py
index b6b7a723..f12ea469 100644
--- a/docs/mkdocs_include_markdown_plugin/mkdocs_include_markdown_plugin/plugin.py
+++ b/docs/mkdocs_include_markdown_plugin/mkdocs_include_markdown_plugin/plugin.py
@@ -1,7 +1,6 @@
import cgi
-import io
-import os
import re
+from pathlib import Path
import mkdocs
@@ -41,13 +40,12 @@ class ImportMarkdownPlugin(mkdocs.plugins.BasePlugin):
def found_include_tag(match):
filename = match.group('filename')
- file_path_abs = os.path.join(os.path.dirname(page_src_path), filename)
+ file_path_abs = Path(page_src_path).parent / filename
- if not os.path.exists(file_path_abs):
+ if not file_path_abs.exists():
raise ValueError('file not found', filename)
- with io.open(file_path_abs, encoding='utf8') as f:
- text_to_include = f.read()
+ text_to_include = file_path_abs.read_text(encoding='utf8')
# Allow good practice of having a final newline in the file
if text_to_include.endswith('\n'):
@@ -60,13 +58,12 @@ class ImportMarkdownPlugin(mkdocs.plugins.BasePlugin):
start = match.group('start')
end = match.group('end')
- file_path_abs = os.path.join(os.path.dirname(page_src_path), filename)
+ file_path_abs = Path(page_src_path).parent / filename
- if not os.path.exists(file_path_abs):
+ if not file_path_abs.exists():
raise ValueError('file not found', filename)
- with io.open(file_path_abs, encoding='utf8') as f:
- text_to_include = f.read()
+ text_to_include = file_path_abs.read_text(encoding='utf8')
if start:
_, _, text_to_include = text_to_include.partition(start)
diff --git a/setup.py b/setup.py
index 68dbff3a..542d29f7 100644
--- a/setup.py
+++ b/setup.py
@@ -1,15 +1,13 @@
# -*- coding: utf-8 -*-
-import io
-import os
+from pathlib import Path
try:
from setuptools import setup
except ImportError:
from distutils.core import setup
-this_directory = os.path.dirname(__file__)
-with io.open(os.path.join(this_directory, 'README.md'), encoding='utf-8') as f:
- long_description = f.read()
+this_directory = Path(__file__).parent
+long_description = (this_directory / 'README.md').read_text(encoding='utf-8')
setup(
name='cibuildwheel',
diff --git a/test/test_dependency_versions.py b/test/test_dependency_versions.py
index 611350f1..2f79bd2a 100644
--- a/test/test_dependency_versions.py
+++ b/test/test_dependency_versions.py
@@ -1,4 +1,3 @@
-import os
import re
import pytest
import textwrap
@@ -39,11 +38,9 @@ VERSION_REGEX = r'([\w-]+)==([^\s]+)'
def get_versions_from_constraint_file(constraint_file):
- with open(constraint_file, encoding='utf8') as f:
- constraint_file_text = f.read()
+ constraint_file_text = constraint_file.read_text(encoding='utf8')
versions = {}
-
for package, version in re.findall(VERSION_REGEX, constraint_file_text):
versions[package] = version
@@ -73,7 +70,7 @@ def test_pinned_versions(tmp_path, python_version):
constraint_filename = 'constraints.txt'
build_pattern = '[cp]p38-*'
- constraint_file = os.path.join(cibuildwheel.util.resources_dir, constraint_filename)
+ constraint_file = cibuildwheel.util.resources_dir / constraint_filename
constraint_versions = get_versions_from_constraint_file(constraint_file)
for package in ['pip', 'setuptools', 'wheel', 'virtualenv']:
diff --git a/test/test_projects/base.py b/test/test_projects/base.py
index 5cac80ad..a5710fdf 100644
--- a/test/test_projects/base.py
+++ b/test/test_projects/base.py
@@ -1,5 +1,7 @@
-import os
+from pathlib import Path
+
import jinja2
+
from typing import Union, Dict, Any
@@ -14,6 +16,8 @@ class TestProject:
Write out to the filesystem using `generate`.
'''
+ __test__ = False # Have pytest ignore this class on `from .test_projects import TestProject`
+
files: FilesDict
template_context: TemplateContext
@@ -21,12 +25,12 @@ class TestProject:
self.files = {}
self.template_context = {}
- def generate(self, path: str):
+ def generate(self, path: Path):
for filename, content in self.files.items():
- file_path = os.path.join(path, filename)
- os.makedirs(os.path.dirname(file_path), exist_ok=True)
+ file_path = path / filename
+ file_path.parent.mkdir(parents=True, exist_ok=True)
- with open(file_path, 'w', encoding='utf8') as f:
+ with file_path.open('w', encoding='utf8') as f:
if isinstance(content, jinja2.Template):
content = content.render(self.template_context)
diff --git a/test/test_subdir_package.py b/test/test_subdir_package.py
index 53bd9520..14a7a58b 100644
--- a/test/test_subdir_package.py
+++ b/test/test_subdir_package.py
@@ -1,4 +1,4 @@
-import os
+from pathlib import Path
import jinja2
@@ -35,7 +35,7 @@ def test(capfd, tmp_path):
project_dir = tmp_path / 'project'
subdir_package_project.generate(project_dir)
- package_dir = os.path.join(project_dir, 'src', 'spam')
+ package_dir = Path('src', 'spam')
# build the wheels
actual_wheels = utils.cibuildwheel_run(project_dir, package_dir=package_dir, add_env={
'CIBW_BEFORE_BUILD': 'python {project}/bin/before_build.py',
diff --git a/test/test_testing.py b/test/test_testing.py
index dda833ea..58a38224 100644
--- a/test/test_testing.py
+++ b/test/test_testing.py
@@ -46,12 +46,15 @@ class TestSpam(TestCase):
self.assertNotEqual(0, spam.system('python -c "exit(1)"'))
def test_virtualenv(self):
- virtualenv_path = os.environ.get("__CIBW_VIRTUALENV_PATH__")
- if not virtualenv_path:
- self.fail("No virtualenv path defined in environment variable __CIBW_VIRTUALENV_PATH__")
+ # sys.prefix is different from sys.base_prefix when running a virtualenv
+ # See https://docs.python.org/3/library/venv.html, which virtualenv seems
+ # to honor in recent releases
+ # Python 2 doesn't have sys.base_prefix by default
+ if not hasattr(sys, 'base_prefix') or sys.prefix == sys.base_prefix:
+ self.fail("Not running in a virtualenv")
- self.assertTrue(path_contains(virtualenv_path, sys.executable))
- self.assertTrue(path_contains(virtualenv_path, spam.__file__))
+ self.assertTrue(path_contains(sys.prefix, sys.executable))
+ self.assertTrue(path_contains(sys.prefix, spam.__file__))
def test_uname(self):
if platform.system() == "Windows":
diff --git a/test/utils.py b/test/utils.py
index 15ce402a..217a616f 100644
--- a/test/utils.py
+++ b/test/utils.py
@@ -10,9 +10,10 @@ import shutil
import subprocess
import sys
from contextlib import contextmanager
+from pathlib import Path
from tempfile import mkdtemp
-IS_WINDOWS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache')
+IS_WINDOWS_RUNNING_ON_AZURE = Path('C:\\hostedtoolcache').exists()
IS_WINDOWS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows'
@@ -66,7 +67,7 @@ def cibuildwheel_run(project_path, package_dir='.', env=None, add_env=None, outp
with TemporaryDirectoryIfNone(output_dir) as _output_dir:
subprocess.check_call(
- [sys.executable, '-m', 'cibuildwheel', '--output-dir', str(_output_dir), package_dir],
+ [sys.executable, '-m', 'cibuildwheel', '--output-dir', str(_output_dir), str(package_dir)],
env=env,
cwd=project_path,
)
diff --git a/unit_test/dependency_constraints_test.py b/unit_test/dependency_constraints_test.py
index b3fb0cb3..5347011c 100644
--- a/unit_test/dependency_constraints_test.py
+++ b/unit_test/dependency_constraints_test.py
@@ -1,30 +1,16 @@
from cibuildwheel.util import DependencyConstraints
-import os
+
+from pathlib import Path
def test_defaults():
dependency_constraints = DependencyConstraints.with_defaults()
- project_root = os.path.dirname(os.path.dirname(__file__))
- resources_dir = os.path.join(project_root, 'cibuildwheel', 'resources')
+ project_root = Path(__file__).parents[1]
+ resources_dir = project_root / 'cibuildwheel' / 'resources'
- assert os.path.samefile(
- dependency_constraints.base_file_path,
- os.path.join(resources_dir, 'constraints.txt')
- )
- assert os.path.samefile(
- dependency_constraints.get_for_python_version('3.8'),
- os.path.join(resources_dir, 'constraints.txt')
- )
- assert os.path.samefile(
- dependency_constraints.get_for_python_version('3.6'),
- os.path.join(resources_dir, 'constraints-python36.txt')
- )
- assert os.path.samefile(
- dependency_constraints.get_for_python_version('3.5'),
- os.path.join(resources_dir, 'constraints-python35.txt')
- )
- assert os.path.samefile(
- dependency_constraints.get_for_python_version('2.7'),
- os.path.join(resources_dir, 'constraints-python27.txt')
- )
+ assert dependency_constraints.base_file_path.samefile(resources_dir / 'constraints.txt')
+ assert dependency_constraints.get_for_python_version('3.8').samefile(resources_dir / 'constraints.txt')
+ assert dependency_constraints.get_for_python_version('3.6').samefile(resources_dir / 'constraints-python36.txt')
+ assert dependency_constraints.get_for_python_version('3.5').samefile(resources_dir / 'constraints-python35.txt')
+ assert dependency_constraints.get_for_python_version('2.7').samefile(resources_dir / 'constraints-python27.txt')
diff --git a/unit_test/main_tests/conftest.py b/unit_test/main_tests/conftest.py
index cd896915..3f684b6c 100644
--- a/unit_test/main_tests/conftest.py
+++ b/unit_test/main_tests/conftest.py
@@ -1,6 +1,6 @@
-import os
import subprocess
import sys
+from pathlib import Path
import pytest
@@ -18,7 +18,7 @@ class ArgsInterceptor:
self.kwargs = kwargs
-MOCK_PACKAGE_DIR = 'some_package_dir'
+MOCK_PACKAGE_DIR = Path('some_package_dir')
@pytest.fixture(autouse=True)
@@ -31,28 +31,33 @@ def mock_protection(monkeypatch):
def fail_on_call(*args, **kwargs):
raise RuntimeError("This should never be called")
+ def ignore_call(*args, **kwargs):
+ pass
+
monkeypatch.setattr(subprocess, 'Popen', fail_on_call)
monkeypatch.setattr(util, 'download', fail_on_call)
monkeypatch.setattr(windows, 'build', fail_on_call)
monkeypatch.setattr(linux, 'build', fail_on_call)
monkeypatch.setattr(macos, 'build', fail_on_call)
+ monkeypatch.setattr(Path, 'mkdir', ignore_call)
+
@pytest.fixture(autouse=True)
def fake_package_dir(monkeypatch):
'''
Monkey-patch enough for the main() function to run
'''
- real_os_path_exists = os.path.exists
+ real_path_exists = Path.exists
- def mock_os_path_exists(path):
- if path == os.path.join(MOCK_PACKAGE_DIR, 'setup.py'):
+ def mock_path_exists(path):
+ if path == MOCK_PACKAGE_DIR / 'setup.py':
return True
else:
- return real_os_path_exists(path)
+ return real_path_exists(path)
- monkeypatch.setattr(os.path, 'exists', mock_os_path_exists)
- monkeypatch.setattr(sys, 'argv', ['cibuildwheel', MOCK_PACKAGE_DIR])
+ monkeypatch.setattr(Path, 'exists', mock_path_exists)
+ monkeypatch.setattr(sys, 'argv', ['cibuildwheel', str(MOCK_PACKAGE_DIR)])
@pytest.fixture(params=['linux', 'macos', 'windows'])
diff --git a/unit_test/main_tests/main_options_test.py b/unit_test/main_tests/main_options_test.py
index 633e9cc5..14252e67 100644
--- a/unit_test/main_tests/main_options_test.py
+++ b/unit_test/main_tests/main_options_test.py
@@ -1,5 +1,6 @@
import sys
from fnmatch import fnmatch
+from pathlib import Path
import pytest
@@ -12,9 +13,9 @@ from cibuildwheel.util import BuildSelector
def test_output_dir(platform, intercepted_build_args, monkeypatch):
- OUTPUT_DIR = 'some_output_dir'
+ OUTPUT_DIR = Path('some_output_dir')
- monkeypatch.setenv('CIBW_OUTPUT_DIR', OUTPUT_DIR)
+ monkeypatch.setenv('CIBW_OUTPUT_DIR', str(OUTPUT_DIR))
main()
@@ -24,14 +25,14 @@ def test_output_dir(platform, intercepted_build_args, monkeypatch):
def test_output_dir_default(platform, intercepted_build_args, monkeypatch):
main()
- assert intercepted_build_args.args[0].output_dir == 'wheelhouse'
+ assert intercepted_build_args.args[0].output_dir == Path('wheelhouse')
@pytest.mark.parametrize('also_set_environment', [False, True])
def test_output_dir_argument(also_set_environment, platform, intercepted_build_args, monkeypatch):
- OUTPUT_DIR = 'some_output_dir'
+ OUTPUT_DIR = Path('some_output_dir')
- monkeypatch.setattr(sys, 'argv', sys.argv + ['--output-dir', OUTPUT_DIR])
+ monkeypatch.setattr(sys, 'argv', sys.argv + ['--output-dir', str(OUTPUT_DIR)])
if also_set_environment:
monkeypatch.setenv('CIBW_OUTPUT_DIR', 'not_this_output_dir')