Use pathlib.Path in main cibuildwheel files

This commit is contained in:
Yannick Jadoul
2020-06-18 00:23:25 +02:00
parent 43fbcfe008
commit a7f73f1593
5 changed files with 139 additions and 135 deletions
+9 -9
View File
@@ -4,6 +4,7 @@ import sys
import textwrap import textwrap
import traceback import traceback
from configparser import ConfigParser from configparser import ConfigParser
from pathlib import Path
from typing import Any, Dict, List, Optional, overload from typing import Any, Dict, List, Optional, overload
@@ -114,8 +115,8 @@ def main() -> None:
file=sys.stderr) file=sys.stderr)
exit(2) exit(2)
package_dir = args.package_dir package_dir = Path(args.package_dir)
output_dir = args.output_dir output_dir = Path(args.output_dir)
if platform == 'linux': if platform == 'linux':
repair_command_default = 'auditwheel repair -w {dest_dir} {wheel}' repair_command_default = 'auditwheel repair -w {dest_dir} {wheel}'
@@ -149,7 +150,8 @@ def main() -> None:
elif dependency_versions == 'latest': elif dependency_versions == 'latest':
dependency_constraints = None dependency_constraints = None
else: else:
dependency_constraints = DependencyConstraints(dependency_versions) dependency_versions_path = Path(dependency_versions)
dependency_constraints = DependencyConstraints(dependency_versions_path)
if test_extras: if test_extras:
test_extras = f'[{test_extras}]' test_extras = f'[{test_extras}]'
@@ -163,7 +165,7 @@ def main() -> None:
# This needs to be passed on to the docker container in linux.py # This needs to be passed on to the docker container in linux.py
os.environ['CIBUILDWHEEL'] = '1' os.environ['CIBUILDWHEEL'] = '1'
if not any(os.path.exists(os.path.join(package_dir, name)) if not any((package_dir / name).exists()
for name in ["setup.py", "setup.cfg", "pyproject.toml"]): for name in ["setup.py", "setup.cfg", "pyproject.toml"]):
print('cibuildwheel: Could not find setup.py, setup.cfg or pyproject.toml at root of package', file=sys.stderr) print('cibuildwheel: Could not find setup.py, setup.cfg or pyproject.toml at root of package', file=sys.stderr)
exit(2) exit(2)
@@ -174,9 +176,7 @@ def main() -> None:
manylinux_images: Optional[Dict[str, str]] = None manylinux_images: Optional[Dict[str, str]] = None
if platform == 'linux': if platform == 'linux':
pinned_docker_images_file = os.path.join( pinned_docker_images_file = Path(__file__).parent / 'resources' / 'pinned_docker_images.cfg'
os.path.dirname(__file__), 'resources', 'pinned_docker_images.cfg'
)
all_pinned_docker_images = ConfigParser() all_pinned_docker_images = ConfigParser()
all_pinned_docker_images.read(pinned_docker_images_file) all_pinned_docker_images.read(pinned_docker_images_file)
# all_pinned_docker_images looks like a dict of dicts, e.g. # all_pinned_docker_images looks like a dict of dicts, e.g.
@@ -224,8 +224,8 @@ def main() -> None:
print_preamble(platform, build_options) print_preamble(platform, build_options)
if not os.path.exists(output_dir): if not output_dir.exists():
os.makedirs(output_dir) output_dir.mkdir(parents=True)
if platform == 'linux': if platform == 'linux':
cibuildwheel.linux.build(build_options) cibuildwheel.linux.build(build_options)
+9 -11
View File
@@ -5,6 +5,7 @@ import subprocess
import sys import sys
import textwrap import textwrap
import uuid import uuid
from pathlib import Path
from typing import List, NamedTuple, Optional, Union from typing import List, NamedTuple, Optional, Union
@@ -104,10 +105,12 @@ def build(options: BuildOptions) -> None:
('pp', 'manylinux_x86_64', options.manylinux_images['pypy_x86_64']), ('pp', 'manylinux_x86_64', options.manylinux_images['pypy_x86_64']),
] ]
if not os.path.realpath(options.package_dir).startswith(os.path.realpath('.')): pwd = Path().resolve()
abs_package_dir = options.package_dir.resolve()
if pwd != abs_package_dir and pwd not in abs_package_dir.parents:
raise Exception('package_dir must be inside the working directory') raise Exception('package_dir must be inside the working directory')
container_package_dir = os.path.join('/project', os.path.relpath(options.package_dir, '.')) container_package_dir = Path('/project') / abs_package_dir.relative_to(pwd)
for implementation, platform_tag, docker_image in platforms: for implementation, platform_tag, docker_image in platforms:
platform_configs = [c for c in python_configurations if c.identifier.startswith(implementation) and c.identifier.endswith(platform_tag)] platform_configs = [c for c in python_configurations if c.identifier.startswith(implementation) and c.identifier.endswith(platform_tag)]
@@ -272,7 +275,7 @@ def build(options: BuildOptions) -> None:
# copy the output back into the host # copy the output back into the host
call(['docker', 'cp', call(['docker', 'cp',
container_name + ':/output/.', container_name + ':/output/.',
os.path.abspath(options.output_dir)]) str(options.output_dir.resolve())])
except subprocess.CalledProcessError as error: except subprocess.CalledProcessError as error:
troubleshoot(options.package_dir, error) troubleshoot(options.package_dir, error)
exit(1) exit(1)
@@ -281,16 +284,11 @@ def build(options: BuildOptions) -> None:
call(['docker', 'rm', '--force', '-v', container_name]) call(['docker', 'rm', '--force', '-v', container_name])
def troubleshoot(package_dir: str, error: Exception) -> None: def troubleshoot(package_dir: Path, error: Exception) -> None:
if (isinstance(error, subprocess.CalledProcessError) and 'exec' in error.cmd): if (isinstance(error, subprocess.CalledProcessError) and 'exec' in error.cmd):
# the bash script failed # the bash script failed
print('Checking for common errors...') print('Checking for common errors...')
so_files = [] so_files = list(package_dir.glob('**/*.so'))
for root, dirs, files in os.walk(package_dir):
for name in files:
_, ext = os.path.splitext(name)
if ext == '.so':
so_files.append(os.path.join(root, name))
if so_files: if so_files:
print(textwrap.dedent(''' print(textwrap.dedent('''
@@ -304,5 +302,5 @@ def troubleshoot(package_dir: str, error: Exception) -> None:
''')) '''))
print(' Files detected:') print(' Files detected:')
print('\n'.join([' ' + f for f in so_files])) print('\n'.join([' ' + str(f) for f in so_files]))
print('') print('')
+50 -47
View File
@@ -5,6 +5,7 @@ import subprocess
import sys import sys
import tempfile import tempfile
from glob import glob from glob import glob
from pathlib import Path
from typing import Dict, List, Optional, NamedTuple, Union from typing import Dict, List, Optional, NamedTuple, Union
@@ -24,7 +25,7 @@ def call(args: Union[str, List[str]], env: Optional[Dict[str, str]] = None, cwd:
if shell: if shell:
print(f'+ {args}') print(f'+ {args}')
else: else:
print('+ ' + ' '.join(shlex.quote(a) for a in args)) print('+ ' + ' '.join(shlex.quote(str(a)) for a in args))
return subprocess.check_call(args, env=env, cwd=cwd, shell=shell) return subprocess.check_call(args, env=env, cwd=cwd, shell=shell)
@@ -52,41 +53,41 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi
return [c for c in python_configurations if build_selector(c.identifier)] return [c for c in python_configurations if build_selector(c.identifier)]
SYMLINKS_DIR = '/tmp/cibw_bin' SYMLINKS_DIR = Path('/tmp/cibw_bin')
def make_symlinks(installation_bin_path: str, python_executable: str, pip_executable: str) -> None: def make_symlinks(installation_bin_path: Path, python_executable: str, pip_executable: str) -> None:
assert os.path.exists(os.path.join(installation_bin_path, python_executable)) assert (installation_bin_path / python_executable).exists()
# Python bin folders on Mac don't symlink `python3` to `python`, and neither # Python bin folders on Mac don't symlink `python3` to `python`, and neither
# does PyPy for `pypy` or `pypy3`, so we do that so `python` and `pip` always # does PyPy for `pypy` or `pypy3`, so we do that so `python` and `pip` always
# point to the active configuration. # point to the active configuration.
if os.path.exists(SYMLINKS_DIR): if SYMLINKS_DIR.exists():
shutil.rmtree(SYMLINKS_DIR) shutil.rmtree(SYMLINKS_DIR)
os.makedirs(SYMLINKS_DIR) SYMLINKS_DIR.mkdir(parents=True)
os.symlink(os.path.join(installation_bin_path, python_executable), os.path.join(SYMLINKS_DIR, 'python')) (SYMLINKS_DIR / 'python').symlink_to(installation_bin_path / python_executable)
os.symlink(os.path.join(installation_bin_path, python_executable + '-config'), os.path.join(SYMLINKS_DIR, 'python-config')) (SYMLINKS_DIR / 'python-config').symlink_to(installation_bin_path / (python_executable + '-config'))
os.symlink(os.path.join(installation_bin_path, pip_executable), os.path.join(SYMLINKS_DIR, 'pip')) (SYMLINKS_DIR / 'pip').symlink_to(installation_bin_path / pip_executable)
def install_cpython(version: str, url: str) -> str: def install_cpython(version: str, url: str) -> Path:
installed_system_packages = subprocess.check_output(['pkgutil', '--pkgs'], universal_newlines=True).splitlines() installed_system_packages = subprocess.check_output(['pkgutil', '--pkgs'], universal_newlines=True).splitlines()
# if this version of python isn't installed, get it from python.org and install # if this version of python isn't installed, get it from python.org and install
python_package_identifier = f'org.python.Python.PythonFramework-{version}' python_package_identifier = f'org.python.Python.PythonFramework-{version}'
if python_package_identifier not in installed_system_packages: if python_package_identifier not in installed_system_packages:
# download the pkg # download the pkg
download(url, '/tmp/Python.pkg') download(url, Path('/tmp/Python.pkg'))
# install # install
call(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/']) call(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/'])
# patch open ssl # patch open ssl
if version == '3.5': if version == '3.5':
open_ssl_patch_url = f'https://github.com/mayeut/patch-macos-python-openssl/releases/download/v1.0.2u/patch-macos-python-{version}-openssl-v1.0.2u.tar.gz' open_ssl_patch_url = f'https://github.com/mayeut/patch-macos-python-openssl/releases/download/v1.0.2u/patch-macos-python-{version}-openssl-v1.0.2u.tar.gz'
download(open_ssl_patch_url, '/tmp/python-patch.tar.gz') download(open_ssl_patch_url, Path('/tmp/python-patch.tar.gz'))
call(['sudo', 'tar', '-C', f'/Library/Frameworks/Python.framework/Versions/{version}/', '-xmf', '/tmp/python-patch.tar.gz']) call(['sudo', 'tar', '-C', f'/Library/Frameworks/Python.framework/Versions/{version}/', '-xmf', '/tmp/python-patch.tar.gz'])
installation_bin_path = f'/Library/Frameworks/Python.framework/Versions/{version}/bin' installation_bin_path = Path(f'/Library/Frameworks/Python.framework/Versions/{version}/bin')
python_executable = 'python3' if version[0] == '3' else 'python' python_executable = 'python3' if version[0] == '3' else 'python'
pip_executable = 'pip3' if version[0] == '3' else 'pip' pip_executable = 'pip3' if version[0] == '3' else 'pip'
make_symlinks(installation_bin_path, python_executable, pip_executable) make_symlinks(installation_bin_path, python_executable, pip_executable)
@@ -94,16 +95,18 @@ def install_cpython(version: str, url: str) -> str:
return installation_bin_path return installation_bin_path
def install_pypy(version: str, url: str) -> str: def install_pypy(version: str, url: str) -> Path:
pypy_tar_bz2 = url.rsplit('/', 1)[-1] pypy_tar_bz2 = url.rsplit('/', 1)[-1]
assert pypy_tar_bz2.endswith(".tar.bz2") extension = ".tar.bz2"
pypy_base_filename = os.path.splitext(os.path.splitext(pypy_tar_bz2)[0])[0] assert pypy_tar_bz2.endswith(extension)
installation_path = os.path.join('/tmp', pypy_base_filename) pypy_base_filename = pypy_tar_bz2[:-len(extension)]
if not os.path.exists(installation_path): installation_path = Path('/tmp') / pypy_base_filename
download(url, os.path.join("/tmp", pypy_tar_bz2)) if not installation_path.exists():
call(['tar', '-C', '/tmp', '-xf', os.path.join("/tmp", pypy_tar_bz2)]) downloaded_tar_bz2 = Path("/tmp") / pypy_tar_bz2
download(url, downloaded_tar_bz2)
call(['tar', '-C', '/tmp', '-xf', str(downloaded_tar_bz2)])
installation_bin_path = os.path.join(installation_path, 'bin') installation_bin_path = installation_path / 'bin'
python_executable = 'pypy3' if version[0] == '3' else 'pypy' python_executable = 'pypy3' if version[0] == '3' else 'pypy'
pip_executable = 'pip3' if version[0] == '3' else 'pip' pip_executable = 'pip3' if version[0] == '3' else 'pip'
make_symlinks(installation_bin_path, python_executable, pip_executable) make_symlinks(installation_bin_path, python_executable, pip_executable)
@@ -121,8 +124,8 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain
env = os.environ.copy() env = os.environ.copy()
env['PATH'] = os.pathsep.join([ env['PATH'] = os.pathsep.join([
SYMLINKS_DIR, str(SYMLINKS_DIR),
installation_bin_path, str(installation_bin_path),
env['PATH'], env['PATH'],
]) ])
@@ -144,8 +147,8 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain
exit(1) exit(1)
# install pip & wheel # install pip & wheel
call(['python', get_pip_script] + dependency_constraint_flags, env=env, cwd="/tmp") call(['python', str(get_pip_script)] + dependency_constraint_flags, env=env, cwd="/tmp")
assert os.path.exists(os.path.join(installation_bin_path, 'pip')) assert (installation_bin_path / 'pip').exists()
call(['which', 'pip'], env=env) call(['which', 'pip'], env=env)
call(['pip', '--version'], env=env) call(['pip', '--version'], env=env)
which_pip = subprocess.check_output(['which', 'pip'], env=env, universal_newlines=True).strip() which_pip = subprocess.check_output(['which', 'pip'], env=env, universal_newlines=True).strip()
@@ -170,9 +173,9 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain
def build(options: BuildOptions) -> None: def build(options: BuildOptions) -> None:
temp_dir = tempfile.mkdtemp(prefix='cibuildwheel') temp_dir = Path(tempfile.mkdtemp(prefix='cibuildwheel'))
built_wheel_dir = os.path.join(temp_dir, 'built_wheel') built_wheel_dir = temp_dir / 'built_wheel'
repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel') repaired_wheel_dir = temp_dir / 'repaired_wheel'
python_configurations = get_python_configurations(options.build_selector) python_configurations = get_python_configurations(options.build_selector)
@@ -180,7 +183,7 @@ def build(options: BuildOptions) -> None:
dependency_constraint_flags = [] dependency_constraint_flags = []
if options.dependency_constraints: if options.dependency_constraints:
dependency_constraint_flags = [ dependency_constraint_flags = [
'-c', options.dependency_constraints.get_for_python_version(config.version) '-c', str(options.dependency_constraints.get_for_python_version(config.version))
] ]
env = setup_python(config, dependency_constraint_flags, options.environment) env = setup_python(config, dependency_constraint_flags, options.environment)
@@ -191,39 +194,39 @@ def build(options: BuildOptions) -> None:
call(before_build_prepared, env=env, shell=True) call(before_build_prepared, env=env, shell=True)
# build the wheel # build the wheel
if os.path.exists(built_wheel_dir): if built_wheel_dir.exists():
shutil.rmtree(built_wheel_dir) shutil.rmtree(built_wheel_dir)
os.makedirs(built_wheel_dir) built_wheel_dir.mkdir(parents=True)
# os.path.abspath is need. Without it pip wheel may try to fetch package from pypi.org # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
# see https://github.com/joerick/cibuildwheel/pull/369 # see https://github.com/joerick/cibuildwheel/pull/369
call(['pip', 'wheel', os.path.abspath(options.package_dir), '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env) call(['pip', 'wheel', str(options.package_dir.resolve()), '-w', str(built_wheel_dir), '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env)
built_wheel = glob(os.path.join(built_wheel_dir, '*.whl'))[0] built_wheel = next(built_wheel_dir.glob('*.whl'))
# repair the wheel # repair the wheel
if os.path.exists(repaired_wheel_dir): if repaired_wheel_dir.exists():
shutil.rmtree(repaired_wheel_dir) shutil.rmtree(repaired_wheel_dir)
os.makedirs(repaired_wheel_dir) repaired_wheel_dir.mkdir(parents=True)
if built_wheel.endswith('none-any.whl') or not options.repair_command: if built_wheel.name.endswith('none-any.whl') or not options.repair_command:
# pure Python wheel or empty repair command # pure Python wheel or empty repair command
shutil.move(built_wheel, repaired_wheel_dir) shutil.move(str(built_wheel), repaired_wheel_dir)
else: else:
repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir)
call(repair_command_prepared, env=env, shell=True) call(repair_command_prepared, env=env, shell=True)
repaired_wheel = glob(os.path.join(repaired_wheel_dir, '*.whl'))[0] repaired_wheel = next(repaired_wheel_dir.glob('*.whl'))
if options.test_command: if options.test_command:
# set up a virtual environment to install and test from, to make sure # set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time. # there are no dependencies that were pulled in at build time.
call(['pip', 'install', 'virtualenv'] + dependency_constraint_flags, env=env) call(['pip', 'install', 'virtualenv'] + dependency_constraint_flags, env=env)
venv_dir = tempfile.mkdtemp() venv_dir = Path(tempfile.mkdtemp())
# Use --no-download to ensure determinism by using seed libraries # Use --no-download to ensure determinism by using seed libraries
# built into virtualenv # built into virtualenv
call(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) call(['python', '-m', 'virtualenv', '--no-download', str(venv_dir)], env=env)
virtualenv_env = env.copy() virtualenv_env = env.copy()
virtualenv_env['PATH'] = os.pathsep.join([ virtualenv_env['PATH'] = os.pathsep.join([
os.path.join(venv_dir, 'bin'), str(venv_dir / 'bin'),
virtualenv_env['PATH'], virtualenv_env['PATH'],
]) ])
@@ -235,7 +238,7 @@ def build(options: BuildOptions) -> None:
call(before_test_prepared, env=virtualenv_env, shell=True) call(before_test_prepared, env=virtualenv_env, shell=True)
# install the wheel # install the wheel
call(['pip', 'install', repaired_wheel + options.test_extras], env=virtualenv_env) call(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env)
# test the wheel # test the wheel
if options.test_requires: if options.test_requires:
@@ -246,8 +249,8 @@ def build(options: BuildOptions) -> None:
# and not the repo code) # and not the repo code)
test_command_prepared = prepare_command( test_command_prepared = prepare_command(
options.test_command, options.test_command,
project=os.path.abspath('.'), project=Path('.').resolve(),
package=os.path.abspath(options.package_dir) package=options.package_dir
) )
call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True) call(test_command_prepared, cwd=os.environ['HOME'], env=virtualenv_env, shell=True)
@@ -255,5 +258,5 @@ def build(options: BuildOptions) -> None:
shutil.rmtree(venv_dir) shutil.rmtree(venv_dir)
# we're all done here; move it to output (overwrite existing) # we're all done here; move it to output (overwrite existing)
dst = os.path.join(options.output_dir, os.path.basename(repaired_wheel)) dst = options.output_dir / repaired_wheel.name
shutil.move(repaired_wheel, dst) shutil.move(str(repaired_wheel), dst)
+22 -22
View File
@@ -1,14 +1,15 @@
import os import os
import urllib.request import urllib.request
from fnmatch import fnmatch from fnmatch import fnmatch
from pathlib import Path
from time import sleep from time import sleep
from typing import Dict, List, NamedTuple, Optional from typing import Dict, List, NamedTuple, Optional, Union
from .environment import ParsedEnvironment from .environment import ParsedEnvironment
def prepare_command(command: str, **kwargs: str) -> str: def prepare_command(command: str, **kwargs: Union[str, os.PathLike]) -> str:
''' '''
Preprocesses a command by expanding variables like {python}. Preprocesses a command by expanding variables like {python}.
@@ -58,11 +59,11 @@ class Unbuffered:
return getattr(self.stream, attr) return getattr(self.stream, attr)
def download(url: str, dest: str) -> None: def download(url: str, dest: Path) -> None:
print('+ Download ' + url + ' to ' + dest) print(f'+ Download {url} to {dest}')
dest_dir = os.path.dirname(dest) dest_dir = dest.parent
if not os.path.exists(dest_dir): if not dest_dir.exists():
os.makedirs(dest_dir) dest_dir.mkdir(parents=True)
repeat_num = 3 repeat_num = 3
for i in range(repeat_num): for i in range(repeat_num):
@@ -76,40 +77,39 @@ def download(url: str, dest: str) -> None:
break break
try: try:
with open(dest, 'wb') as file: dest.write_bytes(response.read())
file.write(response.read())
finally: finally:
response.close() response.close()
class DependencyConstraints: class DependencyConstraints:
def __init__(self, base_file_path: str): def __init__(self, base_file_path: Path):
assert os.path.exists(base_file_path) assert base_file_path.exists()
self.base_file_path = os.path.abspath(base_file_path) self.base_file_path = base_file_path.resolve()
@staticmethod @staticmethod
def with_defaults() -> 'DependencyConstraints': def with_defaults() -> 'DependencyConstraints':
return DependencyConstraints( return DependencyConstraints(
base_file_path=os.path.join(os.path.dirname(__file__), 'resources', 'constraints.txt') base_file_path=Path(__file__).parent / 'resources' / 'constraints.txt'
) )
def get_for_python_version(self, version: str) -> str: def get_for_python_version(self, version: str) -> Path:
version_parts = version.split('.') version_parts = version.split('.')
# try to find a version-specific dependency file e.g. if # try to find a version-specific dependency file e.g. if
# ./constraints.txt is the base, look for ./constraints-python27.txt # ./constraints.txt is the base, look for ./constraints-python27.txt
base, ext = os.path.splitext(self.base_file_path) specific_stem = self.base_file_path.stem + f'-python{version_parts[0]}{version_parts[1]}'
specific = base + f'-python{version_parts[0]}{version_parts[1]}' sepcific_name = specific_stem + self.base_file_path.suffix
specific_file_path = specific + ext specific_file_path = self.base_file_path.with_name(sepcific_name)
if os.path.exists(specific_file_path): if specific_file_path.exists():
return specific_file_path return specific_file_path
else: else:
return self.base_file_path return self.base_file_path
class BuildOptions(NamedTuple): class BuildOptions(NamedTuple):
package_dir: str package_dir: Path
output_dir: str output_dir: Path
build_selector: BuildSelector build_selector: BuildSelector
environment: ParsedEnvironment environment: ParsedEnvironment
before_build: Optional[str] before_build: Optional[str]
@@ -123,5 +123,5 @@ class BuildOptions(NamedTuple):
build_verbosity: int build_verbosity: int
resources_dir = os.path.abspath(os.path.join(os.path.dirname(__file__), 'resources')) resources_dir = Path(__file__).resolve().parent / 'resources'
get_pip_script = os.path.join(resources_dir, 'get-pip.py') get_pip_script = resources_dir / 'get-pip.py'
+49 -46
View File
@@ -4,6 +4,7 @@ import subprocess
import sys import sys
import tempfile import tempfile
from glob import glob from glob import glob
from pathlib import Path
from zipfile import ZipFile from zipfile import ZipFile
from typing import Dict, List, Optional, NamedTuple from typing import Dict, List, Optional, NamedTuple
@@ -19,7 +20,7 @@ from .util import (
) )
IS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache') IS_RUNNING_ON_AZURE = Path('C:\\hostedtoolcache').exists()
IS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows' IS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows'
@@ -71,36 +72,38 @@ def get_python_configurations(build_selector: BuildSelector) -> List[PythonConfi
return python_configurations return python_configurations
def extract_zip(zip_src: str, dest: str) -> None: def extract_zip(zip_src: Path, dest: Path) -> None:
with ZipFile(zip_src) as zip: with ZipFile(zip_src) as zip:
zip.extractall(dest) zip.extractall(dest)
def install_cpython(version: str, arch: str, nuget: str) -> str: def install_cpython(version: str, arch: str, nuget: Path) -> Path:
nuget_args = get_nuget_args(version, arch) nuget_args = get_nuget_args(version, arch)
installation_path = os.path.join(nuget_args[-1], nuget_args[0] + '.' + version, 'tools') installation_path = Path(nuget_args[-1]) / (nuget_args[0] + '.' + version) / 'tools'
shell([nuget, 'install'] + nuget_args) shell([str(nuget), 'install'] + nuget_args)
return installation_path return installation_path
def install_pypy(version: str, arch: str, url: str) -> str: def install_pypy(version: str, arch: str, url: str) -> Path:
assert arch == '32' assert arch == '32'
# Inside the PyPy zip file is a directory with the same name # Inside the PyPy zip file is a directory with the same name
zip_filename = url.rsplit('/', 1)[-1] zip_filename = url.rsplit('/', 1)[-1]
installation_path = os.path.join('C:\\cibw', os.path.splitext(zip_filename)[0]) extension = ".zip"
if not os.path.exists(installation_path): assert zip_filename.endswith(extension)
pypy_zip = os.path.join('C:\\cibw', zip_filename) installation_path = Path('C:\\cibw') / zip_filename[:-len(extension)]
if not installation_path.exists():
pypy_zip = Path('C:\\cibw') / zip_filename
download(url, pypy_zip) download(url, pypy_zip)
# Extract to the parent directory because the zip file still contains a directory # Extract to the parent directory because the zip file still contains a directory
extract_zip(pypy_zip, os.path.dirname(installation_path)) extract_zip(pypy_zip, installation_path.parent)
pypy_exe = 'pypy3.exe' if version[0] == '3' else 'pypy.exe' pypy_exe = 'pypy3.exe' if version[0] == '3' else 'pypy.exe'
shell(['mklink', os.path.join(installation_path, 'python.exe'), os.path.join(installation_path, pypy_exe)]) (installation_path / pypy_exe).symlink_to(installation_path / 'python.exe')
return installation_path return installation_path
def setup_python(python_configuration: PythonConfiguration, dependency_constraint_flags: List[str], environment: ParsedEnvironment) -> Dict[str, str]: def setup_python(python_configuration: PythonConfiguration, dependency_constraint_flags: List[str], environment: ParsedEnvironment) -> Dict[str, str]:
nuget = 'C:\\cibw\\nuget.exe' nuget = Path('C:\\cibw\\nuget.exe')
if not os.path.exists(nuget): if not nuget.exists():
download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget) download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget)
if python_configuration.identifier.startswith('cp'): if python_configuration.identifier.startswith('cp'):
@@ -111,15 +114,15 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain
else: else:
raise ValueError("Unknown Python implementation") raise ValueError("Unknown Python implementation")
assert os.path.exists(os.path.join(installation_path, 'python.exe')) assert (installation_path / 'python.exe').exists()
# set up PATH and environment variables for run_with_env # set up PATH and environment variables for run_with_env
env = os.environ.copy() env = os.environ.copy()
env['PYTHON_VERSION'] = python_configuration.version env['PYTHON_VERSION'] = python_configuration.version
env['PYTHON_ARCH'] = python_configuration.arch env['PYTHON_ARCH'] = python_configuration.arch
env['PATH'] = os.pathsep.join([ env['PATH'] = os.pathsep.join([
installation_path, str(installation_path),
os.path.join(installation_path, 'Scripts'), str(installation_path / 'Scripts'),
env['PATH'] env['PATH']
]) ])
# update env with results from CIBW_ENVIRONMENT # update env with results from CIBW_ENVIRONMENT
@@ -130,16 +133,16 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain
shell(['python', '--version'], env=env) shell(['python', '--version'], env=env)
shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)"'], env=env) shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)"'], env=env)
where_python = subprocess.check_output(['where', 'python'], env=env, universal_newlines=True).splitlines()[0].strip() where_python = subprocess.check_output(['where', 'python'], env=env, universal_newlines=True).splitlines()[0].strip()
if where_python != os.path.join(installation_path, 'python.exe'): if where_python != str(installation_path / 'python.exe'):
print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr) print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr)
exit(1) exit(1)
# make sure pip is installed # make sure pip is installed
if not os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe')): if not (installation_path / 'Scripts' / 'pip.exe').exists():
shell(['python', get_pip_script] + dependency_constraint_flags, env=env, cwd="C:\\cibw") shell(['python', str(get_pip_script)] + dependency_constraint_flags, env=env, cwd="C:\\cibw")
assert os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe')) assert (installation_path / 'Scripts' / 'pip.exe').exists()
where_pip = subprocess.check_output(['where', 'pip'], env=env, universal_newlines=True).splitlines()[0].strip() where_pip = subprocess.check_output(['where', 'pip'], env=env, universal_newlines=True).splitlines()[0].strip()
if where_pip.strip() != os.path.join(installation_path, 'Scripts', 'pip.exe'): if where_pip.strip() != str(installation_path / 'Scripts' / 'pip.exe'):
print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr) print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr)
exit(1) exit(1)
@@ -152,12 +155,12 @@ def setup_python(python_configuration: PythonConfiguration, dependency_constrain
def build(options: BuildOptions) -> None: def build(options: BuildOptions) -> None:
temp_dir = tempfile.mkdtemp(prefix='cibuildwheel') temp_dir = Path(tempfile.mkdtemp(prefix='cibuildwheel'))
built_wheel_dir = os.path.join(temp_dir, 'built_wheel') built_wheel_dir = temp_dir / 'built_wheel'
repaired_wheel_dir = os.path.join(temp_dir, 'repaired_wheel') repaired_wheel_dir = temp_dir / 'repaired_wheel'
# install nuget as best way to provide python # install nuget as best way to provide python
nuget = 'C:\\cibw\\nuget.exe' nuget = Path('C:\\cibw\\nuget.exe')
download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget) download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget)
python_configurations = get_python_configurations(options.build_selector) python_configurations = get_python_configurations(options.build_selector)
@@ -165,7 +168,7 @@ def build(options: BuildOptions) -> None:
dependency_constraint_flags = [] dependency_constraint_flags = []
if options.dependency_constraints: if options.dependency_constraints:
dependency_constraint_flags = [ dependency_constraint_flags = [
'-c', options.dependency_constraints.get_for_python_version(config.version) '-c', str(options.dependency_constraints.get_for_python_version(config.version))
] ]
# install Python # install Python
@@ -177,39 +180,39 @@ def build(options: BuildOptions) -> None:
shell([before_build_prepared], env=env) shell([before_build_prepared], env=env)
# build the wheel # build the wheel
if os.path.exists(built_wheel_dir): if built_wheel_dir.exists():
shutil.rmtree(built_wheel_dir) shutil.rmtree(built_wheel_dir)
os.makedirs(built_wheel_dir) built_wheel_dir.mkdir(parents=True)
# os.path.abspath is need. Without it pip wheel may try to fetch package from pypi.org # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org
# see https://github.com/joerick/cibuildwheel/pull/369 # see https://github.com/joerick/cibuildwheel/pull/369
shell(['pip', 'wheel', os.path.abspath(options.package_dir), '-w', built_wheel_dir, '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env) shell(['pip', 'wheel', str(options.package_dir.resolve()), '-w', str(built_wheel_dir), '--no-deps'] + get_build_verbosity_extra_flags(options.build_verbosity), env=env)
built_wheel = glob(os.path.join(built_wheel_dir, '*.whl'))[0] built_wheel = next(built_wheel_dir.glob('*.whl'))
# repair the wheel # repair the wheel
if os.path.exists(repaired_wheel_dir): if repaired_wheel_dir.exists():
shutil.rmtree(repaired_wheel_dir) shutil.rmtree(repaired_wheel_dir)
os.makedirs(repaired_wheel_dir) repaired_wheel_dir.mkdir(parents=True)
if built_wheel.endswith('none-any.whl') or not options.repair_command: if built_wheel.name.endswith('none-any.whl') or not options.repair_command:
# pure Python wheel or empty repair command # pure Python wheel or empty repair command
shutil.move(built_wheel, repaired_wheel_dir) shutil.move(str(built_wheel), repaired_wheel_dir)
else: else:
repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir) repair_command_prepared = prepare_command(options.repair_command, wheel=built_wheel, dest_dir=repaired_wheel_dir)
shell([repair_command_prepared], env=env) shell([repair_command_prepared], env=env)
repaired_wheel = glob(os.path.join(repaired_wheel_dir, '*.whl'))[0] repaired_wheel = next(repaired_wheel_dir.glob('*.whl'))
if options.test_command: if options.test_command:
# set up a virtual environment to install and test from, to make sure # set up a virtual environment to install and test from, to make sure
# there are no dependencies that were pulled in at build time. # there are no dependencies that were pulled in at build time.
shell(['pip', 'install', 'virtualenv'] + dependency_constraint_flags, env=env) shell(['pip', 'install', 'virtualenv'] + dependency_constraint_flags, env=env)
venv_dir = tempfile.mkdtemp() venv_dir = Path(tempfile.mkdtemp())
# Use --no-download to ensure determinism by using seed libraries # Use --no-download to ensure determinism by using seed libraries
# built into virtualenv # built into virtualenv
shell(['python', '-m', 'virtualenv', '--no-download', venv_dir], env=env) shell(['python', '-m', 'virtualenv', '--no-download', str(venv_dir)], env=env)
virtualenv_env = env.copy() virtualenv_env = env.copy()
virtualenv_env['PATH'] = os.pathsep.join([ virtualenv_env['PATH'] = os.pathsep.join([
os.path.join(venv_dir, 'Scripts'), str(venv_dir / 'Scripts'),
virtualenv_env['PATH'], virtualenv_env['PATH'],
]) ])
@@ -225,7 +228,7 @@ def build(options: BuildOptions) -> None:
shell([before_test_prepared], env=virtualenv_env) shell([before_test_prepared], env=virtualenv_env)
# install the wheel # install the wheel
shell(['pip', 'install', repaired_wheel + options.test_extras], env=virtualenv_env) shell(['pip', 'install', str(repaired_wheel) + options.test_extras], env=virtualenv_env)
# test the wheel # test the wheel
if options.test_requires: if options.test_requires:
@@ -236,8 +239,8 @@ def build(options: BuildOptions) -> None:
# and not the repo code) # and not the repo code)
test_command_prepared = prepare_command( test_command_prepared = prepare_command(
options.test_command, options.test_command,
project=os.path.abspath('.'), project=Path('.').resolve(),
package=os.path.abspath(options.package_dir) package=options.package_dir.resolve()
) )
shell([test_command_prepared], cwd='c:\\', env=virtualenv_env) shell([test_command_prepared], cwd='c:\\', env=virtualenv_env)
@@ -245,7 +248,7 @@ def build(options: BuildOptions) -> None:
shutil.rmtree(venv_dir) shutil.rmtree(venv_dir)
# we're all done here; move it to output (remove if already exists) # we're all done here; move it to output (remove if already exists)
dst = os.path.join(options.output_dir, os.path.basename(repaired_wheel)) dst = options.output_dir / repaired_wheel.name
if os.path.isfile(dst): if dst.is_file():
os.remove(dst) dst.unlink()
shutil.move(repaired_wheel, dst) shutil.move(str(repaired_wheel), dst)