Merge remote-tracking branch 'origin/master' into deterministic-builds
This commit is contained in:
Joe Rickerby
@@ -170,6 +170,10 @@ def main():
|
||||
if platform == 'linux':
|
||||
manylinux_x86_64_image = os.environ.get('CIBW_MANYLINUX_X86_64_IMAGE', 'manylinux2010')
|
||||
manylinux_i686_image = os.environ.get('CIBW_MANYLINUX_I686_IMAGE', 'manylinux2010')
|
||||
manylinux_pypy_x86_64_image = os.environ.get('CIBW_MANYLINUX_PYPY_X86_64_IMAGE', 'manylinux2010')
|
||||
manylinux_aarch64_image = os.environ.get('CIBW_MANYLINUX_AARCH64_IMAGE', 'manylinux2014')
|
||||
manylinux_ppc64le_image = os.environ.get('CIBW_MANYLINUX_PPC64LE_IMAGE', 'manylinux2014')
|
||||
manylinux_s390x_image = os.environ.get('CIBW_MANYLINUX_S390X_IMAGE', 'manylinux2014')
|
||||
|
||||
pinned_docker_images_file = os.path.join(
|
||||
os.path.dirname(__file__), 'resources', 'pinned_docker_images.cfg'
|
||||
@@ -194,6 +198,28 @@ def main():
|
||||
else:
|
||||
manylinux_images['i686'] = manylinux_i686_image
|
||||
|
||||
if manylinux_pypy_x86_64_image in pinned_docker_images:
|
||||
manylinux_images['pypy_x86_64'] = pinned_docker_images[manylinux_pypy_x86_64_image]['pypy_x86_64']
|
||||
else:
|
||||
manylinux_images['pypy_x86_64'] = manylinux_pypy_x86_64_image
|
||||
|
||||
if manylinux_aarch64_image in pinned_docker_images:
|
||||
manylinux_images['aarch64'] = pinned_docker_images[manylinux_aarch64_image]['aarch64']
|
||||
else:
|
||||
manylinux_images['aarch64'] = manylinux_aarch64_image
|
||||
|
||||
if manylinux_ppc64le_image in pinned_docker_images:
|
||||
manylinux_images['ppc64le'] = pinned_docker_images[manylinux_ppc64le_image]['ppc64le']
|
||||
else:
|
||||
manylinux_images['ppc64le'] = manylinux_ppc64le_image
|
||||
|
||||
if manylinux_s390x_image in pinned_docker_images:
|
||||
manylinux_images['x390x'] = pinned_docker_images[manylinux_s390x_image]['x390x']
|
||||
else:
|
||||
manylinux_images['x390x'] = manylinux_s390x_image
|
||||
|
||||
manylinux_images['pypy_x86_64'] = ''
|
||||
|
||||
build_options.update(
|
||||
manylinux_images=manylinux_images
|
||||
)
|
||||
|
||||
+66
-22
@@ -1,4 +1,5 @@
|
||||
import os
|
||||
import platform
|
||||
import shlex
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -12,25 +13,59 @@ from .util import (
|
||||
)
|
||||
|
||||
|
||||
def matches_platform(identifier):
|
||||
pm = platform.machine()
|
||||
if pm == "x86_64":
|
||||
# x86_64 machines can run i686 docker containers
|
||||
if identifier.endswith('x86_64') or identifier.endswith('i686'):
|
||||
return True
|
||||
elif pm == "i686":
|
||||
if identifier.endswith('i686'):
|
||||
return True
|
||||
elif pm == "aarch64":
|
||||
if identifier.endswith('aarch64'):
|
||||
return True
|
||||
elif pm == "ppc64le":
|
||||
if identifier.endswith('ppc64le'):
|
||||
return True
|
||||
elif pm == "s390x":
|
||||
if identifier.endswith('s390x'):
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def get_python_configurations(build_selector):
|
||||
PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'path'])
|
||||
python_configurations = [
|
||||
PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path='/opt/python/cp27-cp27m'),
|
||||
PythonConfiguration(version='2.7', identifier='cp27-manylinux_x86_64', path='/opt/python/cp27-cp27mu'),
|
||||
PythonConfiguration(version='3.5', identifier='cp35-manylinux_x86_64', path='/opt/python/cp35-cp35m'),
|
||||
PythonConfiguration(version='3.6', identifier='cp36-manylinux_x86_64', path='/opt/python/cp36-cp36m'),
|
||||
PythonConfiguration(version='3.7', identifier='cp37-manylinux_x86_64', path='/opt/python/cp37-cp37m'),
|
||||
PythonConfiguration(version='3.8', identifier='cp38-manylinux_x86_64', path='/opt/python/cp38-cp38'),
|
||||
PythonConfiguration(version='2.7', identifier='cp27-manylinux_i686', path='/opt/python/cp27-cp27m'),
|
||||
PythonConfiguration(version='2.7', identifier='cp27-manylinux_i686', path='/opt/python/cp27-cp27mu'),
|
||||
PythonConfiguration(version='3.5', identifier='cp35-manylinux_i686', path='/opt/python/cp35-cp35m'),
|
||||
PythonConfiguration(version='3.6', identifier='cp36-manylinux_i686', path='/opt/python/cp36-cp36m'),
|
||||
PythonConfiguration(version='3.7', identifier='cp37-manylinux_i686', path='/opt/python/cp37-cp37m'),
|
||||
PythonConfiguration(version='3.8', identifier='cp38-manylinux_i686', path='/opt/python/cp38-cp38'),
|
||||
PythonConfiguration(identifier='cp27-manylinux_x86_64', path='/opt/python/cp27-cp27m'),
|
||||
PythonConfiguration(identifier='cp27-manylinux_x86_64', path='/opt/python/cp27-cp27mu'),
|
||||
PythonConfiguration(identifier='cp35-manylinux_x86_64', path='/opt/python/cp35-cp35m'),
|
||||
PythonConfiguration(identifier='cp36-manylinux_x86_64', path='/opt/python/cp36-cp36m'),
|
||||
PythonConfiguration(identifier='cp37-manylinux_x86_64', path='/opt/python/cp37-cp37m'),
|
||||
PythonConfiguration(identifier='cp38-manylinux_x86_64', path='/opt/python/cp38-cp38'),
|
||||
PythonConfiguration(identifier='cp27-manylinux_i686', path='/opt/python/cp27-cp27m'),
|
||||
PythonConfiguration(identifier='cp27-manylinux_i686', path='/opt/python/cp27-cp27mu'),
|
||||
PythonConfiguration(identifier='cp35-manylinux_i686', path='/opt/python/cp35-cp35m'),
|
||||
PythonConfiguration(identifier='cp36-manylinux_i686', path='/opt/python/cp36-cp36m'),
|
||||
PythonConfiguration(identifier='cp37-manylinux_i686', path='/opt/python/cp37-cp37m'),
|
||||
PythonConfiguration(identifier='cp38-manylinux_i686', path='/opt/python/cp38-cp38'),
|
||||
PythonConfiguration(identifier='pp27-manylinux_x86_64', path='/opt/python/pp27-pypy_73'),
|
||||
PythonConfiguration(identifier='pp36-manylinux_x86_64', path='/opt/python/pp36-pypy36_pp73'),
|
||||
PythonConfiguration(identifier='cp35-manylinux_aarch64', path='/opt/python/cp35-cp35m'),
|
||||
PythonConfiguration(identifier='cp36-manylinux_aarch64', path='/opt/python/cp36-cp36m'),
|
||||
PythonConfiguration(identifier='cp37-manylinux_aarch64', path='/opt/python/cp37-cp37m'),
|
||||
PythonConfiguration(identifier='cp38-manylinux_aarch64', path='/opt/python/cp38-cp38'),
|
||||
PythonConfiguration(identifier='cp35-manylinux_ppc64le', path='/opt/python/cp35-cp35m'),
|
||||
PythonConfiguration(identifier='cp36-manylinux_ppc64le', path='/opt/python/cp36-cp36m'),
|
||||
PythonConfiguration(identifier='cp37-manylinux_ppc64le', path='/opt/python/cp37-cp37m'),
|
||||
PythonConfiguration(identifier='cp38-manylinux_ppc64le', path='/opt/python/cp38-cp38'),
|
||||
PythonConfiguration(identifier='cp35-manylinux_s390x', path='/opt/python/cp35-cp35m'),
|
||||
PythonConfiguration(identifier='cp36-manylinux_s390x', path='/opt/python/cp36-cp36m'),
|
||||
PythonConfiguration(identifier='cp37-manylinux_s390x', path='/opt/python/cp37-cp37m'),
|
||||
PythonConfiguration(identifier='cp38-manylinux_s390x', path='/opt/python/cp38-cp38'),
|
||||
]
|
||||
|
||||
# skip builds as required
|
||||
return [c for c in python_configurations if build_selector(c.identifier)]
|
||||
return [c for c in python_configurations if matches_platform(c.identifier) and build_selector(c.identifier)]
|
||||
|
||||
|
||||
def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, manylinux_images, dependency_constraints):
|
||||
@@ -45,12 +80,16 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
|
||||
|
||||
python_configurations = get_python_configurations(build_selector)
|
||||
platforms = [
|
||||
('manylinux_x86_64', manylinux_images['x86_64']),
|
||||
('manylinux_i686', manylinux_images['i686']),
|
||||
('cp', 'manylinux_x86_64', manylinux_images['x86_64']),
|
||||
('cp', 'manylinux_i686', manylinux_images['i686']),
|
||||
('cp', 'manylinux_aarch64', manylinux_images['aarch64']),
|
||||
('cp', 'manylinux_ppc64le', manylinux_images['ppc64le']),
|
||||
('cp', 'manylinux_s390x', manylinux_images['s390x']),
|
||||
('pp', 'manylinux_x86_64', manylinux_images['pypy_x86_64']),
|
||||
]
|
||||
|
||||
for platform_tag, docker_image in platforms:
|
||||
platform_configs = [c for c in python_configurations if c.identifier.endswith(platform_tag)]
|
||||
for implementation, platform_tag, docker_image in platforms:
|
||||
platform_configs = [c for c in python_configurations if c.identifier.startswith(implementation) and c.identifier.endswith(platform_tag)]
|
||||
if not platform_configs:
|
||||
continue
|
||||
|
||||
@@ -84,14 +123,19 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
|
||||
cd /project
|
||||
|
||||
PYBIN="{config_python_bin}"
|
||||
export PATH="$PYBIN:$PATH"
|
||||
|
||||
export PATH="$PYBIN:$PATH"
|
||||
{environment_exports}
|
||||
|
||||
# check the active python and pip are in PYBIN
|
||||
# if `test` returns false, the script will exit due to errexit
|
||||
test "$(which pip)" = "$PYBIN/pip"
|
||||
test "$(which python)" = "$PYBIN/python"
|
||||
if [ "$(which pip)" != "$PYBIN/pip" ]; then
|
||||
echo "cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it."
|
||||
exit 1
|
||||
fi
|
||||
if [ "$(which python)" != "$PYBIN/python" ]; then
|
||||
echo "cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -z {before_build} ]; then
|
||||
sh -c {before_build}
|
||||
|
||||
+103
-43
@@ -2,6 +2,7 @@ import os
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from collections import namedtuple
|
||||
from glob import glob
|
||||
@@ -14,6 +15,16 @@ from .util import (
|
||||
)
|
||||
|
||||
|
||||
def call(args, env=None, cwd=None, shell=False):
|
||||
# print the command executing for the logs
|
||||
if shell:
|
||||
print('+ %s' % args)
|
||||
else:
|
||||
print('+ ' + ' '.join(shlex.quote(a) for a in args))
|
||||
|
||||
return subprocess.check_call(args, env=env, cwd=cwd, shell=shell)
|
||||
|
||||
|
||||
def get_python_configurations(build_selector):
|
||||
PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'url'])
|
||||
python_configurations = [
|
||||
@@ -22,12 +33,79 @@ def get_python_configurations(build_selector):
|
||||
PythonConfiguration(version='3.6', identifier='cp36-macosx_x86_64', url='https://www.python.org/ftp/python/3.6.8/python-3.6.8-macosx10.9.pkg'),
|
||||
PythonConfiguration(version='3.7', identifier='cp37-macosx_x86_64', url='https://www.python.org/ftp/python/3.7.6/python-3.7.6-macosx10.9.pkg'),
|
||||
PythonConfiguration(version='3.8', identifier='cp38-macosx_x86_64', url='https://www.python.org/ftp/python/3.8.1/python-3.8.1-macosx10.9.pkg'),
|
||||
PythonConfiguration(version='2.7-v7.3.0', identifier='pp27-macosx_x86_64', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.0-osx64.tar.bz2'),
|
||||
PythonConfiguration(version='3.6-v7.3.0', identifier='pp36-macosx_x86_64', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.3.0-osx64.tar.bz2'),
|
||||
]
|
||||
|
||||
# skip builds as required
|
||||
return [c for c in python_configurations if build_selector(c.identifier)]
|
||||
|
||||
|
||||
SYMLINKS_DIR = '/tmp/cibw_bin'
|
||||
|
||||
|
||||
def make_symlinks(installation_bin_path, python_executable, pip_executable):
|
||||
assert os.path.exists(os.path.join(installation_bin_path, python_executable))
|
||||
|
||||
# Python bin folders on Mac don't symlink `python3` to `python`, and neither
|
||||
# does PyPy for `pypy` or `pypy3`, so we do that so `python` and `pip` always
|
||||
# point to the active configuration.
|
||||
if os.path.exists(SYMLINKS_DIR):
|
||||
shutil.rmtree(SYMLINKS_DIR)
|
||||
os.makedirs(SYMLINKS_DIR)
|
||||
|
||||
os.symlink(os.path.join(installation_bin_path, python_executable), os.path.join(SYMLINKS_DIR, 'python'))
|
||||
os.symlink(os.path.join(installation_bin_path, python_executable + '-config'), os.path.join(SYMLINKS_DIR, 'python-config'))
|
||||
os.symlink(os.path.join(installation_bin_path, pip_executable), os.path.join(SYMLINKS_DIR, 'pip'))
|
||||
|
||||
|
||||
def install_cpython(version, url):
|
||||
installed_system_packages = subprocess.check_output(['pkgutil', '--pkgs'], universal_newlines=True).splitlines()
|
||||
|
||||
# if this version of python isn't installed, get it from python.org and install
|
||||
python_package_identifier = 'org.python.Python.PythonFramework-{}'.format(version)
|
||||
if python_package_identifier not in installed_system_packages:
|
||||
# download the pkg
|
||||
download(url, '/tmp/Python.pkg')
|
||||
# install
|
||||
call(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/'])
|
||||
# patch open ssl
|
||||
if version == '3.5':
|
||||
open_ssl_patch_url = 'https://github.com/mayeut/patch-macos-python-openssl/releases/download/v1.0.2t/patch-macos-python-%s-openssl-v1.0.2t.tar.gz' % version
|
||||
download(open_ssl_patch_url, '/tmp/python-patch.tar.gz')
|
||||
call(['sudo', 'tar', '-C', '/Library/Frameworks/Python.framework/Versions/{}/'.format(version), '-xmf', '/tmp/python-patch.tar.gz'])
|
||||
|
||||
installation_bin_path = '/Library/Frameworks/Python.framework/Versions/{}/bin'.format(version)
|
||||
python_executable = 'python3' if version[0] == '3' else 'python'
|
||||
pip_executable = 'pip3' if version[0] == '3' else 'pip'
|
||||
make_symlinks(installation_bin_path, python_executable, pip_executable)
|
||||
|
||||
return installation_bin_path
|
||||
|
||||
|
||||
def install_pypy(version, url):
|
||||
pypy_tar_bz2 = url.rsplit('/', 1)[-1]
|
||||
assert pypy_tar_bz2.endswith(".tar.bz2")
|
||||
pypy_base_filename = os.path.splitext(os.path.splitext(pypy_tar_bz2)[0])[0]
|
||||
installation_path = os.path.join('/tmp', pypy_base_filename)
|
||||
if not os.path.exists(installation_path):
|
||||
download(url, os.path.join("/tmp", pypy_tar_bz2))
|
||||
call(['tar', '-C', '/tmp', '-xf', os.path.join("/tmp", pypy_tar_bz2)])
|
||||
|
||||
# fix PyPy 7.3.0 bug resulting in wrong macOS platform tag
|
||||
if version.endswith("-v7.3.0") and version[0] == '3':
|
||||
patch_file = os.path.abspath(os.path.join(os.path.dirname(__file__), 'resources', 'pypy3.6.patch'))
|
||||
sysconfigdata_file = os.path.join(installation_path, 'lib_pypy', '_sysconfigdata.py')
|
||||
call(['patch', sysconfigdata_file, patch_file, '-N']) # Always has nonzero return code
|
||||
|
||||
installation_bin_path = os.path.join(installation_path, 'bin')
|
||||
python_executable = 'pypy3' if version[0] == '3' else 'pypy'
|
||||
pip_executable = 'pip3' if version[0] == '3' else 'pip'
|
||||
make_symlinks(installation_bin_path, python_executable, pip_executable)
|
||||
|
||||
return installation_bin_path
|
||||
|
||||
|
||||
def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, dependency_constraints):
|
||||
abs_project_dir = os.path.abspath(project_dir)
|
||||
temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
|
||||
@@ -36,57 +114,37 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
|
||||
|
||||
python_configurations = get_python_configurations(build_selector)
|
||||
|
||||
pkgs_output = subprocess.check_output(['pkgutil', '--pkgs'], universal_newlines=True)
|
||||
installed_system_packages = pkgs_output.splitlines()
|
||||
|
||||
def call(args, env=None, cwd=None, shell=False):
|
||||
# print the command executing for the logs
|
||||
if shell:
|
||||
print('+ %s' % args)
|
||||
else:
|
||||
print('+ ' + ' '.join(shlex.quote(a) for a in args))
|
||||
|
||||
return subprocess.check_call(args, env=env, cwd=cwd, shell=shell)
|
||||
|
||||
for config in python_configurations:
|
||||
# if this version of python isn't installed, get it from python.org and install
|
||||
python_package_identifier = 'org.python.Python.PythonFramework-%s' % config.version
|
||||
if python_package_identifier not in installed_system_packages:
|
||||
# download the pkg
|
||||
download(config.url, '/tmp/Python.pkg')
|
||||
# install
|
||||
call(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/'])
|
||||
# patch open ssl
|
||||
if config.version == '3.5':
|
||||
open_ssl_patch_url = 'https://github.com/mayeut/patch-macos-python-openssl/releases/download/v1.0.2t/patch-macos-python-%s-openssl-v1.0.2t.tar.gz' % config.version
|
||||
download(open_ssl_patch_url, '/tmp/python-patch.tar.gz')
|
||||
call(['sudo', 'tar', '-C', '/Library/Frameworks/Python.framework/Versions/%s/' % config.version, '-xmf', '/tmp/python-patch.tar.gz'])
|
||||
|
||||
installation_bin_path = '/Library/Frameworks/Python.framework/Versions/{}/bin'.format(config.version)
|
||||
assert os.path.exists(os.path.join(installation_bin_path, 'python3' if config.version[0] == '3' else 'python'))
|
||||
|
||||
# Python bin folders on Mac don't symlink python3 to python, so we do that
|
||||
# so `python` and `pip` always point to the active configuration.
|
||||
if os.path.exists('/tmp/cibw_bin'):
|
||||
shutil.rmtree('/tmp/cibw_bin')
|
||||
os.makedirs('/tmp/cibw_bin')
|
||||
|
||||
if config.version[0] == '3':
|
||||
os.symlink(os.path.join(installation_bin_path, 'python3'), '/tmp/cibw_bin/python')
|
||||
os.symlink(os.path.join(installation_bin_path, 'python3-config'), '/tmp/cibw_bin/python-config')
|
||||
os.symlink(os.path.join(installation_bin_path, 'pip3'), '/tmp/cibw_bin/pip')
|
||||
if config.identifier.startswith('cp'):
|
||||
installation_bin_path = install_cpython(config.version, config.url)
|
||||
elif config.identifier.startswith('pp'):
|
||||
installation_bin_path = install_pypy(config.version, config.url)
|
||||
else:
|
||||
raise ValueError("Unknown Python implementation")
|
||||
|
||||
env = os.environ.copy()
|
||||
env['PATH'] = os.pathsep.join([
|
||||
'/tmp/cibw_bin',
|
||||
SYMLINKS_DIR,
|
||||
installation_bin_path,
|
||||
env['PATH'],
|
||||
])
|
||||
|
||||
# Fix issue with site.py setting the wrong `sys.prefix`, `sys.exec_prefix`,
|
||||
# `sys.path`, ... for PyPy: https://foss.heptapod.net/pypy/pypy/issues/3175
|
||||
# Also fix an issue with the shebang of installed scripts inside the
|
||||
# testing virtualenv- see https://github.com/theacodes/nox/issues/44 and
|
||||
# https://github.com/pypa/virtualenv/issues/620
|
||||
# Also see https://github.com/python/cpython/pull/9516
|
||||
env.pop('__PYVENV_LAUNCHER__', None)
|
||||
env = environment.as_dictionary(prev_environment=env)
|
||||
|
||||
# check what version we're on
|
||||
call(['which', 'python'], env=env)
|
||||
call(['python', '--version'], env=env)
|
||||
which_python = subprocess.check_output(['which', 'python'], env=env, universal_newlines=True).strip()
|
||||
if which_python != '/tmp/cibw_bin/python':
|
||||
print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr)
|
||||
exit(1)
|
||||
|
||||
dependency_constraint_flags = []
|
||||
if dependency_constraints:
|
||||
@@ -95,9 +153,14 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
|
||||
]
|
||||
|
||||
# install pip & wheel
|
||||
call(['python', get_pip_script, '--no-setuptools', '--no-wheel'] + dependency_constraint_flags, env=env, cwd="/tmp")
|
||||
call(['python', get_pip_script] + dependency_constraint_flags, env=env, cwd="/tmp")
|
||||
assert os.path.exists(os.path.join(installation_bin_path, 'pip'))
|
||||
call(['which', 'pip'], env=env)
|
||||
call(['pip', '--version'], env=env)
|
||||
which_pip = subprocess.check_output(['which', 'pip'], env=env, universal_newlines=True).strip()
|
||||
if which_pip != '/tmp/cibw_bin/pip':
|
||||
print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr)
|
||||
exit(1)
|
||||
call(['pip', 'install', '--upgrade', 'setuptools', 'wheel', 'delocate'] + dependency_constraint_flags, env=env)
|
||||
|
||||
# setup target platform, only required for python 3.5
|
||||
@@ -150,9 +213,6 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
|
||||
os.path.join(venv_dir, 'bin'),
|
||||
virtualenv_env['PATH'],
|
||||
])
|
||||
# Fix some weird issue with the shebang of installed scripts
|
||||
# See https://github.com/theacodes/nox/issues/44 and https://github.com/pypa/virtualenv/issues/620
|
||||
virtualenv_env.pop('__PYVENV_LAUNCHER__', None)
|
||||
|
||||
# check that we are using the Python from the virtual environment
|
||||
call(['which', 'python'], env=virtualenv_env)
|
||||
|
||||
@@ -0,0 +1,7 @@
|
||||
--- a/lib_pypy/_sysconfigdata.py Tue Jan 28 22:54:59 2020 +0200
|
||||
+++ b/lib_pypy/_sysconfigdata.py Wed Jan 29 19:21:23 2020 +0200
|
||||
@@ -47,4 +47,5 @@
|
||||
build_time_vars['CC'] += ' -arch %s' % (arch,)
|
||||
if "CXX" in build_time_vars:
|
||||
build_time_vars['CXX'] += ' -arch %s' % (arch,)
|
||||
+ build_time_vars['MACOSX_DEPLOYMENT_TARGET'] = '10.7'
|
||||
+89
-47
@@ -1,9 +1,11 @@
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from collections import namedtuple
|
||||
from glob import glob
|
||||
from zipfile import ZipFile
|
||||
|
||||
from .util import (
|
||||
download,
|
||||
@@ -17,37 +19,54 @@ IS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache')
|
||||
IS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows'
|
||||
|
||||
|
||||
def get_python_path(config):
|
||||
nuget_args = get_nuget_args(config)
|
||||
return os.path.join(nuget_args[-1], nuget_args[0] + "." + config.version, "tools")
|
||||
def simple_shell(args, env=None, cwd=None):
|
||||
print('+ ' + ' '.join(args))
|
||||
args = ['cmd', '/E:ON', '/V:ON', '/C'] + args
|
||||
return subprocess.check_call(' '.join(args), env=env, cwd=cwd)
|
||||
|
||||
|
||||
def get_nuget_args(configuration):
|
||||
python_name = "python" if configuration.version[0] == '3' else "python2"
|
||||
if configuration.arch == "32":
|
||||
python_name = python_name + "x86"
|
||||
return [python_name, "-Version", configuration.version, "-OutputDirectory", "C:/cibw/python"]
|
||||
if IS_RUNNING_ON_AZURE or IS_RUNNING_ON_TRAVIS:
|
||||
shell = simple_shell
|
||||
else:
|
||||
run_with_env = os.path.abspath(os.path.join(os.path.dirname(__file__), 'resources', 'appveyor_run_with_env.cmd'))
|
||||
|
||||
# run_with_env is a cmd file that sets the right environment variables
|
||||
# to build on AppVeyor.
|
||||
def shell(args, env=None, cwd=None):
|
||||
# print the command executing for the logs
|
||||
print('+ ' + ' '.join(args))
|
||||
args = ['cmd', '/E:ON', '/V:ON', '/C', run_with_env] + args
|
||||
return subprocess.check_call(' '.join(args), env=env, cwd=cwd)
|
||||
|
||||
|
||||
def get_nuget_args(version, arch):
|
||||
python_name = 'python' if version[0] == '3' else 'python2'
|
||||
if arch == '32':
|
||||
python_name = python_name + 'x86'
|
||||
return [python_name, '-Version', version, '-OutputDirectory', 'C:\\cibw\\python']
|
||||
|
||||
|
||||
def get_python_configurations(build_selector):
|
||||
PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier'])
|
||||
PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier', 'url'])
|
||||
python_configurations = [
|
||||
PythonConfiguration(version='2.7.17', arch="32", identifier='cp27-win32'),
|
||||
PythonConfiguration(version='2.7.17', arch="64", identifier='cp27-win_amd64'),
|
||||
PythonConfiguration(version='3.5.4', arch="32", identifier='cp35-win32'),
|
||||
PythonConfiguration(version='3.5.4', arch="64", identifier='cp35-win_amd64'),
|
||||
PythonConfiguration(version='3.6.8', arch="32", identifier='cp36-win32'),
|
||||
PythonConfiguration(version='3.6.8', arch="64", identifier='cp36-win_amd64'),
|
||||
PythonConfiguration(version='3.7.6', arch="32", identifier='cp37-win32'),
|
||||
PythonConfiguration(version='3.7.6', arch="64", identifier='cp37-win_amd64'),
|
||||
PythonConfiguration(version='3.8.1', arch="32", identifier='cp38-win32'),
|
||||
PythonConfiguration(version='3.8.1', arch="64", identifier='cp38-win_amd64'),
|
||||
PythonConfiguration(version='2.7.17', arch='32', identifier='cp27-win32', url=None),
|
||||
PythonConfiguration(version='2.7.17', arch='64', identifier='cp27-win_amd64', url=None),
|
||||
PythonConfiguration(version='3.5.4', arch='32', identifier='cp35-win32', url=None),
|
||||
PythonConfiguration(version='3.5.4', arch='64', identifier='cp35-win_amd64', url=None),
|
||||
PythonConfiguration(version='3.6.8', arch='32', identifier='cp36-win32', url=None),
|
||||
PythonConfiguration(version='3.6.8', arch='64', identifier='cp36-win_amd64', url=None),
|
||||
PythonConfiguration(version='3.7.6', arch='32', identifier='cp37-win32', url=None),
|
||||
PythonConfiguration(version='3.7.6', arch='64', identifier='cp37-win_amd64', url=None),
|
||||
PythonConfiguration(version='3.8.1', arch='32', identifier='cp38-win32', url=None),
|
||||
PythonConfiguration(version='3.8.1', arch='64', identifier='cp38-win_amd64', url=None),
|
||||
PythonConfiguration(version='2.7-v7.3.0', arch='32', identifier='pp27-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.0-win32.zip'),
|
||||
PythonConfiguration(version='3.6-v7.3.0', arch='32', identifier='pp36-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.3.0-win32.zip'),
|
||||
]
|
||||
|
||||
if IS_RUNNING_ON_TRAVIS:
|
||||
# cannot install VCForPython27.msi which is needed for compiling C software
|
||||
# try with (and similar): msiexec /i VCForPython27.msi ALLUSERS=1 ACCEPT=YES /passive
|
||||
python_configurations = [c for c in python_configurations if not c.version.startswith('2.7.')]
|
||||
python_configurations = [c for c in python_configurations if not c.version.startswith('2.7')]
|
||||
|
||||
# skip builds as required
|
||||
python_configurations = [c for c in python_configurations if build_selector(c.identifier)]
|
||||
@@ -55,25 +74,35 @@ def get_python_configurations(build_selector):
|
||||
return python_configurations
|
||||
|
||||
|
||||
def extract_zip(zip_src, dest):
|
||||
with ZipFile(zip_src) as zip:
|
||||
zip.extractall(dest)
|
||||
|
||||
|
||||
def install_cpython(version, arch, nuget):
|
||||
nuget_args = get_nuget_args(version, arch)
|
||||
installation_path = os.path.join(nuget_args[-1], nuget_args[0] + '.' + version, 'tools')
|
||||
simple_shell([nuget, 'install'] + nuget_args)
|
||||
return installation_path
|
||||
|
||||
|
||||
def install_pypy(version, arch, url):
|
||||
assert arch == '32'
|
||||
# Inside the PyPy zip file is a directory with the same name
|
||||
zip_filename = url.rsplit('/', 1)[-1]
|
||||
installation_path = os.path.join('C:\\cibw', os.path.splitext(zip_filename)[0])
|
||||
if not os.path.exists(installation_path):
|
||||
pypy_zip = os.path.join('C:\\cibw', zip_filename)
|
||||
download(url, pypy_zip)
|
||||
# Extract to the parent directory because the zip file still contains a directory
|
||||
extract_zip(pypy_zip, os.path.dirname(installation_path))
|
||||
pypy_exe = 'pypy3.exe' if version[0] == '3' else 'pypy.exe'
|
||||
simple_shell(['mklink', os.path.join(installation_path, 'python.exe'), os.path.join(installation_path, pypy_exe)])
|
||||
simple_shell(['mklink', '/d', os.path.join(installation_path, 'Scripts'), os.path.join(installation_path, 'bin')])
|
||||
return installation_path
|
||||
|
||||
|
||||
def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, dependency_constraints):
|
||||
def simple_shell(args, env=None, cwd=None):
|
||||
print('+ ' + ' '.join(args))
|
||||
args = ['cmd', '/E:ON', '/V:ON', '/C'] + args
|
||||
return subprocess.check_call(' '.join(args), env=env, cwd=cwd)
|
||||
|
||||
if IS_RUNNING_ON_AZURE or IS_RUNNING_ON_TRAVIS:
|
||||
shell = simple_shell
|
||||
else:
|
||||
run_with_env = os.path.abspath(os.path.join(os.path.dirname(__file__), 'resources', 'appveyor_run_with_env.cmd'))
|
||||
|
||||
# run_with_env is a cmd file that sets the right environment variables
|
||||
# to build on AppVeyor.
|
||||
def shell(args, env=None, cwd=None):
|
||||
# print the command executing for the logs
|
||||
print('+ ' + ' '.join(args))
|
||||
args = ['cmd', '/E:ON', '/V:ON', '/C', run_with_env] + args
|
||||
return subprocess.check_call(' '.join(args), env=env, cwd=cwd)
|
||||
|
||||
abs_project_dir = os.path.abspath(project_dir)
|
||||
temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
|
||||
built_wheel_dir = os.path.join(temp_dir, 'built_wheel')
|
||||
@@ -86,17 +115,22 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
|
||||
python_configurations = get_python_configurations(build_selector)
|
||||
for config in python_configurations:
|
||||
# install Python
|
||||
config_python_path = get_python_path(config)
|
||||
simple_shell([nuget, "install"] + get_nuget_args(config))
|
||||
assert os.path.exists(os.path.join(config_python_path, 'python.exe'))
|
||||
if config.identifier.startswith('cp'):
|
||||
installation_path = install_cpython(config.version, config.arch, nuget)
|
||||
elif config.identifier.startswith('pp'):
|
||||
installation_path = install_pypy(config.version, config.arch, config.url)
|
||||
else:
|
||||
raise ValueError("Unknown Python implementation")
|
||||
|
||||
assert os.path.exists(os.path.join(installation_path, 'python.exe'))
|
||||
|
||||
# set up PATH and environment variables for run_with_env
|
||||
env = os.environ.copy()
|
||||
env['PYTHON_VERSION'] = config.version
|
||||
env['PYTHON_ARCH'] = config.arch
|
||||
env['PATH'] = os.pathsep.join([
|
||||
config_python_path,
|
||||
os.path.join(config_python_path, 'Scripts'),
|
||||
installation_path,
|
||||
os.path.join(installation_path, 'Scripts'),
|
||||
env['PATH']
|
||||
])
|
||||
# update env with results from CIBW_ENVIRONMENT
|
||||
@@ -105,7 +139,11 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
|
||||
# for the logs - check we're running the right version of python
|
||||
simple_shell(['where', 'python'], env=env)
|
||||
simple_shell(['python', '--version'], env=env)
|
||||
simple_shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)\"'], env=env)
|
||||
simple_shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)"'], env=env)
|
||||
where_python = subprocess.check_output(['where', 'python'], env=env, universal_newlines=True).splitlines()[0].strip()
|
||||
if where_python != os.path.join(installation_path, 'python.exe'):
|
||||
print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr)
|
||||
exit(1)
|
||||
|
||||
dependency_constraint_flags = []
|
||||
if dependency_constraints:
|
||||
@@ -114,9 +152,13 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
|
||||
]
|
||||
|
||||
# make sure pip is installed
|
||||
if not os.path.exists(os.path.join(config_python_path, 'Scripts', 'pip.exe')):
|
||||
simple_shell(['python', get_pip_script] + dependency_constraint_flags, env=env, cwd="C:\\cibw")
|
||||
assert os.path.exists(os.path.join(config_python_path, 'Scripts', 'pip.exe'))
|
||||
if not os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe')):
|
||||
simple_shell(['python', get_pip_script], env=env, cwd="C:\\cibw")
|
||||
assert os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe'))
|
||||
where_pip = subprocess.check_output(['where', 'pip'], env=env, universal_newlines=True).splitlines()[0].strip()
|
||||
if where_pip.strip() != os.path.join(installation_path, 'Scripts', 'pip.exe'):
|
||||
print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr)
|
||||
exit(1)
|
||||
|
||||
# prepare the Python environment
|
||||
simple_shell(['python', '-m', 'pip', 'install', '--upgrade', 'pip'] + dependency_constraint_flags, env=env)
|
||||
|
||||
Reference in New Issue
Block a user