Merge remote-tracking branch 'origin/master' into deterministic-builds
This commit is contained in:
Joe Rickerby
+89
-47
@@ -1,9 +1,11 @@
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from collections import namedtuple
|
||||
from glob import glob
|
||||
from zipfile import ZipFile
|
||||
|
||||
from .util import (
|
||||
download,
|
||||
@@ -17,37 +19,54 @@ IS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache')
|
||||
IS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows'
|
||||
|
||||
|
||||
def get_python_path(config):
|
||||
nuget_args = get_nuget_args(config)
|
||||
return os.path.join(nuget_args[-1], nuget_args[0] + "." + config.version, "tools")
|
||||
def simple_shell(args, env=None, cwd=None):
|
||||
print('+ ' + ' '.join(args))
|
||||
args = ['cmd', '/E:ON', '/V:ON', '/C'] + args
|
||||
return subprocess.check_call(' '.join(args), env=env, cwd=cwd)
|
||||
|
||||
|
||||
def get_nuget_args(configuration):
|
||||
python_name = "python" if configuration.version[0] == '3' else "python2"
|
||||
if configuration.arch == "32":
|
||||
python_name = python_name + "x86"
|
||||
return [python_name, "-Version", configuration.version, "-OutputDirectory", "C:/cibw/python"]
|
||||
if IS_RUNNING_ON_AZURE or IS_RUNNING_ON_TRAVIS:
|
||||
shell = simple_shell
|
||||
else:
|
||||
run_with_env = os.path.abspath(os.path.join(os.path.dirname(__file__), 'resources', 'appveyor_run_with_env.cmd'))
|
||||
|
||||
# run_with_env is a cmd file that sets the right environment variables
|
||||
# to build on AppVeyor.
|
||||
def shell(args, env=None, cwd=None):
|
||||
# print the command executing for the logs
|
||||
print('+ ' + ' '.join(args))
|
||||
args = ['cmd', '/E:ON', '/V:ON', '/C', run_with_env] + args
|
||||
return subprocess.check_call(' '.join(args), env=env, cwd=cwd)
|
||||
|
||||
|
||||
def get_nuget_args(version, arch):
|
||||
python_name = 'python' if version[0] == '3' else 'python2'
|
||||
if arch == '32':
|
||||
python_name = python_name + 'x86'
|
||||
return [python_name, '-Version', version, '-OutputDirectory', 'C:\\cibw\\python']
|
||||
|
||||
|
||||
def get_python_configurations(build_selector):
|
||||
PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier'])
|
||||
PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier', 'url'])
|
||||
python_configurations = [
|
||||
PythonConfiguration(version='2.7.17', arch="32", identifier='cp27-win32'),
|
||||
PythonConfiguration(version='2.7.17', arch="64", identifier='cp27-win_amd64'),
|
||||
PythonConfiguration(version='3.5.4', arch="32", identifier='cp35-win32'),
|
||||
PythonConfiguration(version='3.5.4', arch="64", identifier='cp35-win_amd64'),
|
||||
PythonConfiguration(version='3.6.8', arch="32", identifier='cp36-win32'),
|
||||
PythonConfiguration(version='3.6.8', arch="64", identifier='cp36-win_amd64'),
|
||||
PythonConfiguration(version='3.7.6', arch="32", identifier='cp37-win32'),
|
||||
PythonConfiguration(version='3.7.6', arch="64", identifier='cp37-win_amd64'),
|
||||
PythonConfiguration(version='3.8.1', arch="32", identifier='cp38-win32'),
|
||||
PythonConfiguration(version='3.8.1', arch="64", identifier='cp38-win_amd64'),
|
||||
PythonConfiguration(version='2.7.17', arch='32', identifier='cp27-win32', url=None),
|
||||
PythonConfiguration(version='2.7.17', arch='64', identifier='cp27-win_amd64', url=None),
|
||||
PythonConfiguration(version='3.5.4', arch='32', identifier='cp35-win32', url=None),
|
||||
PythonConfiguration(version='3.5.4', arch='64', identifier='cp35-win_amd64', url=None),
|
||||
PythonConfiguration(version='3.6.8', arch='32', identifier='cp36-win32', url=None),
|
||||
PythonConfiguration(version='3.6.8', arch='64', identifier='cp36-win_amd64', url=None),
|
||||
PythonConfiguration(version='3.7.6', arch='32', identifier='cp37-win32', url=None),
|
||||
PythonConfiguration(version='3.7.6', arch='64', identifier='cp37-win_amd64', url=None),
|
||||
PythonConfiguration(version='3.8.1', arch='32', identifier='cp38-win32', url=None),
|
||||
PythonConfiguration(version='3.8.1', arch='64', identifier='cp38-win_amd64', url=None),
|
||||
PythonConfiguration(version='2.7-v7.3.0', arch='32', identifier='pp27-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.0-win32.zip'),
|
||||
PythonConfiguration(version='3.6-v7.3.0', arch='32', identifier='pp36-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.3.0-win32.zip'),
|
||||
]
|
||||
|
||||
if IS_RUNNING_ON_TRAVIS:
|
||||
# cannot install VCForPython27.msi which is needed for compiling C software
|
||||
# try with (and similar): msiexec /i VCForPython27.msi ALLUSERS=1 ACCEPT=YES /passive
|
||||
python_configurations = [c for c in python_configurations if not c.version.startswith('2.7.')]
|
||||
python_configurations = [c for c in python_configurations if not c.version.startswith('2.7')]
|
||||
|
||||
# skip builds as required
|
||||
python_configurations = [c for c in python_configurations if build_selector(c.identifier)]
|
||||
@@ -55,25 +74,35 @@ def get_python_configurations(build_selector):
|
||||
return python_configurations
|
||||
|
||||
|
||||
def extract_zip(zip_src, dest):
|
||||
with ZipFile(zip_src) as zip:
|
||||
zip.extractall(dest)
|
||||
|
||||
|
||||
def install_cpython(version, arch, nuget):
|
||||
nuget_args = get_nuget_args(version, arch)
|
||||
installation_path = os.path.join(nuget_args[-1], nuget_args[0] + '.' + version, 'tools')
|
||||
simple_shell([nuget, 'install'] + nuget_args)
|
||||
return installation_path
|
||||
|
||||
|
||||
def install_pypy(version, arch, url):
|
||||
assert arch == '32'
|
||||
# Inside the PyPy zip file is a directory with the same name
|
||||
zip_filename = url.rsplit('/', 1)[-1]
|
||||
installation_path = os.path.join('C:\\cibw', os.path.splitext(zip_filename)[0])
|
||||
if not os.path.exists(installation_path):
|
||||
pypy_zip = os.path.join('C:\\cibw', zip_filename)
|
||||
download(url, pypy_zip)
|
||||
# Extract to the parent directory because the zip file still contains a directory
|
||||
extract_zip(pypy_zip, os.path.dirname(installation_path))
|
||||
pypy_exe = 'pypy3.exe' if version[0] == '3' else 'pypy.exe'
|
||||
simple_shell(['mklink', os.path.join(installation_path, 'python.exe'), os.path.join(installation_path, pypy_exe)])
|
||||
simple_shell(['mklink', '/d', os.path.join(installation_path, 'Scripts'), os.path.join(installation_path, 'bin')])
|
||||
return installation_path
|
||||
|
||||
|
||||
def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment, dependency_constraints):
|
||||
def simple_shell(args, env=None, cwd=None):
|
||||
print('+ ' + ' '.join(args))
|
||||
args = ['cmd', '/E:ON', '/V:ON', '/C'] + args
|
||||
return subprocess.check_call(' '.join(args), env=env, cwd=cwd)
|
||||
|
||||
if IS_RUNNING_ON_AZURE or IS_RUNNING_ON_TRAVIS:
|
||||
shell = simple_shell
|
||||
else:
|
||||
run_with_env = os.path.abspath(os.path.join(os.path.dirname(__file__), 'resources', 'appveyor_run_with_env.cmd'))
|
||||
|
||||
# run_with_env is a cmd file that sets the right environment variables
|
||||
# to build on AppVeyor.
|
||||
def shell(args, env=None, cwd=None):
|
||||
# print the command executing for the logs
|
||||
print('+ ' + ' '.join(args))
|
||||
args = ['cmd', '/E:ON', '/V:ON', '/C', run_with_env] + args
|
||||
return subprocess.check_call(' '.join(args), env=env, cwd=cwd)
|
||||
|
||||
abs_project_dir = os.path.abspath(project_dir)
|
||||
temp_dir = tempfile.mkdtemp(prefix='cibuildwheel')
|
||||
built_wheel_dir = os.path.join(temp_dir, 'built_wheel')
|
||||
@@ -86,17 +115,22 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
|
||||
python_configurations = get_python_configurations(build_selector)
|
||||
for config in python_configurations:
|
||||
# install Python
|
||||
config_python_path = get_python_path(config)
|
||||
simple_shell([nuget, "install"] + get_nuget_args(config))
|
||||
assert os.path.exists(os.path.join(config_python_path, 'python.exe'))
|
||||
if config.identifier.startswith('cp'):
|
||||
installation_path = install_cpython(config.version, config.arch, nuget)
|
||||
elif config.identifier.startswith('pp'):
|
||||
installation_path = install_pypy(config.version, config.arch, config.url)
|
||||
else:
|
||||
raise ValueError("Unknown Python implementation")
|
||||
|
||||
assert os.path.exists(os.path.join(installation_path, 'python.exe'))
|
||||
|
||||
# set up PATH and environment variables for run_with_env
|
||||
env = os.environ.copy()
|
||||
env['PYTHON_VERSION'] = config.version
|
||||
env['PYTHON_ARCH'] = config.arch
|
||||
env['PATH'] = os.pathsep.join([
|
||||
config_python_path,
|
||||
os.path.join(config_python_path, 'Scripts'),
|
||||
installation_path,
|
||||
os.path.join(installation_path, 'Scripts'),
|
||||
env['PATH']
|
||||
])
|
||||
# update env with results from CIBW_ENVIRONMENT
|
||||
@@ -105,7 +139,11 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
|
||||
# for the logs - check we're running the right version of python
|
||||
simple_shell(['where', 'python'], env=env)
|
||||
simple_shell(['python', '--version'], env=env)
|
||||
simple_shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)\"'], env=env)
|
||||
simple_shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)"'], env=env)
|
||||
where_python = subprocess.check_output(['where', 'python'], env=env, universal_newlines=True).splitlines()[0].strip()
|
||||
if where_python != os.path.join(installation_path, 'python.exe'):
|
||||
print("cibuildwheel: python available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert python above it.", file=sys.stderr)
|
||||
exit(1)
|
||||
|
||||
dependency_constraint_flags = []
|
||||
if dependency_constraints:
|
||||
@@ -114,9 +152,13 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
|
||||
]
|
||||
|
||||
# make sure pip is installed
|
||||
if not os.path.exists(os.path.join(config_python_path, 'Scripts', 'pip.exe')):
|
||||
simple_shell(['python', get_pip_script] + dependency_constraint_flags, env=env, cwd="C:\\cibw")
|
||||
assert os.path.exists(os.path.join(config_python_path, 'Scripts', 'pip.exe'))
|
||||
if not os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe')):
|
||||
simple_shell(['python', get_pip_script], env=env, cwd="C:\\cibw")
|
||||
assert os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe'))
|
||||
where_pip = subprocess.check_output(['where', 'pip'], env=env, universal_newlines=True).splitlines()[0].strip()
|
||||
if where_pip.strip() != os.path.join(installation_path, 'Scripts', 'pip.exe'):
|
||||
print("cibuildwheel: pip available on PATH doesn't match our installed instance. If you have modified PATH, ensure that you don't overwrite cibuildwheel's entry or insert pip above it.", file=sys.stderr)
|
||||
exit(1)
|
||||
|
||||
# prepare the Python environment
|
||||
simple_shell(['python', '-m', 'pip', 'install', '--upgrade', 'pip'] + dependency_constraint_flags, env=env)
|
||||
|
||||
Reference in New Issue
Block a user