Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Cleaning up Python installations on Windows and macOS, and creating PyPy executable python symlinks outside of installation directory

Browse Source
This commit is contained in:
Yannick Jadoul
2020-02-15 23:24:51 +01:00
parent 17137c3c2f
commit aac5db8a46
3 changed files with 94 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cibuildwheel/macos.py
+41 -31
View File
@@ -52,49 +52,59 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
return subprocess.check_call(args, env=env, cwd=cwd, shell=shell) return subprocess.check_call(args, env=env, cwd=cwd, shell=shell)
def install_cpython(version, url):
# if this version of python isn't installed, get it from python.org and install
python_package_identifier = 'org.python.Python.PythonFramework-{}'.format(version)
if python_package_identifier not in installed_system_packages:
# download the pkg
download(url, '/tmp/Python.pkg')
# install
call(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/'])
# patch open ssl
if version == '3.5':
open_ssl_patch_url = 'https://github.com/mayeut/patch-macos-python-openssl/releases/download/v1.0.2t/patch-macos-python-%s-openssl-v1.0.2t.tar.gz' % config.version
download(open_ssl_patch_url, '/tmp/python-patch.tar.gz')
call(['sudo', 'tar', '-C', '/Library/Frameworks/Python.framework/Versions/{}/'.format(version), '-xmf', '/tmp/python-patch.tar.gz'])
return '/Library/Frameworks/Python.framework/Versions/{}/bin'.format(version)
def install_pypy(version, url):
pypy_tar_bz2 = url.rsplit('/', 1)[-1]
assert pypy_tar_bz2.endswith(".tar.bz2")
pypy_base_filename = os.path.splitext(os.path.splitext(pypy_tar_bz2)[0])[0]
installation_path = os.path.join('/tmp', pypy_base_filename)
if not os.path.exists(installation_path):
download(url, os.path.join("/tmp", pypy_tar_bz2))
call(['tar', '-C', '/tmp', '-xf', os.path.join("/tmp", pypy_tar_bz2)])
return os.path.join(installation_path, 'bin')
# get latest pip once and for all # get latest pip once and for all
download(get_pip_url, get_pip_script) download(get_pip_url, get_pip_script)
for config in python_configurations: for config in python_configurations:
if config.identifier.startswith('cp'): if config.identifier.startswith('cp'):
# if this version of python isn't installed, get it from python.org and install installation_bin_path = install_cpython(config.version, config.url)
python_package_identifier = 'org.python.Python.PythonFramework-%s' % config.version python_executable = 'python3' if config.version[0] == '3' else 'python'
if python_package_identifier not in installed_system_packages: pip_executable = 'pip3' if config.version[0] == '3' else 'pip'
# download the pkg
download(config.url, '/tmp/Python.pkg')
# install
call(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/'])
# patch open ssl
if config.version == '3.5':
open_ssl_patch_url = 'https://github.com/mayeut/patch-macos-python-openssl/releases/download/v1.0.2t/patch-macos-python-%s-openssl-v1.0.2t.tar.gz' % config.version
download(open_ssl_patch_url, '/tmp/python-patch.tar.gz')
call(['sudo', 'tar', '-C', '/Library/Frameworks/Python.framework/Versions/%s/' % config.version, '-xmf', '/tmp/python-patch.tar.gz'])
installation_bin_path = '/Library/Frameworks/Python.framework/Versions/{}/bin'.format(config.version)
elif config.identifier.startswith('pp'): elif config.identifier.startswith('pp'):
pypy_tar_bz2 = config.url.rsplit('/', 1)[-1] installation_bin_path = install_pypy(config.version, config.url)
assert pypy_tar_bz2.endswith(".tar.bz2") python_executable = 'pypy3' if config.version[0] == '3' else 'pypy'
pypy_base_filename = os.path.splitext(os.path.splitext(pypy_tar_bz2)[0])[0] pip_executable = 'pip3' if config.version[0] == '3' else 'pip'
installation_bin_path = os.path.join('/tmp', pypy_base_filename, 'bin') else:
if not os.path.exists(installation_bin_path): raise ValueError("Unknown Python implementation")
download(config.url, os.path.join("/tmp", pypy_tar_bz2))
call(['tar', '-C', '/tmp', '-xf', os.path.join("/tmp", pypy_tar_bz2)])
pypy_executable = 'pypy' if config.version[0] == '2' else 'pypy3'
python_symlink = 'python' if config.version[0] == '2' else 'python3'
os.symlink(os.path.join(installation_bin_path, pypy_executable), os.path.join(installation_bin_path, python_symlink))
assert os.path.exists(os.path.join(installation_bin_path, 'python3' if config.version[0] == '3' else 'python')) assert os.path.exists(os.path.join(installation_bin_path, python_executable))
# Python bin folders on Mac don't symlink python3 to python, so we do that # Python bin folders on Mac don't symlink `python3` to `python`, and neither
# so `python` and `pip` always point to the active configuration. # does PyPy for `pypy` or `pypy3`, so we do that so `python` and `pip` always
# point to the active configuration.
if os.path.exists('/tmp/cibw_bin'): if os.path.exists('/tmp/cibw_bin'):
shutil.rmtree('/tmp/cibw_bin') shutil.rmtree('/tmp/cibw_bin')
os.makedirs('/tmp/cibw_bin') os.makedirs('/tmp/cibw_bin')
if config.version[0] == '3': os.symlink(os.path.join(installation_bin_path, python_executable), '/tmp/cibw_bin/python')
os.symlink(os.path.join(installation_bin_path, 'python3'), '/tmp/cibw_bin/python') os.symlink(os.path.join(installation_bin_path, python_executable + '-config'), '/tmp/cibw_bin/python-config')
os.symlink(os.path.join(installation_bin_path, 'python3-config'), '/tmp/cibw_bin/python-config') os.symlink(os.path.join(installation_bin_path, pip_executable), '/tmp/cibw_bin/pip')
os.symlink(os.path.join(installation_bin_path, 'pip3'), '/tmp/cibw_bin/pip')
env = os.environ.copy() env = os.environ.copy()
env['PATH'] = os.pathsep.join([ env['PATH'] = os.pathsep.join([
cibuildwheel/windows.py
+51 -43
View File
@@ -17,38 +17,28 @@ IS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache')
IS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows' IS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows'
def get_python_path(config): def get_nuget_args(version, arch):
if config.identifier.startswith('cp'): python_name = 'python' if version[0] == '3' else 'python2'
nuget_args = get_nuget_args(config) if arch == '32':
return os.path.join(nuget_args[-1], nuget_args[0] + "." + config.version, "tools") python_name = python_name + 'x86'
elif config.identifier.startswith('pp'): return [python_name, '-Version', version, '-OutputDirectory', 'C:\\cibw\\python']
# Inside the PyPy zip file is a directory with the same name
filename = config.url.rsplit('/', 1)[-1]
return os.path.join("C:\\cibw", os.path.splitext(filename)[0])
def get_nuget_args(configuration):
python_name = "python" if configuration.version[0] == '3' else "python2"
if configuration.arch == "32":
python_name = python_name + "x86"
return [python_name, "-Version", configuration.version, "-OutputDirectory", "C:/cibw/python"]
def get_python_configurations(build_selector): def get_python_configurations(build_selector):
PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier', 'url']) PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier', 'url'])
python_configurations = [ python_configurations = [
PythonConfiguration(version='2.7.17', arch="32", identifier='cp27-win32', url=None), PythonConfiguration(version='2.7.17', arch='32', identifier='cp27-win32', url=None),
PythonConfiguration(version='2.7.17', arch="64", identifier='cp27-win_amd64', url=None), PythonConfiguration(version='2.7.17', arch='64', identifier='cp27-win_amd64', url=None),
PythonConfiguration(version='3.5.4', arch="32", identifier='cp35-win32', url=None), PythonConfiguration(version='3.5.4', arch='32', identifier='cp35-win32', url=None),
PythonConfiguration(version='3.5.4', arch="64", identifier='cp35-win_amd64', url=None), PythonConfiguration(version='3.5.4', arch='64', identifier='cp35-win_amd64', url=None),
PythonConfiguration(version='3.6.8', arch="32", identifier='cp36-win32', url=None), PythonConfiguration(version='3.6.8', arch='32', identifier='cp36-win32', url=None),
PythonConfiguration(version='3.6.8', arch="64", identifier='cp36-win_amd64', url=None), PythonConfiguration(version='3.6.8', arch='64', identifier='cp36-win_amd64', url=None),
PythonConfiguration(version='3.7.6', arch="32", identifier='cp37-win32', url=None), PythonConfiguration(version='3.7.6', arch='32', identifier='cp37-win32', url=None),
PythonConfiguration(version='3.7.6', arch="64", identifier='cp37-win_amd64', url=None), PythonConfiguration(version='3.7.6', arch='64', identifier='cp37-win_amd64', url=None),
PythonConfiguration(version='3.8.1', arch="32", identifier='cp38-win32', url=None), PythonConfiguration(version='3.8.1', arch='32', identifier='cp38-win32', url=None),
PythonConfiguration(version='3.8.1', arch="64", identifier='cp38-win_amd64', url=None), PythonConfiguration(version='3.8.1', arch='64', identifier='cp38-win_amd64', url=None),
PythonConfiguration(version='2.7-v7.3.0', arch="32", identifier='pp27_73-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.0-win32.zip'), PythonConfiguration(version='2.7-v7.3.0', arch='32', identifier='pp27_73-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.0-win32.zip'),
PythonConfiguration(version='3.6-v7.3.0', arch="32", identifier='pp36_73-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.3.0-win32.zip'), PythonConfiguration(version='3.6-v7.3.0', arch='32', identifier='pp36_73-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.3.0-win32.zip'),
] ]
if IS_RUNNING_ON_TRAVIS: if IS_RUNNING_ON_TRAVIS:
@@ -72,6 +62,28 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
with ZipFile(zip_src) as zip: with ZipFile(zip_src) as zip:
zip.extractall(dest) zip.extractall(dest)
def install_cpython(version, arch):
nuget_args = get_nuget_args(version, arch)
installation_path = os.path.join(nuget_args[-1], nuget_args[0] + '.' + version, 'tools')
simple_shell([nuget, 'install'] + nuget_args)
return installation_path
def install_pypy(version, arch, url):
assert arch == '32'
# Inside the PyPy zip file is a directory with the same name
zip_filename = url.rsplit('/', 1)[-1]
installation_path = os.path.join('C:\\cibw', os.path.splitext(zip_filename)[0])
if not os.path.exists(installation_path):
pypy_zip = os.path.join('C:\\cibw', zip_filename)
download(url, pypy_zip)
# Extract to the parent directory because the zip file still contains a directory
extract_zip(pypy_zip, os.path.dirname(installation_path))
pypy_exe = 'pypy3.exe' if config.version[0] == '3' else 'pypy.exe'
simple_shell(['mklink', os.path.join(installation_path, 'python.exe'), os.path.join(installation_path, pypy_exe)])
simple_shell(['mklink', '/d', os.path.join(installation_path, 'Scripts'), os.path.join(installation_path, 'bin')])
return installation_path
if IS_RUNNING_ON_AZURE or IS_RUNNING_ON_TRAVIS: if IS_RUNNING_ON_AZURE or IS_RUNNING_ON_TRAVIS:
shell = simple_shell shell = simple_shell
else: else:
@@ -100,26 +112,22 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
python_configurations = get_python_configurations(build_selector) python_configurations = get_python_configurations(build_selector)
for config in python_configurations: for config in python_configurations:
# install Python # install Python
config_python_path = get_python_path(config)
if config.identifier.startswith('cp'): if config.identifier.startswith('cp'):
simple_shell([nuget, "install"] + get_nuget_args(config)) installation_path = install_cpython(config.version, config.arch)
elif config.identifier.startswith('pp') and not os.path.exists(config_python_path): elif config.identifier.startswith('pp'):
pypy_zip = os.path.join("C:\\cibw", config.url.rsplit('/', 1)[-1]) installation_path = install_pypy(config.version, config.arch, config.url)
download(config.url, pypy_zip) else:
# Extract to the parent of config_python_path because the zip file still contains a directory raise ValueError("Unknown Python implementation")
extract_zip(pypy_zip, os.path.dirname(config_python_path))
pypy_exe = 'pypy.exe' if config.version[0] == '2' else 'pypy3.exe' assert os.path.exists(os.path.join(installation_path, 'python.exe'))
simple_shell(['mklink', os.path.join(config_python_path, 'python.exe'), os.path.join(config_python_path, pypy_exe)])
simple_shell(['mklink', '/d', os.path.join(config_python_path, 'Scripts'), os.path.join(config_python_path, 'bin')])
assert os.path.exists(os.path.join(config_python_path, 'python.exe'))
# set up PATH and environment variables for run_with_env # set up PATH and environment variables for run_with_env
env = os.environ.copy() env = os.environ.copy()
env['PYTHON_VERSION'] = config.version env['PYTHON_VERSION'] = config.version
env['PYTHON_ARCH'] = config.arch env['PYTHON_ARCH'] = config.arch
env['PATH'] = os.pathsep.join([ env['PATH'] = os.pathsep.join([
config_python_path, installation_path,
os.path.join(config_python_path, 'Scripts'), os.path.join(installation_path, 'Scripts'),
env['PATH'] env['PATH']
]) ])
# update env with results from CIBW_ENVIRONMENT # update env with results from CIBW_ENVIRONMENT
@@ -128,12 +136,12 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
# for the logs - check we're running the right version of python # for the logs - check we're running the right version of python
simple_shell(['where', 'python'], env=env) simple_shell(['where', 'python'], env=env)
simple_shell(['python', '--version'], env=env) simple_shell(['python', '--version'], env=env)
simple_shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)\"'], env=env) simple_shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)"'], env=env)
# make sure pip is installed # make sure pip is installed
if not os.path.exists(os.path.join(config_python_path, 'Scripts', 'pip.exe')): if not os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe')):
simple_shell(['python', get_pip_script], env=env, cwd="C:\\cibw") simple_shell(['python', get_pip_script], env=env, cwd="C:\\cibw")
assert os.path.exists(os.path.join(config_python_path, 'Scripts', 'pip.exe')) assert os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe'))
# prepare the Python environment # prepare the Python environment
simple_shell(['python', '-m', 'pip', 'install', '--upgrade', 'pip'], env=env) simple_shell(['python', '-m', 'pip', 'install', '--upgrade', 'pip'], env=env)
docs/options.md
+2 -2
View File
@@ -117,7 +117,7 @@ CIBW_BUILD: cp3?-*
CIBW_SKIP: "*-win32 *-manylinux_i686" CIBW_SKIP: "*-win32 *-manylinux_i686"
# Only build PyPy and CPython 3 # Only build PyPy and CPython 3
CIBW_BUILD: pp??_??-* cp3?-* CIBW_BUILD: pp*-* cp3?-*
``` ```
<style> <style>
@@ -260,7 +260,7 @@ Note that `manylinux2014` doesn't support builds with Python 2.7 - when building
# skip PyPy, since there is no PyPy manylinux1 image # skip PyPy, since there is no PyPy manylinux1 image
CIBW_MANYLINUX_X86_64_IMAGE: manylinux1 CIBW_MANYLINUX_X86_64_IMAGE: manylinux1
CIBW_MANYLINUX_I686_IMAGE: manylinux1 CIBW_MANYLINUX_I686_IMAGE: manylinux1
CIBW_SKIP: pp??_??-* CIBW_SKIP: pp*-*
# build using the manylinux2014 image # build using the manylinux2014 image
CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014 CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014