Hongru/cibuildwheel
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Cleaning up Python installations on Windows and macOS, and creating PyPy executable python symlinks outside of installation directory

Browse Source
This commit is contained in:
Yannick Jadoul
2020-02-15 23:24:51 +01:00
parent 17137c3c2f
commit aac5db8a46
3 changed files with 94 additions and 76 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cibuildwheel/macos.py
+41 -31
View File
@@ -51,50 +51,60 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
print('+ ' + ' '.join(shlex.quote(a) for a in args))
return subprocess.check_call(args, env=env, cwd=cwd, shell=shell)
def install_cpython(version, url):
# if this version of python isn't installed, get it from python.org and install
python_package_identifier = 'org.python.Python.PythonFramework-{}'.format(version)
if python_package_identifier not in installed_system_packages:
# download the pkg
download(url, '/tmp/Python.pkg')
# install
call(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/'])
# patch open ssl
if version == '3.5':
open_ssl_patch_url = 'https://github.com/mayeut/patch-macos-python-openssl/releases/download/v1.0.2t/patch-macos-python-%s-openssl-v1.0.2t.tar.gz' % config.version
download(open_ssl_patch_url, '/tmp/python-patch.tar.gz')
call(['sudo', 'tar', '-C', '/Library/Frameworks/Python.framework/Versions/{}/'.format(version), '-xmf', '/tmp/python-patch.tar.gz'])
return '/Library/Frameworks/Python.framework/Versions/{}/bin'.format(version)
def install_pypy(version, url):
pypy_tar_bz2 = url.rsplit('/', 1)[-1]
assert pypy_tar_bz2.endswith(".tar.bz2")
pypy_base_filename = os.path.splitext(os.path.splitext(pypy_tar_bz2)[0])[0]
installation_path = os.path.join('/tmp', pypy_base_filename)
if not os.path.exists(installation_path):
download(url, os.path.join("/tmp", pypy_tar_bz2))
call(['tar', '-C', '/tmp', '-xf', os.path.join("/tmp", pypy_tar_bz2)])
return os.path.join(installation_path, 'bin')
# get latest pip once and for all
download(get_pip_url, get_pip_script)
for config in python_configurations:
if config.identifier.startswith('cp'):
# if this version of python isn't installed, get it from python.org and install
python_package_identifier = 'org.python.Python.PythonFramework-%s' % config.version
if python_package_identifier not in installed_system_packages:
# download the pkg
download(config.url, '/tmp/Python.pkg')
# install
call(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/'])
# patch open ssl
if config.version == '3.5':
open_ssl_patch_url = 'https://github.com/mayeut/patch-macos-python-openssl/releases/download/v1.0.2t/patch-macos-python-%s-openssl-v1.0.2t.tar.gz' % config.version
download(open_ssl_patch_url, '/tmp/python-patch.tar.gz')
call(['sudo', 'tar', '-C', '/Library/Frameworks/Python.framework/Versions/%s/' % config.version, '-xmf', '/tmp/python-patch.tar.gz'])
installation_bin_path = '/Library/Frameworks/Python.framework/Versions/{}/bin'.format(config.version)
installation_bin_path = install_cpython(config.version, config.url)
python_executable = 'python3' if config.version[0] == '3' else 'python'
pip_executable = 'pip3' if config.version[0] == '3' else 'pip'
elif config.identifier.startswith('pp'):
pypy_tar_bz2 = config.url.rsplit('/', 1)[-1]
assert pypy_tar_bz2.endswith(".tar.bz2")
pypy_base_filename = os.path.splitext(os.path.splitext(pypy_tar_bz2)[0])[0]
installation_bin_path = os.path.join('/tmp', pypy_base_filename, 'bin')
if not os.path.exists(installation_bin_path):
download(config.url, os.path.join("/tmp", pypy_tar_bz2))
call(['tar', '-C', '/tmp', '-xf', os.path.join("/tmp", pypy_tar_bz2)])
pypy_executable = 'pypy' if config.version[0] == '2' else 'pypy3'
python_symlink = 'python' if config.version[0] == '2' else 'python3'
os.symlink(os.path.join(installation_bin_path, pypy_executable), os.path.join(installation_bin_path, python_symlink))
installation_bin_path = install_pypy(config.version, config.url)
python_executable = 'pypy3' if config.version[0] == '3' else 'pypy'
pip_executable = 'pip3' if config.version[0] == '3' else 'pip'
else:
raise ValueError("Unknown Python implementation")
assert os.path.exists(os.path.join(installation_bin_path, 'python3' if config.version[0] == '3' else 'python'))
assert os.path.exists(os.path.join(installation_bin_path, python_executable))
# Python bin folders on Mac don't symlink python3 to python, so we do that
# so `python` and `pip` always point to the active configuration.
# Python bin folders on Mac don't symlink `python3` to `python`, and neither
# does PyPy for `pypy` or `pypy3`, so we do that so `python` and `pip` always
# point to the active configuration.
if os.path.exists('/tmp/cibw_bin'):
shutil.rmtree('/tmp/cibw_bin')
os.makedirs('/tmp/cibw_bin')
if config.version[0] == '3':
os.symlink(os.path.join(installation_bin_path, 'python3'), '/tmp/cibw_bin/python')
os.symlink(os.path.join(installation_bin_path, 'python3-config'), '/tmp/cibw_bin/python-config')
os.symlink(os.path.join(installation_bin_path, 'pip3'), '/tmp/cibw_bin/pip')
os.symlink(os.path.join(installation_bin_path, python_executable), '/tmp/cibw_bin/python')
os.symlink(os.path.join(installation_bin_path, python_executable + '-config'), '/tmp/cibw_bin/python-config')
os.symlink(os.path.join(installation_bin_path, pip_executable), '/tmp/cibw_bin/pip')
env = os.environ.copy()
env['PATH'] = os.pathsep.join([
cibuildwheel/windows.py
+51 -43
View File
@@ -17,38 +17,28 @@ IS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache')
IS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows'
def get_python_path(config):
if config.identifier.startswith('cp'):
nuget_args = get_nuget_args(config)
return os.path.join(nuget_args[-1], nuget_args[0] + "." + config.version, "tools")
elif config.identifier.startswith('pp'):
# Inside the PyPy zip file is a directory with the same name
filename = config.url.rsplit('/', 1)[-1]
return os.path.join("C:\\cibw", os.path.splitext(filename)[0])
def get_nuget_args(configuration):
python_name = "python" if configuration.version[0] == '3' else "python2"
if configuration.arch == "32":
python_name = python_name + "x86"
return [python_name, "-Version", configuration.version, "-OutputDirectory", "C:/cibw/python"]
def get_nuget_args(version, arch):
python_name = 'python' if version[0] == '3' else 'python2'
if arch == '32':
python_name = python_name + 'x86'
return [python_name, '-Version', version, '-OutputDirectory', 'C:\\cibw\\python']
def get_python_configurations(build_selector):
PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier', 'url'])
python_configurations = [
PythonConfiguration(version='2.7.17', arch="32", identifier='cp27-win32', url=None),
PythonConfiguration(version='2.7.17', arch="64", identifier='cp27-win_amd64', url=None),
PythonConfiguration(version='3.5.4', arch="32", identifier='cp35-win32', url=None),
PythonConfiguration(version='3.5.4', arch="64", identifier='cp35-win_amd64', url=None),
PythonConfiguration(version='3.6.8', arch="32", identifier='cp36-win32', url=None),
PythonConfiguration(version='3.6.8', arch="64", identifier='cp36-win_amd64', url=None),
PythonConfiguration(version='3.7.6', arch="32", identifier='cp37-win32', url=None),
PythonConfiguration(version='3.7.6', arch="64", identifier='cp37-win_amd64', url=None),
PythonConfiguration(version='3.8.1', arch="32", identifier='cp38-win32', url=None),
PythonConfiguration(version='3.8.1', arch="64", identifier='cp38-win_amd64', url=None),
PythonConfiguration(version='2.7-v7.3.0', arch="32", identifier='pp27_73-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.0-win32.zip'),
PythonConfiguration(version='3.6-v7.3.0', arch="32", identifier='pp36_73-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.3.0-win32.zip'),
PythonConfiguration(version='2.7.17', arch='32', identifier='cp27-win32', url=None),
PythonConfiguration(version='2.7.17', arch='64', identifier='cp27-win_amd64', url=None),
PythonConfiguration(version='3.5.4', arch='32', identifier='cp35-win32', url=None),
PythonConfiguration(version='3.5.4', arch='64', identifier='cp35-win_amd64', url=None),
PythonConfiguration(version='3.6.8', arch='32', identifier='cp36-win32', url=None),
PythonConfiguration(version='3.6.8', arch='64', identifier='cp36-win_amd64', url=None),
PythonConfiguration(version='3.7.6', arch='32', identifier='cp37-win32', url=None),
PythonConfiguration(version='3.7.6', arch='64', identifier='cp37-win_amd64', url=None),
PythonConfiguration(version='3.8.1', arch='32', identifier='cp38-win32', url=None),
PythonConfiguration(version='3.8.1', arch='64', identifier='cp38-win_amd64', url=None),
PythonConfiguration(version='2.7-v7.3.0', arch='32', identifier='pp27_73-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.0-win32.zip'),
PythonConfiguration(version='3.6-v7.3.0', arch='32', identifier='pp36_73-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.3.0-win32.zip'),
]
if IS_RUNNING_ON_TRAVIS:
@@ -71,6 +61,28 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
def extract_zip(zip_src, dest):
with ZipFile(zip_src) as zip:
zip.extractall(dest)
def install_cpython(version, arch):
nuget_args = get_nuget_args(version, arch)
installation_path = os.path.join(nuget_args[-1], nuget_args[0] + '.' + version, 'tools')
simple_shell([nuget, 'install'] + nuget_args)
return installation_path
def install_pypy(version, arch, url):
assert arch == '32'
# Inside the PyPy zip file is a directory with the same name
zip_filename = url.rsplit('/', 1)[-1]
installation_path = os.path.join('C:\\cibw', os.path.splitext(zip_filename)[0])
if not os.path.exists(installation_path):
pypy_zip = os.path.join('C:\\cibw', zip_filename)
download(url, pypy_zip)
# Extract to the parent directory because the zip file still contains a directory
extract_zip(pypy_zip, os.path.dirname(installation_path))
pypy_exe = 'pypy3.exe' if config.version[0] == '3' else 'pypy.exe'
simple_shell(['mklink', os.path.join(installation_path, 'python.exe'), os.path.join(installation_path, pypy_exe)])
simple_shell(['mklink', '/d', os.path.join(installation_path, 'Scripts'), os.path.join(installation_path, 'bin')])
return installation_path
if IS_RUNNING_ON_AZURE or IS_RUNNING_ON_TRAVIS:
shell = simple_shell
@@ -100,26 +112,22 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
python_configurations = get_python_configurations(build_selector)
for config in python_configurations:
# install Python
config_python_path = get_python_path(config)
if config.identifier.startswith('cp'):
simple_shell([nuget, "install"] + get_nuget_args(config))
elif config.identifier.startswith('pp') and not os.path.exists(config_python_path):
pypy_zip = os.path.join("C:\\cibw", config.url.rsplit('/', 1)[-1])
download(config.url, pypy_zip)
# Extract to the parent of config_python_path because the zip file still contains a directory
extract_zip(pypy_zip, os.path.dirname(config_python_path))
pypy_exe = 'pypy.exe' if config.version[0] == '2' else 'pypy3.exe'
simple_shell(['mklink', os.path.join(config_python_path, 'python.exe'), os.path.join(config_python_path, pypy_exe)])
simple_shell(['mklink', '/d', os.path.join(config_python_path, 'Scripts'), os.path.join(config_python_path, 'bin')])
assert os.path.exists(os.path.join(config_python_path, 'python.exe'))
installation_path = install_cpython(config.version, config.arch)
elif config.identifier.startswith('pp'):
installation_path = install_pypy(config.version, config.arch, config.url)
else:
raise ValueError("Unknown Python implementation")
assert os.path.exists(os.path.join(installation_path, 'python.exe'))
# set up PATH and environment variables for run_with_env
env = os.environ.copy()
env['PYTHON_VERSION'] = config.version
env['PYTHON_ARCH'] = config.arch
env['PATH'] = os.pathsep.join([
config_python_path,
os.path.join(config_python_path, 'Scripts'),
installation_path,
os.path.join(installation_path, 'Scripts'),
env['PATH']
])
# update env with results from CIBW_ENVIRONMENT
@@ -128,12 +136,12 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef
# for the logs - check we're running the right version of python
simple_shell(['where', 'python'], env=env)
simple_shell(['python', '--version'], env=env)
simple_shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)\"'], env=env)
simple_shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)"'], env=env)
# make sure pip is installed
if not os.path.exists(os.path.join(config_python_path, 'Scripts', 'pip.exe')):
if not os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe')):
simple_shell(['python', get_pip_script], env=env, cwd="C:\\cibw")
assert os.path.exists(os.path.join(config_python_path, 'Scripts', 'pip.exe'))
assert os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe'))
# prepare the Python environment
simple_shell(['python', '-m', 'pip', 'install', '--upgrade', 'pip'], env=env)
docs/options.md
+2 -2
View File
@@ -117,7 +117,7 @@ CIBW_BUILD: cp3?-*
CIBW_SKIP: "*-win32 *-manylinux_i686"
# Only build PyPy and CPython 3
CIBW_BUILD: pp??_??-* cp3?-*
CIBW_BUILD: pp*-* cp3?-*
```
<style>
@@ -260,7 +260,7 @@ Note that `manylinux2014` doesn't support builds with Python 2.7 - when building
# skip PyPy, since there is no PyPy manylinux1 image
CIBW_MANYLINUX_X86_64_IMAGE: manylinux1
CIBW_MANYLINUX_I686_IMAGE: manylinux1
CIBW_SKIP: pp??_??-*
CIBW_SKIP: pp*-*
# build using the manylinux2014 image
CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014