From aac5db8a469d051e0d86d9e207024c97be729883 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Thu, 26 Dec 2019 20:30:58 +0100 Subject: [PATCH] Cleaning up Python installations on Windows and macOS, and creating PyPy executable `python` symlinks outside of installation directory --- cibuildwheel/macos.py | 72 +++++++++++++++++-------------- cibuildwheel/windows.py | 94 ++++++++++++++++++++++------------------- docs/options.md | 4 +- 3 files changed, 94 insertions(+), 76 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 6e41ac95..95554baf 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -51,50 +51,60 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef print('+ ' + ' '.join(shlex.quote(a) for a in args)) return subprocess.check_call(args, env=env, cwd=cwd, shell=shell) + + def install_cpython(version, url): + # if this version of python isn't installed, get it from python.org and install + python_package_identifier = 'org.python.Python.PythonFramework-{}'.format(version) + if python_package_identifier not in installed_system_packages: + # download the pkg + download(url, '/tmp/Python.pkg') + # install + call(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/']) + # patch open ssl + if version == '3.5': + open_ssl_patch_url = 'https://github.com/mayeut/patch-macos-python-openssl/releases/download/v1.0.2t/patch-macos-python-%s-openssl-v1.0.2t.tar.gz' % config.version + download(open_ssl_patch_url, '/tmp/python-patch.tar.gz') + call(['sudo', 'tar', '-C', '/Library/Frameworks/Python.framework/Versions/{}/'.format(version), '-xmf', '/tmp/python-patch.tar.gz']) + return '/Library/Frameworks/Python.framework/Versions/{}/bin'.format(version) + + def install_pypy(version, url): + pypy_tar_bz2 = url.rsplit('/', 1)[-1] + assert pypy_tar_bz2.endswith(".tar.bz2") + pypy_base_filename = os.path.splitext(os.path.splitext(pypy_tar_bz2)[0])[0] + installation_path = os.path.join('/tmp', pypy_base_filename) + if not os.path.exists(installation_path): + download(url, os.path.join("/tmp", pypy_tar_bz2)) + call(['tar', '-C', '/tmp', '-xf', os.path.join("/tmp", pypy_tar_bz2)]) + return os.path.join(installation_path, 'bin') + # get latest pip once and for all download(get_pip_url, get_pip_script) for config in python_configurations: if config.identifier.startswith('cp'): - # if this version of python isn't installed, get it from python.org and install - python_package_identifier = 'org.python.Python.PythonFramework-%s' % config.version - if python_package_identifier not in installed_system_packages: - # download the pkg - download(config.url, '/tmp/Python.pkg') - # install - call(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/']) - # patch open ssl - if config.version == '3.5': - open_ssl_patch_url = 'https://github.com/mayeut/patch-macos-python-openssl/releases/download/v1.0.2t/patch-macos-python-%s-openssl-v1.0.2t.tar.gz' % config.version - download(open_ssl_patch_url, '/tmp/python-patch.tar.gz') - call(['sudo', 'tar', '-C', '/Library/Frameworks/Python.framework/Versions/%s/' % config.version, '-xmf', '/tmp/python-patch.tar.gz']) - - installation_bin_path = '/Library/Frameworks/Python.framework/Versions/{}/bin'.format(config.version) + installation_bin_path = install_cpython(config.version, config.url) + python_executable = 'python3' if config.version[0] == '3' else 'python' + pip_executable = 'pip3' if config.version[0] == '3' else 'pip' elif config.identifier.startswith('pp'): - pypy_tar_bz2 = config.url.rsplit('/', 1)[-1] - assert pypy_tar_bz2.endswith(".tar.bz2") - pypy_base_filename = os.path.splitext(os.path.splitext(pypy_tar_bz2)[0])[0] - installation_bin_path = os.path.join('/tmp', pypy_base_filename, 'bin') - if not os.path.exists(installation_bin_path): - download(config.url, os.path.join("/tmp", pypy_tar_bz2)) - call(['tar', '-C', '/tmp', '-xf', os.path.join("/tmp", pypy_tar_bz2)]) - pypy_executable = 'pypy' if config.version[0] == '2' else 'pypy3' - python_symlink = 'python' if config.version[0] == '2' else 'python3' - os.symlink(os.path.join(installation_bin_path, pypy_executable), os.path.join(installation_bin_path, python_symlink)) + installation_bin_path = install_pypy(config.version, config.url) + python_executable = 'pypy3' if config.version[0] == '3' else 'pypy' + pip_executable = 'pip3' if config.version[0] == '3' else 'pip' + else: + raise ValueError("Unknown Python implementation") - assert os.path.exists(os.path.join(installation_bin_path, 'python3' if config.version[0] == '3' else 'python')) + assert os.path.exists(os.path.join(installation_bin_path, python_executable)) - # Python bin folders on Mac don't symlink python3 to python, so we do that - # so `python` and `pip` always point to the active configuration. + # Python bin folders on Mac don't symlink `python3` to `python`, and neither + # does PyPy for `pypy` or `pypy3`, so we do that so `python` and `pip` always + # point to the active configuration. if os.path.exists('/tmp/cibw_bin'): shutil.rmtree('/tmp/cibw_bin') os.makedirs('/tmp/cibw_bin') - if config.version[0] == '3': - os.symlink(os.path.join(installation_bin_path, 'python3'), '/tmp/cibw_bin/python') - os.symlink(os.path.join(installation_bin_path, 'python3-config'), '/tmp/cibw_bin/python-config') - os.symlink(os.path.join(installation_bin_path, 'pip3'), '/tmp/cibw_bin/pip') + os.symlink(os.path.join(installation_bin_path, python_executable), '/tmp/cibw_bin/python') + os.symlink(os.path.join(installation_bin_path, python_executable + '-config'), '/tmp/cibw_bin/python-config') + os.symlink(os.path.join(installation_bin_path, pip_executable), '/tmp/cibw_bin/pip') env = os.environ.copy() env['PATH'] = os.pathsep.join([ diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index f4c499f7..3a1edb9d 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -17,38 +17,28 @@ IS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache') IS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows' -def get_python_path(config): - if config.identifier.startswith('cp'): - nuget_args = get_nuget_args(config) - return os.path.join(nuget_args[-1], nuget_args[0] + "." + config.version, "tools") - elif config.identifier.startswith('pp'): - # Inside the PyPy zip file is a directory with the same name - filename = config.url.rsplit('/', 1)[-1] - return os.path.join("C:\\cibw", os.path.splitext(filename)[0]) - - -def get_nuget_args(configuration): - python_name = "python" if configuration.version[0] == '3' else "python2" - if configuration.arch == "32": - python_name = python_name + "x86" - return [python_name, "-Version", configuration.version, "-OutputDirectory", "C:/cibw/python"] +def get_nuget_args(version, arch): + python_name = 'python' if version[0] == '3' else 'python2' + if arch == '32': + python_name = python_name + 'x86' + return [python_name, '-Version', version, '-OutputDirectory', 'C:\\cibw\\python'] def get_python_configurations(build_selector): PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'arch', 'identifier', 'url']) python_configurations = [ - PythonConfiguration(version='2.7.17', arch="32", identifier='cp27-win32', url=None), - PythonConfiguration(version='2.7.17', arch="64", identifier='cp27-win_amd64', url=None), - PythonConfiguration(version='3.5.4', arch="32", identifier='cp35-win32', url=None), - PythonConfiguration(version='3.5.4', arch="64", identifier='cp35-win_amd64', url=None), - PythonConfiguration(version='3.6.8', arch="32", identifier='cp36-win32', url=None), - PythonConfiguration(version='3.6.8', arch="64", identifier='cp36-win_amd64', url=None), - PythonConfiguration(version='3.7.6', arch="32", identifier='cp37-win32', url=None), - PythonConfiguration(version='3.7.6', arch="64", identifier='cp37-win_amd64', url=None), - PythonConfiguration(version='3.8.1', arch="32", identifier='cp38-win32', url=None), - PythonConfiguration(version='3.8.1', arch="64", identifier='cp38-win_amd64', url=None), - PythonConfiguration(version='2.7-v7.3.0', arch="32", identifier='pp27_73-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.0-win32.zip'), - PythonConfiguration(version='3.6-v7.3.0', arch="32", identifier='pp36_73-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.3.0-win32.zip'), + PythonConfiguration(version='2.7.17', arch='32', identifier='cp27-win32', url=None), + PythonConfiguration(version='2.7.17', arch='64', identifier='cp27-win_amd64', url=None), + PythonConfiguration(version='3.5.4', arch='32', identifier='cp35-win32', url=None), + PythonConfiguration(version='3.5.4', arch='64', identifier='cp35-win_amd64', url=None), + PythonConfiguration(version='3.6.8', arch='32', identifier='cp36-win32', url=None), + PythonConfiguration(version='3.6.8', arch='64', identifier='cp36-win_amd64', url=None), + PythonConfiguration(version='3.7.6', arch='32', identifier='cp37-win32', url=None), + PythonConfiguration(version='3.7.6', arch='64', identifier='cp37-win_amd64', url=None), + PythonConfiguration(version='3.8.1', arch='32', identifier='cp38-win32', url=None), + PythonConfiguration(version='3.8.1', arch='64', identifier='cp38-win_amd64', url=None), + PythonConfiguration(version='2.7-v7.3.0', arch='32', identifier='pp27_73-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy2.7-v7.3.0-win32.zip'), + PythonConfiguration(version='3.6-v7.3.0', arch='32', identifier='pp36_73-win32', url='https://bitbucket.org/pypy/pypy/downloads/pypy3.6-v7.3.0-win32.zip'), ] if IS_RUNNING_ON_TRAVIS: @@ -71,6 +61,28 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef def extract_zip(zip_src, dest): with ZipFile(zip_src) as zip: zip.extractall(dest) + + def install_cpython(version, arch): + nuget_args = get_nuget_args(version, arch) + installation_path = os.path.join(nuget_args[-1], nuget_args[0] + '.' + version, 'tools') + simple_shell([nuget, 'install'] + nuget_args) + return installation_path + + def install_pypy(version, arch, url): + assert arch == '32' + # Inside the PyPy zip file is a directory with the same name + zip_filename = url.rsplit('/', 1)[-1] + installation_path = os.path.join('C:\\cibw', os.path.splitext(zip_filename)[0]) + if not os.path.exists(installation_path): + pypy_zip = os.path.join('C:\\cibw', zip_filename) + download(url, pypy_zip) + # Extract to the parent directory because the zip file still contains a directory + extract_zip(pypy_zip, os.path.dirname(installation_path)) + pypy_exe = 'pypy3.exe' if config.version[0] == '3' else 'pypy.exe' + simple_shell(['mklink', os.path.join(installation_path, 'python.exe'), os.path.join(installation_path, pypy_exe)]) + simple_shell(['mklink', '/d', os.path.join(installation_path, 'Scripts'), os.path.join(installation_path, 'bin')]) + return installation_path + if IS_RUNNING_ON_AZURE or IS_RUNNING_ON_TRAVIS: shell = simple_shell @@ -100,26 +112,22 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef python_configurations = get_python_configurations(build_selector) for config in python_configurations: # install Python - config_python_path = get_python_path(config) if config.identifier.startswith('cp'): - simple_shell([nuget, "install"] + get_nuget_args(config)) - elif config.identifier.startswith('pp') and not os.path.exists(config_python_path): - pypy_zip = os.path.join("C:\\cibw", config.url.rsplit('/', 1)[-1]) - download(config.url, pypy_zip) - # Extract to the parent of config_python_path because the zip file still contains a directory - extract_zip(pypy_zip, os.path.dirname(config_python_path)) - pypy_exe = 'pypy.exe' if config.version[0] == '2' else 'pypy3.exe' - simple_shell(['mklink', os.path.join(config_python_path, 'python.exe'), os.path.join(config_python_path, pypy_exe)]) - simple_shell(['mklink', '/d', os.path.join(config_python_path, 'Scripts'), os.path.join(config_python_path, 'bin')]) - assert os.path.exists(os.path.join(config_python_path, 'python.exe')) + installation_path = install_cpython(config.version, config.arch) + elif config.identifier.startswith('pp'): + installation_path = install_pypy(config.version, config.arch, config.url) + else: + raise ValueError("Unknown Python implementation") + + assert os.path.exists(os.path.join(installation_path, 'python.exe')) # set up PATH and environment variables for run_with_env env = os.environ.copy() env['PYTHON_VERSION'] = config.version env['PYTHON_ARCH'] = config.arch env['PATH'] = os.pathsep.join([ - config_python_path, - os.path.join(config_python_path, 'Scripts'), + installation_path, + os.path.join(installation_path, 'Scripts'), env['PATH'] ]) # update env with results from CIBW_ENVIRONMENT @@ -128,12 +136,12 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef # for the logs - check we're running the right version of python simple_shell(['where', 'python'], env=env) simple_shell(['python', '--version'], env=env) - simple_shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)\"'], env=env) + simple_shell(['python', '-c', '"import struct; print(struct.calcsize(\'P\') * 8)"'], env=env) # make sure pip is installed - if not os.path.exists(os.path.join(config_python_path, 'Scripts', 'pip.exe')): + if not os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe')): simple_shell(['python', get_pip_script], env=env, cwd="C:\\cibw") - assert os.path.exists(os.path.join(config_python_path, 'Scripts', 'pip.exe')) + assert os.path.exists(os.path.join(installation_path, 'Scripts', 'pip.exe')) # prepare the Python environment simple_shell(['python', '-m', 'pip', 'install', '--upgrade', 'pip'], env=env) diff --git a/docs/options.md b/docs/options.md index 15d79927..1cabb8ea 100644 --- a/docs/options.md +++ b/docs/options.md @@ -117,7 +117,7 @@ CIBW_BUILD: cp3?-* CIBW_SKIP: "*-win32 *-manylinux_i686" # Only build PyPy and CPython 3 -CIBW_BUILD: pp??_??-* cp3?-* +CIBW_BUILD: pp*-* cp3?-* ```