From fd0a789988f5b8d444cd45b0bb5c88f83cd36ee7 Mon Sep 17 00:00:00 2001 From: Yannick Jadoul Date: Tue, 11 Feb 2020 14:44:53 +0100 Subject: [PATCH] Fix minor review comment from @Czaki and @joerick --- README.md | 18 ++++---- cibuildwheel/macos.py | 78 +++++++++++++++++----------------- cibuildwheel/windows.py | 94 +++++++++++++++++++++-------------------- docs/options.md | 4 +- 4 files changed, 100 insertions(+), 94 deletions(-) diff --git a/README.md b/README.md index b00b5c8a..74497bbf 100644 --- a/README.md +++ b/README.md @@ -15,20 +15,18 @@ Python wheels are great. Building them across **Mac, Linux, Windows**, on **mult What does it do? ---------------- -| | macOS 10.9+ x86_64 | manylinux i686 | manylinux x86_64 | Windows 32bit | Windows 64bit | +| | macOS x86_64 | manylinux i686 | manylinux x86_64 | Windows 32bit | Windows 64bit | |---|---|---|---|---|---| -| Python 2.7 | ✅ | ✅ | ✅ | ✅¹ | ✅¹ | -| Python 3.5 | ✅ | ✅ | ✅ | ✅ | ✅ | -| Python 3.6 | ✅ | ✅ | ✅ | ✅ | ✅ | -| Python 3.7 | ✅ | ✅ | ✅ | ✅ | ✅ | -| Python 3.8 | ✅ | ✅ | ✅ | ✅ | ✅ | -| PyPy 2.7 v7.3.0 | ✅² | | ✅ | ✅ | | -| PyPy 3.6 v7.3.0 | ✅² | | ✅ | ✅ | | +| Python 2.7 | ✅ | ✅ | ✅ | ✅¹ | ✅¹ | +| Python 3.5 | ✅ | ✅ | ✅ | ✅ | ✅ | +| Python 3.6 | ✅ | ✅ | ✅ | ✅ | ✅ | +| Python 3.7 | ✅ | ✅ | ✅ | ✅ | ✅ | +| Python 3.8 | ✅ | ✅ | ✅ | ✅ | ✅ | +| PyPy 2.7 v7.3.0 | ✅ | | ✅ | ✅ | | +| PyPy 3.6 v7.3.0 | ✅ | | ✅ | ✅ | | > ¹ Not supported on Travis -> ² For PyPy 2.7, the minimally supported OS X/macOS version is 10.7; for PyPy 3.6, macOS 10.13 or higher is required. - - Builds manylinux, macOS and Windows (32 and 64bit) wheels for CPython and PyPy using Azure Pipelines, Travis CI, AppVeyor, and CircleCI - Bundles shared library dependencies on Linux and macOS through [auditwheel](https://github.com/pypa/auditwheel) and [delocate](https://github.com/matthew-brett/delocate) - Runs the library test suite against the wheel-installed version of your library diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 6a246057..a3ff0524 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -13,6 +13,16 @@ from .util import ( ) +def call(args, env=None, cwd=None, shell=False): + # print the command executing for the logs + if shell: + print('+ %s' % args) + else: + print('+ ' + ' '.join(shlex.quote(a) for a in args)) + + return subprocess.check_call(args, env=env, cwd=cwd, shell=shell) + + def get_python_configurations(build_selector): PythonConfiguration = namedtuple('PythonConfiguration', ['version', 'identifier', 'url']) python_configurations = [ @@ -29,6 +39,36 @@ def get_python_configurations(build_selector): return [c for c in python_configurations if build_selector(c.identifier)] +def install_cpython(version, url): + installed_system_packages = subprocess.check_output(['pkgutil', '--pkgs'], universal_newlines=True).splitlines() + + # if this version of python isn't installed, get it from python.org and install + python_package_identifier = 'org.python.Python.PythonFramework-{}'.format(version) + if python_package_identifier not in installed_system_packages: + # download the pkg + download(url, '/tmp/Python.pkg') + # install + call(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/']) + # patch open ssl + if version == '3.5': + open_ssl_patch_url = 'https://github.com/mayeut/patch-macos-python-openssl/releases/download/v1.0.2t/patch-macos-python-%s-openssl-v1.0.2t.tar.gz' % version + download(open_ssl_patch_url, '/tmp/python-patch.tar.gz') + call(['sudo', 'tar', '-C', '/Library/Frameworks/Python.framework/Versions/{}/'.format(version), '-xmf', '/tmp/python-patch.tar.gz']) + + return '/Library/Frameworks/Python.framework/Versions/{}/bin'.format(version) + + +def install_pypy(version, url): + pypy_tar_bz2 = url.rsplit('/', 1)[-1] + assert pypy_tar_bz2.endswith(".tar.bz2") + pypy_base_filename = os.path.splitext(os.path.splitext(pypy_tar_bz2)[0])[0] + installation_path = os.path.join('/tmp', pypy_base_filename) + if not os.path.exists(installation_path): + download(url, os.path.join("/tmp", pypy_tar_bz2)) + call(['tar', '-C', '/tmp', '-xf', os.path.join("/tmp", pypy_tar_bz2)]) + return os.path.join(installation_path, 'bin') + + def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment): abs_project_dir = os.path.abspath(project_dir) temp_dir = tempfile.mkdtemp(prefix='cibuildwheel') @@ -40,44 +80,6 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef get_pip_url = 'https://bootstrap.pypa.io/get-pip.py' get_pip_script = '/tmp/get-pip.py' - pkgs_output = subprocess.check_output(['pkgutil', '--pkgs'], universal_newlines=True) - installed_system_packages = pkgs_output.splitlines() - - def call(args, env=None, cwd=None, shell=False): - # print the command executing for the logs - if shell: - print('+ %s' % args) - else: - print('+ ' + ' '.join(shlex.quote(a) for a in args)) - - return subprocess.check_call(args, env=env, cwd=cwd, shell=shell) - - def install_cpython(version, url): - # if this version of python isn't installed, get it from python.org and install - python_package_identifier = 'org.python.Python.PythonFramework-{}'.format(version) - if python_package_identifier not in installed_system_packages: - # download the pkg - download(url, '/tmp/Python.pkg') - # install - call(['sudo', 'installer', '-pkg', '/tmp/Python.pkg', '-target', '/']) - # patch open ssl - if version == '3.5': - open_ssl_patch_url = 'https://github.com/mayeut/patch-macos-python-openssl/releases/download/v1.0.2t/patch-macos-python-%s-openssl-v1.0.2t.tar.gz' % config.version - download(open_ssl_patch_url, '/tmp/python-patch.tar.gz') - call(['sudo', 'tar', '-C', '/Library/Frameworks/Python.framework/Versions/{}/'.format(version), '-xmf', '/tmp/python-patch.tar.gz']) - return '/Library/Frameworks/Python.framework/Versions/{}/bin'.format(version) - - def install_pypy(version, url): - pypy_tar_bz2 = url.rsplit('/', 1)[-1] - assert pypy_tar_bz2.endswith(".tar.bz2") - pypy_base_filename = os.path.splitext(os.path.splitext(pypy_tar_bz2)[0])[0] - installation_path = os.path.join('/tmp', pypy_base_filename) - if not os.path.exists(installation_path): - download(url, os.path.join("/tmp", pypy_tar_bz2)) - call(['tar', '-C', '/tmp', '-xf', os.path.join("/tmp", pypy_tar_bz2)]) - return os.path.join(installation_path, 'bin') - - # get latest pip once and for all download(get_pip_url, get_pip_script) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 3a1edb9d..1bd436b7 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -17,6 +17,26 @@ IS_RUNNING_ON_AZURE = os.path.exists('C:\\hostedtoolcache') IS_RUNNING_ON_TRAVIS = os.environ.get('TRAVIS_OS_NAME') == 'windows' +def simple_shell(args, env=None, cwd=None): + print('+ ' + ' '.join(args)) + args = ['cmd', '/E:ON', '/V:ON', '/C'] + args + return subprocess.check_call(' '.join(args), env=env, cwd=cwd) + + +if IS_RUNNING_ON_AZURE or IS_RUNNING_ON_TRAVIS: + shell = simple_shell +else: + run_with_env = os.path.abspath(os.path.join(os.path.dirname(__file__), 'resources', 'appveyor_run_with_env.cmd')) + + # run_with_env is a cmd file that sets the right environment variables + # to build on AppVeyor. + def shell(args, env=None, cwd=None): + # print the command executing for the logs + print('+ ' + ' '.join(args)) + args = ['cmd', '/E:ON', '/V:ON', '/C', run_with_env] + args + return subprocess.check_call(' '.join(args), env=env, cwd=cwd) + + def get_nuget_args(version, arch): python_name = 'python' if version[0] == '3' else 'python2' if arch == '32': @@ -52,51 +72,35 @@ def get_python_configurations(build_selector): return python_configurations +def extract_zip(zip_src, dest): + with ZipFile(zip_src) as zip: + zip.extractall(dest) + + +def install_cpython(version, arch, nuget): + nuget_args = get_nuget_args(version, arch) + installation_path = os.path.join(nuget_args[-1], nuget_args[0] + '.' + version, 'tools') + simple_shell([nuget, 'install'] + nuget_args) + return installation_path + + +def install_pypy(version, arch, url): + assert arch == '32' + # Inside the PyPy zip file is a directory with the same name + zip_filename = url.rsplit('/', 1)[-1] + installation_path = os.path.join('C:\\cibw', os.path.splitext(zip_filename)[0]) + if not os.path.exists(installation_path): + pypy_zip = os.path.join('C:\\cibw', zip_filename) + download(url, pypy_zip) + # Extract to the parent directory because the zip file still contains a directory + extract_zip(pypy_zip, os.path.dirname(installation_path)) + pypy_exe = 'pypy3.exe' if version[0] == '3' else 'pypy.exe' + simple_shell(['mklink', os.path.join(installation_path, 'python.exe'), os.path.join(installation_path, pypy_exe)]) + simple_shell(['mklink', '/d', os.path.join(installation_path, 'Scripts'), os.path.join(installation_path, 'bin')]) + return installation_path + + def build(project_dir, output_dir, test_command, test_requires, test_extras, before_build, build_verbosity, build_selector, repair_command, environment): - def simple_shell(args, env=None, cwd=None): - print('+ ' + ' '.join(args)) - args = ['cmd', '/E:ON', '/V:ON', '/C'] + args - return subprocess.check_call(' '.join(args), env=env, cwd=cwd) - - def extract_zip(zip_src, dest): - with ZipFile(zip_src) as zip: - zip.extractall(dest) - - def install_cpython(version, arch): - nuget_args = get_nuget_args(version, arch) - installation_path = os.path.join(nuget_args[-1], nuget_args[0] + '.' + version, 'tools') - simple_shell([nuget, 'install'] + nuget_args) - return installation_path - - def install_pypy(version, arch, url): - assert arch == '32' - # Inside the PyPy zip file is a directory with the same name - zip_filename = url.rsplit('/', 1)[-1] - installation_path = os.path.join('C:\\cibw', os.path.splitext(zip_filename)[0]) - if not os.path.exists(installation_path): - pypy_zip = os.path.join('C:\\cibw', zip_filename) - download(url, pypy_zip) - # Extract to the parent directory because the zip file still contains a directory - extract_zip(pypy_zip, os.path.dirname(installation_path)) - pypy_exe = 'pypy3.exe' if config.version[0] == '3' else 'pypy.exe' - simple_shell(['mklink', os.path.join(installation_path, 'python.exe'), os.path.join(installation_path, pypy_exe)]) - simple_shell(['mklink', '/d', os.path.join(installation_path, 'Scripts'), os.path.join(installation_path, 'bin')]) - return installation_path - - - if IS_RUNNING_ON_AZURE or IS_RUNNING_ON_TRAVIS: - shell = simple_shell - else: - run_with_env = os.path.abspath(os.path.join(os.path.dirname(__file__), 'resources', 'appveyor_run_with_env.cmd')) - - # run_with_env is a cmd file that sets the right environment variables - # to build on AppVeyor. - def shell(args, env=None, cwd=None): - # print the command executing for the logs - print('+ ' + ' '.join(args)) - args = ['cmd', '/E:ON', '/V:ON', '/C', run_with_env] + args - return subprocess.check_call(' '.join(args), env=env, cwd=cwd) - abs_project_dir = os.path.abspath(project_dir) temp_dir = tempfile.mkdtemp(prefix='cibuildwheel') built_wheel_dir = os.path.join(temp_dir, 'built_wheel') @@ -113,7 +117,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef for config in python_configurations: # install Python if config.identifier.startswith('cp'): - installation_path = install_cpython(config.version, config.arch) + installation_path = install_cpython(config.version, config.arch, nuget) elif config.identifier.startswith('pp'): installation_path = install_pypy(config.version, config.arch, config.url) else: diff --git a/docs/options.md b/docs/options.md index 1cabb8ea..f6b74438 100644 --- a/docs/options.md +++ b/docs/options.md @@ -56,7 +56,7 @@ Default: `auto` `auto` will auto-detect platform using environment variables, such as `TRAVIS_OS_NAME`/`APPVEYOR`/`CIRCLECI`. -For `linux` you need Docker running, on macOS or Linux. For `macos`, you need a Mac machine, and note that this script is going to automatically install MacPython on your system, so don't run on your development machine. For `windows`, you need to run in Windows, and it will build and test for all versions of Python under `C:\cibw\python`. +For `linux` you need Docker running, on macOS or Linux. For `macos`, you need a Mac machine, and note that this script is going to automatically install MacPython on your system, so don't run on your development machine. For `windows`, you need to run in Windows, and `cibuildwheel` will install required versions of Python to `C:\cibw\python` using NuGet. This option can also be set using the command-line option `--platform`. @@ -85,6 +85,8 @@ When setting the options, you can use shell-style globbing syntax (as per [`fnma The list of supported and currently selected build identifiers can also be retrieved by passing the `--print-build-identifiers` flag to `cibuildwheel`. The format is `python_tag-platform_tag`, with tags similar to those in [PEP 425](https://www.python.org/dev/peps/pep-0425/#details). +For CPython, the minimally supported OS X/macOS version is 10.9; for PyPy 2.7 and PyPy 3.6, respectively macOS 10.7 and 10.13 or higher is required. + #### Examples ```yaml