From a1832ae6c692e20ed6ad717b7393a1c2ed2a7f20 Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Tue, 17 Dec 2019 01:08:14 +0100 Subject: [PATCH 1/7] Repeat get-pip download on fail --- cibuildwheel/macos.py | 10 +++++++++- cibuildwheel/windows.py | 11 ++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 3bd0df79..ffea3163 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -1,4 +1,5 @@ from __future__ import print_function +from time import sleep import tempfile import os, subprocess, shlex, sys, shutil from collections import namedtuple @@ -50,7 +51,14 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef return subprocess.check_call(args, env=env, cwd=cwd, shell=shell) # get latest pip once and for all - call(['curl', '-L', '-o', get_pip_script, get_pip_url]) + for _ in range(10): + try: + call(['curl', '-L', '-o', get_pip_script, get_pip_url]) + except subprocess.CalledProcessError: + sleep(3) + continue + break + assert os.path.exists(get_pip_script) for config in python_configurations: # if this version of python isn't installed, get it from python.org and install diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 7c03f815..56561fba 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -1,5 +1,6 @@ from __future__ import print_function import os, tempfile, subprocess, shutil, sys +from time import sleep from collections import namedtuple from glob import glob @@ -96,7 +97,15 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget) # get pip fo this installation which not have. get_pip_script = 'C:\\cibw\\get-pip.py' - download('https://bootstrap.pypa.io/get-pip.py', get_pip_script) + + for _ in range(10): + try: + download('https://bootstrap.pypa.io/get-pip.py', get_pip_script) + except: + sleep(3) + continue + break + assert os.path.exists(get_pip_script) python_configurations = get_python_configurations(build_selector) for config in python_configurations: From e0b385a520788022e238bc2118937ad7c66fbe28 Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Tue, 17 Dec 2019 10:55:16 +0100 Subject: [PATCH 2/7] Move windows repetition download to download function. --- cibuildwheel/windows.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 56561fba..d046461c 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -63,17 +63,29 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef print('+ ' + ' '.join(args)) args = ['cmd', '/E:ON', '/V:ON', '/C'] + args return subprocess.check_call(' '.join(args), env=env, cwd=cwd) + def download(url, dest): print('+ Download ' + url + ' to ' + dest) dest_dir = os.path.dirname(dest) if not os.path.exists(dest_dir): os.makedirs(dest_dir) - response = urlopen(url) + for _ in range(10): + try: + response = urlopen(url) + except: + sleep(3) + continue + break + else: + print("Download from url " + url + "failed", file=sys.stderr) + sys.exit(1) + try: with open(dest, 'wb') as file: file.write(response.read()) finally: response.close() + if IS_RUNNING_ON_AZURE or IS_RUNNING_ON_TRAVIS: shell = simple_shell else: @@ -97,15 +109,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef download('https://dist.nuget.org/win-x86-commandline/latest/nuget.exe', nuget) # get pip fo this installation which not have. get_pip_script = 'C:\\cibw\\get-pip.py' - - for _ in range(10): - try: - download('https://bootstrap.pypa.io/get-pip.py', get_pip_script) - except: - sleep(3) - continue - break - assert os.path.exists(get_pip_script) + download('https://bootstrap.pypa.io/get-pip.py', get_pip_script) python_configurations = get_python_configurations(build_selector) for config in python_configurations: From e00f03047a8bf5fd62ca9c18a278be2caa6c8cdd Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Tue, 17 Dec 2019 11:53:57 +0100 Subject: [PATCH 3/7] reduce repetition in download retry --- cibuildwheel/macos.py | 2 +- cibuildwheel/windows.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index ffea3163..280fdd91 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -51,7 +51,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef return subprocess.check_call(args, env=env, cwd=cwd, shell=shell) # get latest pip once and for all - for _ in range(10): + for _ in range(3): try: call(['curl', '-L', '-o', get_pip_script, get_pip_url]) except subprocess.CalledProcessError: diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index d046461c..86fce758 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -69,7 +69,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef dest_dir = os.path.dirname(dest) if not os.path.exists(dest_dir): os.makedirs(dest_dir) - for _ in range(10): + for _ in range(3): try: response = urlopen(url) except: From dd9b665c1d39b266a282b786ce7ea02b6330f4ae Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Tue, 17 Dec 2019 21:34:26 +0100 Subject: [PATCH 4/7] retry on macos with cli arguments --- cibuildwheel/macos.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 280fdd91..74e072b1 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -51,14 +51,8 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef return subprocess.check_call(args, env=env, cwd=cwd, shell=shell) # get latest pip once and for all - for _ in range(3): - try: - call(['curl', '-L', '-o', get_pip_script, get_pip_url]) - except subprocess.CalledProcessError: - sleep(3) - continue - break - assert os.path.exists(get_pip_script) + + call(['curl', '-L', '-o', '--retry', '3', '--retry-delay', '3', get_pip_script, get_pip_url]) for config in python_configurations: # if this version of python isn't installed, get it from python.org and install From ebcda4d049dd93c860b74c0bcc1288a3a4b5ef4f Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Tue, 17 Dec 2019 22:40:00 +0100 Subject: [PATCH 5/7] Update cibuildwheel/macos.py Co-Authored-By: Matthieu Darbois --- cibuildwheel/macos.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index 74e072b1..d9723e8b 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -52,7 +52,7 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef # get latest pip once and for all - call(['curl', '-L', '-o', '--retry', '3', '--retry-delay', '3', get_pip_script, get_pip_url]) + call(['curl', '-L', '--retry', '3', '--retry-delay', '3', '-o', get_pip_script, get_pip_url]) for config in python_configurations: # if this version of python isn't installed, get it from python.org and install From 32182f3edce2d7ea4249d09b7f4708a37485f431 Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Tue, 17 Dec 2019 23:23:57 +0100 Subject: [PATCH 6/7] re raise last exception in windows --- cibuildwheel/windows.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index 86fce758..61999fcc 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -69,16 +69,16 @@ def build(project_dir, output_dir, test_command, test_requires, test_extras, bef dest_dir = os.path.dirname(dest) if not os.path.exists(dest_dir): os.makedirs(dest_dir) - for _ in range(3): + repeat_num = 3 + for i in range(repeat_num): try: response = urlopen(url) except: + if i == repeat_num - 1: + raise sleep(3) continue break - else: - print("Download from url " + url + "failed", file=sys.stderr) - sys.exit(1) try: with open(dest, 'wb') as file: From 0af635fb868e711f33e78495b0f914701db152ad Mon Sep 17 00:00:00 2001 From: Grzegorz Bokota Date: Wed, 18 Dec 2019 23:22:19 +0100 Subject: [PATCH 7/7] Update cibuildwheel/macos.py Co-Authored-By: Matthieu Darbois --- cibuildwheel/macos.py | 1 - 1 file changed, 1 deletion(-) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index d9723e8b..5ada8ab2 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -1,5 +1,4 @@ from __future__ import print_function -from time import sleep import tempfile import os, subprocess, shlex, sys, shutil from collections import namedtuple