amprolla

commit e57b27612f2df0ade2d8f4dba62943712d6b74b4
parent f82aba29785dd49fa0ec1425798de134366e2abd
Author: parazyd <parazyd@dyne.org>
Date:   Sun, 23 Jul 2017 17:58:47 +0200

fix multiprocessing issue with download()

the process should not forcefully exit with die() since it causes
a deadlock for multiprocessing.Pool

this commit also reimplements handling of
requests.exceptions.ReadTimeout

Diffstat:
M
lib/net.py
 | 
10
++++------

1 file changed, 4 insertions(+), 6 deletions(-)
diff --git a/lib/net.py b/lib/net.py
@@ -21,16 +21,14 @@ def download(uris):
 
     try:
         r = requests.get(url, stream=True, timeout=20)
-    # TODO: investigate also requests.exceptions.ReadTimeout
-    except requests.exceptions.ConnectionError as e:
+    except (requests.exceptions.ConnectionError,
+            requests.exceptions.ReadTimeout) as e:
         warn('Caught exception: "%s". Retrying...' % e)
         return download(uris)
 
-    if r.status_code == 404:
-        warn('failed: 404 not found!')
+    if r.status_code != 200:
+        warn('%s failed: %d' % (url, r.status_code))
         return
-    elif r.status_code != 200:
-        die('failed: %d' % r.status_code)
 
     makedirs(dirname(path), exist_ok=True)
     f = open(path, 'wb')