net.py (1031B)
1 # See LICENSE file for copyright and license details. 2 3 """ 4 Network functions/helpers 5 """ 6 7 from os import makedirs 8 from os.path import dirname 9 import requests 10 11 from lib.log import info, warn 12 13 14 def download(uris): 15 """ 16 Downloads a file by providing it an url and a write path in a tuple 17 """ 18 url = uris[0] 19 path = uris[1] 20 info("dl: %s" % url) 21 22 try: 23 rfile = requests.get(url, stream=True, timeout=20) 24 except (requests.exceptions.ConnectionError, 25 requests.exceptions.ReadTimeout, 26 ConnectionResetError) as err: 27 warn('Caught exception: "%s". Retrying...' % err) 28 return download(uris) 29 30 if rfile.status_code != 200: 31 warn('%s failed: %d' % (url, rfile.status_code)) 32 return 33 34 makedirs(dirname(path), exist_ok=True) 35 lfile = open(path, 'wb') 36 # chunk_size {sh,c}ould be more on gbit servers 37 for chunk in rfile.iter_content(chunk_size=1024): 38 if chunk: 39 lfile.write(chunk) 40 # f.flush() 41 lfile.close()