amprolla

devuan's apt repo merger
git clone https://git.parazyd.org/amprolla
Log | Files | Refs | README | LICENSE

amprolla_init.py (3140B)

      1 #!/usr/bin/env python3
      2 # See LICENSE file for copyright and license details.
      3 
      4 """
      5 This module will download the initial Release files used to populate
      6 the spooldir, along with all the files hashed inside the Release files
      7 """
      8 
      9 from os.path import join
     10 from multiprocessing import Pool
     11 from time import time
     12 
     13 from lib.config import (aliases, arches, categories, cpunm, mainrepofiles,
     14                         repos, spooldir, suites, skips)
     15 from lib.lock import check_lock, free_lock
     16 from lib.net import download
     17 from lib.parse import parse_release
     18 from lib.log import die, info
     19 
     20 
     21 def pop_dirs(repo):
     22     """
     23     Crawls through the directories to come up with complete needed
     24     directory structure.
     25     Returns a list of tuples holding the remote and local locations
     26     of the files
     27 
     28     Example:
     29     (http://deb.debian.org/debian/dists/jessie/main/binary-all/Packages.gz,
     30      ./spool/debian/dists/jessie/main/binary-all/Packages.gz)
     31     """
     32     repodata = repos[repo]
     33 
     34     urls = []
     35 
     36     for i in suites:
     37         for j in suites[i]:
     38             baseurl = join(repodata['host'], repodata['dists'])
     39             suite = j
     40             if repodata['aliases'] is True:
     41                 if j in aliases[repodata['name']]:
     42                     suite = aliases[repodata['name']][j]
     43                 elif repodata['skipmissing'] is True:
     44                     continue
     45                 if repo == 'debian' and j in skips:
     46                     continue
     47             pair = (join(baseurl, suite),
     48                     join(baseurl.replace(repodata['host'],
     49                                          spooldir), suite))
     50             urls.append(pair)
     51 
     52     return urls
     53 
     54 
     55 def main():
     56     """
     57     Loops through all repositories, and downloads their Release files, along
     58     with all the files listed within those Release files.
     59     """
     60     for dist in repos:
     61         print('Downloading %s directory structure' % dist)
     62         dlurls = pop_dirs(dist)
     63         for url in dlurls:
     64             tpl = []
     65             for file in mainrepofiles:
     66                 urls = (join(url[0], file), join(url[1], file))
     67                 tpl.append(urls)
     68             dlpool = Pool(cpunm)
     69             dlpool.map(download, tpl)
     70             dlpool.close()
     71 
     72             release_contents = open(join(url[1], 'Release')).read()
     73             release_contents = parse_release(release_contents)
     74             tpl = []
     75             for k in release_contents:
     76                 # if k.endswith('/binary-armhf/Packages.gz'):
     77                 # for a in arches:
     78                 #     for c in categories:
     79                 #        if a in k and ("/%s/" % c) in k:
     80                 #            urls = (join(url[0], k), join(url[1], k))
     81                 #            tpl.append(urls)
     82                 urls = (join(url[0], k), join(url[1], k))
     83                 tpl.append(urls)
     84             dlpool = Pool(cpunm)
     85             dlpool.map(download, tpl)
     86             dlpool.close()
     87 
     88 
     89 if __name__ == '__main__':
     90     try:
     91         t1 = time()
     92         check_lock()
     93         main()
     94         free_lock()
     95         t2 = time()
     96         info('Total init time: %s' % (t2 - t1), tofile=True)
     97     except Exception as e:
     98         die(e)