amprolla

commit cf78750a6a7d40fe6025a81a2738f71a19a2a042
parent 89497bf3ee81e2b724808fed0f409aa3c41c2444
Author: parazyd <parazyd@dyne.org>
Date:   Tue,  6 Jun 2017 20:25:16 +0200

parallelize Release generation

Diffstat:
M
amprolla_merge.py
 | 
62
++++++++++++++++++++++++++++++++++----------------------------

1 file changed, 34 insertions(+), 28 deletions(-)
diff --git a/amprolla_merge.py b/amprolla_merge.py
@@ -102,35 +102,33 @@ def merge(packages_list):
         write_packages(new_pkgs, new_out)
 
 
-def gen_release(s):
+def gen_release(suite):
     """
-    Generates a Release file for a given main suite (jessie/ascii/unstable)
+    Generates a Release file for a given suite (jessie/ascii/unstable)
     """
 
-    for suite in suites[s]:
-        filelist = []
-        print('Crawling %s' % suite)
-        rootdir = join(mergedir, mergesubdir, suite)
-        for cat in categories:
-            for arch in arches:
-                if arch == 'source':
-                    flist = srcfiles
-                else:
-                    flist = pkgfiles
+    filelist = []
+    print('Crawling %s' % suite)
+    rootdir = join(mergedir, mergesubdir, suite)
+    for cat in categories:
+        for arch in arches:
+            if arch == 'source':
+                flist = srcfiles
+            else:
+                flist = pkgfiles
 
-                for fl in flist:
-                    filelist.append(join(rootdir, cat, arch, fl))
-                    if arch != 'source':
-                        filelist.append(join(rootdir, cat,
-                                             'debian-installer', arch, fl))
+            for i in flist:
+                filelist.append(join(rootdir, cat, arch, i))
+                if arch != 'source':
+                    filelist.append(join(rootdir, cat,
+                                         'debian-installer', arch, i))
 
-        newrfl = join(rootdir, 'Release')
-        oldrfl = newrfl.replace(join(mergedir, mergesubdir),
-                                join(spooldir, repos['devuan']['dists']))
+    newrfl = join(rootdir, 'Release')
+    oldrfl = newrfl.replace(join(mergedir, mergesubdir),
+                            join(spooldir, repos['devuan']['dists']))
 
-        print('Writing Release')
-        write_release(oldrfl, newrfl, filelist, rootdir)
-        # break
+    print('Writing Release')
+    write_release(oldrfl, newrfl, filelist, rootdir)
 
 
 def main_merge(packages_file):
@@ -167,11 +165,19 @@ def main():
             pkg.append(join(j, i, mrgfile))
 
     # pprint(pkg)
-    p = Pool(4)  # Set it to the number of CPUs you want to use
-    p.map(main_merge, pkg)
-
-    for st in suites:
-        gen_release(st)
+    mrgpool = Pool(4)  # Set it to the number of CPUs you want to use
+    mrgpool.map(main_merge, pkg)
+    mrgpool.close()
+
+    rel_list = []
+    for i in suites:
+        for j in suites[i]:
+            rel_list.append(j)
+            # gen_release(j)
+
+    relpool = Pool(4)  # Set it to the number of CPUs you want to use
+    relpool.map(gen_release, rel_list)
+    relpool.close()
 
 
 if __name__ == '__main__':