amprolla

commit fcae27d990e48253848006c52f79247a30ccd9f7
parent 4eb2e225dc9cfdb272b1cbda5e5f7d4f5afec01c
Author: parazyd <parazyd@dyne.org>
Date:   Mon,  5 Jun 2017 14:38:49 +0200

merge sources files as well

Diffstat:
M
amprolla_merge.py
 | 
22
++++++++++++++++------
M
lib/config.py
 | 
45
+++++++++++++++++++++++++++++++++++++++++++++
M
lib/package.py
 | 
32
++++++++++++++++++++------------
M
orchestrate.py
 | 
9
++++++---

4 files changed, 87 insertions(+), 21 deletions(-)
diff --git a/amprolla_merge.py b/amprolla_merge.py
@@ -6,7 +6,7 @@ Amprolla main module
 """
 
 from sys import argv
-from os.path import join
+from os.path import basename, join
 from time import time
 
 from lib.package import (write_packages, load_packages_file,
@@ -62,7 +62,7 @@ def devuan_rewrite(pkg, repo_name):
 
 def merge(packages_list):
     """
-    Merges the Packages files given in the package list
+    Merges the Packages/Sources files given in the package list
     """
     t1 = time()
 
@@ -81,16 +81,26 @@ def merge(packages_list):
     if debian:
         all_repos.append({'name': 'debian', 'packages': debian})
 
-    print('Merging packages')
-    new_pkgs = merge_packages_many(all_repos, banned_packages=banpkgs,
-                                   rewriter=devuan_rewrite)
+
+    if basename(packages_list[0]) == 'Packages.gz':
+        print('Merging packages')
+        src = False
+        new_pkgs = merge_packages_many(all_repos, banned_packages=banpkgs,
+                                       rewriter=devuan_rewrite)
+    elif basename(packages_list[0]) == 'Sources.gz':
+        print('Merging sources')
+        src = True
+        new_pkgs = merge_packages_many(all_repos)
 
     print('Writing packages')
     # replace the devuan subdir with our mergedir that we plan to fill
     new_out = packages_list[0].replace(join(spooldir,
                                             repos['devuan']['dists']),
                                        join(mergedir, mergesubdir))
-    write_packages(new_pkgs, new_out)
+    if src:
+        write_packages(new_pkgs, new_out, sources=True)
+    else:
+        write_packages(new_pkgs, new_out)
 
     t2 = time()
     print('time:', t2-t1)
diff --git a/lib/config.py b/lib/config.py
@@ -83,6 +83,7 @@ aliases = {
 categories = ['main', 'contrib', 'non-free']
 
 arches = [
+    'source',
     # 'binary-all',
     # 'binary-alpha',
     # 'binary-amd64',
@@ -170,3 +171,47 @@ packages_keys = [
     'SHA1',
     'SHA256'
 ]
+
+sources_keys = [
+    'Package',
+    'Binary',
+    'Version',
+    'Maintainer',
+    'Uploaders',
+    'Build-Depends',
+    'Architecture',
+    'Standards-Version',
+    'Format',
+    'Files',
+    'Vcs-Browser',
+    'Vcs-Svn',
+    'Checksums-Sha1',
+    'Checksums-Sha256',
+    'Homepage',
+    'Package-List',
+    'Directory',
+    'Priority',
+    'Section',
+    'Vcs-Git',
+    'Dm-Upload-Allowed',
+    'Build-Conflicts',
+    'Testsuite',
+    'Build-Depends-Indep',
+    'Vcs-Bzr',
+    'Vcs-Mtn',
+    'Vcs-Hg',
+    'Ruby-Versions',
+    'Dgit',
+    'Vcs-Darcs',
+    'Extra-Source-Only',
+    'Python-Version',
+    'Testsuite-Triggers',
+    'Autobuild',
+    'Build-Conflicts-Indep',
+    'Vcs-Cvs',
+    'Comment',
+    'Origin',
+    'Vcs-Arch',
+    'Original-Maintainer',
+    'Python3-Version'
+]
diff --git a/lib/package.py b/lib/package.py
@@ -11,10 +11,10 @@ from lzma import open as lzma_open
 from shutil import copyfile
 
 from lib.parse import (parse_packages, parse_dependencies)
-from lib.config import packages_keys, mergedir, spooldir
+from lib.config import packages_keys, sources_keys, mergedir, spooldir
 
 
-def write_packages(packages, filename, sort=True):
+def write_packages(packages, filename, sort=True, sources=False):
     """
     Writes `packages` to a file (per debian Packages format)
     If sort=True, the packages are sorted by name.
@@ -22,32 +22,40 @@ def write_packages(packages, filename, sort=True):
     os.makedirs(os.path.dirname(filename), exist_ok=True)
 
     # Copy the arch-specific Release file from devuan if it's not there
-    rl = filename.replace('Packages.gz', 'Release')
+    bsnm = 'Packages.gz'
+    if sources:
+        bsnm = 'Sources.gz'
+    rl = filename.replace(bsnm, 'Release')
     if not os.path.isfile(rl):
         copyfile(rl.replace(mergedir, join(spooldir, 'devuan')), rl)
 
     gzf = gzip_open(filename, 'w')
-    xzf = lzma_open(filename.replace('.gz', '.xz'), 'w')
-    f = open(filename.replace('.gz', ''), 'w')
+    #xzf = lzma_open(filename.replace('.gz', '.xz'), 'w')
+    #f = open(filename.replace('.gz', ''), 'w')
 
     pkg_items = packages.items()
     if sort:
         pkg_items = sorted(pkg_items, key=lambda x: x[0])
 
+    if sources:
+        keylist = sources_keys
+    else:
+        keylist = packages_keys
+
     for pkg_name, pkg_contents in pkg_items:
-        for key in packages_keys:
+        for key in keylist:
             if key in pkg_contents:
                 s = '%s: %s\n' % (key, pkg_contents[key])
                 gzf.write(s.encode('utf-8'))
-                xzf.write(s.encode('utf-8'))
-                f.write(s)
+                #xzf.write(s.encode('utf-8'))
+                #f.write(s)
         gzf.write(b'\n')
-        xzf.write(b'\n')
-        f.write('\n')
+        #xzf.write(b'\n')
+        #f.write('\n')
 
     gzf.close()
-    xzf.close()
-    f.close()
+    #xzf.close()
+    #f.close()
 
 
 def load_packages_file(filename):
diff --git a/orchestrate.py b/orchestrate.py
@@ -9,11 +9,14 @@ from multiprocessing import Pool
 pkg = []
 for i in arches:
     for j in categories:
-        pkg.append(join(j, i, 'Packages.gz'))
-        pkg.append(join(j, 'debian-installer', i, 'Packages.gz'))
+        if i == 'source':
+            mrgfile = 'Sources.gz'
+        else:
+            mrgfile = 'Packages.gz'
+            pkg.append(join(j, 'debian-installer', i, mrgfile))
 
+        pkg.append(join(j, i, mrgfile))
 
-#print(pkg)
 am = __import__('amprolla_merge')
 
 p = Pool(4)