amprolla

devuan's apt repo merger
git clone git://parazyd.org/amprolla.git
Log | Files | Refs | README | LICENSE

commit fcae27d990e48253848006c52f79247a30ccd9f7
parent 4eb2e225dc9cfdb272b1cbda5e5f7d4f5afec01c
Author: parazyd <parazyd@dyne.org>
Date:   Mon,  5 Jun 2017 14:38:49 +0200

merge sources files as well

Diffstat:
Mamprolla_merge.py | 22++++++++++++++++------
Mlib/config.py | 45+++++++++++++++++++++++++++++++++++++++++++++
Mlib/package.py | 32++++++++++++++++++++------------
Morchestrate.py | 9++++++---
4 files changed, 87 insertions(+), 21 deletions(-)

diff --git a/amprolla_merge.py b/amprolla_merge.py @@ -6,7 +6,7 @@ Amprolla main module """ from sys import argv -from os.path import join +from os.path import basename, join from time import time from lib.package import (write_packages, load_packages_file, @@ -62,7 +62,7 @@ def devuan_rewrite(pkg, repo_name): def merge(packages_list): """ - Merges the Packages files given in the package list + Merges the Packages/Sources files given in the package list """ t1 = time() @@ -81,16 +81,26 @@ def merge(packages_list): if debian: all_repos.append({'name': 'debian', 'packages': debian}) - print('Merging packages') - new_pkgs = merge_packages_many(all_repos, banned_packages=banpkgs, - rewriter=devuan_rewrite) + + if basename(packages_list[0]) == 'Packages.gz': + print('Merging packages') + src = False + new_pkgs = merge_packages_many(all_repos, banned_packages=banpkgs, + rewriter=devuan_rewrite) + elif basename(packages_list[0]) == 'Sources.gz': + print('Merging sources') + src = True + new_pkgs = merge_packages_many(all_repos) print('Writing packages') # replace the devuan subdir with our mergedir that we plan to fill new_out = packages_list[0].replace(join(spooldir, repos['devuan']['dists']), join(mergedir, mergesubdir)) - write_packages(new_pkgs, new_out) + if src: + write_packages(new_pkgs, new_out, sources=True) + else: + write_packages(new_pkgs, new_out) t2 = time() print('time:', t2-t1) diff --git a/lib/config.py b/lib/config.py @@ -83,6 +83,7 @@ aliases = { categories = ['main', 'contrib', 'non-free'] arches = [ + 'source', # 'binary-all', # 'binary-alpha', # 'binary-amd64', @@ -170,3 +171,47 @@ packages_keys = [ 'SHA1', 'SHA256' ] + +sources_keys = [ + 'Package', + 'Binary', + 'Version', + 'Maintainer', + 'Uploaders', + 'Build-Depends', + 'Architecture', + 'Standards-Version', + 'Format', + 'Files', + 'Vcs-Browser', + 'Vcs-Svn', + 'Checksums-Sha1', + 'Checksums-Sha256', + 'Homepage', + 'Package-List', + 'Directory', + 'Priority', + 'Section', + 'Vcs-Git', + 'Dm-Upload-Allowed', + 'Build-Conflicts', + 'Testsuite', + 'Build-Depends-Indep', + 'Vcs-Bzr', + 'Vcs-Mtn', + 'Vcs-Hg', + 'Ruby-Versions', + 'Dgit', + 'Vcs-Darcs', + 'Extra-Source-Only', + 'Python-Version', + 'Testsuite-Triggers', + 'Autobuild', + 'Build-Conflicts-Indep', + 'Vcs-Cvs', + 'Comment', + 'Origin', + 'Vcs-Arch', + 'Original-Maintainer', + 'Python3-Version' +] diff --git a/lib/package.py b/lib/package.py @@ -11,10 +11,10 @@ from lzma import open as lzma_open from shutil import copyfile from lib.parse import (parse_packages, parse_dependencies) -from lib.config import packages_keys, mergedir, spooldir +from lib.config import packages_keys, sources_keys, mergedir, spooldir -def write_packages(packages, filename, sort=True): +def write_packages(packages, filename, sort=True, sources=False): """ Writes `packages` to a file (per debian Packages format) If sort=True, the packages are sorted by name. @@ -22,32 +22,40 @@ def write_packages(packages, filename, sort=True): os.makedirs(os.path.dirname(filename), exist_ok=True) # Copy the arch-specific Release file from devuan if it's not there - rl = filename.replace('Packages.gz', 'Release') + bsnm = 'Packages.gz' + if sources: + bsnm = 'Sources.gz' + rl = filename.replace(bsnm, 'Release') if not os.path.isfile(rl): copyfile(rl.replace(mergedir, join(spooldir, 'devuan')), rl) gzf = gzip_open(filename, 'w') - xzf = lzma_open(filename.replace('.gz', '.xz'), 'w') - f = open(filename.replace('.gz', ''), 'w') + #xzf = lzma_open(filename.replace('.gz', '.xz'), 'w') + #f = open(filename.replace('.gz', ''), 'w') pkg_items = packages.items() if sort: pkg_items = sorted(pkg_items, key=lambda x: x[0]) + if sources: + keylist = sources_keys + else: + keylist = packages_keys + for pkg_name, pkg_contents in pkg_items: - for key in packages_keys: + for key in keylist: if key in pkg_contents: s = '%s: %s\n' % (key, pkg_contents[key]) gzf.write(s.encode('utf-8')) - xzf.write(s.encode('utf-8')) - f.write(s) + #xzf.write(s.encode('utf-8')) + #f.write(s) gzf.write(b'\n') - xzf.write(b'\n') - f.write('\n') + #xzf.write(b'\n') + #f.write('\n') gzf.close() - xzf.close() - f.close() + #xzf.close() + #f.close() def load_packages_file(filename): diff --git a/orchestrate.py b/orchestrate.py @@ -9,11 +9,14 @@ from multiprocessing import Pool pkg = [] for i in arches: for j in categories: - pkg.append(join(j, i, 'Packages.gz')) - pkg.append(join(j, 'debian-installer', i, 'Packages.gz')) + if i == 'source': + mrgfile = 'Sources.gz' + else: + mrgfile = 'Packages.gz' + pkg.append(join(j, 'debian-installer', i, mrgfile)) + pkg.append(join(j, i, mrgfile)) -#print(pkg) am = __import__('amprolla_merge') p = Pool(4)