amprolla

devuan's apt repo merger
git clone git://parazyd.org/amprolla.git
Log | Files | Refs | README | LICENSE

commit 7b234b5febe9b32148c60a7ff5b5f582e5882d58
parent 385ad0a13cbf41188892e9880db84353a19657b6
Author: parazyd <parazyd@dyne.org>
Date:   Wed, 31 May 2017 23:07:42 +0200

add a orchestrate merge example

Diffstat:
Damprolla-merge | 107-------------------------------------------------------------------------------
Aamprolla-merge.py | 116+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Mlib/config.py | 2++
Aorchestrate.py | 19+++++++++++++++++++
4 files changed, 137 insertions(+), 107 deletions(-)

diff --git a/amprolla-merge b/amprolla-merge @@ -1,107 +0,0 @@ -#!/usr/bin/env python3 -""" -Amprolla main module -""" - -from os.path import join -from time import time - -from lib.package import (write_packages, load_packages_file, - merge_packages_many) -from lib.config import (aliases, banpkgs, repo_order, repos, - spooldir, suites , mergedir, mergesubdir) - - -def prepare_merge_dict(): - """ - This function will prepare a dict of lists that contain the repos - that need to be merged in an ordered fashion. Orders them using the - repo_order list found in lib.config - Example output: - {'ascii': ['ascii', None, 'stretch']}, - """ - merge_dict = {} - - for suite in suites: - for i in suites[suite]: - merge_dict[i] = [] - - for suite in merge_dict: - for repo in repo_order: - tmpsuite = suite - if repos[repo]['aliases'] is True: - if tmpsuite in aliases[repos[repo]['name']]: - tmpsuite = aliases[repos[repo]['name']][suite] - elif repos[repo]['skipmissing'] is True: - tmpsuite = None - skips = ['jessie-security', 'ascii-security'] - if repo == 'debian' and suite in skips: - tmpsuite = None - if tmpsuite: # make it a proper path - tmpsuite = join(spooldir, repos[repo]['dists'], tmpsuite) - merge_dict[suite].append(tmpsuite) - - return merge_dict - - -def devuan_rewrite(pkg, repo_name): - """ - Function to be called when including a certain package. Allows for changing - any attributes. Currently only changes the filename if we include a package - when repo_name == 'devuan'. - """ - if repo_name == 'devuan': - pkg['Filename'] = pkg['Filename'].replace('pool/', 'pool/%s/' % - repos[repo_name]['name']) - - return pkg - - -def merge(packages_list): - t1 = time() - - all_repos = [] - print('Loading packages: %s' % packages_list) - - devuan = load_packages_file(packages_list[0]) - if devuan: - all_repos.append({'name': 'devuan', 'packages': devuan}) - - debian_sec = load_packages_file(packages_list[1]) - if debian_sec: - all_repos.append({'name': 'debian-sec', 'packages': debian_sec}) - - debian = load_packages_file(packages_list[2]) - if debian: - all_repos.append({'name': 'debian', 'packages': debian}) - - print('Merging packages') - new_pkgs = merge_packages_many(all_repos, banned_packages=banpkgs, rewriter=devuan_rewrite) - - print('Writing packages') - # replace the devuan subdir with our mergedir that we plan to fill - new_out = packages_list[0].replace(join(spooldir, - repos['devuan']['dists']), - join(mergedir, mergesubdir)) - write_packages(new_pkgs, new_out) - - t2 = time() - print('time:', t2-t1) - - -packages_file = 'main/binary-armhf/Packages.gz' -to_merge = prepare_merge_dict() - -tt1 = time() -for suite in to_merge: - pkg_list = [] - for rep in to_merge[suite]: - if rep: - pkg_list.append(join(rep, packages_file)) - else: - pkg_list.append(None) - - merge(pkg_list) - -tt2 = time() -print('total time:', tt2-tt1) diff --git a/amprolla-merge.py b/amprolla-merge.py @@ -0,0 +1,116 @@ +#!/usr/bin/env python3 +""" +Amprolla main module +""" + +import sys +from os.path import join +from time import time + +from lib.package import (write_packages, load_packages_file, + merge_packages_many) +from lib.config import (aliases, banpkgs, repo_order, repos, + spooldir, suites , mergedir, mergesubdir) + + +def prepare_merge_dict(): + """ + This function will prepare a dict of lists that contain the repos + that need to be merged in an ordered fashion. Orders them using the + repo_order list found in lib.config + Example output: + {'ascii': ['ascii', None, 'stretch']}, + """ + merge_dict = {} + + for suite in suites: + for i in suites[suite]: + merge_dict[i] = [] + + for suite in merge_dict: + for repo in repo_order: + tmpsuite = suite + if repos[repo]['aliases'] is True: + if tmpsuite in aliases[repos[repo]['name']]: + tmpsuite = aliases[repos[repo]['name']][suite] + elif repos[repo]['skipmissing'] is True: + tmpsuite = None + skips = ['jessie-security', 'ascii-security'] + if repo == 'debian' and suite in skips: + tmpsuite = None + if tmpsuite: # make it a proper path + tmpsuite = join(spooldir, repos[repo]['dists'], tmpsuite) + merge_dict[suite].append(tmpsuite) + + return merge_dict + + +def devuan_rewrite(pkg, repo_name): + """ + Function to be called when including a certain package. Allows for changing + any attributes. Currently only changes the filename if we include a package + when repo_name == 'devuan'. + """ + if repo_name == 'devuan': + pkg['Filename'] = pkg['Filename'].replace('pool/', 'pool/%s/' % + repos[repo_name]['name']) + + return pkg + + +def merge(packages_list): + t1 = time() + + all_repos = [] + print('Loading packages: %s' % packages_list) + + devuan = load_packages_file(packages_list[0]) + if devuan: + all_repos.append({'name': 'devuan', 'packages': devuan}) + + debian_sec = load_packages_file(packages_list[1]) + if debian_sec: + all_repos.append({'name': 'debian-sec', 'packages': debian_sec}) + + debian = load_packages_file(packages_list[2]) + if debian: + all_repos.append({'name': 'debian', 'packages': debian}) + + print('Merging packages') + new_pkgs = merge_packages_many(all_repos, banned_packages=banpkgs, rewriter=devuan_rewrite) + + print('Writing packages') + # replace the devuan subdir with our mergedir that we plan to fill + new_out = packages_list[0].replace(join(spooldir, + repos['devuan']['dists']), + join(mergedir, mergesubdir)) + write_packages(new_pkgs, new_out) + + t2 = time() + print('time:', t2-t1) + + +def main(packages_file): + #packages_file = 'main/binary-armhf/Packages.gz' + #packages_file = argv[1] + print(packages_file) + to_merge = prepare_merge_dict() + + tt1 = time() + for suite in to_merge: + pkg_list = [] + for rep in to_merge[suite]: + if rep: + pkg_list.append(join(rep, packages_file)) + else: + pkg_list.append(None) + + # print(pkg_list) + merge(pkg_list) + + tt2 = time() + print('total time:', tt2-tt1) + + +if __name__ == '__main__': + main(sys.argv[1]) diff --git a/lib/config.py b/lib/config.py @@ -73,6 +73,8 @@ aliases = { categories = ['main', 'contrib', 'non-free'] +arches = ['binary-armhf'] + mainrepofiles = [ 'InRelease', 'Release', diff --git a/orchestrate.py b/orchestrate.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 + +from os.path import join +from lib.config import arches, categories +from multiprocessing import Pool + + +pkg = [] +for i in arches: + for j in categories: + pkg.append(join(j, i, 'Packages.gz')) + pkg.append(join(j, 'debian-installer', i, 'Packages.gz')) + + +#print(pkg) +am = __import__('amprolla-merge') + +p = Pool(4) +p.map(am.main, pkg)