amprolla

devuan's apt repo merger
git clone git://parazyd.org/amprolla.git
Log | Files | Refs | README | LICENSE

commit 447efaed180df30074f5722963ebf28e91911efd
parent c81a62a15175afaa1e706eec7b97924c0df8fe2c
Author: parazyd <parazyd@dyne.org>
Date:   Tue,  6 Jun 2017 00:30:06 +0200

merge orchestrate.py with amprolla_merge.py

Diffstat:
MREADME.md | 11++++++-----
Mamprolla_merge.py | 75+++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--------------
Dorchestrate.py | 72------------------------------------------------------------------------
3 files changed, 67 insertions(+), 91 deletions(-)

diff --git a/README.md b/README.md @@ -31,11 +31,12 @@ app-crypt/gnupg dev-python/requests dev-python/python-gnupg Basic usage ----------- -Edit `lib/config.py` to your needs, and then run `amprolla_init.py`. This -will download the repositories we will merge afterwards. When this is done, -you can now run `orchestrate.py` which will perform the merge, and finally -sign the Release files needed. +Edit `lib/config.py` to your needs, and then run `amprolla_init.py`. +This will download the repositories we will merge afterwards. When this +is done, you can now run `amprolla_merge.py` which will perform the +merge, and finally sign the Release files needed. -A `nginx` configuration for the amprolla server can be found in `contrib`. +A `nginx` configuration for the amprolla server can be found in +`contrib`. More information on amprolla should be found in the `doc` directory. diff --git a/amprolla_merge.py b/amprolla_merge.py @@ -5,14 +5,16 @@ Amprolla main module """ -from sys import argv from os.path import basename, join -# from time import time +from multiprocessing import Pool +# from pprint import pprint from lib.package import (write_packages, load_packages_file, merge_packages_many) -from lib.config import (aliases, banpkgs, repo_order, repos, - spooldir, suites, mergedir, mergesubdir) +from lib.config import (aliases, banpkgs, repo_order, repos, spooldir, suites, + mergedir, mergesubdir, pkgfiles, srcfiles, categories, + arches) +from lib.release import write_release def prepare_merge_dict(): @@ -64,8 +66,6 @@ def merge(packages_list): """ Merges the Packages/Sources files given in the package list """ - # t1 = time() - all_repos = [] print('Loading packages: %s' % packages_list) @@ -101,18 +101,44 @@ def merge(packages_list): else: write_packages(new_pkgs, new_out) - # t2 = time() - # print('time:', t2-t1) +def gen_release(s): + """ + Generates a Release file for a given main suite (jessie/ascii/unstable) + """ -def main(packages_file): + for suite in suites[s]: + filelist = [] + print('Crawling %s' % suite) + rootdir = join(mergedir, mergesubdir, suite) + for cat in categories: + for arch in arches: + if arch == 'source': + flist = srcfiles + else: + flist = pkgfiles + + for fl in flist: + filelist.append(join(rootdir, cat, arch, fl)) + if arch != 'source': + filelist.append(join(rootdir, cat, + 'debian-installer', arch, fl)) + + newrfl = join(rootdir, 'Release') + oldrfl = newrfl.replace(join(mergedir, mergesubdir), + join(spooldir, repos['devuan']['dists'])) + + print('Writing Release') + write_release(oldrfl, newrfl, filelist, rootdir) + # break + + +def main_merge(packages_file): """ Main function that calls the actual merge """ - # print(packages_file) to_merge = prepare_merge_dict() - # tt1 = time() for suite in to_merge: pkg_list = [] for rep in to_merge[suite]: @@ -123,9 +149,30 @@ def main(packages_file): merge(pkg_list) - # tt2 = time() - # print('total time:', tt2-tt1) + +def main(): + """ + Crawls the entire directory structure and orchestrates the merge + in a queue using multiprocessing + """ + pkg = [] + for i in arches: + for j in categories: + if i == 'source': + mrgfile = 'Sources.gz' + else: + mrgfile = 'Packages.gz' + pkg.append(join(j, 'debian-installer', i, mrgfile)) + + pkg.append(join(j, i, mrgfile)) + + # pprint(pkg) + p = Pool(4) # Set it to the number of CPUs you want to use + p.map(main_merge, pkg) + + for st in suites: + gen_release(st) if __name__ == '__main__': - main(argv[1]) + main() diff --git a/orchestrate.py b/orchestrate.py @@ -1,72 +0,0 @@ -#!/usr/bin/env python3 -# see LICENSE file for copyright and license details - -""" -Module used to orchestrate the entire amprolla merge -""" - -from os.path import join -from multiprocessing import Pool - -from lib.config import (arches, categories, suites, mergedir, mergesubdir, - pkgfiles, srcfiles, spooldir, repos) -from lib.release import write_release - - -def do_merge(): - """ - Crawls the entire directory structure and orchestrates the merge - in a queue using multiprocessing - """ - pkg = [] - for i in arches: - for j in categories: - if i == 'source': - mrgfile = 'Sources.gz' - else: - mrgfile = 'Packages.gz' - pkg.append(join(j, 'debian-installer', i, mrgfile)) - - pkg.append(join(j, i, mrgfile)) - - am = __import__('amprolla_merge') - - p = Pool(4) # Set it to the number of CPUs you want to use - p.map(am.main, pkg) - - -def gen_release(s): - """ - Generates a Release file for a given main suite (jessie/ascii/unstable) - """ - - for suite in suites[s]: - filelist = [] - print('Crawling %s' % suite) - rootdir = join(mergedir, mergesubdir, suite) - for cat in categories: - for arch in arches: - if arch == 'source': - flist = srcfiles - else: - flist = pkgfiles - - for fl in flist: - filelist.append(join(rootdir, cat, arch, fl)) - if arch != 'source': - filelist.append(join(rootdir, cat, - 'debian-installer', arch, fl)) - - newrfl = join(rootdir, 'Release') - oldrfl = newrfl.replace(join(mergedir, mergesubdir), - join(spooldir, repos['devuan']['dists'])) - - print('Writing Release') - write_release(oldrfl, newrfl, filelist, rootdir) - # break - - -do_merge() - -for st in suites: - gen_release(st)