amprolla

devuan's apt repo merger
git clone git://parazyd.org/amprolla.git
Log | Files | Refs | README | LICENSE

commit 6be10ca7af5168dcfbc39b64c50fdcaee251ca75
parent 1d9670ade4cc7c28dfd1c6de9bc14ca099be0c9d
Author: parazyd <parazyd@dyne.org>
Date:   Mon,  5 Jun 2017 22:46:46 +0200

pep8/pylint fixes, safety checks in write_packages() and write_release()

Diffstat:
MREADME.md | 6++++--
Mamprolla_init.py | 6+++---
Mamprolla_merge.py | 15+++++++--------
Mlib/config.py | 9++++-----
Mlib/package.py | 11++++++-----
Mlib/parse.py | 2+-
Mlib/release.py | 10++++++----
7 files changed, 31 insertions(+), 28 deletions(-)

diff --git a/README.md b/README.md @@ -1,6 +1,8 @@ amprolla ======== +<img src="contrib/amprolla.png" width="64"> + amprolla is an apt repository merger originally intended for use with the [Devuan](https://devuan.org) infrastructure. This version is the third iteration of the software. The original version of amprolla was @@ -10,10 +12,10 @@ finished - therefore this version has emerged. Dependencies ------------ -### Devuan +### Devuan Ascii/Debian Stretch ``` -gnupg2 python3-requests, python3-gnupg +gnupg2 python3-requests python3-gnupg ``` ### Gentoo: diff --git a/amprolla_init.py b/amprolla_init.py @@ -67,9 +67,9 @@ def main(): for k in release_contents: # if k.endswith('/binary-armhf/Packages.gz'): # if k.endswith('Packages.gz'): - remote = join(url[0], k) - local = join(url[1], k) - download(remote, local) + remote = join(url[0], k) + local = join(url[1], k) + download(remote, local) if __name__ == '__main__': diff --git a/amprolla_merge.py b/amprolla_merge.py @@ -7,7 +7,7 @@ Amprolla main module from sys import argv from os.path import basename, join -from time import time +# from time import time from lib.package import (write_packages, load_packages_file, merge_packages_many) @@ -64,7 +64,7 @@ def merge(packages_list): """ Merges the Packages/Sources files given in the package list """ - t1 = time() + # t1 = time() all_repos = [] print('Loading packages: %s' % packages_list) @@ -81,7 +81,6 @@ def merge(packages_list): if debian: all_repos.append({'name': 'debian', 'packages': debian}) - if basename(packages_list[0]) == 'Packages.gz': print('Merging packages') src = False @@ -102,8 +101,8 @@ def merge(packages_list): else: write_packages(new_pkgs, new_out) - t2 = time() - print('time:', t2-t1) + # t2 = time() + # print('time:', t2-t1) def main(packages_file): @@ -113,7 +112,7 @@ def main(packages_file): # print(packages_file) to_merge = prepare_merge_dict() - #tt1 = time() + # tt1 = time() for suite in to_merge: pkg_list = [] for rep in to_merge[suite]: @@ -124,8 +123,8 @@ def main(packages_file): merge(pkg_list) - #tt2 = time() - #print('total time:', tt2-tt1) + # tt2 = time() + # print('total time:', tt2-tt1) if __name__ == '__main__': diff --git a/lib/config.py b/lib/config.py @@ -7,7 +7,7 @@ amprolla configuration file from hashlib import md5, sha1, sha256 spooldir = './spool' -signingkey = 'B876CB44FA1B0274' +signingkey = 'CA608125' mergedir = './merged' mergesubdir = 'dists' banpkgs = {'systemd', 'systemd-sysv'} @@ -83,15 +83,15 @@ categories = ['main', 'contrib', 'non-free'] arches = [ 'source', - # 'binary-all', + 'binary-all', # 'binary-alpha', - # 'binary-amd64', + 'binary-amd64', # 'binary-arm64', # 'binary-armel', 'binary-armhf', # 'binary-hppa', # 'binary-hurd-i386', - # 'binary-i386', + 'binary-i386', # 'binary-ia64', # 'binary-kfreebsd-amd64', # 'binary-kfreebsd-i386', @@ -175,7 +175,6 @@ packages_keys = [ 'Npp-File', 'Npp-Mimetype', 'Npp-Name', - 'Origin', 'Original-Maintainer', 'Original-Source-Maintainer', 'Package-Type', diff --git a/lib/package.py b/lib/package.py @@ -26,12 +26,13 @@ def write_packages(packages, filename, sort=True, sources=False): if sources: bsnm = 'Sources.gz' rl = filename.replace(bsnm, 'Release') - if not os.path.isfile(rl): - copyfile(rl.replace(mergedir, join(spooldir, 'devuan')), rl) + sprl = rl.replace(mergedir, join(spooldir, 'devuan')) + if not os.path.isfile(rl) and os.path.isfile(sprl): + copyfile(sprl, rl) gzf = gzip_open(filename, 'w') xzf = lzma_open(filename.replace('.gz', '.xz'), 'w') - f = open(filename.replace('.gz', ''), 'w') + f = open(filename.replace('.gz', ''), 'wb') pkg_items = packages.items() if sort: @@ -48,10 +49,10 @@ def write_packages(packages, filename, sort=True, sources=False): s = '%s: %s\n' % (key, pkg_contents[key]) gzf.write(s.encode('utf-8')) xzf.write(s.encode('utf-8')) - f.write(s) + f.write(s.encode('utf-8')) gzf.write(b'\n') xzf.write(b'\n') - f.write('\n') + f.write(b'\n') gzf.close() xzf.close() diff --git a/lib/parse.py b/lib/parse.py @@ -48,6 +48,7 @@ def parse_release(reltext): return hashes + def parse_release_head(reltext): """ Parses the header of the release file to grab potentially needed @@ -71,7 +72,6 @@ def parse_release_head(reltext): return metadata - def parse_release_re(reltext): """ Parses a Release file using regular expressions and returns a dict diff --git a/lib/release.py b/lib/release.py @@ -5,7 +5,7 @@ Release file functions and helpers """ from datetime import datetime, timedelta -from os.path import getsize +from os.path import getsize, isfile import gnupg from lib.config import release_keys, checksums, signingkey @@ -43,14 +43,16 @@ def write_release(oldrel, newrel, filelist, r): for csum in checksums: new.write('%s:\n' % csum['name']) for f in filelist: - cont = open(f, 'rb').read() - new.write(' %s %8s %s\n' % (csum['f'](cont).hexdigest(), - getsize(f), f.replace(r+'/', ''))) + if isfile(f): + cont = open(f, 'rb').read() + new.write(' %s %8s %s\n' % (csum['f'](cont).hexdigest(), + getsize(f), f.replace(r+'/', ''))) new.close() sign_release(newrel) + def sign_release(infile): """ Signs both the clearsign and the detached signature of a Release file