package.py (7166B)
1 # See LICENSE file for copyright and license details. 2 3 """ 4 Package merging functions and helpers 5 """ 6 7 from os import makedirs 8 from os.path import dirname, isfile, join 9 from gzip import open as gzip_open 10 from lzma import open as lzma_open 11 from shutil import copyfile 12 13 import lib.globalvars as globalvars 14 from lib.config import mergedir, packages_keys, sources_keys, spooldir 15 from lib.log import logtofile 16 from lib.parse import parse_dependencies, parse_packages, cmppkgver 17 18 19 def write_packages(packages, filename, sort=True, sources=False): 20 """ 21 Writes `packages` to a file (per debian Packages format) 22 If sort=True, the packages are sorted by name. 23 """ 24 makedirs(dirname(filename), exist_ok=True) 25 26 # Copy the arch-specific Release file from devuan if it's not there 27 bsnm = 'Packages.gz' 28 if sources: 29 bsnm = 'Sources.gz' 30 rel = filename.replace(bsnm, 'Release') 31 sprl = rel.replace(mergedir, join(spooldir, 'devuan')) 32 if not isfile(rel) and isfile(sprl): 33 copyfile(sprl, rel) 34 35 gzf = gzip_open(filename, 'w') 36 xzf = lzma_open(filename.replace('.gz', '.xz'), 'w') 37 # f = open(filename.replace('.gz', ''), 'wb') 38 39 pkg_items = packages.items() 40 if sort: 41 pkg_items = sorted(pkg_items, key=lambda x: x[0]) 42 43 if sources: 44 keylist = sources_keys 45 else: 46 keylist = packages_keys 47 48 for pkg_name, pkg_contents in pkg_items: 49 for key in keylist: 50 if key in pkg_contents: 51 sin = '%s: %s\n' % (key, pkg_contents[key]) 52 gzf.write(sin.encode('utf-8')) 53 xzf.write(sin.encode('utf-8')) 54 # f.write(sin.encode('utf-8')) 55 gzf.write(b'\n') 56 xzf.write(b'\n') 57 # f.write(b'\n') 58 59 gzf.close() 60 xzf.close() 61 # f.close() 62 63 64 def load_packages_file(filename): 65 """ 66 Load a gzip'd packages file. 67 Returns a dictionary of package name and package key-values. 68 """ 69 # TODO: should we skip files like this if they don't exist? 70 if filename is not None and isfile(filename): 71 packages_contents = gzip_open(filename).read() 72 packages_contents = packages_contents.decode('utf-8') 73 return parse_packages(packages_contents) 74 75 return None 76 77 78 def depends_on_all_banned(deps, banned_pkgs): 79 """ 80 Gets a list of dicts of dep alternatives and a set of banned packages and 81 returns True if any of the dicts consist exclusively of banned_pkgs. 82 """ 83 for dep in deps: 84 alt = set(dep.keys()) 85 if len(alt.intersection(banned_pkgs)) == len(alt): 86 # All the alternatives are banned 87 return True 88 return False 89 90 91 def depends_on(deps, package_set): 92 """ 93 Gets a list of dicts of dep alternatives and a set of packages and returns 94 True if any of the dicts include at least one of the elements in 95 package_set. 96 """ 97 for dep in deps: 98 alt = set(dep.keys()) 99 if alt.intersection(package_set): 100 # At least one alternative is in package_set 101 return True 102 return False 103 104 105 def package_banned(pkg, banned_pkgs): 106 """ 107 Returns True is the package contains a banned dependency. 108 Currently checks and parses both the 'Depends:' and the 'Pre-Depends' 109 fields of the package. 110 """ 111 if pkg.get('Package') in banned_pkgs: 112 logtofile('bannedpackages.txt', '%s,%s\n' % (globalvars.suite, 113 pkg.get('Package'))) 114 return True 115 116 depends = parse_dependencies(pkg.get('Depends', '')) 117 pre_depends = parse_dependencies(pkg.get('Pre-Depends', '')) 118 119 depends.extend(pre_depends) 120 121 if depends_on(depends, set(['libsystemd0'])): 122 logtofile('libsystemd.txt', '%s,%s\n' % (globalvars.suite, 123 pkg.get('Package'))) 124 125 isbanned = depends_on_all_banned(depends, banned_pkgs) 126 if isbanned: 127 logtofile('bannedpackages.txt', '%s,%s\n' % (globalvars.suite, 128 pkg.get('Package'))) 129 return isbanned 130 131 132 def package_newer(pkg1, pkg2): 133 """ 134 Checks whether the package of a lower priority has a higher version than 135 the package of the higher priority and returns True if so. 136 137 Ref: https://www.debian.org/doc/debian-policy/#version 138 """ 139 # Hardcoded list of packages we don't want to check 140 _skips = [] 141 if pkg1.get('Package') in _skips: 142 return False 143 144 if cmppkgver(pkg1.get('Version'), pkg2.get('Version')) < 0: 145 return True 146 147 return False 148 149 150 def merge_packages(pkg1, pkg2, name1, name2, banned_packages=set(), 151 rewriter=None): 152 """ 153 Merges two previously loaded/parsed (using load_packages_file) packages 154 dictionaries, preferring `pkg1` over `pkg2`, and optionally discarding any 155 banned packages. 156 """ 157 new_pkgs = {} 158 package_names = set(pkg1.keys()).union(set(pkg2.keys())) 159 obsoletepkgs = [] 160 161 for pkg in package_names: 162 pkg1_pkg = pkg1.get(pkg) 163 pkg2_pkg = pkg2.get(pkg) 164 165 if pkg1_pkg and pkg2_pkg: 166 if rewriter: 167 pkg1_pkg = rewriter(pkg1_pkg, name1) 168 new_pkgs[pkg] = pkg1_pkg 169 if package_newer(pkg1_pkg, pkg2_pkg): 170 obsoletepkgs.append('%s,%s,%s,%s' % (globalvars.suite, 171 pkg1_pkg.get('Package'), 172 pkg1_pkg.get('Version'), 173 pkg2_pkg.get('Version'))) 174 elif pkg1_pkg: 175 if not package_banned(pkg1_pkg, banned_packages): 176 if rewriter: 177 pkg1_pkg = rewriter(pkg1_pkg, name1) 178 new_pkgs[pkg] = pkg1_pkg 179 elif pkg2_pkg: 180 if not package_banned(pkg2_pkg, banned_packages): 181 if rewriter: 182 pkg2_pkg = rewriter(pkg2_pkg, name2) 183 new_pkgs[pkg] = pkg2_pkg 184 else: 185 assert False, 'Impossibru' 186 187 if obsoletepkgs: 188 obsoletepkgs = '\n'.join(obsoletepkgs) + '\n' 189 logtofile('%s-oldpackages.txt' % globalvars.suite, obsoletepkgs, 190 redo=False) 191 192 return new_pkgs 193 194 195 def merge_packages_many(packages, banned_packages=set(), rewriter=None): 196 """ 197 Merges two (or more) previously loaded/parsed (using load_packages_file) 198 packages dictionaries, priority is defined by the order of the `packages` 199 list, optionally discarding any banned packages. 200 """ 201 if not len(packages) > 1: 202 while not len(packages) > 1: 203 packages.append({'name': '', 'packages': {}}) 204 205 new_pkgs = {} 206 207 pkg1 = packages[0] 208 pkg2 = packages[1] 209 210 new_pkgs = merge_packages(pkg1['packages'], pkg2['packages'], 211 pkg1['name'], pkg2['name'], 212 banned_packages=banned_packages, 213 rewriter=rewriter) 214 215 for pkg in packages[2:]: 216 new_pkgs = merge_packages(new_pkgs, pkg['packages'], '', pkg['name'], 217 banned_packages=banned_packages) 218 219 return new_pkgs