Today I have written a program to filter out common library
dependencies of the three programs exposing this bug, i.e. cheese, vlc
and firefox-esr. For one library depending upon another only the top
level dependency is outputted:
elm@system76-18:~/projects/pydeps$ ./fetchupd.py commdep cheese vlc
common dependencies:
libasound2-data all
libevdev2 amd64
libblkid1 amd64
libaa1 amd64
libdrm-common all
libasound2 amd64
libmtdev1 amd64
libmount1 amd64
libaom0 amd64
libinput10 amd64
libdrm2 amd64
libglib2.0-0 amd64
libgcc1 amd64
libass9 amd64
libvorbisfile3 amd64
libc6 amd64
libgpm2 amd64
elm@system76-18:~/projects/pydeps$ ./fetchupd.py commdep cheese firefox-esr
common dependencies:
libncursesw6 amd64
libatk1.0-0 amd64
libasound2-data all
libasound2 amd64
libice6 amd64
lsb-base all
x11-common all
libncurses6 amd64
libatk1.0-data all
libc6 amd64
libgcc1 amd64
elm@system76-18:~/projects/pydeps$ ./fetchupd.py commdep vlc firefox-esr
common dependencies:
libwayland-client0 amd64
libdbus-1-3 amd64
libsystemd0 amd64
xkb-data all
libxkbcommon0 amd64
libgcc1 amd64
libxcb-util0 amd64
libc6 amd64
elm@system76-18:~/projects/pydeps$ ./fetchupd.py commdep vlc firefox-esr
cheese
common dependencies:
libdbus-1-3 amd64
libxkbcommon0 amd64
libgcc1 amd64
libwayland-client0 amd64
xkb-data all
libc6 amd64
libsystemd0 amd64
elm@system76-18:~/projects/pydeps$ ./fetchupd.py commdep cheese guvcview
common dependencies:
libcap2 amd64
libasound2-data all
libavcodec58 amd64
libaom0 amd64
libusb-1.0-0 amd64
libc6 amd64
libgcc1 amd64
libatk1.0-0 amd64
libogg0 amd64
libasound2 amd64
libatk1.0-data all
However I still do not know which library is causing the fault. A
possible candidate would be libaom0 which implements an internet video
codec. However I would suppose that the hardware uses a different
protocol. Additionally libaom0 and libatk1.0 are also used by guvcview
which works well. I do not know whether libinput10 only implements input
devices like mice and keyboards or whether things like webcams are also
supported. Another possible candidate would be libatk1.0. Unfortunately
I do not really know what these libraries do or whether the bug is
evoked by a totally different component. Only you, the developers or
maintainers of cheese will know so please have a look and tell me about it!
#!/usr/bin/python3
# -*- coding: utf-8
# vim: expandtab;
import gzip, bz2, lzma, urllib.request, urllib.error, hashlib;
from itertools import count; from functools import reduce;
import sys, os, re;
repos = [ ("http://security.debian.org/debian-security";, "stable/updates", ["main"] ),
("http://deb.debian.org/debian";, "stable", ["main"] ) ];
agent="apt";
architectures=["amd64"];
blocksize=8192;
max_retries=3;
doCache = True; dryRun = False; topOnly = False;
def get_file(url):
headers = { 'User-Agent': agent }
req = urllib.request.Request(url, headers = headers)
resp = urllib.request.urlopen(req)
return resp;
def get_file_content_cached(url):
cachefile = url.translate({ord(c):'_' for c in ":/"})+".cache"
try:
with open(cachefile,'rb') as rfd:
return rfd.read();
except Exception as ex:
data = get_file(url).read();
try:
with open(cachefile,'wb') as wfd:
wfd.write(data);
except Exception as ex:
print(ex);
return data;
def print_urllib_error(url,e):
if type(e) == urllib.error.URLError:
print("error downloading %s: %s" % ( url, e.reason.strerror ), file=sys.stderr );
elif type(e) == urllib.error.HTTPError:
print("error downloading %s: %i %s" % ( url, e.code, e.msg ), file=sys.stderr );
else:
print("unknown error downloading %s: %s" % ( url, str(e) ), file=sys.stderr );
sys.exit(2);
def get_file_content(url):
try:
if doCache: return get_file_content_cached(url);
return get_file(url).read();
except Exception as e:
print_urllib_error(url,e);
def save_file(url,filename):
try:
resp = get_file(url);
h = hashlib.sha256();
with open(filename,'wb') as wfd:
block = resp.read(blocksize);
while block:
wfd.write(block);
h.update(block);
block = resp.read(blocksize);
return h.hexdigest();
except Exception as e:
print_urllib_error(url,e);
def find(elm,lis):
i = 0;
for e in lis:
if e == elm: return i;
i += 1;
return i;
def sgn(val):
return -1 if val < 0 else ( 1 if val > 0 else 0 );
hash_prefs = ['SHA512','SHA384','SHA256','SHA224','SHA1','MD5'];
good_hash_prefs = 3;
def get_hash(content,hashtype):
if hashtype=='SHA256':
h = hashlib.sha256();
elif hashtype=='SHA512':
h = hashlib.sha512();
elif hashtype=='MD5SUM' or hashtype=='MD5':
h = hashlib.md5();
elif hashtype=='SHA1':
h = hashlib.sha1();
elif hashtype=='SHA384':
h = hashlib.sha384();
elif hashtype=='SHA224':
h = hashlib.sha224();
else:
raise Exception("hash of type %s unknown!" % hashtype);
h.update(content);
return h.hexdigest();
has_ver_sep = re.compile("[+~.:-]");
#is_ver = re.compile("[0-9+~.:-]*");
#print("letters:",a[:ia],b[:ib]);
#print("numbers:",a[:ia],b[:ib]);
def numidx(s,search4num):
i = 0;
if search4num:
for c in s:
if '0' <= c and c <= '9': break;
i += 1;
else:
for c in s:
if c < '0' or '9' < c: break;
i += 1;
return i;
def cmp_ver(a,b):
def do_cmp_ver(a,b,seplis):
if not seplis or ( len(seplis) <= 4 and not has_ver_sep.search(a) and not has_ver_sep.search(b)):
while len(a) > 0 and len(b) > 0:
ia = numidx(a,True); ib = numidx(b,True);
if a[:ia] != b[:ib]:
return -1 if a[:ia] < b[:ib] else +1;
a = a[ia:]; b = b[ib:];
ia = numidx(a,False); ib = numidx(b,False);
a_int = int(a[:ia]) if ia!=0 else -1;
b_int = int(b[:ib]) if ib!=0 else -1;
if a_int != b_int:
return -1 if a_int < b_int else +1;
a = a[ia:]; b = b[ib:];
if a == b: return 0;
if a == '': return -1;
return +1;
splitby = seplis.pop();
a = a.split(splitby); b = b.split(splitby); maxlen = max(len(a),len(b));
# chr(254): 1:3.1+dfsg-8+deb10u2~ "<<" 1:3.1+dfsg-8+deb10u2
if splitby == ':':
if len(a) < maxlen: a = [''] * (maxlen-len(a)) + a;
if len(b) < maxlen: b = [''] * (maxlen-len(b)) + b;
elif splitby == '~':
if len(a) < maxlen: a = a + [chr(254)] * (maxlen-len(a));
if len(b) < maxlen: b = b + [chr(254)] * (maxlen-len(b));
else:
if len(a) < maxlen: a = a + [''] * (maxlen-len(a));
if len(b) < maxlen: b = b + [''] * (maxlen-len(b));
for tok_a, tok_b in zip( a, b ):
res = do_cmp_ver( tok_a, tok_b, seplis.copy() )
if res != 0: return res;
return 0;
return do_cmp_ver(a,b,['.','-','+','~',':']);
def do_cmp_ver(cmp_type,a,b):
res = cmp_ver(a,b);
if res == 0 and cmp_type in ['=','<=','>=']: return True;
elif res < 0 and cmp_type in [ '<=', '<<' ]: return True;
elif res > 0 and cmp_type in [ '>=', '>>' ]: return True;
return False;
#print(cmp_ver( "12:13.15+16", "12:13.15+17" ));
#print(cmp_ver( "2.2", "1:2.6.1-1" ));
#print(cmp_ver( "1:3.1+dfsg-8+deb10u2~", "1:3.1+dfsg-8+deb10u2" )); # -1
#print(cmp_ver( "1:3.1+dfsg-8+deb10u2~1", "1:3.1+dfsg-8+deb10u2" )); # -1
#print(cmp_ver( "1.18-3", "1.18.1-4" ));
#print(cmp_ver( "1.18", "1.18.1" ));
#print(cmp_ver( "1.18-3", "1.18-4" ));
#sys.exit(0);
def conc_url(lis):
url="";
for tok in lis:
url += tok;
if not url.endswith('/'): url += '/';
return url[:-1];
sumline = re.compile("^ *(?P<sum>[0-9a-fA-F]*) +(?P<size>[0-9]+) +(?P<filename>.*)$");
pkgs = dict();
class package:
def __init__(self,pname,ver,arch,repoidx,filename,sha256,size,depends,depends_or,conflicts):
self.pname = pname; self.ver = ver; self.arch=arch; self.repoidx = repoidx; self.filename=filename; self.sha256=sha256; self.size=size;
self.depends=depends; self.depends_or=depends_or; self.conflicts=conflicts; self.dependents = [];
def __hash__(self): return hash( ( self.pname, self.ver, self.arch, self.repoidx ) ); # sets shall only use __hash__ and __eq__
def __repr__(self):
return "(ver=%s,arch=%s,repo=%i)" % (self.ver,self.arch,self.repoidx);
# igitt, python3!
def __cmp__(self,other):
result = cmp_ver(self.ver,other.ver);
if result != 0: return result;
result = self.repoidx - other.repoidx;
if result != 0: return -sgn(result); # descending sort order
result = find(self.arch,architectures) - find(other.arch,architectures);
return -sgn(result); # descending sort order
def __lt__(self,other): return self.__cmp__(other) < 0; # the only that shall be used for sorting
def __le__(self,other): return self.__cmp__(other) <= 0;
def __eq__(self,other): return self.pname == other.pname and self.ver == other.ver and self.arch == other.arch and self.repoidx == other.repoidx;
def __ne__(self,other): return not self.__eq__(other);
def __ge__(self,other): return self.__cmp__(other) >= 0;
def __gt__(self,other): return self.__cmp__(other) > 0;
# ~~~~~~~~~~~~~~ actual part of the program ~~~~~~~~~~~~~~
instfile = None;
justprn = []; tryupd = {}; updseq = []; dependents = {}; # tryupd is index by (pname,arch), dependents only by pname
def arch_is_compatible(arch1,arch2):
return arch1 == arch2 or arch1 == 'all' or arch2 == 'all';
def get_with_arch_like(pkghash,pname,arch):
inst = pkghash.get((pname,arch),None);
if inst: return (inst,arch);
if arch != 'all':
return ( pkghash.get((pname,'all'),None), 'all' );
for arch in architectures:
inst = pkghash.get((pname,arch),None);
if inst: return (inst,arch);
return (None,None);
def create_tryupd_entry( pname, compat_with_arch ):
global tryupd;
availables = []; base_arch = None;
for a in pkgs.get(pname,[]):
if ( not base_arch and arch_is_compatible( a.arch, compat_with_arch ) ) or ( a.arch == base_arch ):
availables.append(a); base_arch = a.arch;
if not availables:
return (None,None);
base = [ -1, availables ]; assert(base_arch); # base must not be a tuple to keep base[0] assignable
tryupd[(pname,base_arch)] = base;
return ( base, base_arch );
def get_tryupd_entry( provides, compat_with_arch ):
global tryupd, what_provides;
baselis = [];
for pname in [ provides ] + what_provides.get(provides,[]):
( base, base_arch ) = get_with_arch_like( tryupd, pname, compat_with_arch );
if base:
if base[0] != -1: availables = [ base[1][base[0]] ];
else: availables = base[1];
else:
( base, base_arch ) = create_tryupd_entry( pname, compat_with_arch );
if base != None: availables = base[1];
if base != None:
baselis.append( ( base, availables ) );
return baselis;
def get_installed(pname):
found = None;
for try_arch in architectures + ['all']:
found = installed.get((pname,try_arch),None);
if found: return found;
return found;
def upd( candidate, parent_candidate, always_visit ):
global tryupd, justprn, dependents;
remember_idx = [];
all_installs = [];
depts = dependents.get(candidate.pname,[]);
if depts: always_visit.add(candidate); # in case of dt_* own update will be performed as part of a new call by dt_xx instead of this call
did_upd_dependent = False;
#print(candidate.pname,list(depts)[:10]);
for dt_pname, dt_arch in depts:
if parent_candidate and dt_pname == parent_candidate.pname and dt_arch == parent_candidate.arch: continue;
# first see if that package is already to be updated
dt_good = True; # set to false on unresolvable conflict
pname_arch = (dt_pname,dt_arch);
base = tryupd.get(pname_arch);
if base == None or base[0] != -1:
#print("newly visited",candidate.pname,dt_pname);
if base == None:
inst = installed.get(pname_arch);
if not inst:
# that should not happen: either already installed or when created by an update a base record must already be here
print("warning: strange dependent ignored:",dt_pname,file=sys.stderr);
continue;
else:
inst = base[1][base[0]]
dep_lis = inst.depends.get(candidate.pname,[]) or [];
for cmpop, ver in dep_lis:
if not do_cmp_ver( cmpop, candidate.ver, ver ):
print("update of %s would break dependency of %s; can not update" % (candidate.pname,dt_pname),file=sys.stderr);
dt_good = False;
break;
else:
result = False;
for i, pkg in zip(count(),base[1]):
remember_idx.append((base,base[0]));
#print(candidate.pname,"->",pkg.pname);
base[0] = i; ( result, installs ) = upd( pkg, candidate, always_visit );
if result:
did_upd_dependent = True;
all_installs += installs;
break;
if not result:
print("no update candidate for %s; can not update %s" % (dt_pname,candidate.pname),file=sys.stderr);
dt_good = False;
if not dt_good:
for base, prev_idx in remember_idx:
base[0] = prev_idx;
return ( False, [] );
if did_upd_dependent:
base = tryupd.get((candidate.pname,candidate.arch));
assert(base and base[0] != -1 and base[1][base[0]] == candidate);
return ( True, all_installs );
if depts: always_visit.remove(candidate);
#print("done");
choices = []; # dependency fullfillment choices for packages
# and choice lists
for dep_pname, dep_lis in candidate.depends.items():
baselis = get_tryupd_entry( dep_pname, candidate.arch );
# base=baselis[i]: base[0] is idx for remember_idx into base[1]-list
# base[1] is list of availabel candidates
# if idx in base[0] is set: then availables has only one entry; otherwise it equals base[1]
if not baselis:
print("no arch-deps for", dep_pname, candidate.arch, file=sys.stderr );
return False;
satisfying = [];
for base, availables in baselis:
for i, a in zip(count(max(0,base[0])),availables):
dep_fulfilled = True;
for cmpop, ver in dep_lis or []:
if not do_cmp_ver( cmpop, a.ver, ver ):
dep_fulfilled = False;
break;
if dep_fulfilled:
satisfying.append((base,i,a));
if base[0] == -1: # not yet chosen idx, will be chosen by this upd-call
remember_idx.append((base,base[0]));
choices.append(satisfying); # whole base list is added to get reference instead of value for base[0];
if not satisfying and dep_pname not in justprn:
print("no deps for", dep_pname, dep_lis, availables, "~ by ~", candidate.pname, candidate.ver, file=sys.stderr );
#if len(pb) > 1:
# print(candidate.pname,dep_pname,pb,file=sys.stderr);
justprn.append(dep_pname);
return ( False, [] );
# or choice lists; select one out of each sub-list
for or_lis in candidate.depends_or:
this_select = [];
# put in front: already installed packet
for ( dep_pname, cmpop, ver ), i in zip(or_lis,count()):
if get_installed(dep_pname):
if i: or_lis = [ ( dep_pname, cmpop, ver ) ] + or_lis[:i] + or_lis[i+1:];
if i: print(or_lis);
break;
# convert into choices format and select packages where version fits
for dep_pname, cmpop, ver in or_lis:
baselis = get_tryupd_entry( dep_pname, candidate.arch );
if not baselis: continue;
for base, availables in baselis:
for i, a in zip(count(max(0,base[0])),availables):
if cmpop == None or do_cmp_ver( cmpop, a.ver, ver ):
this_select.append((base,i,a));
if base[0] == -1: # not yet chosen idx, will be chosen by this upd-call
remember_idx.append((base,base[0]));
if not this_select:
print("no deps available for or-list %s" % repr(or_lis));
return ( False, [] );
choices.append(this_select);
#print(candidate.pname);
for this_select in choices:
result = True; # we have already asserted that there is at least one choice; if base[0]>=0 (will be updated) or if it is already installed then result shall be True
for base, i, pkg in this_select:
qq = pkg in always_visit;
if qq: assert( base[1][base[0]] == pkg );
if base[0] == -1 or qq:
base[0] = i;
if pkg.repoidx >= 0: # -1 ... already installed
( result, installs ) = upd( pkg, candidate, always_visit );
if result: all_installs += installs;
break;
if not result:
for base, prev_idx in remember_idx:
base[0] = prev_idx;
return ( False, [] );
do_with = 'i'; # install, dpkg -i
if get_installed(candidate.pname): do_with = 'u'; # dpkg --update-avail
all_installs.append((do_with,candidate));
for dep_pname in candidate.depends.keys():
dts = dependents.get(dep_pname,None);
if dts: dts.add((candidate.pname,candidate.arch)); #print(inst.pname,inst.dependents,pname_arch[0]);
else: dependents[dep_pname] = set([ (candidate.pname,candidate.arch) ]);
return ( True, all_installs );
instFile = None; dldFile = None; sha256File = None;
def get_packages_4_update(instfile_param):
global tryupd, updseq, topOnly; global already_selected, already_selected_at_top;
global instfile, repos, verbose; instfile = instfile_param;
already_selected = set(); already_selected_at_top = set();
def get_updates(pname_arch,inst_pkg,first_level):
global tryupd, updseq, topOnly; global already_selected, already_selected_at_top;
if pname_arch in already_selected:
if pname_arch in already_selected_at_top: # removed from here: if not first_level and ...
updseq.remove(pname_arch);
if topOnly:
already_selected_at_top.remove(pname_arch);
else:
updseq.append(pname_arch);
#print("deleted:",pname_arch);
return;
already_selected.add(pname_arch);
availables = pkgs.get(pname_arch[0],[]);
choices = []; deps = set();
for a in availables:
if arch_is_compatible(inst_pkg.arch,a.arch) and cmp_ver(inst_pkg.ver,a.ver) < 0:
choices.append(a);
for dep_pname in a.depends:
deps.add(dep_pname);
if choices:
tryupd[pname_arch] = [ -1, choices ];
#print(first_level,pname_arch,deps);
if first_level or not topOnly:
updseq.append(pname_arch);
already_selected_at_top.add(pname_arch);
for dep_pname in deps:
( inst, arch ) = get_with_arch_like( installed, dep_pname, pname_arch[1] );
if inst:
#assert(pname_arch not in inst.dependents);
get_updates((dep_pname,arch),inst,False);
else: print("not installed:",dep_pname,file=sys.stderr);
#print("trying to update",pname_arch[0]);
for pname_arch, inst_pkg in installed.items():
get_updates( pname_arch, inst_pkg, True );
already_selected = None; already_selected_at_top = None;
#updseq.reverse(); # for testing downward propagation only
#print(updseq);
all_dlds = []; errdld = 0;
for pname_arch in updseq:
(idx,lis) = tryupd[pname_arch];
if idx==-1:
print("trying to update",pname_arch[0]);
for i, candidate in zip(count(),lis):
tryupd[pname_arch][0] = i;
(result,installs) = upd(candidate,None,set());
if result:
print(repr(installs));
if verbose: print(" "+reduce(lambda x,y: x+"\n "+y,map(lambda inst: repr((inst[0],inst[1].pname,inst[1].ver,inst[1].repoidx)),installs)));
if not dryRun:
for itype, pkg in installs:
r = repos[pkg.repoidx>>3]; urlpraefix = r[0];
dldurl = conc_url([urlpraefix,pkg.filename]); basename = os.path.basename(pkg.filename);
if dldFile: print("torsocks wget -c "+dldurl,file=dldFile);
if instFile:
if itype=='i': print("dpkg -i "+os.path.basename(pkg.filename),file=instFile);
else: print("dpkg --update-avail "+basename,file=instFile);
if sha256File: print(pkg.sha256+" "+basename,file=sha256File);
all_dlds.append((dldurl,basename,pkg.sha256));
break;
else:
tryupd[pname_arch][0] = -1;
else:
print(" already being updated",pname_arch[0]);
if shallDownload:
if verbose: print("starting downloads");
for dldurl, filename, sha256 in all_dlds:
print("downloading",filename,end="");
real_sha256 = save_file(dldurl,filename);
if real_sha256 == sha256: print("\t\tOK");
else: print("\t\tERROR"); errdld+=1;
if errdld: print("%i erroneous downloads" % errdld);
else: print("all dlds ok");
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def calcCommDep(commpkgs):
global installed, already_seen, topmost_deps;
def get_dependencies(pname,pkg):
global installed, already_seen;
pkgdesc = (pname,pkg.arch);
if pkgdesc in already_seen:
return {};
already_seen.add(pkgdesc);
dependencies = {};
for dep_pname in pkg.depends:
for candidate_pname in [ dep_pname ] + what_provides.get(dep_pname,[]):
( dep_pkg, dep_arch ) = get_with_arch_like(installed,candidate_pname,pkg.arch);
if dep_pkg: break;
if not dep_pkg:
print("required package dependency of %s seems not installed: %s." % (pname,dep_pname),file=sys.stderr);
continue;
dependencies[(dep_pname,dep_arch)] = pkg;
new_dependencies = get_dependencies(dep_pname,dep_pkg);
dependencies.update(new_dependencies);
return dependencies;
deps_by_pkg = []; any_pkg_not_found = False;
for pname in commpkgs:
pname_arch = pname.split(':',1);
if len(pname_arch) <= 1:
for candidate_pname in [ pname ] + what_provides.get(pname,[]):
pkg = get_installed(candidate_pname);
if pkg: break;
else:
pkg = installed.get(tuple(pname_arch));
if not pkg:
any_pkg_not_found = True;
print("package '%s' not installed." % pname, file=sys.stderr);
continue;
already_seen = set([]);
deps_by_pkg.append( get_dependencies(pname,pkg) );
def filter_topmost_deps(desc,pkg,topmost):
global installed, topmost_deps, already_seen;
if desc in already_seen:
return;
already_seen.add(desc);
if topmost: topmost_deps.add(desc);
elif desc in topmost_deps: topmost_deps.remove(desc);
for dep_pname in pkg.depends:
( dep_pkg, dep_arch ) = get_with_arch_like(installed,dep_pname,pkg.arch);
if not dep_pkg: continue;
filter_topmost_deps( (dep_pname,dep_arch), dep_pkg, False );
if any_pkg_not_found: print(file=sys.stderr); return;
commdeps = deps_by_pkg[0];
for deps in deps_by_pkg[1:]:
#commdeps = commdeps.intersection(deps);
for pkgdesc in list(commdeps.keys()):
if pkgdesc not in deps:
del commdeps[pkgdesc];
topmost_deps = set([]); already_seen = set([]);
for (desc,pkg) in commdeps.items():
filter_topmost_deps(desc,pkg,True);
print("common dependencies:");
for (pname,arch) in topmost_deps:
print("",pname,arch);
# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
def addpkg(pkgname,pkg):
global pkgs;
pkglis = pkgs.get(pkgname,[]);
pkglis.append(pkg);
pkgs[pkgname] = pkglis;
def prepare_pkgs_sort_dependents():
global pkgs, dependents;
for name, speclis in pkgs.items():
speclis.sort(reverse=True);
#if name in ['libaom0','libdvbpsi10','apache2','vlc']:
#if name in ['libarchive13']:
#print(name,repr(speclis),speclis[0].depends,speclis[0].conflicts);
#sys.exit(0);
for ( pname, arch ), pkg in installed.items():
for dep_pname in pkg.depends:
dts = dependents.get(dep_pname,None);
if dts: dts.add((pname,arch)); #print(inst.pname,inst.dependents,pname_arch[0]);
else: dependents[dep_pname] = set([ (pname,arch) ]);
#
# output like (list of versions is and-ed; i.e. it may depict a range of versions):
# apache2 {'apache2-bin': [('=', '2.4.38-3+deb10u1')], 'apache2-data': [('=', '2.4.38-3+deb10u1')], 'lsb-base': None, 'mime-support': None, ... }
#
def process_deps(depends,depends_or,s):
def split_pkg_ver(dep):
( dep_pkg, dep_ver ) = ( dep.strip().split(' ',1) + ["()"] )[:2];
dep_pkg = dep_pkg.split(':',1)[0];
dep_ver = dep_ver.strip('()').split(' ',1);
if dep_ver != ['']: dep_ver = tuple(dep_ver);
else: dep_ver = None;
return (dep_pkg,dep_ver);
if s == "": return;
for deplis in s.split(','):
deplis = deplis.split('|');
if len(deplis) == 0:
pass;
elif len(deplis) <= 1:
( dep_pkg, dep_ver ) = split_pkg_ver(deplis[0]);
pkg_dep = depends.get(dep_pkg,None);
if pkg_dep == None:
if dep_ver: depends[dep_pkg] = [ dep_ver ];
else: depends[dep_pkg] = None;
else:
if dep_ver: pkg_dep.append(dep_ver);
else:
deps_or = [];
for dep in deplis:
( dep_pkg, dep_ver ) = split_pkg_ver(dep);
deps_or.append( (dep_pkg,)+dep_ver if dep_ver else (dep_pkg,None,None) );
depends_or.append(deps_or);
what_provides = {};
def register_what_provides(pname,provides_list):
global what_provides;
for provides in provides_list.split(','):
provides = provides.strip();
existing = what_provides.get(provides,None);
if existing: existing.append(pname);
else: what_provides[provides] = [ pname ];
def fetch_repo_desc(repos):
#if not repo.endswith('/'): repo += '/';
repoidx = 0;
for url, subdir, sections in repos:
if len(sections) > 8: print("at most 8 sections allowed per repo!",sys.stderr);
for section in sections[:8]:
baseurl = conc_url([url,'dists',subdir]);
release_file = get_file_content(baseurl+'/Release').decode('latin_1').split('\n');
relfiles = {}; sumtype=None;
for line in release_file:
if not line: continue;
if line[-1]==':':
sumtype=line[:-1];
if sumtype == "size": sumtype = None;
this_pos = find(sumtype,hash_prefs);
mo = sumline.match(line);
if mo and sumtype:
filename = mo.group('filename');
hashsum = mo.group('sum');
if filename not in relfiles: relfiles[filename] = { 'size': int(mo.group('size')), 'hashtype': sumtype, 'hash': hashsum };
else:
prev_sumtype = relfiles[filename]['hashtype'];
prev_pos = find(prev_sumtype,hash_prefs);
if this_pos < prev_pos:
relfiles[filename]['hashtype'] = sumtype;
relfiles[filename]['hash'] = hashsum;
if relfiles[filename]['size'] != int(mo.group('size')):
print("error: Release file states different file sizes for file %s." % filename,file=sys.stderr);
#print(relfiles);
release_file = None;
for architecture in architectures:
idxfile = None;
for suffix in [".bz2",".xz",".gz",""]:
this_idxfile = 'main/binary-'+architecture+'/Packages'+suffix;
if this_idxfile in relfiles:
idxfile = this_idxfile;
break;
if idxfile == None:
print("Indexfile '"+'main/binary-'+architecture+'/Packages(.bz2|.xz|.gz)'+"' not found for repo '%s'!" % baseurl,file=sys.stderr);
sys.exit(2);
goodIdx = False;
for i in range(max_retries):
pkgs_file = get_file_content(baseurl+'/'+idxfile);
file_hash = get_hash( pkgs_file, relfiles[idxfile]['hashtype'] );
if file_hash == relfiles[idxfile]['hash']:
goodIdx = True;
if find(relfiles[idxfile]['hashtype'],hash_prefs) >= good_hash_prefs:
print("warning: unsafe checksum (%s) for file %s." % (relfiles[idxfile]['hashtype'],idxfile),file=sys.stderr);
break;
print("wrong hash for file %s/%s: expected %s, found %s." % (baseurl,idxfile,relfiles[idxfile]['hash'],file_hash) );
if not goodIdx:
sys.exit(2);
if suffix == ".bz2": pkgs_file =bz2.decompress(pkgs_file);
elif suffix == ".xz": pkgs_file = lzma.decompress(pkgs_file);
elif suffix == ".gz": pkgs_file = gzip.decompress(pkgs_file);
pname = None; depends = {}; depends_or = []; conflicts = {};
for line in pkgs_file.decode('latin1').split('\n'):
#print(line);
if line == "" and pname:
addpkg(pname,package(pname,version,arch,repoidx,filename,sha256,size,depends,depends_or,conflicts));
pname = None; version = None; arch = None; filename = None; size = None; sha256 = None; depends = {}; depends_or = []; conflicts = {};
toks = line.split(':',1);
if len(toks) >= 2:
(tag,rest) = toks;
rest = rest.strip(' \t');
if tag == "Package": pname = rest;
elif tag == "Version": version = rest;
elif tag == "Architecture": arch = rest;
elif tag == "Filename": filename = rest;
elif tag == "SHA256": sha256 = rest;
elif tag == "Size": size = rest;
elif tag == "Depends":
process_deps(depends,depends_or,rest);
elif tag == "Conflicts" or tag == "Breaks":
process_deps(conflicts,None,rest);
elif tag == "Provides": register_what_provides(pname,rest);
if pname:
addpkg(pname,package(pname,version,arch,repoidx,filename,sha256,size,depends,depends_or,conflicts));
repoidx += 1;
repoidx = ( repoidx & ~7 ) + 8;
installed = {};
def addinst(pname,version,arch,depends_txt,conflicts_txt):
global installed;
depends = {}; depends_or = []; conflicts = {};
process_deps( depends, depends_or, depends_txt );
process_deps( conflicts, None, conflicts_txt );
pkg = package(pname,version,arch,-1,None,None,None,depends,depends_or,conflicts);
if pname in installed: raise Exception("package %s installed two times!" % pname)
installed[(pname,arch)] = pkg;
addpkg(pname,pkg);
def getinst(idbfd=None):
pname = None; depends_txt = ""; conflicts_txt = ""; provides_txt = "";
with open("/var/lib/dpkg/status","r") as fd:
for line in fd:
line = line.rstrip(' \t\r\n');
if line == "" and pname:
if idbfd != None: print(pname,version,arch,depends_txt,"#",conflicts_txt,"#",provides_txt,file=idbfd);
else: addinst(pname,version,arch,depends_txt,conflicts_txt); register_what_provides(pname,provides_txt);
pname = None; version = None; arch = None; depends_txt = ""; conflicts_txt = ""; provides_txt = "";
toks = line.split(':',1);
if len(toks) >= 2:
(tag,rest) = toks; rest = rest.lstrip(' \t')
if tag == "Package": pname = rest;
elif tag == "Version": version = rest;
elif tag == "Architecture": arch = rest;
elif tag == "Depends": depends_txt = rest;
elif tag == "Conflicts" or tag == "Breaks":
if conflicts_txt: conflicts_txt += ", " + rest;
else: conflicts_txt = rest;
elif tag == "Provides": provides_txt = rest;
if pname:
if idbfd != None: print(pname,version,arch,depends_txt,"#",conflicts_txt,"#",provides_txt,file=idbfd);
else: addinst(pname,version,arch,depends_txt,conflicts_txt); register_what_provides(pname,provides_txt);
def readinstdb(filename):
global installed;
with open(filename,"r") as fd:
for line in fd:
line = line.rstrip(' \t\r\n');
(pname,version,arch,rest) = line.split(' ',3);
try:
(depends_txt,conflicts_txt,provides_txt) = rest.split(' #',2);
except Exception as e:
print("unable to split 3 tags by #:",line,file=sys.stderr);
raise e;
addinst(pname,version,arch,depends_txt,conflicts_txt);
register_what_provides(pname,provides_txt);
if len(sys.argv) >= 3 and sys.argv[1] == "getinst":
if len(sys.argv) > 3: print("spurious params: "+" ".join(sys.argv[3:]),file=sys.stderr);
with os.fdopen(os.open(sys.argv[2],os.O_CREAT|os.O_EXCL|os.O_WRONLY),'w') as idbfd:
getinst(idbfd);
sys.exit(0);
pkglis_file = None; verbose = False; shallDownload = True;
if len(sys.argv) >= 4 and sys.argv[1] == "cmpver":
print(cmp_ver(sys.argv[2],sys.argv[3]));
sys.exit(0);
if len(sys.argv) >= 3 and sys.argv[1] == "commdep":
if sys.argv[2] == "--pkglis":
pkglis_file = sys.argv[3];
readinstdb(pkglis_file);
commpkgs = sys.argv[4:];
else:
getinst();
commpkgs = sys.argv[2:];
calcCommDep(commpkgs);
sys.exit(0);
while len(sys.argv) >= 2:
if sys.argv[1] == "--agent":
agent = sys.argv[2];
sys.argv = sys.argv[:1] + sys.argv[3:];
elif sys.argv[1] == "--pkglis":
pkglis_file = sys.argv[2];
sys.argv = sys.argv[:1] + sys.argv[3:];
elif sys.argv[1] == "--arches":
architectures = sys.argv[2].split(':');
sys.argv = sys.argv[:1] + sys.argv[3:];
elif sys.argv[1] == "--verbose":
verbose = True;
sys.argv = sys.argv[:1] + sys.argv[2:];
elif sys.argv[1] == "--no-download":
shallDownload = False;
sys.argv = sys.argv[:1] + sys.argv[2:];
elif sys.argv[1] == "--upd-repo-only":
repos = repos[:1];
sys.argv = sys.argv[:1] + sys.argv[2:];
elif sys.argv[1] == "--dry-run":
dryRun = True;
sys.argv = sys.argv[:1] + sys.argv[2:];
elif sys.argv[1] == "--top-only":
topOnly = True;
sys.argv = sys.argv[:1] + sys.argv[2:];
else:
break;
agent_strs = { "apt": "Debian APT-HTTP/1.3 (1.8.2)", "wget": "Wget/1.20.1 (linux-gnu)", "mozilla": "Mozilla/5.0 (X11; Linux x86_64; rv:60.0) Gecko/20100101 Firefox/60.0" }
def main():
if verbose: print("reading installation database",file=sys.stderr);
instfile = sys.stdout;
if pkglis_file:
readinstdb(pkglis_file);
else:
getinst();
if verbose: print("fetching repo descriptions",file=sys.stderr);
fetch_repo_desc(repos);
prepare_pkgs_sort_dependents();
if verbose: print("calculating updates",file=sys.stderr);
get_packages_4_update(instfile);
if len(sys.argv) <= 1 or len(sys.argv) > 4 or sys.argv[1] == "--help":
print("torsocks fetchupd.py [--agent browserid] [--arches amd64:i386] [--pkglis installed.pkgs] [other options] [--upd-repo-only] targetdir",file=sys.stdout);
print(" other options: --no-download --dry-run --top-only (only top level dependencies, no libraries)",file=sys.stdout);
print("fetchupd getinst installed.pkgs",file=sys.stdout);
print("fetchupd cmpver ver1 ver2",file=sys.stdout);
print("fetchupd commdep [--pkglis installed.pkgs] pkg1 pkg2 pkg3:arch3 ...",file=sys.stdout);
print(" change fetchupd.py by hand in order to adjust repos as found in /etc/apt/sources.list",file=sys.stdout);
print(" user agent string may be one of 'apt, 'wget', '' or 'mozilla'; default is 'apt', '' uses the python3 default. You may also specify a full string like 'Debian APT-HTTP/1.3 (1.8.2)'.",file=sys.stdout);
print(" creates scripts with name targetdir/install-updates and targetdir/download-updates and a SHA-file with name targetdir/SHA256SUMS",file=sys.stdout);
print("",file=sys.stdout);
else:
putdir = sys.argv[1];
if( len(sys.argv) > 2 ): subdir = sys.argv[2];
if( len(sys.argv) > 3 ): repo = sys.argv[3];
if( len(sys.argv) > 4 ): agent = sys.argv[4];
agent = agent_strs.get(agent,agent);
if verbose:
print("repos:", len(repos));
if(agent): print("agent:",agent);
print();
prevdir = os.getcwd();
os.chdir(putdir);
try:
if dryRun:
main();
else:
with os.fdopen(os.open("install-updates",os.O_CREAT|os.O_EXCL|os.O_WRONLY,0o777),'w') as this_instfile:
with os.fdopen(os.open("download-updates",os.O_CREAT|os.O_EXCL|os.O_WRONLY,0o777),'w') as this_dldfile:
with open("SHA256SUMS",'a') as this_shafile:
instFile = this_instfile;
dldFile = this_dldfile;
sha256File = this_shafile;
main();
finally:
os.chdir(prevdir);
print();
