Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package MirrorCache for openSUSE:Factory checked in at 2022-10-08 01:23:49 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/MirrorCache (Old) and /work/SRC/openSUSE:Factory/.MirrorCache.new.2275 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "MirrorCache" Sat Oct 8 01:23:49 2022 rev:19 rq:1008401 version:1.042 Changes: -------- --- /work/SRC/openSUSE:Factory/MirrorCache/MirrorCache.changes 2022-09-09 18:29:13.921221291 +0200 +++ /work/SRC/openSUSE:Factory/.MirrorCache.new.2275/MirrorCache.changes 2022-10-08 01:23:53.646046506 +0200 @@ -1,0 +2,26 @@ +Thu Sep 29 09:47:07 UTC 2022 - Andrii Nikitin <[email protected]> + +- Update to version 1.042: + * Add support for metalink4 format (#303) + * Add url parameter IP to simulate geo experience (#304) + * t: Fix CI after changes in async rendering in Mojo (#303) + +------------------------------------------------------------------- +Tue Sep 27 11:42:51 UTC 2022 - Andrii Nikitin <[email protected]> + +- Update to version 1.041: + * Fix metalink missing 'verification' tag (#302) + +------------------------------------------------------------------- +Thu Sep 22 14:41:52 UTC 2022 - Andrii Nikitin <[email protected]> + +- Update to version 1.040: + * Add parameter METALINK_GREEDY (#300) + * Optimization: assume package managers never want to render dir (#300) + * Fix mirror picked despite disabled in server_project table (#301) + * Avoid by and ru mirrors for ua requests (#298) + +- Remove temporary patch, it is now included in sources + * 0001-Avoid-by-and-ru-mrrors-for-ua-requests.patch + +------------------------------------------------------------------- Old: ---- 0001-Avoid-by-and-ru-mrrors-for-ua-requests.patch MirrorCache-1.039.obscpio New: ---- MirrorCache-1.042.obscpio ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ MirrorCache.spec ++++++ --- /var/tmp/diff_new_pack.zstGEG/_old 2022-10-08 01:23:54.194047763 +0200 +++ /var/tmp/diff_new_pack.zstGEG/_new 2022-10-08 01:23:54.198047772 +0200 @@ -22,7 +22,7 @@ %define main_requires %{assetpack_requires} perl(Carp) perl(DBD::Pg) >= 3.7.4 perl(DBI) >= 1.632 perl(DBIx::Class) >= 0.082801 perl(DBIx::Class::DynamicDefault) perl(DateTime) perl(Encode) perl(Time::Piece) perl(Time::Seconds) perl(Time::ParseDate) perl(DateTime::Format::Pg) perl(Exporter) perl(File::Basename) perl(LWP::UserAgent) perl(Mojo::Base) perl(Mojo::ByteStream) perl(Mojo::IOLoop) perl(Mojo::JSON) perl(Mojo::Pg) perl(Mojo::URL) perl(Mojo::Util) perl(Mojolicious::Commands) perl(Mojolicious::Plugin) perl(Mojolicious::Plugin::RenderFile) perl(Mojolicious::Static) perl(Net::OpenID::Consumer) perl(POSIX) perl(Sort::Versions) perl(URI::Escape) perl(XML::Writer) perl(base) perl(constant) perl(diagnostics) perl(strict) perl(warnings) shadow rubygem(sass) perl(Net::DNS) perl(LWP::Protocol::https) perl(Digest::SHA) %define build_requires %{assetpack_requires} rubygem(sass) tidy sysuser-shadow sysuser-tools Name: MirrorCache -Version: 1.039 +Version: 1.042 Release: 0 Summary: WebApp to redirect and manage mirrors License: GPL-2.0-or-later @@ -34,7 +34,6 @@ Source3: %{name}-tmpfilesd.conf # use update-cache (or tools/generate-packed-assets) to generate/update cache.tar.xz Source101: update-cache.sh -Patch0: 0001-Avoid-by-and-ru-mrrors-for-ua-requests.patch BuildRequires: %{build_requires} Requires: %{main_requires} Requires: perl(Minion) >= 10.0 @@ -46,7 +45,6 @@ %prep %setup -q -a1 -%patch0 -p1 %build # make {?_smp_mflags} ++++++ MirrorCache-1.039.obscpio -> MirrorCache-1.042.obscpio ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.039/lib/MirrorCache/Datamodule.pm new/MirrorCache-1.042/lib/MirrorCache/Datamodule.pm --- old/MirrorCache-1.039/lib/MirrorCache/Datamodule.pm 2022-08-26 15:13:31.000000000 +0200 +++ new/MirrorCache-1.042/lib/MirrorCache/Datamodule.pm 2022-09-29 11:41:23.000000000 +0200 @@ -27,6 +27,7 @@ has [ '_route', '_route_len' ]; # this is '/download' has [ 'route', 'route_len' ]; # this may be '/download' or empty if one of TOP_FOLDERS present has [ 'metalink', 'metalink_accept' ]; +has [ 'meta4', 'meta4_accept' ]; has [ '_ip', '_country', '_region', '_lat', '_lng', '_vpn' ]; has [ '_avoid_countries' ]; has [ '_pedantic' ]; @@ -78,6 +79,10 @@ } $self->c($c); $self->_ip(undef); + $self->metalink(undef); + $self->metalink_accept(undef); + $self->meta4(undef); + $self->meta4_accept(undef); } sub ip_sha1($self) { @@ -107,7 +112,7 @@ } sub extra($self) { - return ($self->metalink || $self->mirrorlist || $self->zsync || $self->magnet || $self->torrent || $self->btih ); + return ($self->metalink || $self->meta4 || $self->mirrorlist || $self->zsync || $self->magnet || $self->torrent || $self->btih ); } sub region($self) { @@ -260,7 +265,7 @@ sub redirect($self, $url) { my $xtra = ''; - if ($self->_original_path =~ m/(\.metalink|\.mirrorlist|\.torrent|\.magnet|\.btih)$/) { + if ($self->_original_path =~ m/(\.metalink|\.meta4|\.mirrorlist|\.torrent|\.magnet|\.btih)$/) { $xtra = $1; } @@ -268,6 +273,7 @@ } sub _init_headers($self) { + $self->_agent(''); my $headers = $self->c->req->headers; return unless $headers; $self->_agent($headers->user_agent ? $headers->user_agent : ''); @@ -275,6 +281,10 @@ $self->metalink(1); $self->metalink_accept(1); } + if ($headers->accept && $headers->accept =~ m/\bapplication\/metalink4/) { + $self->meta4(1); + $self->meta4_accept(1); + } } sub _init_req($self) { @@ -307,10 +317,14 @@ } sub _init_location($self) { + my $query = $self->c->req->url->query; + if (my $p = $query->param('IP')) { + $self->_ip($p); + } my ($lat, $lng, $country, $region) = $self->c->geodb->location($self->ip); + $region = 'eu' if $country && $country eq 'tr'; $self->_lat($lat); $self->_lng($lng); - my $query = $self->c->req->url->query; if (my $p = $query->param('COUNTRY')) { if (length($p) == 2 ) { $country = $p; @@ -322,18 +336,25 @@ $region = lc($p); } } - if (my $p = $query->param('AVOID_COUNTRY')) { - my @avoid_countries = (); + if (!$country) { + $country = ''; + } else { + $country = substr(lc($country), 0, 2) ; + } + my $p = $query->param('AVOID_COUNTRY'); + my @avoid_countries = (); + @avoid_countries = ('by', 'ru') if $country eq 'ua'; + if ($p) { for my $c (split ',', $p) { next unless length($c) == 2; $c = lc($c); push @avoid_countries, $c; $country = '' if $c eq lc($country // ''); - } - $self->_avoid_countries(\@avoid_countries); + }; } - $country = substr($country, 0, 2) if $country; - $self->_country($country // ''); + $self->_avoid_countries(\@avoid_countries); + + $self->_country($country); $self->_region($region // ''); } @@ -386,6 +407,12 @@ $path = substr($path,0,$pos); } } + if(!$trailing_slash && ((my $pos = length($path)-length('.meta4')) > 1)) { + if ('.meta4' eq substr($path,$pos)) { + $self->meta4(1); + $path = substr($path,0,$pos); + } + } if (!$trailing_slash && ((my $pos = length($path) - length('.mirrorlist')) > 1)) { if ('.mirrorlist' eq substr($path, $pos)) { $self->mirrorlist(1); @@ -432,6 +459,7 @@ $self->must_render_from_root(1) if !$self->mirrorlist && ( !$self->metalink || $self->metalink_accept ) + && ( !$self->meta4 || $self->meta4_accept ) && !$self->zsync && $path =~ m/.*\/(repodata\/repomd.xml[^\/]*|media\.1\/media|.*\.sha256(\.asc)|Release(.key|.gpg)?|InRelease|Packages(.gz)?|Sources(.gz)?|.*_Arch\.(files|db|key)(\.(sig|tar\.gz(\.sig)?))?|(files|primary|other).xml.gz|[Pp]ackages(\.[A-Z][A-Z])?\.(xz|gz)|gpg-pubkey.*\.asc|CHECKSUMS)$/; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.039/lib/MirrorCache/Schema/ResultSet/Server.pm new/MirrorCache-1.042/lib/MirrorCache/Schema/ResultSet/Server.pm --- old/MirrorCache-1.039/lib/MirrorCache/Schema/ResultSet/Server.pm 2022-08-26 15:13:31.000000000 +0200 +++ new/MirrorCache-1.042/lib/MirrorCache/Schema/ResultSet/Server.pm 2022-09-29 11:41:23.000000000 +0200 @@ -128,6 +128,8 @@ where fd.folder_id = ? and fdf.file_id is NULL ) s join folder f on f.id = ? +left join project p on f.path like concat(p.path, '%') +left join server_project sp on (sp.server_id, sp.project_id) = (s.id, p.id) left join server_capability_declaration scd on s.id = scd.server_id and scd.capability = '$capability' and NOT scd.enabled left join server_capability_force scf on s.id = scf.server_id and scf.capability = '$capability' left join server_capability_declaration scd2 on s.id = scd2.server_id and scd.capability = '$ipv' and NOT scd.enabled @@ -136,6 +138,7 @@ left join server_stability stability_schemex on s.id = stability_schemex.server_id and stability_schemex.capability = '$capabilityx' left join server_stability stability_ipv on s.id = stability_ipv.server_id and stability_ipv.capability = '$ipv' left join server_stability stability_ipvx on s.id = stability_ipvx.server_id and stability_ipvx.capability = '$ipvx' +where sp.state is NULL or sp.state > 0 ) x WHERE not_disabled $extra order by rating_country desc, (dist/100)::int, support_scheme desc, rating_scheme desc, support_ipv desc, rating_ipv desc, score, random() diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.039/lib/MirrorCache/Schema/ResultSet/Stat.pm new/MirrorCache-1.042/lib/MirrorCache/Schema/ResultSet/Stat.pm --- old/MirrorCache-1.039/lib/MirrorCache/Schema/ResultSet/Stat.pm 2022-08-26 15:13:31.000000000 +0200 +++ new/MirrorCache-1.042/lib/MirrorCache/Schema/ResultSet/Stat.pm 2022-09-29 11:41:23.000000000 +0200 @@ -136,7 +136,7 @@ my $sql = << "END_SQL"; select stat.id, mirror_id, trim(stat.country), - concat(case when secure then 'https://' else 'http://' end, CASE WHEN length(server.hostname_vpn)>0 THEN server.hostname_vpn ELSE server.hostname END, server.urldir, case when metalink then regexp_replace(path, '(.*)\.metalink', E'\\1') else path end) as url, + concat(case when secure then 'https://' else 'http://' end, CASE WHEN length(server.hostname_vpn)>0 THEN server.hostname_vpn ELSE server.hostname END, server.urldir, case when metalink then regexp_replace(path, '(.*)\.(metalink|meta4)', E'\\1') else path end) as url, regexp_replace(path, '(^.*)/[^/]*', E'\\1') as folder, folder_id from stat join server on mirror_id = server.id where stat.id > $prev_stat_id diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.039/lib/MirrorCache/WebAPI/Plugin/Dir.pm new/MirrorCache-1.042/lib/MirrorCache/WebAPI/Plugin/Dir.pm --- old/MirrorCache-1.039/lib/MirrorCache/WebAPI/Plugin/Dir.pm 2022-08-26 15:13:31.000000000 +0200 +++ new/MirrorCache-1.042/lib/MirrorCache/WebAPI/Plugin/Dir.pm 2022-09-29 11:41:23.000000000 +0200 @@ -47,6 +47,9 @@ return $app; } +# Assume UserAgent of such pattern never lists directories and only needs to render files +my $PACKAGE_MANAGER_PATTERN = 'ZYpp .*|Debian APT.*|libdnf.*|osc.*'; + sub indx { my $c = shift; my $reqpath = $c->req->url->path; @@ -75,6 +78,11 @@ || _render_from_db($dm) || _local_render($dm, 1); # check if we should render local when metalink cannot be provided + if ($dm->agent =~ qr/$PACKAGE_MANAGER_PATTERN/) { + my ($path, $trailing_slash) = $dm->path; + return $root->render_file($dm, $path . $trailing_slash); + } + my $tx = $c->render_later->tx; my $rendered; my $handle_error = sub { @@ -192,7 +200,8 @@ my ($path, $trailing_slash, $original_path) = $dm->path; return undef if $path eq '/'; $path = $path . '.metalink' if $dm->metalink && !$dm->metalink_accept; - return $dm->c->redirect_to($dm->route . $path . $trailing_slash . $dm->query1) unless $original_path eq $path || ($dm->extra && !$dm->metalink); + $path = $path . '.meta4' if $dm->meta4 && !$dm->meta4_accept; + return $dm->c->redirect_to($dm->route . $path . $trailing_slash . $dm->query1) unless $original_path eq $path || ($dm->extra && !$dm->metalink && !$dm->meta4); return undef; } @@ -253,10 +262,13 @@ sub _local_render { my $dm = shift; my $accept = shift; - return undef if $dm->extra && (!$accept || !$dm->metalink_accept); + return undef if $dm->extra && (!$accept || (!$dm->metalink_accept && !$dm->meta4_accept)); my ($path, $trailing_slash) = $dm->path; - - return $root->render_file_if_nfs($dm, $path) if $root->is_remote; + if ($root->is_remote) { + # we can just render top folders + return _render_top_folders($dm) if $ENV{MIRRORCACHE_TOP_FOLDERS} && $path eq '/'; + return $root->render_file_if_nfs($dm, $path); + } if ($root->is_dir($path)) { return $dm->redirect($dm->route . $path . '/') if !$trailing_slash && $path ne '/'; @@ -317,7 +329,7 @@ my ($path, $trailing_slash) = $dm->path; if ($dm->extra) { - return $root->render_file($dm, $path) if $dm->metalink_accept; + return $root->render_file($dm, $path) if $dm->metalink_accept || $dm->meta4_accept; # the file is unknown, we cannot show generate meither mirrorlist or metalink my $res = $c->render(status => 425, text => "The file is unknown, retry later"); # log miss here even thoough we haven't rendered anything @@ -366,7 +378,7 @@ # this should happen only if $url is a valid file or non-existing path return $root->render_file($dm, $path . $trailing_slash); })->catch(sub { - my $res = $root->render_file($dm, $path . $trailing_slash); + $root->render_file($dm, $path . $trailing_slash); my $msg = "Error while guessing how to render $url: "; if (1 == scalar(@_)) { $msg = $msg . $_[0]; @@ -384,6 +396,33 @@ versioncmp(lc($a->{name}), lc($b->{name})); } +sub _render_top_folders { + my $dm = shift; + my $dir = '/'; + my $c = $dm->c; + my @files; + my $json = $dm->json; + + for my $basename ( @top_folders ) { + if ($json) { + push @files, { + name => $basename, + }; + next; + } + my $encoded = Encode::decode_utf8( './' . $basename ); + + push @files, { + url => $encoded, + name => $basename, + dir => 1, + }; + } + my @items = sort _by_filename @files; + return $c->render( json => \@items) if $json; + return $c->render( 'dir', files => \@items, cur_path => $dir, folder_id => undef ); +} + sub _render_dir_from_db { my $dm = shift; my $id = shift; @@ -482,7 +521,7 @@ return undef unless $SMALL_FILE_SIZE && ($ROOT_NFS || !$root->is_remote ); my $dm = shift; $dm->_init_path; - return undef if (($dm->metalink && !$dm->metalink_accept) || $dm->mirrorlist || $dm->zsync); + return undef if (($dm->metalink && !$dm->metalink_accept) || ($dm->meta4 && !$dm->meta4_accept) || $dm->mirrorlist || $dm->zsync); my ($path, undef) = $dm->path; my $full; return $root->render_file_if_small($dm, $path, $SMALL_FILE_SIZE) unless $ROOT_NFS; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.039/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm new/MirrorCache-1.042/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm --- old/MirrorCache-1.039/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm 2022-08-26 15:13:31.000000000 +0200 +++ new/MirrorCache-1.042/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm 2022-09-29 11:41:23.000000000 +0200 @@ -30,7 +30,8 @@ sub register { my ($self, $app) = @_; - $app->types->type(metalink => 'application/metalink+xml; charset=UTF-8'); + $app->types->type(metalink => 'application/metalink+xml; charset=UTF-8'); + $app->types->type(meta4 => 'application/metalink4+xml; charset=UTF-8'); $app->helper( 'mirrorcache.render_file' => sub { my ($c, $filepath, $dm, $file)= @_; @@ -83,6 +84,7 @@ my $region = $dm->region; if (!$folder || !$file) { return $root->render_file($dm, $filepath . '.metalink') if ($dm->metalink && !$file && !$dm->metalink_accept); # file is unknown - cannot generate metalink + return $root->render_file($dm, $filepath . '.meta4') if ($dm->meta4 && !$file && !$dm->meta4_accept); # file is unknown - cannot generate meta4 return $root->render_file($dm, $filepath) if !$dm->extra || $dm->metalink_accept; # TODO we still can check file on mirrors even if it is missing in DB } @@ -92,7 +94,7 @@ } my $baseurl; # just hostname + eventual urldir (without folder and file) my $fullurl; # baseurl with path and filename - if ($dm->metalink || $dm->torrent || $dm->zsync || $dm->magnet) { + if ($dm->metalink || $dm->meta4 || $dm->torrent || $dm->zsync || $dm->magnet) { $baseurl = $root->is_remote ? $root->location($dm) : $root->redirect($dm, $folder->path) # we must pass $path here because it potenially has impact } if ($dm->torrent || $dm->zsync || $dm->magnet) { @@ -140,9 +142,7 @@ } } - return _render_torrent($dm, $file, \@mirrors_country, \@mirrors_region, \@mirrors_rest, $fullurl) if $dm->torrent; - - if ($dm->metalink && !($dm->metalink_accept && 'media.1/media' eq substr($filepath,length($filepath)-length('media.1/media')))) { + if (($dm->metalink || $dm->meta4) && !(($dm->metalink_accept || $dm->meta4_accept) && 'media.1/media' eq substr($filepath,length($filepath)-length('media.1/media')))) { my $origin; if (my $publisher_url = $ENV{MIRRORCACHE_METALINK_PUBLISHER_URL}) { $publisher_url =~ s/^https?:\/\///; @@ -154,7 +154,16 @@ $origin = $origin . $dm->route; } $origin = $origin . $filepath; - my $xml = _build_metalink( + my $xml; + if ($dm->meta4) { + $xml = _build_meta4( + $dm, $folder->path, $file, $country, $region, \@mirrors_country, \@mirrors_region, + \@mirrors_rest, $origin, 'MirrorCache', $baseurl); + $c->res->headers->content_disposition('attachment; filename="' .$basename. '.meta4"'); + $c->render(data => $xml, format => 'meta4'); + return 1; + } + $xml = _build_metalink( $dm, $folder->path, $file, $country, $region, \@mirrors_country, \@mirrors_region, \@mirrors_rest, $origin, 'MirrorCache', $baseurl); $c->res->headers->content_disposition('attachment; filename="' .$basename. '.metalink"'); @@ -162,6 +171,8 @@ return 1; } + return _render_torrent($dm, $file, \@mirrors_country, \@mirrors_region, \@mirrors_rest, $fullurl) if $dm->torrent; + if ($dm->mirrorlist) { my @mirrordata; if ($country and !$dm->avoid_countries || !(grep { $country eq $_ } $dm->avoid_countries)) { @@ -352,7 +363,13 @@ return $app; } -sub _build_metalink() { +# metalink should not include root url in mirror list if mirror count exceeds METALINK_GREEDY parameter +my $METALINK_GREEDY = int( $ENV{MIRRORCACHE_METALINK_GREEDY} // 0 ) // 0; +my $publisher = $ENV{MIRRORCACHE_METALINK_PUBLISHER} || 'openSUSE'; +my $publisher_url = $ENV{MIRRORCACHE_METALINK_PUBLISHER_URL} || 'http://download.opensuse.org'; + + +sub _build_meta4() { my ( $dm, $path, $file, $country, $region, $mirrors_country, $mirrors_region, $mirrors_rest, $origin, $generator, $rooturl @@ -361,8 +378,121 @@ $country = uc($country) if $country; $region = uc($region) if $region; - my $publisher = $ENV{MIRRORCACHE_METALINK_PUBLISHER} || 'openSUSE'; - my $publisher_url = $ENV{MIRRORCACHE_METALINK_PUBLISHER_URL} || 'http://download.opensuse.org'; + my $writer = XML::Writer->new(OUTPUT => 'self', DATA_MODE => 1, DATA_INDENT => 2, ); + $writer->xmlDecl('UTF-8'); + $writer->startTag('metalink', xmlns => 'urn:ietf:params:xml:ns:metalink'); + $writer->dataElement( generator => $generator ) if $generator; + $writer->dataElement( origin => $origin, dynamic => 'true') if $origin; + $writer->dataElement( published => strftime('%Y-%m-%dT%H:%M:%SZ', localtime time)); + + $writer->startTag('publisher'); + $writer->dataElement( name => $publisher ) if $publisher; + $writer->dataElement( url => $publisher_url) if $publisher_url; + $writer->endTag('publisher'); + + $writer->startTag('file', name => $basename); + $writer->dataElement( size => $file->{size} ) if $file->{size}; + $writer->comment('<mtime>' . $file->{mtime} . '</mtime>') if ($file->{mtime}); + if (my $md5 = $file->{md5}) { + $writer->startTag('hash', type => 'md5'); + $writer->characters($md5); + $writer->endTag('hash'); + } + if (my $sha1 = $file->{sha1}) { + $writer->startTag('hash', type => 'sha-1'); + $writer->characters($sha1); + $writer->endTag('hash'); + } + if (my $sha256 = $file->{sha256}) { + $writer->startTag('hash', type => 'sha-256'); + $writer->characters($sha256); + $writer->endTag('hash'); + } + if (my $piece_size = $file->{piece_size}) { + $writer->startTag('pieces', length => $piece_size, type => 'sha-1'); + for my $piece (grep {$_} split /(.{40})/, $file->{pieces}) { + $writer->dataElement( hash => $piece ); + } + $writer->endTag('pieces'); + } + + my $priority = 1; + my $fullname = $path . '/' . $basename; + my $root_included = 0; + my $print_root = sub { + return unless $rooturl; + + my $print = shift; + return if $root_included and !$print; + + $writer->comment("File origin location: ") if $print; + if ($METALINK_GREEDY && $METALINK_GREEDY <= $priority) { + $writer->comment($rooturl . $fullname); + } else { + $writer->startTag('url', location => uc($dm->root_country), priority => $priority); + $writer->characters($rooturl . $fullname); + $writer->endTag('url'); + } + $root_included = 1; + $priority++; + }; + $writer->comment("Mirrors which handle this country ($country): "); + for my $m (@$mirrors_country) { + my $url = $m->{url}; + + $print_root->() if $country ne uc($m->{country}) && $dm->root_is_better($m->{region}, $m->{lng}); + $writer->startTag('url', location => uc($m->{country}), priority => $priority); + $writer->characters($url); + $writer->endTag('url'); + $priority++; + } + $print_root->() if $dm->root_country eq lc($country); + + $writer->comment("Mirrors in the same continent ($region): "); + for my $m (@$mirrors_region) { + my $url = $m->{url}; + + $print_root->() if $dm->root_is_better($m->{region}, $m->{lng}); + $writer->startTag( + 'url', + location => uc($m->{country}), + priority => $priority + ); + $writer->characters($url); + $writer->endTag('url'); + $priority++; + } + $print_root->() if $dm->root_is_hit; + + $writer->comment("Mirrors in other parts of the world: "); + for my $m (@$mirrors_rest) { + my $url = $m->{url}; + $print_root->() if $dm->root_is_better($m->{region}, $m->{lng}); + $writer->startTag( + 'url', + location => uc($m->{country}), + priority => $priority + ); + $writer->characters($url); + $writer->endTag('url'); + $priority++; + } + + $print_root->(1); + $writer->endTag('file'); + $writer->endTag('metalink'); + + return $writer->end(); +} + +sub _build_metalink() { + my ( + $dm, $path, $file, $country, $region, $mirrors_country, + $mirrors_region, $mirrors_rest, $origin, $generator, $rooturl + ) = @_; + my $basename = $file->{name}; + $country = uc($country) if $country; + $region = uc($region) if $region; my $writer = XML::Writer->new(OUTPUT => 'self', DATA_MODE => 1, DATA_INDENT => 2, ); $writer->xmlDecl('UTF-8'); @@ -387,27 +517,36 @@ $writer->startTag('file', name => $basename); $writer->dataElement( size => $file->{size} ) if $file->{size}; $writer->comment('<mtime>' . $file->{mtime} . '</mtime>') if ($file->{mtime}); - if (my $md5 = $file->{md5}) { - $writer->startTag('hash', type => 'md5'); - $writer->characters($md5); - $writer->endTag('hash'); - } - if (my $sha1 = $file->{sha1}) { - $writer->startTag('hash', type => 'sha-1'); - $writer->characters($sha1); - $writer->endTag('hash'); - } - if (my $sha256 = $file->{sha256}) { - $writer->startTag('hash', type => 'sha-256'); - $writer->characters($sha256); - $writer->endTag('hash'); - } - if (my $piece_size = $file->{piece_size}) { - $writer->startTag('pieces', length => $piece_size, type => 'sha-1'); - for my $piece (grep {$_} split /(.{40})/, $file->{pieces}) { - $writer->dataElement( hash => $piece ); + my $md5 = $file->{md5}; + my $sha1 = $file->{sha1}; + my $sha256 = $file->{sha256}; + if ($md5 || $sha1 || $sha256) { + $writer->startTag('verification'); + if ($md5) { + $writer->startTag('hash', type => 'md5'); + $writer->characters($md5); + $writer->endTag('hash'); + } + if ($sha1) { + $writer->startTag('hash', type => 'sha-1'); + $writer->characters($sha1); + $writer->endTag('hash'); + } + if ($sha256) { + $writer->startTag('hash', type => 'sha-256'); + $writer->characters($sha256); + $writer->endTag('hash'); + } + if (my $piece_size = $file->{piece_size}) { + $writer->startTag('pieces', length => $piece_size, type => 'sha-1'); + my $piecen = 0; + for my $piece (grep {$_} split /(.{40})/, $file->{pieces}) { + $writer->dataElement( hash => $piece, piece => $piecen); + $piecen++; + } + $writer->endTag('pieces'); } - $writer->endTag('pieces'); + $writer->endTag('verification'); } my $colon = $rooturl ? index(substr($rooturl,0,6),':') : ''; @@ -423,9 +562,13 @@ return if $root_included and !$print; $writer->comment("File origin location: ") if $print; - $writer->startTag('url', type => substr($rooturl,0,$colon), location => uc($dm->root_country), preference => $preference); - $writer->characters($rooturl . $fullname); - $writer->endTag('url'); + if ($METALINK_GREEDY && $METALINK_GREEDY <= (100 - $preference)) { + $writer->comment($rooturl . $fullname); + } else { + $writer->startTag('url', type => substr($rooturl,0,$colon), location => uc($dm->root_country), preference => $preference); + $writer->characters($rooturl . $fullname); + $writer->endTag('url'); + } $root_included = 1; $preference--; }; @@ -503,7 +646,7 @@ my $avoid_countries = $dm->avoid_countries; my $mirrorlist = $dm->mirrorlist; my $ipvstrict = $dm->ipvstrict; - my $metalink = $dm->metalink; + my $metalink = $dm->metalink || $dm->meta4; my $limit = $mirrorlist ? 100 : (( $metalink || $dm->pedantic )? 10 : 1); my $rs = $dm->c->schemaR->resultset('Server'); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.039/lib/MirrorCache/WebAPI/Plugin/Stat.pm new/MirrorCache-1.042/lib/MirrorCache/WebAPI/Plugin/Stat.pm --- old/MirrorCache-1.039/lib/MirrorCache/WebAPI/Plugin/Stat.pm 2022-08-26 15:13:31.000000000 +0200 +++ new/MirrorCache-1.042/lib/MirrorCache/WebAPI/Plugin/Stat.pm 2022-09-29 11:41:23.000000000 +0200 @@ -68,7 +68,7 @@ $path = $dm->root_subtree . $path; my $rows = $self->rows; my @rows = defined $rows? @$rows : (); - push @rows, [ $dm->ip_sha1, scalar $dm->agent, scalar ($path . $trailing_slash), $dm->country, datetime_now(), $mirror_id, $dm->folder_id, $dm->file_id, $dm->is_secure, $dm->is_ipv4, $dm->metalink? 1 : 0, $dm->mirrorlist? 1 : 0, $dm->is_head, $$, int($dm->elapsed*1000), $dm->file_age, $dm->folder_scan_last, $dm->mirror_country ]; + push @rows, [ $dm->ip_sha1, scalar $dm->agent, scalar ($path . $trailing_slash), $dm->country, datetime_now(), $mirror_id, $dm->folder_id, $dm->file_id, $dm->is_secure, $dm->is_ipv4, ($dm->metalink || $dm->meta4)? 1 : 0, $dm->mirrorlist? 1 : 0, $dm->is_head, $$, int($dm->elapsed*1000), $dm->file_age, $dm->folder_scan_last, $dm->mirror_country ]; my $cnt = @rows; if ($cnt >= $FLUSH_COUNT) { $self->rows(undef); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.039/t/environ/01-smoke-mirror-hasall-remote.sh new/MirrorCache-1.042/t/environ/01-smoke-mirror-hasall-remote.sh --- old/MirrorCache-1.039/t/environ/01-smoke-mirror-hasall-remote.sh 2022-08-26 15:13:31.000000000 +0200 +++ new/MirrorCache-1.042/t/environ/01-smoke-mirror-hasall-remote.sh 2022-09-29 11:41:23.000000000 +0200 @@ -10,6 +10,7 @@ $mc/gen_env \ MIRRORCACHE_ROOT=http://$($ap9/print_address) \ + MIRRORCACHE_METALINK_GREEDY=3 \ MIRRORCACHE_REDIRECT=$FAKEURL2 $mc/start @@ -31,7 +32,7 @@ $ap8/start $ap8/curl /folder1/ | grep file1.1.dat - +rm $ap7/dt/folder1/file2.1.dat # remove a file from ap7 $mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap7/print_address)','','t','us','na'" $mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap8/print_address)','','t','de','eu'" @@ -55,4 +56,11 @@ rc=0 $mc/curl -I /download/folder1/file1.1.dat?"COUNTRY=it&PEDANTIC=1" | grep "${FAKEURL}" || rc=$? test $rc -gt 0 + +echo When METALINK_GREEDY is set, REDIRECT url will appear only as comment if mirror count exceeds value of METALINK_GREEDY +$mc/curl /download/folder1/file1.1.dat.metalink | grep -A1 'File origin' | grep '<!-- http://'${FAKEURL2}'/folder1/file1.1.dat -->' + +echo for file2 REDIRECT still appears in metalink from folder2, because mirror count doesnt exceed value of METALINK_GREEDY +$mc/curl /download/folder1/file2.1.dat.metalink | grep -A1 'File origin' | grep '<url type="http" location="" preference="98">http://'${FAKEURL2}'/folder1/file2.1.dat</url>' + echo success diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.039/t/environ/01-smoke.sh new/MirrorCache-1.042/t/environ/01-smoke.sh --- old/MirrorCache-1.039/t/environ/01-smoke.sh 2022-08-26 15:13:31.000000000 +0200 +++ new/MirrorCache-1.042/t/environ/01-smoke.sh 2022-09-29 11:41:23.000000000 +0200 @@ -122,3 +122,7 @@ $mc/sql 'select * from stat_agg' $mc/curl /rest/stat | grep '"hour":{"bot":2,"hit":24,"miss":6,"prev_bot":2,"prev_hit":24,"prev_miss":6}' +$mc/curl /download/folder3/file1.1.dat.metalink | xmllint --noout --format - +$mc/curl /download/folder3/file1.1.dat.meta4 | xmllint --noout --format - +$mc/curl /download/folder3/file1.1.dat.meta4 | grep '<url location="US" priority="1">http://127.0.0.1:1304/folder3/file1.1.dat</url>' + diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.039/t/environ/02-files-hashes.sh new/MirrorCache-1.042/t/environ/02-files-hashes.sh --- old/MirrorCache-1.039/t/environ/02-files-hashes.sh 2022-08-26 15:13:31.000000000 +0200 +++ new/MirrorCache-1.042/t/environ/02-files-hashes.sh 2022-09-29 11:41:23.000000000 +0200 @@ -56,7 +56,8 @@ $mc/curl /download/folder1/file1.1.dat.metalink | grep -o "<mtime>$(date +%s -r $mc/dt/folder1/file1.1.dat)</mtime>" $mc/curl /download/folder1/file1.1.dat.metalink | grep -o '<hash type="md5">b2c5860a03d2c4f1f049a3b2409b39a8</hash>' $mc/curl /download/folder1/file1.1.dat.metalink | grep -o '<hash type="sha-256">63d19a99ef7db94ddbb1e4a5083062226551cd8197312e3aa0aa7c369ac3e458</hash>' -$mc/curl /download/folder1/file1.1.dat.metalink | grep -o '<hash>5179db3d4263c9cb4ecf0edbc653ca460e3678b7</hash>' +$mc/curl /download/folder1/file1.1.dat.meta4 | grep -o '<hash>5179db3d4263c9cb4ecf0edbc653ca460e3678b7</hash>' +$mc/curl /download/folder1/file1.1.dat.metalink | grep -o '<hash piece="0">5179db3d4263c9cb4ecf0edbc653ca460e3678b7</hash>' $mc/curl -I /download/folder1/file1.1.dat.btih | grep '200 OK' $mc/curl /download/folder1/file1.1.dat.btih @@ -65,4 +66,7 @@ $mc/curl -I /download/folder1/file1.1.dat.torrent | grep '200 OK' $mc/curl /download/folder1/file1.1.dat.torrent +$mc/curl /download/folder1/file1.1.dat.metalink | xmllint --noout --format - +$mc/curl /download/folder1/file1.1.dat.meta4 | xmllint --noout --format - + echo success diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.039/t/environ/03-geo.sh new/MirrorCache-1.042/t/environ/03-geo.sh --- old/MirrorCache-1.039/t/environ/03-geo.sh 2022-08-26 15:13:31.000000000 +0200 +++ new/MirrorCache-1.042/t/environ/03-geo.sh 2022-09-29 11:41:23.000000000 +0200 @@ -36,6 +36,9 @@ $mc/curl --interface 127.0.0.4 -I /download/folder1/file1.1.dat | grep 1324 $mc/curl --interface 127.0.0.3 -I /download/folder1/file1.1.dat | grep 1314 $mc/curl --interface 127.0.0.2 -I /download/folder1/file1.1.dat | grep 1304 +$mc/curl -I /download/folder1/file1.1.dat?IP=127.0.0.4 | grep 1324 +$mc/curl -I /download/folder1/file1.1.dat?IP=127.0.0.3 | grep 1314 +$mc/curl -I /download/folder1/file1.1.dat?IP=127.0.0.2 | grep 1304 # check same continent $mc/curl --interface 127.0.0.4 -Is /download/folder1/file1.1.dat?COUNTRY=jp | grep 1324 @@ -66,3 +69,4 @@ # check continent $mc/curl -H "Accept: */*, application/metalink+xml" --interface 127.0.0.2 -s /download/folder1/file1.1.dat?COUNTRY=fr | grep -B20 127.0.0.3 +$mc/curl -H "Accept: */*, application/metalink4+xml" --interface 127.0.0.2 -s /download/folder1/file1.1.dat?COUNTRY=fr | grep -B20 127.0.0.3 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.039/t/environ/14-project-capability.sh new/MirrorCache-1.042/t/environ/14-project-capability.sh --- old/MirrorCache-1.039/t/environ/14-project-capability.sh 2022-08-26 15:13:31.000000000 +0200 +++ new/MirrorCache-1.042/t/environ/14-project-capability.sh 2022-09-29 11:41:23.000000000 +0200 @@ -51,6 +51,10 @@ $mc/sql_test 0 == "select state from server_project where server_id = 2 and project_id = 1" $mc/sql_test -1 == "select state from server_project where server_id = 3 and project_id = 1" +echo metalink doesnt have ap8, because project was disabled on server +rc=0 +$mc/curl /download/project1/folder1/file.1.1.dat.metalink | grep $($ap8/print_address) || rc=$? +test $rc -gt 0 $mc/sql "update server_project set state = 0 where server_id = 3 and project_id = 1" $mc/backstage/job -e mirror_probe_projects @@ -58,5 +62,16 @@ $mc/sql_test 0 == "select state from server_project where server_id = 2 and project_id = 1" $mc/sql_test 1 == "select state from server_project where server_id = 3 and project_id = 1" +$mc/backstage/job -e mirror_scan -a '["/project1/folder1"]' +$mc/backstage/shoot + +echo now metalink has ap8, because project has been enabled on server_id 3 +$mc/curl /download/project1/folder1/file1.1.dat.metalink | grep $($ap8/print_address) + +echo disable again, make sure it disappeared in mirrorlist +$mc/sql "update server_project set state = 0 where server_id = 3 and project_id = 1" +rc=0 +$mc/curl /download/project1/folder1/file1.1.dat.metalink | grep $($ap8/print_address) || rc=$? +test $rc -gt 0 echo success diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.039/t/lib/Dockerfile.environ.mariadb new/MirrorCache-1.042/t/lib/Dockerfile.environ.mariadb --- old/MirrorCache-1.039/t/lib/Dockerfile.environ.mariadb 2022-08-26 15:13:31.000000000 +0200 +++ new/MirrorCache-1.042/t/lib/Dockerfile.environ.mariadb 2022-09-29 11:41:23.000000000 +0200 @@ -14,7 +14,7 @@ apache2 perl-Digest-MD4 tidy nginx bbe perl-DBD-mysql perl-Mojo-mysql perl-Minion-Backend-mysql perl-DateTime-HiRes # optional dependencies used in testing -RUN zypper -vvv -n install perl-Geo-IP2Location perl-Inline-C gcc perl-DateTime-Format-MySQL +RUN zypper -vvv -n install perl-Geo-IP2Location perl-Inline-C gcc perl-DateTime-Format-MySQL libxml2-tools # this hack is needed because old nginx versions cannot run as non-root RUN bbe -e 's,/var/log/nginx/error.log,/tmp/log_nginx_error.log,' /usr/sbin/nginx > /usr/sbin/nginx.hacked diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.039/t/lib/Dockerfile.environ.postgresql new/MirrorCache-1.042/t/lib/Dockerfile.environ.postgresql --- old/MirrorCache-1.039/t/lib/Dockerfile.environ.postgresql 2022-08-26 15:13:31.000000000 +0200 +++ new/MirrorCache-1.042/t/lib/Dockerfile.environ.postgresql 2022-09-29 11:41:23.000000000 +0200 @@ -13,7 +13,7 @@ apache2 perl-Digest-MD4 tidy nginx bbe perl-DateTime-HiRes # optional dependencies used in testing -RUN zypper -vvv -n install perl-Geo-IP2Location perl-Inline-C gcc +RUN zypper -vvv -n install perl-Geo-IP2Location perl-Inline-C gcc libxml2-tools # this hack is needed because old nginx versions cannot run as non-root RUN bbe -e 's,/var/log/nginx/error.log,/tmp/log_nginx_error.log,' /usr/sbin/nginx > /usr/sbin/nginx.hacked diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.039/templates/mirrorlist.html.ep new/MirrorCache-1.042/templates/mirrorlist.html.ep --- old/MirrorCache-1.039/templates/mirrorlist.html.ep 2022-08-26 15:13:31.000000000 +0200 +++ new/MirrorCache-1.042/templates/mirrorlist.html.ep 2022-09-29 11:41:23.000000000 +0200 @@ -52,6 +52,7 @@ <li>MD5 Hash: <%= $file->{md5} %></li> % } <li><a href="<%= $bc_url . $bc_last %>.metalink">Metalink (v3.0)</a></li> + <li><a href="<%= $bc_url . $bc_last %>.meta4">Metalink (v4.0)</a></li> % if ($file->{url}) { <li>Origin: <a href="<%= $file->{url} %>"><%= $file->{url} %></a></li> % } ++++++ MirrorCache.obsinfo ++++++ --- /var/tmp/diff_new_pack.zstGEG/_old 2022-10-08 01:23:54.498048461 +0200 +++ /var/tmp/diff_new_pack.zstGEG/_new 2022-10-08 01:23:54.498048461 +0200 @@ -1,5 +1,5 @@ name: MirrorCache -version: 1.039 -mtime: 1661519611 -commit: 056db65ee5963af81f5b04c526dcc3f8df31e682 +version: 1.042 +mtime: 1664444483 +commit: b79e2f052f85c455161a636b56db7adc31d3ca67
