Script 'mail_helper' called by obssrc
Hello community,
here is the log from the commit of package MirrorCache for openSUSE:Factory
checked in at 2022-08-19 17:56:13
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/MirrorCache (Old)
and /work/SRC/openSUSE:Factory/.MirrorCache.new.2083 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "MirrorCache"
Fri Aug 19 17:56:13 2022 rev:17 rq:998020 version:1.037
Changes:
--------
--- /work/SRC/openSUSE:Factory/MirrorCache/MirrorCache.changes 2022-06-30
13:18:39.521546715 +0200
+++ /work/SRC/openSUSE:Factory/.MirrorCache.new.2083/MirrorCache.changes
2022-08-19 17:59:01.320420212 +0200
@@ -1,0 +2,15 @@
+Thu Aug 11 15:10:05 UTC 2022 - Andrii Nikitin <andrii.niki...@suse.com>
+
+- Update to version 1.037:
+ * Add configuration for redirect for projects (#293)
+
+-------------------------------------------------------------------
+Fri Aug 05 12:14:42 UTC 2022 - Andrii Nikitin <andrii.niki...@suse.com>
+
+- Update to version 1.036:
+ * Fix mirrors report when missing folders (#288)
+ * Tweak systemd files (#291)
+ * Rework mirrors report (#290)
+ * Rescan if no country hit (#292)
+
+-------------------------------------------------------------------
Old:
----
MirrorCache-1.035.obscpio
New:
----
MirrorCache-1.037.obscpio
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ MirrorCache.spec ++++++
--- /var/tmp/diff_new_pack.o67KlN/_old 2022-08-19 17:59:01.756421125 +0200
+++ /var/tmp/diff_new_pack.o67KlN/_new 2022-08-19 17:59:01.760421134 +0200
@@ -22,7 +22,7 @@
%define main_requires %{assetpack_requires} perl(Carp) perl(DBD::Pg) >= 3.7.4
perl(DBI) >= 1.632 perl(DBIx::Class) >= 0.082801
perl(DBIx::Class::DynamicDefault) perl(DateTime) perl(Encode) perl(Time::Piece)
perl(Time::Seconds) perl(Time::ParseDate) perl(DateTime::Format::Pg)
perl(Exporter) perl(File::Basename) perl(LWP::UserAgent) perl(Mojo::Base)
perl(Mojo::ByteStream) perl(Mojo::IOLoop) perl(Mojo::JSON) perl(Mojo::Pg)
perl(Mojo::URL) perl(Mojo::Util) perl(Mojolicious::Commands)
perl(Mojolicious::Plugin) perl(Mojolicious::Plugin::RenderFile)
perl(Mojolicious::Static) perl(Net::OpenID::Consumer) perl(POSIX)
perl(Sort::Versions) perl(URI::Escape) perl(XML::Writer) perl(base)
perl(constant) perl(diagnostics) perl(strict) perl(warnings) shadow
rubygem(sass) perl(Net::DNS) perl(LWP::Protocol::https) perl(Digest::SHA)
%define build_requires %{assetpack_requires} rubygem(sass) tidy sysuser-shadow
sysuser-tools
Name: MirrorCache
-Version: 1.035
+Version: 1.037
Release: 0
Summary: WebApp to redirect and manage mirrors
License: GPL-2.0-or-later
++++++ MirrorCache-1.035.obscpio -> MirrorCache-1.037.obscpio ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.035/dist/salt/profile/mirrorcache/files/usr/share/mirrorcache/sql/projects.sql
new/MirrorCache-1.037/dist/salt/profile/mirrorcache/files/usr/share/mirrorcache/sql/projects.sql
---
old/MirrorCache-1.035/dist/salt/profile/mirrorcache/files/usr/share/mirrorcache/sql/projects.sql
2022-06-23 06:47:51.000000000 +0200
+++
new/MirrorCache-1.037/dist/salt/profile/mirrorcache/files/usr/share/mirrorcache/sql/projects.sql
2022-08-11 15:29:02.000000000 +0200
@@ -9,18 +9,18 @@
insert into project(name,path) select '15.2
repo','/distribution/leap/15.2/repo';
insert into project(name,path) select '15.2
debug','/debug/distribution/leap/15.2/repo';
insert into project(name,path) select '15.2
source','/source/distribution/leap/15.2/repo';
-insert into project(name,path) select '15.2
update','/update/distribution/leap/15.2/repo';
+insert into project(name,path) select '15.2
update','/update/distribution/leap/15.2';
insert into project(name,path) select '15.3 ISO','/distribution/leap/15.3/iso';
insert into project(name,path) select '15.3
repo','/distribution/leap/15.3/repo';
insert into project(name,path) select '15.3
debug','/debug/distribution/leap/15.3/repo';
insert into project(name,path) select '15.3
source','/source/distribution/leap/15.3/repo';
-insert into project(name,path) select '15.3
update','/update/distribution/leap/15.3/repo';
-insert into project(name,path) select '15.3
port','/port/aarch64/distribution/leap/15.3';
+insert into project(name,path) select '15.3
update','/update/distribution/leap/15.3';
+insert into project(name,path) select '15.3
port','/ports/aarch64/distribution/leap/15.3';
insert into project(name,path) select '15.4 ISO','/distribution/leap/15.4/iso';
insert into project(name,path) select '15.4
repo','/distribution/leap/15.4/repo';
insert into project(name,path) select '15.4
debug','/debug/distribution/leap/15.4/repo';
insert into project(name,path) select '15.4
source','/source/distribution/leap/15.4/repo';
-insert into project(name,path) select '15.4
update','/update/distribution/leap/15.4/repo';
-insert into project(name,path) select '15.4
port','/port/aarch64/distribution/leap/15.4';
+insert into project(name,path) select '15.4
update','/update/distribution/leap/15.4';
+insert into project(name,path) select '15.4
port','/ports/aarch64/distribution/leap/15.4';
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.035/dist/systemd/mirrorcache-hypnotoad.service
new/MirrorCache-1.037/dist/systemd/mirrorcache-hypnotoad.service
--- old/MirrorCache-1.035/dist/systemd/mirrorcache-hypnotoad.service
2022-06-23 06:47:51.000000000 +0200
+++ new/MirrorCache-1.037/dist/systemd/mirrorcache-hypnotoad.service
2022-08-11 15:29:02.000000000 +0200
@@ -1,7 +1,7 @@
[Unit]
Description=MirrorCache webApp
Before=apache2.service
-After=postgresql.service nss-lookup.target
+After=postgresql.service mariadb.service nss-lookup.target
[Service]
User=mirrorcache
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.035/dist/systemd/mirrorcache.service
new/MirrorCache-1.037/dist/systemd/mirrorcache.service
--- old/MirrorCache-1.035/dist/systemd/mirrorcache.service 2022-06-23
06:47:51.000000000 +0200
+++ new/MirrorCache-1.037/dist/systemd/mirrorcache.service 2022-08-11
15:29:02.000000000 +0200
@@ -1,7 +1,7 @@
[Unit]
Description=MirrorCache webApp
Before=apache2.service
-After=postgresql.service nss-lookup.target
+After=postgresql.service mariadb.service nss-lookup.target
[Service]
User=mirrorcache
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.035/lib/MirrorCache/Datamodule.pm
new/MirrorCache-1.037/lib/MirrorCache/Datamodule.pm
--- old/MirrorCache-1.035/lib/MirrorCache/Datamodule.pm 2022-06-23
06:47:51.000000000 +0200
+++ new/MirrorCache-1.037/lib/MirrorCache/Datamodule.pm 2022-08-11
15:29:02.000000000 +0200
@@ -53,6 +53,7 @@
has 'at';
has '_mime';
+has 'mirror_country';
my $TYPES = Mojolicious::Types->new;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.035/lib/MirrorCache/Schema/Result/Project.pm
new/MirrorCache-1.037/lib/MirrorCache/Schema/Result/Project.pm
--- old/MirrorCache-1.035/lib/MirrorCache/Schema/Result/Project.pm
2022-06-23 06:47:51.000000000 +0200
+++ new/MirrorCache-1.037/lib/MirrorCache/Schema/Result/Project.pm
2022-08-11 15:29:02.000000000 +0200
@@ -23,6 +23,8 @@
{ data_type => "varchar", is_nullable => 0, size => 64 },
"path",
{ data_type => "varchar", is_nullable => 0, size => 512 },
+ "redirect",
+ { data_type => "varchar", is_nullable => 0, size => 512 },
db_sync_last => {
data_type => 'timestamp',
is_nullable => 1
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.035/lib/MirrorCache/Schema/ResultSet/Project.pm
new/MirrorCache-1.037/lib/MirrorCache/Schema/ResultSet/Project.pm
--- old/MirrorCache-1.035/lib/MirrorCache/Schema/ResultSet/Project.pm
2022-06-23 06:47:51.000000000 +0200
+++ new/MirrorCache-1.037/lib/MirrorCache/Schema/ResultSet/Project.pm
2022-08-11 15:29:02.000000000 +0200
@@ -13,20 +13,23 @@
my $dbh = $self->result_source->schema->storage->dbh;
my $sql = <<"END_SQL";
-select sum(case when diff = 0 then 1 else 0 end) as current, sum(case when
diff = 0 then 0 else 1 end) as outdated
+ select sum(case when diff = 0 then 1 else 0 end) as current, sum(case when
diff = 0 or diff is null then 0 else 1 end) as outdated
from (
-select server_id, max(case when d1 <> d2 then 1 else 0 end) diff
+select
+ server_id,
+ max(case when d1 <> d2 or d1 is null then 1 else 0 end) diff
from (
-select prj.etalon, f.path, f.id, fd.folder_id, fds.server_id,
fds.folder_diff_id as d1, fds2.folder_diff_id as d2
-from project prj
+select s.id as server_id, prj.name, fds.folder_diff_id d1, fds2.folder_diff_id
d2
+from server s
+join project prj on prj.name = ?
join folder f on f.path like concat(prj.path,'/%')
join folder_diff fd on fd.folder_id = f.id
-join folder_diff fd2 on fd2.folder_id = fd.folder_id
-join folder_diff_server fds on fd.id = fds.folder_diff_id
-join folder_diff_server fds2 on fd2.id = fds2.folder_diff_id and
fds2.server_id = prj.etalon
-where prj.name = ?
+left join folder_diff_server fds on fds.folder_diff_id = fd.id and
fds.server_id = s.id
+join folder_diff_server fds2 on fds2.folder_diff_id = fd.id and fds2.server_id
= prj.etalon
) x
-group by server_id) xx
+group by server_id
+having sum(case when d1 is not null then 1 else 0 end) > 0
+) xx
END_SQL
my $prep = $dbh->prepare($sql);
$prep->execute($name);
@@ -38,19 +41,22 @@
my $dbh = $self->result_source->schema->storage->dbh;
my $sql = <<"END_SQL";
-select server_id, min(case when d1 = d2 then 1 else 0 end) as current,
concat(s.hostname, s.urldir) as url
+select
+ s.id server_id,
+ min(case when d1 = d2 then 1 else 0 end) as current,
+ concat(s.hostname, s.urldir) as url
from (
-select fds.server_id, fds.folder_diff_id as d1, fds2.folder_diff_id as d2
-from project prj
+select s.id as server_id, prj.name, fds.folder_diff_id d1, fds2.folder_diff_id
d2
+from server s
+join project prj on prj.name = ?
join folder f on f.path like concat(prj.path,'/%')
join folder_diff fd on fd.folder_id = f.id
-join folder_diff fd2 on fd2.folder_id = fd.folder_id
-join folder_diff_server fds on fd.id = fds.folder_diff_id
-join folder_diff_server fds2 on fd2.id = fds2.folder_diff_id and
fds2.server_id = prj.etalon
-where prj.name = ?
+left join folder_diff_server fds on fds.folder_diff_id = fd.id and
fds.server_id = s.id
+join folder_diff_server fds2 on fds2.folder_diff_id = fd.id and fds2.server_id
= prj.etalon
) x
join server s on x.server_id = s.id
-group by x.server_id, s.id
+group by s.id, s.hostname, s.urldir
+having sum(case when d1 is not null then 1 else 0 end) > 0
order by current desc, url
END_SQL
my $prep = $dbh->prepare($sql);
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.035/lib/MirrorCache/Schema/ResultSet/Server.pm
new/MirrorCache-1.037/lib/MirrorCache/Schema/ResultSet/Server.pm
--- old/MirrorCache-1.035/lib/MirrorCache/Schema/ResultSet/Server.pm
2022-06-23 06:47:51.000000000 +0200
+++ new/MirrorCache-1.037/lib/MirrorCache/Schema/ResultSet/Server.pm
2022-08-11 15:29:02.000000000 +0200
@@ -254,33 +254,57 @@
my $dbh = $schema->storage->dbh;
my $where1 = '';
- my $where2 = '';
$where1 = "WHERE REPLACE(prj.name,' ','') = ?" if $project;
- $where2 = "WHERE s.region = ?" if $region;
my $sql = <<"END_SQL";
-select s.region, s.country, x.name project, s.id, min(case when s1 = 0 then 1
else 0 end) as curr, round((sum(s2)*1.0)/(sum(s1)+sum(s2))*100) as score,
concat(s.hostname, s.urldir) as url, max(example) victim
+with
+etalon as (
+ select prj.id project_id, prj.name, f.id as folder_id, fd2.id as diff_id,
f.path
+ from project prj
+ join folder f on f.path like concat(prj.path,'%')
+ join folder_diff fd2 on fd2.folder_id = f.id
+ join folder_diff_server fds2 on fd2.id = fds2.folder_diff_id and
fds2.server_id = prj.etalon
+ -- where prj.name = 'repositories'
+ $where1
+),
+project_folder_count as (
+ select project_id, count(*) cnt
+ from etalon
+ group by project_id
+)
+select s.id, s.region, s.country,
+ concat(s.hostname, s.urldir) as url,
+ project,
+ round(case when project_folder_count.cnt > 3 then s_eq * 100 /
project_folder_count.cnt when s_eq = project_folder_count.cnt then 100 else 50
end, 0) score,
+ s_eq, s_ne, victim, project_folder_count.cnt
from (
-select prj.name, fds.server_id,
- sum(case when fds.folder_diff_id != fds2.folder_diff_id then 1 else 0 end)
s1,
- sum(case when fds.folder_diff_id = fds2.folder_diff_id then 1 else 0 end)
s2,
- max(case when fds.folder_diff_id != fds2.folder_diff_id then f.path else
'' end) example
-from project prj
-join folder f on f.path like concat(prj.path,'%')
-join folder_diff fd on fd.folder_id = f.id
-join folder_diff fd2 on fd2.folder_id = fd.folder_id
-join folder_diff_server fds on fd.id = fds.folder_diff_id
-join folder_diff_server fds2 on fd2.id = fds2.folder_diff_id and
fds2.server_id = prj.etalon
-$where1
-group by fds2.server_id, prj.name, fds.server_id ) x
-join server s on x.server_id = s.id
-$where2
-group by s.id, x.name
-order by region, country, score, url;
+select
+ cmp.project_id,
+ cmp.name project,
+ cmp.server_id,
+ sum(s_eq) s_eq, sum(s_ne) s_ne,
+ max(example) victim
+from (
+ select
+ etalon.project_id,
+ etalon.name,
+ case when etalon.diff_id = fd.id then 1 else 0 end as s_eq,
+ case when etalon.diff_id != fd.id or fd.id is null then 1 else 0 end
as s_ne,
+ case when etalon.diff_id != fd.id or fd.id is null then path else ''
end example,
+ fds.server_id
+ from etalon
+ left join folder_diff fd on fd.folder_id = etalon.folder_id
+ left join folder_diff_server fds on fds.folder_diff_id = fd.id
+) cmp
+group by server_id, project_id, name
+) smry
+join project_folder_count on project_folder_count.project_id = smry.project_id
+join server s on smry.server_id = s.id
+order by region, country, score, url, project;
END_SQL
my $prep = $dbh->prepare($sql);
if ($project && $region) {
- $prep->execute($project, $region);
+ $prep->execute($project, $project, $region);
} elsif ($project) {
$prep->execute($project);
} elsif ($region) {
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.035/lib/MirrorCache/WebAPI/Controller/Report/Mirrors.pm
new/MirrorCache-1.037/lib/MirrorCache/WebAPI/Controller/Report/Mirrors.pm
--- old/MirrorCache-1.035/lib/MirrorCache/WebAPI/Controller/Report/Mirrors.pm
1970-01-01 01:00:00.000000000 +0100
+++ new/MirrorCache-1.037/lib/MirrorCache/WebAPI/Controller/Report/Mirrors.pm
2022-08-11 15:29:02.000000000 +0200
@@ -0,0 +1,57 @@
+# Copyright (C) 2022 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package MirrorCache::WebAPI::Controller::Report::Mirrors;
+use Mojo::Base 'Mojolicious::Controller';
+use Mojo::JSON qw(decode_json encode_json);
+
+sub index {
+ my ($self) = @_;
+ my $project = $self->param('project');
+ my $projects = $self->mcproject->list_full;
+
+ if ($project && $project ne "all") {
+ my @projects_new;
+ for my $p (@$projects) {
+ push @projects_new, $p if (0 == rindex $p->{name}, $project,
0) ||
+ (0 == rindex $p->{alias}, $project,
0) ||
+ (0 == rindex $p->{alias}, "c$project",
0) ||
+ ( lc($project) eq 'tumbleweed' && 0 ==
rindex $p->{alias}, 'tw', 0 );
+ }
+ $projects = \@projects_new if scalar(@projects_new);
+ }
+
+ my $sql = 'select dt, body from report_body where report_id = 1 order by
dt desc limit 1';
+
+ eval {
+ my @res = $self->schema->storage->dbh->selectrow_array($sql);
+ my $body = $res[1];
+ my $hash = decode_json($body);
+
+ $self->stash;
+ $self->render(
+ "report/mirrors/index",
+ mirrors => $hash,
+ projects => $projects
+ );
+ };
+ my $error = $@;
+ if ($error) {
+ print STDERR "RESPMIRRORSREPORT : " . $error . "\n";
+ return $self->render(json => {error => $error}, status => 404);
+ }
+}
+
+1;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.035/lib/MirrorCache/WebAPI/Controller/Rest/Project.pm
new/MirrorCache-1.037/lib/MirrorCache/WebAPI/Controller/Rest/Project.pm
--- old/MirrorCache-1.035/lib/MirrorCache/WebAPI/Controller/Rest/Project.pm
2022-06-23 06:47:51.000000000 +0200
+++ new/MirrorCache-1.037/lib/MirrorCache/WebAPI/Controller/Rest/Project.pm
2022-08-11 15:29:02.000000000 +0200
@@ -1,4 +1,4 @@
-# Copyright (C) 2021 SUSE LLC
+# Copyright (C) 2022 SUSE LLC
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
@@ -44,4 +44,12 @@
$self->render(json => { current => $rec->{current}, outdated =>
$rec->{outdated} });
}
+sub list {
+ my ($self) = @_;
+
+ my $list = $self->mcproject->list_full;
+
+ $self->render(json => $list);
+}
+
1;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.035/lib/MirrorCache/WebAPI/Plugin/Dir.pm
new/MirrorCache-1.037/lib/MirrorCache/WebAPI/Plugin/Dir.pm
--- old/MirrorCache-1.035/lib/MirrorCache/WebAPI/Plugin/Dir.pm 2022-06-23
06:47:51.000000000 +0200
+++ new/MirrorCache-1.037/lib/MirrorCache/WebAPI/Plugin/Dir.pm 2022-08-11
15:29:02.000000000 +0200
@@ -68,7 +68,7 @@
return $c
if _render_hashes($dm)
|| _render_small($dm)
- || _redirect_geo($dm)
+ || _redirect_project_ln_geo($dm)
|| _redirect_normalized($dm)
|| _render_stats($dm)
|| _local_render($dm, 0) # check if we should render local
@@ -151,17 +151,25 @@
return $c->render(status => 425, text => "Waiting in queue, at " .
strftime("%Y-%m-%d %H:%M:%S", gmtime time) . " position: $pos");
}
-sub _redirect_geo {
+# this combines similar checks for redirecting as specified in DB links and
projects, as well as subsidiaries
+sub _redirect_project_ln_geo {
my $dm = shift;
- my $route = $dm->route;
my ($path, $trailing_slash) = $dm->path;
my $c = $dm->c;
- my $ln;
- $ln = $root->detect_ln($path);
+ # each project may have a redirect defined in DB, so all requests are
redirected for it
+ my $redirect = $c->mcproject->redirect($path, $dm->region);
+ if ($redirect) {
+ $dm->redirect($dm->scheme . '://' . $redirect . $path);
+ $c->stat->redirect_to_region($dm);
+ return 1;
+ }
+
+ my $ln = $root->detect_ln($path);
if ($ln) {
# redirect to the symlink
$dm->redirect($dm->route . $ln);
+ $c->stat->redirect_to_region($dm);
return 1;
}
return undef if $trailing_slash || $path eq '/' || $dm->mirrorlist;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.035/lib/MirrorCache/WebAPI/Plugin/Helpers.pm
new/MirrorCache-1.037/lib/MirrorCache/WebAPI/Plugin/Helpers.pm
--- old/MirrorCache-1.035/lib/MirrorCache/WebAPI/Plugin/Helpers.pm
2022-06-23 06:47:51.000000000 +0200
+++ new/MirrorCache-1.037/lib/MirrorCache/WebAPI/Plugin/Helpers.pm
2022-08-11 15:29:02.000000000 +0200
@@ -143,6 +143,21 @@
}
$app->helper( trusted_addr => sub { %trust_addr{
shift->tx->original_remote_address } });
}
+
+ $app->helper(
+ 'region_name' => sub {
+ shift;
+ my $reg = shift;
+ return 'Unknown' unless $reg;
+ return 'Oceania' if $reg eq 'oc';
+ return 'North America' if $reg eq 'na';
+ return 'South America' if $reg eq 'sa';
+ return 'Africa' if $reg eq 'af';
+ return 'Europe' if $reg eq 'eu';
+ return 'Asia' if $reg eq 'as';
+ return 'Unknown';
+ });
+
}
sub _current_user {
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.035/lib/MirrorCache/WebAPI/Plugin/Project.pm
new/MirrorCache-1.037/lib/MirrorCache/WebAPI/Plugin/Project.pm
--- old/MirrorCache-1.035/lib/MirrorCache/WebAPI/Plugin/Project.pm
2022-06-23 06:47:51.000000000 +0200
+++ new/MirrorCache-1.037/lib/MirrorCache/WebAPI/Plugin/Project.pm
2022-08-11 15:29:02.000000000 +0200
@@ -22,11 +22,15 @@
my @projects;
my %projects_path;
my %projects_alias;
+my %projects_redirect;
+my %projects_region_redirect;
sub register {
my ($self, $app) = @_;
$app->helper('mcproject.list' => \&_list);
+ $app->helper('mcproject.list_full' => \&_list_full);
+ $app->helper('mcproject.redirect' => \&_redirect);
return $self;
}
@@ -39,14 +43,27 @@
1;
} or $c->log->error(Dumper("Cannot load projects", @_));
-
-
for my $p (@projects) {
my $name = $p->name;
my $alias = $name;
- $alias =~ tr/ //ds; # remove spaces
+ $alias =~ tr/ //ds; # remove spaces
+ $alias =~ tr/\.//ds; # remove dots
+ $alias = "c$alias" if $alias =~ /^\d/;
+ $alias = lc($alias);
$projects_path{$name} = $p->path;
$projects_alias{$name} = $alias;
+ my $redirect = $p->redirect;
+ next unless $redirect;
+ my @parts = split ';', $redirect;
+ for my $r (@parts) {
+ my $prefix = substr($r,0,3);
+ my $c = chop($prefix);
+ if (':' eq $c) {
+ $projects_region_redirect{$prefix}{$name} = substr($r,3);
+ } else {
+ $projects_redirect{$name} = $r;
+ }
+ }
}
}
@@ -61,4 +78,37 @@
return \@res;
}
+sub _list_full {
+ my ($c) = @_;
+ _init_if_needed($c);
+
+ my @res;
+ for my $p (@projects) {
+ my $name = $p->name;
+ my $alias = $projects_alias{$name};
+ my $path = $projects_path{$name};
+
+ my %prj = ( name => $name, alias => $alias, path => $path );
+ push @res, \%prj;
+ }
+ return \@res;
+}
+
+sub _redirect {
+ my ($c, $path, $region) = @_;
+ _init_if_needed($c);
+
+ for my $p (@projects) {
+ my $name = $p->name;
+ my $redirect;
+ $redirect = $projects_region_redirect{$region}{$name} if $region;
+ $redirect = $projects_redirect{$name} unless $redirect;
+ next unless $redirect;
+ my $ppath = $projects_path{$name};
+ return $redirect if (0 == rindex($path, $ppath, 0));
+ }
+ return '';
+}
+
+
1;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.035/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
new/MirrorCache-1.037/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
--- old/MirrorCache-1.035/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
2022-06-23 06:47:51.000000000 +0200
+++ new/MirrorCache-1.037/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
2022-08-11 15:29:02.000000000 +0200
@@ -131,6 +131,7 @@
$mirror = $mirrors_region[0] if !$mirror && @mirrors_region;
$mirror = $mirrors_rest[0] if !$mirror && @mirrors_rest;
+ $dm->mirror_country($mirror->{country}) if $mirror;
if ($dm->extra) {
if ($mirror) {
$c->stat->redirect_to_mirror($mirror->{mirror_id}, $dm);
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.035/lib/MirrorCache/WebAPI/Plugin/Stat.pm
new/MirrorCache-1.037/lib/MirrorCache/WebAPI/Plugin/Stat.pm
--- old/MirrorCache-1.035/lib/MirrorCache/WebAPI/Plugin/Stat.pm 2022-06-23
06:47:51.000000000 +0200
+++ new/MirrorCache-1.037/lib/MirrorCache/WebAPI/Plugin/Stat.pm 2022-08-11
15:29:02.000000000 +0200
@@ -68,7 +68,7 @@
$path = $dm->root_subtree . $path;
my $rows = $self->rows;
my @rows = defined $rows? @$rows : ();
- push @rows, [ $dm->ip_sha1, scalar $dm->agent, scalar ($path .
$trailing_slash), $dm->country, datetime_now(), $mirror_id, $dm->folder_id,
$dm->file_id, $dm->is_secure, $dm->is_ipv4, $dm->metalink? 1 : 0,
$dm->mirrorlist? 1 : 0, $dm->is_head, $$, int($dm->elapsed*1000),
$dm->file_age, $dm->folder_scan_last ];
+ push @rows, [ $dm->ip_sha1, scalar $dm->agent, scalar ($path .
$trailing_slash), $dm->country, datetime_now(), $mirror_id, $dm->folder_id,
$dm->file_id, $dm->is_secure, $dm->is_ipv4, $dm->metalink? 1 : 0,
$dm->mirrorlist? 1 : 0, $dm->is_head, $$, int($dm->elapsed*1000),
$dm->file_age, $dm->folder_scan_last, $dm->mirror_country ];
my $cnt = @rows;
if ($cnt >= $FLUSH_COUNT) {
$self->rows(undef);
@@ -108,13 +108,15 @@
$dbh = $self->schema->storage->dbh;
my $prep = $dbh->prepare($sql);
for my $row (@rows) {
- my $folder_id = $row->[6];
+ my $folder_id = $row->[6];
my $mirror_id = $row->[5];
+ my $country = $row->[3];
+ my $country_m = pop @$row;
my $scan_last = pop @$row;
my $file_age = pop @$row;
$prep->execute(@$row);
if ($folder_id) {
- next if $mirror_id > 0;
+ next if $mirror_id > 0 && (!$country || !$country_m ||
($country eq $country_m));
next if $mirror_id < -1;
my $agent = $row->[1];
next unless -1 == index($agent, 'bot');
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.035/lib/MirrorCache/WebAPI/Plugin/Subsidiary.pm
new/MirrorCache-1.037/lib/MirrorCache/WebAPI/Plugin/Subsidiary.pm
--- old/MirrorCache-1.035/lib/MirrorCache/WebAPI/Plugin/Subsidiary.pm
2022-06-23 06:47:51.000000000 +0200
+++ new/MirrorCache-1.037/lib/MirrorCache/WebAPI/Plugin/Subsidiary.pm
2022-08-11 15:29:02.000000000 +0200
@@ -152,7 +152,7 @@
my ($c, $region, $country) = @_;
$region = $country if ($country && $subsidiary_country{$country});
my $url;
- $url = $subsidiary_urls{$region} if $region;
+ $url = $subsidiary_urls{$region} if $region && !$subsidiary_local{$region};
my @res = ($url? $region : '');
for my $s (@regions) {
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.035/lib/MirrorCache/WebAPI.pm
new/MirrorCache-1.037/lib/MirrorCache/WebAPI.pm
--- old/MirrorCache-1.035/lib/MirrorCache/WebAPI.pm 2022-06-23
06:47:51.000000000 +0200
+++ new/MirrorCache-1.037/lib/MirrorCache/WebAPI.pm 2022-08-11
15:29:02.000000000 +0200
@@ -161,6 +161,7 @@
my $rest_r = $rest->any('/')->to(namespace =>
'MirrorCache::WebAPI::Controller::Rest');
$rest_r->get('/server')->name('rest_server')->to('table#list', table =>
'Server');
$rest_r->get('/server/:id')->to('table#list', table => 'Server');
+ $rest_r->get('/project')->to('project#list');
$rest_r->get('/project/:name')->to('project#show');
$rest_r->get('/project/:name/mirror_summary')->to('project#mirror_summary');
$rest_r->get('/project/:name/mirror_list')->to('project#mirror_list');
@@ -195,6 +196,8 @@
my $report_r = $r->any('/report')->to(namespace =>
'MirrorCache::WebAPI::Controller::Report');
$report_r->get('/mirror')->name('report_mirror')->to('mirror#index');
+ $report_r->get('/mirrors')->name('report_mirrors')->to('mirrors#index');
+
$report_r->get('/mirrors/:project')->name('report_mirrors_project')->to('mirrors#index');
my $app_r = $r->any('/app')->to(namespace =>
'MirrorCache::WebAPI::Controller::App');
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.035/lib/MirrorCache/resources/migrations/Pg.sql
new/MirrorCache-1.037/lib/MirrorCache/resources/migrations/Pg.sql
--- old/MirrorCache-1.035/lib/MirrorCache/resources/migrations/Pg.sql
2022-06-23 06:47:51.000000000 +0200
+++ new/MirrorCache-1.037/lib/MirrorCache/resources/migrations/Pg.sql
2022-08-11 15:29:02.000000000 +0200
@@ -294,3 +294,6 @@
body text
);
create index if not exists report_content_dt_inx on report_body(report_id, dt);
+-- 25 up
+alter table project add column if not exists redirect varchar(512);
+
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.035/lib/MirrorCache/resources/migrations/mysql.sql
new/MirrorCache-1.037/lib/MirrorCache/resources/migrations/mysql.sql
--- old/MirrorCache-1.035/lib/MirrorCache/resources/migrations/mysql.sql
2022-06-23 06:47:51.000000000 +0200
+++ new/MirrorCache-1.037/lib/MirrorCache/resources/migrations/mysql.sql
2022-08-11 15:29:02.000000000 +0200
@@ -291,3 +291,6 @@
constraint `fk_report_body_report` FOREIGN KEY(report_id) references
report(id) on delete cascade
);
create index if not exists report_content_dt_inx on report_body(report_id, dt);
+-- 25 up
+alter table project add column if not exists redirect varchar(512);
+
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.035/t/environ/14-project-hq.sh
new/MirrorCache-1.037/t/environ/14-project-hq.sh
--- old/MirrorCache-1.035/t/environ/14-project-hq.sh 2022-06-23
06:47:51.000000000 +0200
+++ new/MirrorCache-1.037/t/environ/14-project-hq.sh 2022-08-11
15:29:02.000000000 +0200
@@ -77,8 +77,33 @@
mc$i/backstage/shoot
done
+echo project1 to eu if we contacted hq from na
+$mc9/sql "update project set redirect = 'na:$eu_address/download' where id = 1"
+echo project2 to na even if we contacted hq from as or eu
+$mc9/sql "update project set redirect =
'as:$na_address/download;eu:$na_address/download' where id = 2"
+echo test project1 redirects
+$mc9/curl --interface $na_interface -I /download/project1/folder1/file1.1.dat
| grep "$eu_address"
+$mc9/curl --interface $eu_interface -I /download/project1/folder1/file1.1.dat
| grep "$eu_address"
+
+echo test project2 redirects
+$mc9/curl --interface $eu_interface -I /download/project2/folder1/file1.1.dat
| grep "$na_address"
+$mc9/curl --interface $as_interface -I /download/project2/folder1/file1.1.dat
| grep "$na_address"
+$mc9/curl --interface $as_interface -IL /download/project2/folder1/file1.1.dat
| grep -E "$($ap3/print_address)|$($ap4/print_address)"
+$mc9/curl --interface $na_interface -I /download/project2/folder1/file1.1.dat
| grep "$na_address"
+
+echo "Let's pretend proj 2 has no good mirrors in as and we redirect all
requests from it to na subsidiary"
+$mc8/sql "update project set redirect = '$na_address/download' where id = 2"
+
+echo project1 redirects to regular mirror
+$mc8/curl -I /download/project1/folder1/file1.1.dat | grep -E
"$($ap7/print_address)|$($ap8/print_address)"
+
+echo project2 redirects to na
+$mc8/curl -I /download/project2/folder1/file1.1.dat | grep
$na_address/download/project2/folder1/file1.1.dat
+$mc8/curl -IL /download/project2/folder1/file1.1.dat | grep -E
"$($ap3/print_address)|$($ap4/print_address)"
+
+
# all countries present in report
-mc9/curl -s /rest/repmirror \
+$mc9/curl -s /rest/repmirror \
| grep '"country":"br"' \
| grep '"country":"de"' \
| grep '"country":"dk"' \
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.035/t/environ/14-project-report.sh
new/MirrorCache-1.037/t/environ/14-project-report.sh
--- old/MirrorCache-1.035/t/environ/14-project-report.sh 1970-01-01
01:00:00.000000000 +0100
+++ new/MirrorCache-1.037/t/environ/14-project-report.sh 2022-08-11
15:29:02.000000000 +0200
@@ -0,0 +1,66 @@
+#!lib/test-in-container-environ.sh
+set -exo pipefail
+
+mc=$(environ mc $(pwd))
+
+$mc/start
+
+ap8=$(environ ap8)
+ap7=$(environ ap7)
+ap6=$(environ ap6)
+ap5=$(environ ap5)
+ap4=$(environ ap4)
+
+for x in $mc $ap7 $ap8 $ap6 $ap5 $ap4; do
+ mkdir -p $x/dt/{folder1,folder2,folder3}
+ mkdir -p $x/dt/project1/{folder1,folder2,folder3}
+ mkdir -p $x/dt/project2/{folder1,folder2,folder3}
+ echo $x/dt/{folder1,folder2,folder3}/{file1.1,file2.1}.dat | xargs -n 1
touch
+ echo $x/dt/project1/{folder1,folder2,folder3}/{file1.1,file2.1}.dat |
xargs -n 1 touch
+ echo $x/dt/project2/{folder1,folder2,folder3}/{file1.1,file2.1}.dat |
xargs -n 1 touch
+done
+
+$ap4/start
+$ap5/start
+$ap6/start
+$ap7/start
+$ap8/start
+
+# remove some files and folders
+rm $ap7/dt/project1/folder2/file2.1.dat
+rm $ap7/dt/project2/folder2/file2.1.dat
+rm -r $ap5/dt/project2/folder2/
+rm -r $ap5/dt/project1/
+rm -r $ap4/dt/project2/
+
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select
'$($ap6/print_address)','','t','us','na'"
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select
'$($ap7/print_address)','','t','us','na'"
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select
'$($ap8/print_address)','','t','de','eu'"
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select
'$($ap5/print_address)','','t','cn','as'"
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select
'$($ap4/print_address)','','t','jp','as'"
+
+$mc/sql "insert into project(name,path,etalon) select 'proj1','/project1', 3"
+$mc/sql "insert into project(name,path,etalon) select '2.0
1','/project2/folder1', 3"
+$mc/sql "insert into project(name,path,etalon) select '2.0
2','/project2/folder2', 3"
+
+$mc/backstage/job -e folder_sync -a '["/project1/folder1"]'
+$mc/backstage/job -e mirror_scan -a '["/project1/folder1"]'
+$mc/backstage/job -e folder_sync -a '["/project1/folder2"]'
+$mc/backstage/job -e mirror_scan -a '["/project1/folder2"]'
+$mc/backstage/job -e folder_sync -a '["/project2/folder1"]'
+$mc/backstage/job -e mirror_scan -a '["/project2/folder1"]'
+$mc/backstage/job -e folder_sync -a '["/project2/folder2"]'
+$mc/backstage/job -e mirror_scan -a '["/project2/folder2"]'
+$mc/backstage/shoot
+
+$mc/backstage/job -e report -a '["once"]'
+$mc/backstage/shoot
+
+$mc/curl /report/mirrors | tidy --drop-empty-elements no | \
+ grep -A5 -F '<div class="repo">' | \
+ grep -A4 -F '<a class="repouncertain"' | \
+ grep -A3 -F '"diff in: /project2/folder2"' | \
+ grep -A2 -F '"http://127.0.0.1:1304/project2/folder2";>' | \
+ grep -C3 '\b2\b' | \
+ grep -C3 -F '</a>'
+
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.035/t/environ/14-project.sh
new/MirrorCache-1.037/t/environ/14-project.sh
--- old/MirrorCache-1.035/t/environ/14-project.sh 2022-06-23
06:47:51.000000000 +0200
+++ new/MirrorCache-1.037/t/environ/14-project.sh 2022-08-11
15:29:02.000000000 +0200
@@ -8,8 +8,10 @@
ap8=$(environ ap8)
ap7=$(environ ap7)
ap6=$(environ ap6)
+ap5=$(environ ap5)
+ap4=$(environ ap4)
-for x in $mc $ap7 $ap8 $ap6; do
+for x in $mc $ap7 $ap8 $ap6 $ap5 $ap4; do
mkdir -p $x/dt/{folder1,folder2,folder3}
mkdir -p $x/dt/project1/{folder1,folder2,folder3}
mkdir -p $x/dt/project2/{folder1,folder2,folder3}
@@ -18,29 +20,34 @@
echo $x/dt/project2/{folder1,folder2,folder3}/{file1.1,file2.1}.dat |
xargs -n 1 touch
done
+$ap4/start
+$ap5/start
$ap6/start
$ap7/start
$ap8/start
# remove a file from ap7
rm $ap7/dt/project1/folder2/file2.1.dat
+rm -r $ap5/dt/project1/folder2/
+rm -r $ap4/dt/project1/
$mc/sql "insert into server(hostname,urldir,enabled,country,region) select
'$($ap6/print_address)','','t','us','na'"
$mc/sql "insert into server(hostname,urldir,enabled,country,region) select
'$($ap7/print_address)','','t','us','na'"
$mc/sql "insert into server(hostname,urldir,enabled,country,region) select
'$($ap8/print_address)','','t','de','eu'"
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select
'$($ap5/print_address)','','t','cn','as'"
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select
'$($ap4/print_address)','','t','jp','as'"
$mc/sql "insert into project(name,path,etalon) select 'proj1','/project1', 3"
$mc/sql "insert into project(name,path,etalon) select 'proj 2','/project2', 3"
$mc/backstage/job -e folder_sync -a '["/project1/folder1"]'
-$mc/backstage/job -e mirror_scan -a '["/project1/folder1","us"]'
+$mc/backstage/job -e mirror_scan -a '["/project1/folder1"]'
$mc/backstage/shoot
$mc/curl -s /rest/project/proj1
$mc/curl -s /rest/project/proj1/mirror_summary
-$mc/curl -s /rest/project/proj1/mirror_summary | grep -E '"current":"?3' |
grep -E '"outdated":"?0'
-
+$mc/curl -s /rest/project/proj1/mirror_summary | grep -E '"current":"?4' |
grep -E '"outdated":"?0'
$mc/backstage/job -e folder_sync -a '["/project1/folder2"]'
$mc/backstage/job -e mirror_scan -a '["/project1/folder2"]'
@@ -53,10 +60,16 @@
$mc/backstage/job -e report -a '["once"]'
$mc/backstage/shoot
-$mc/curl -s /rest/repmirror | grep
'[{"country":"de","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"eu","url":"127.0.0.1:1314"},{"country":"us","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1294"},{"country":"us","proj1score":"50","proj1victim":"\/project1\/folder2","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1304"}]'
+rc=0
+$mc/curl -s /rest/repmirror | grep -F
'"country":"jp","proj1score":"0","proj1victim":"","proj2score":"100","proj2victim":"","region":"as","url":"'$($ap4/print_address)'"'
|| rc=$?
+echo proj1 is not on ap4, so it shouldnt appear in repmirror at all
+test $rc -gt 0
+
+$mc/curl -s /rest/repmirror | grep -F
'{"country":"cn","proj1score":"50","proj1victim":"","proj2score":"100","proj2victim":"","region":"as","url":"127.0.0.1:1284"},{"country":"jp","proj2score":"100","proj2victim":"","region":"as","url":"127.0.0.1:1274"},{"country":"de","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"eu","url":"127.0.0.1:1314"},{"country":"us","proj1score":"100","proj1victim":"","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1294"},{"country":"us","proj1score":"50","proj1victim":"\/project1\/folder2","proj2score":"100","proj2victim":"","region":"na","url":"127.0.0.1:1304"}'
$mc/curl -s /rest/project/proj1/mirror_summary
-$mc/curl -s /rest/project/proj1/mirror_summary | grep -E '"current":"?2' |
grep -E '"outdated":"?1'
+$mc/curl -s /rest/project/proj1/mirror_summary | grep -E '"current":"?2' |
grep -E '"outdated":"?2'
+
+$mc/curl -s /rest/project/proj1/mirror_list | grep -E
'{"current":"?1"?,"server_id":"?1"?,"url":"127.0.0.1:1294"},{"current":"?1"?,"server_id":"?3"?,"url":"127.0.0.1:1314"},{"current":"?0"?,"server_id":"?4"?,"url":"127.0.0.1:1284"},{"current":"?0"?,"server_id":"?2"?,"url":"127.0.0.1:1304"}'
-# $mc/curl -s /rest/project/proj1/mirror_list | grep -E
'{"current":"?1"?,"server_id":"?1"?,"url":"127.0.0.1:1294"},{"current":"?1"?,"server_id":"?3"?,"url":"127.0.0.1:1314"},{"current":"?0"?,"server_id":"?2"?,"url":"127.0.0.1:1304"}'
-$mc/curl -s /rest/project/proj1/mirror_list | grep
'{"current":1,"server_id":1,"url":"127.0.0.1:1294"},{"current":1,"server_id":3,"url":"127.0.0.1:1314"},{"current":0,"server_id":2,"url":"127.0.0.1:1304"}'
+echo success
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.035/t/environ/16-rescan-no-country.sh
new/MirrorCache-1.037/t/environ/16-rescan-no-country.sh
--- old/MirrorCache-1.035/t/environ/16-rescan-no-country.sh 1970-01-01
01:00:00.000000000 +0100
+++ new/MirrorCache-1.037/t/environ/16-rescan-no-country.sh 2022-08-11
15:29:02.000000000 +0200
@@ -0,0 +1,118 @@
+#!lib/test-in-container-environ.sh
+set -ex
+
+# This test is identical to 16-rescan, just it has hit in another country
instead of miss
+
+mc=$(environ mc $(pwd))
+MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=0
+
+MIRRORCACHE_RESCAN_INTERVAL=$((7 * 24 * 60 * 60)) # set one week to avoid
automatic rescan
+
+$mc/gen_env
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=$MIRRORCACHE_SCHEDULE_RETRY_INTERVAL \
+ MIRRORCACHE_RESCAN_INTERVAL=$MIRRORCACHE_RESCAN_INTERVAL \
+ MIRRORCACHE_RECKLESS=0
+
+$mc/start
+$mc/status
+
+ap8=$(environ ap8)
+$ap8/start
+$ap8/status
+
+for x in $mc $ap8; do
+ mkdir -p $x/dt/folder{1,2,3,4,5,6}
+ echo $x/dt/folder{2,3,4,5,6}/file1.1.dat | xargs -n 1 touch
+done
+
+for x in {1,2,3,4,5,6} ; do
+ $mc/backstage/job -e folder_sync -a '["/folder'$x'"]'
+done
+# folder1 must have some file, so scan is triggered
+touch $mc/dt/folder1/x
+
+$mc/sql "insert into server(hostname,urldir,enabled,country,region) select
'$($ap8/print_address)','','t','de','eu'"
+
+$mc/backstage/shoot
+$mc/sql_test 6 == "select count(*) from minion_jobs where task='folder_sync'"
+$mc/backstage/job mirror_scan_schedule
+$mc/backstage/shoot
+$mc/sql_test 6 == "select count(*) from minion_jobs where task='mirror_scan'"
+
+# handling of misses:
+# if file is unknown - schedule resync unless it was performed up to 15 min ago
+# if file is earlier than 1h - schedule rescan unless it was performed up to
30 min ago
+# if file is earlier than 4h - schedule rescan unless it was performed up to
1 hour ago
+# if file is earlier than 24h - schedule rescan unless it was performed up to
4 hours ago
+# if file is earlier than 72h - schedule rescan unless it was performed up to
8 hours ago
+# if file is older than 72 hours - schedule rescan unless it was performed up
to 1 24 hours ago
+
+S=10
+S1=$((60-$S))
+$mc/sql "update file set dt = now() - interval '1 hour' where folder_id in
(select id from folder where path = '/folder2')"
+$mc/sql "update file set dt = now() - interval '4 hour' where folder_id in
(select id from folder where path = '/folder3')"
+$mc/sql "update file set dt = now() - interval '24 hour' where folder_id in
(select id from folder where path = '/folder4')"
+$mc/sql "update file set dt = now() - interval '72 hour' where folder_id in
(select id from folder where path = '/folder5')"
+
+# should cover both Pg and MariaDB
+$mc/sql "update file set dt = dt + 2* interval '$S second'" || \
+ $mc/sql "update file set dt = date_add(dt, interval 2*$S second)"
+
+$mc/sql "update file set dt = dt - interval '72 hour' where folder_id in
(select id from folder where path = '/folder6')"
+
+touch $mc/dt/folder1/file1.1.dat
+cp $mc/dt/folder1/file1.1.dat $ap8/dt/folder1/file1.1.dat
+
+$mc/sql "update folder set scan_last = now() - interval '14 minute $S1 second'
where path = '/folder1'"
+$mc/sql "update folder set scan_last = now() - interval '29 minute $S1 second'
where path = '/folder2'"
+$mc/sql "update folder set scan_last = now() - interval '59 minute $S1 second'
where path = '/folder3'"
+$mc/sql "update folder set scan_last = now() - interval '3 hour 59 minute $S1
second' where path = '/folder4'"
+$mc/sql "update folder set scan_last = now() - interval '7 hour 59 minute $S1
second' where path = '/folder5'"
+$mc/sql "update folder set scan_last = now() - interval '24 hour' where path =
'/folder6'"
+
+$mc/sql "update folder set scan_requested = scan_last - interval '2 second',
scan_scheduled = scan_last - interval '1 second'"
+$mc/sql 'select * from folder'
+
+
+$mc/curl -I /download/folder1/file1.1.dat | grep '200 OK'
+
+for x in {2,3,4,5,6} ; do
+ $mc/curl -IL /download/folder$x/file1.1.dat | grep -A20 '302 Found' | grep
'200 OK'
+done
+
+$mc/backstage/job folder_sync_schedule
+$mc/backstage/shoot
+$mc/backstage/job mirror_scan_schedule
+$mc/backstage/shoot
+
+echo folder1 was resynced because it got new file
+$mc/sql_test 7 == "select count(*) from minion_jobs where task='folder_sync'"
+echo still nothing was scheduled, except folder1 which got new files and
folder6 which reaches max refresh time 24 hours
+$mc/sql_test 8 == "select count(*) from minion_jobs where task='mirror_scan'"
+
+sleep $S
+# c/sql "update folder set scan_last = scan_last + interval '$S second'"
+$mc/backstage/job folder_sync_schedule
+$mc/backstage/job mirror_scan_schedule
+$mc/backstage/shoot
+
+# no new jobs were submitted yet
+$mc/sql_test 7 == "select count(*) from minion_jobs where task='folder_sync'"
+$mc/sql_test 8 == "select count(*) from minion_jobs where task='mirror_scan'"
+
+for x in {1,2,3,4,5,6} ; do
+ $mc/curl -IL /download/folder$x/file1.1.dat | grep -A20 '302 Found' | grep
'200 OK'
+done
+
+$mc/backstage/job mirror_scan_schedule
+$mc/backstage/shoot
+
+# Now all fodlers are scanned twice
+$mc/sql_test 12 == "select count(*) from minion_jobs where task='mirror_scan'"
+$mc/sql_test 7 == "select count(*) from minion_jobs where task='folder_sync'"
+
+echo only first request was miss
+$mc/sql_test 1 == "select count(*) from stat where mirror_id < 1"
+echo all other requests are hits for the mirror
+$mc/sql_test 11 == "select count(*) from stat where mirror_id = 1"
+
+echo success
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.035/t/environ/16-rescan.sh
new/MirrorCache-1.037/t/environ/16-rescan.sh
--- old/MirrorCache-1.035/t/environ/16-rescan.sh 2022-06-23
06:47:51.000000000 +0200
+++ new/MirrorCache-1.037/t/environ/16-rescan.sh 2022-08-11
15:29:02.000000000 +0200
@@ -71,9 +71,9 @@
$mc/backstage/job mirror_scan_schedule
$mc/backstage/shoot
-# folder1 was resynced because it got new file
+echo folder1 was resynced because it got new file
$mc/sql_test 7 == "select count(*) from minion_jobs where task='folder_sync'"
-# still nothing was scheduled, except folder1 which got new files and folder6
which reaches max refresh time 24 hours
+echo still nothing was scheduled, except folder1 which got new files and
folder6 which reaches max refresh time 24 hours
$mc/sql_test 8 == "select count(*) from minion_jobs where task='mirror_scan'"
sleep $S
@@ -96,3 +96,5 @@
# Now all fodlers are scanned twice
$mc/sql_test 12 == "select count(*) from minion_jobs where task='mirror_scan'"
$mc/sql_test 7 == "select count(*) from minion_jobs where task='folder_sync'"
+
+echo success
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.035/templates/app/folder/show.html.ep
new/MirrorCache-1.037/templates/app/folder/show.html.ep
--- old/MirrorCache-1.035/templates/app/folder/show.html.ep 2022-06-23
06:47:51.000000000 +0200
+++ new/MirrorCache-1.037/templates/app/folder/show.html.ep 2022-08-11
15:29:02.000000000 +0200
@@ -60,7 +60,7 @@
<div><span>Path: </span><%= $folder->{path} %></div>
<div><span>Last sync: </span><%= $folder->{sync_last} %></div>
<div><span>Sync Scheduled: </span><%= $folder->{sync_scheduled}
%></div>
- <div><span>Sync Requested: </span><%= $folder->{sync_requested}
%></div>
+ <div><span>Sync Requested: </span><%= $folder->{sync_requested}
%></div>
</div>
</div>
</div>
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.035/templates/branding/openSUSE/footer.html.ep
new/MirrorCache-1.037/templates/branding/openSUSE/footer.html.ep
--- old/MirrorCache-1.035/templates/branding/openSUSE/footer.html.ep
2022-06-23 06:47:51.000000000 +0200
+++ new/MirrorCache-1.037/templates/branding/openSUSE/footer.html.ep
2022-08-11 15:29:02.000000000 +0200
@@ -2,7 +2,7 @@
<div class="container">
<div class="d-flex justify-content-between">
<div class="footer-copyright">
- © 2021 SUSE LLC., openSUSE contributors
+ © 2021,2022 SUSE LLC., openSUSE contributors
</div>
<div class="list-inline">
<a class="list-inline-item"
href="https://en.opensuse.org/Imprint";>Legal notice</a>
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.035/templates/branding/openSUSE/header.html.ep
new/MirrorCache-1.037/templates/branding/openSUSE/header.html.ep
--- old/MirrorCache-1.035/templates/branding/openSUSE/header.html.ep
2022-06-23 06:47:51.000000000 +0200
+++ new/MirrorCache-1.037/templates/branding/openSUSE/header.html.ep
2022-08-11 15:29:02.000000000 +0200
@@ -58,6 +58,22 @@
% }
</div>
</li>
+ <li class="nav-item dropdown" id="reports">
+ <a href="#" class="nav-link dropdown-toggle" data-toggle="dropdown"
role="button" aria-haspopup="true" aria-expanded="false"
data-submenu>Reports<span class="caret"></span></a>
+ <ul class="dropdown-menu">
+ <li>
+ %= link_to 'All Mirrors' => url_for('/report/mirrors') => class =>
'dropdown-item'
+ </li><li>
+ %= link_to '15.4 Mirrors' => url_for('/report/mirrors/154') =>
class => 'dropdown-item'
+ </li><li>
+ %= link_to '15.3 Mirrors' => url_for('/report/mirrors/153') =>
class => 'dropdown-item'
+ </li><li>
+ %= link_to 'TW Mirrors' => url_for('/report/mirrors/tumbleweed') =>
class => 'dropdown-item'
+ </li><li>
+ %= link_to 'Build Service Mirrors' =>
url_for('/report/mirrors/repositories') => class => 'dropdown-item'
+ </li>
+ </ul>
+ </li>
</ul>
</div>
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.035/templates/branding/openSUSE/report_mirrors_banner.html.ep
new/MirrorCache-1.037/templates/branding/openSUSE/report_mirrors_banner.html.ep
---
old/MirrorCache-1.035/templates/branding/openSUSE/report_mirrors_banner.html.ep
1970-01-01 01:00:00.000000000 +0100
+++
new/MirrorCache-1.037/templates/branding/openSUSE/report_mirrors_banner.html.ep
2022-08-11 15:29:02.000000000 +0200
@@ -0,0 +1,32 @@
+<div class="report-mirrors-banner">
+
+ <p class="alert alert-danger">
+ The information on this page is generated directly off the
+ database used by the
+ <a href="http://download.opensuse.org/";>master download server</a>
+ every 30 minutes.
+ Under normal circumstances, there <em>should</em> be no need to use
+ mirrors directly, and look at this table for that purpose.
+ If you <em>do</em> see the need, there might be something wrong
+ and it would be appreciated if you notify the admins about it,
+ so it can be fixed for the benefit of yourself, and possibly
+ many other users.
+ </p>
+
+ <p class="alert alert-warning">
+ This page lists mirror servers where you can download openSUSE.
+ For specific distributions, we recommend <a
href="http://get.opensuse.org/";>get.opensuse.org</a>
+ to help with choosing the correct download, which will automatically
choose the best mirror.
+ The report uses heuristic scoring system to show how outdated the
mirrors are. Scoring is done for tree of folders,
+ so a mirror with a bad score can still have recent files in some
locations and may be useful.
+ Meaning of colors:
+ <span style="color:green">up to date</span>, <span
style="color:olive">quite up to date</span>, <span style="color:grey">might be
outdated</span>, <span style="color:red">outdated</span>.
+ The state is also dynamic, so the score and the colors may change on
next refresh.
+ </p>
+
+ <p class="alert alert-info">
+ If you have a server with some space left, and want to help with
making the openSUSE experience better for other users,
+ <a href="https://en.opensuse.org/openSUSE:Mirror_howto";>become a
mirror</a>!
+ </p>
+
+</div>
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.035/templates/report/mirrors/index.html.ep
new/MirrorCache-1.037/templates/report/mirrors/index.html.ep
--- old/MirrorCache-1.035/templates/report/mirrors/index.html.ep
1970-01-01 01:00:00.000000000 +0100
+++ new/MirrorCache-1.037/templates/report/mirrors/index.html.ep
2022-08-11 15:29:02.000000000 +0200
@@ -0,0 +1,162 @@
+% layout 'bootstrap';
+
+% my $first_old = '';
+% my $first;
+% for my $project (@$projects) {
+% my $name = $project->{name};
+% ($first) = split ' ', $name, 2;
+% if ($first_old && $first_old ne $first) {
+% $first = '';
+% last;
+% }
+% $first_old = $first;
+% }
+% if ($first) {
+% title 'Mirrors Report for ' . $first;
+% } else {
+% title 'Mirrors Report';
+% }
+
+% content_for 'head' => begin
+
+<style type='text/css'>
+
+a.repoperfect {
+ color: green
+}
+
+a.repogood {
+ color: olive
+}
+
+a.repouncertain {
+ color: grey
+}
+
+a.repooutdated {
+ color: red
+}
+
+div.repo {
+ float: left;
+ margin-right: 2px;
+}
+
+tr:nth-child(even) {
+ background-color: #88888822;
+}
+
+tr:hover td {
+ background-color: #88888840;
+}
+
+tr:nth-child(even):hover td {
+ background-color: #88888820;
+}
+
+td.newregion, tr:hover td.newregion {
+ background: #fff;
+ font-size: 1.2rem;
+ padding: 1.5rem 0.5rem;
+}
+
+@media (prefers-color-scheme: dark) {
+ td.newregion, tr:hover td.newregion {
+ background: #212529;
+ }
+ thead th {
+ background: linear-gradient(to top, #787878 0%, #363a3c 1px, #363a3c
100%);
+ }
+ thead th:nth-child(even), thead th:nth-child(1), thead th:nth-child(3) {
+ background: linear-gradient(to top, #787878 0%, #2f3236 1px, #2f3236
100%);
+ }
+}
+
+</style>
+% end
+
+%= include_branding 'report_mirrors_banner'
+
+<form>
+<div class="row">
+ <div class="col-sm-12">
+ <h2><%= title %></h2>
+ %= include 'layouts/info'
+ <table id="servers" class="reporttable table table-striped">
+ <thead>
+ <tr>
+ <th class="col_value">Country</th>
+ <th class="col_value">Url</th>
+ % my $first_old = '';
+ % for my $project (@$projects) {
+ % my $name = $project->{name};
+ % (my $first) = split ' ', $name, 2;
+ % $first = "Leap $first" if 0 == rindex $first,
"15.", 0;
+ % $first = "Build Service" if $first eq 'repositories';
+ % $first = "Tumbleweed" if $first eq 'TW';
+ % if ($first_old ne $first) {
+ % if ($first_old) {
+ </th>
+ % }
+ <th class="col_value"><%= $first %>
+ % }
+ % $first_old = $first;
+ % }
+ </th>
+ </tr>
+ </thead>
+ <tbody>
+ % my $prev_region = '';
+ % for my $m (@$mirrors) {
+ % my $region = $m->{region};
+ % if ($region ne $prev_region) {
+ % my $region_name = region_name($region);
+ <tr><td colspan="24" class="newregion"><%= $region_name
%>:</td></tr>
+ % }
+ % $prev_region = $region;
+ <tr>
+ % my $url = $m->{url};
+ <td><%= $m->{country} %></td>
+ <td><a href="http://<%= $url %>"><%=$url%></a></td>
+ % my $inner = '';
+ % $first_old = '';
+ <td>
+ % for my $project (@$projects) {
+ % my $name = $project->{name};
+ % my $alias = $project->{alias};
+ % my $path = $project->{path};
+ % my $column1 = $alias . "score";
+ % my $column2 = $alias . "victim";
+ % my ($first, $second) = split ' ', $name;
+ % my $victim = $m->{$column2};
+ % if ($first_old && $first_old ne $first) {
+ </td><td>
+ % }
+ % $first_old = $first;
+ % my $x = $m->{$column1};
+ % if (!defined $x) {
+ % next;
+ % }
+ % $second = 'repo' unless $second;
+ % my $style = 'repooutdated';
+ % $style = 'repouncertain' if $x >= 30;
+ % $style = 'repogood' if $x >= 75;
+ % $style = 'repoperfect' if $x == 100;
+ <div class="repo">
+ <a class="<%=$style%>"
+ % if ($victim) {
+ title="diff in: <%=$victim%>"
+ % }
+ % $second = 'iso' if $second eq 'ISO' || $second eq
'ISOs';
+ href="http://<%=$url . $path%>">
+ <%= $second %>
+ </a>
+ </div>
+ % }
+ </td>
+ % }
+ </tbody>
+ </table>
+ </div>
+</div>
+</form>
++++++ MirrorCache.obsinfo ++++++
--- /var/tmp/diff_new_pack.o67KlN/_old 2022-08-19 17:59:02.052421746 +0200
+++ /var/tmp/diff_new_pack.o67KlN/_new 2022-08-19 17:59:02.056421755 +0200
@@ -1,5 +1,5 @@
name: MirrorCache
-version: 1.035
-mtime: 1655959671
-commit: 04f97244031e34226a586781f8a35cca0ec01c4a
+version: 1.037
+mtime: 1660224542
+commit: 5281a9c79509f79aec972de25f37d12b0e099b85
