Script 'mail_helper' called by obssrc
Hello community,
here is the log from the commit of package MirrorCache for openSUSE:Factory
checked in at 2021-12-09 19:45:42
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/MirrorCache (Old)
and /work/SRC/openSUSE:Factory/.MirrorCache.new.2520 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "MirrorCache"
Thu Dec 9 19:45:42 2021 rev:6 rq:937394 version:1.019
Changes:
--------
--- /work/SRC/openSUSE:Factory/MirrorCache/MirrorCache.changes 2021-11-20
02:39:47.104652476 +0100
+++ /work/SRC/openSUSE:Factory/.MirrorCache.new.2520/MirrorCache.changes
2021-12-09 19:46:22.509155376 +0100
@@ -1,0 +2,16 @@
+Thu Dec 02 05:33:04 UTC 2021 - Andrii Nikitin <andrii.niki...@suse.com>
+
+- Update to version 1.019:
+ * Optimize sql for selecting mirror misses (#232)
+ * Import hashes (#231)
+ * Fix mirrorlist for subsidiary when in the same db with headquarter (#230)
+
+-------------------------------------------------------------------
+Thu Nov 25 08:56:58 UTC 2021 - Andrii Nikitin <andrii.niki...@suse.com>
+
+- Update to version 1.018:
+ * Fix occasional 404 on subtree mirrorlist
+ * Fix not selecting mirrors for unknown country
+ * Let Task MirrorProbe work without GeoIP database
+
+-------------------------------------------------------------------
Old:
----
MirrorCache-1.017.obscpio
New:
----
MirrorCache-1.019.obscpio
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ MirrorCache.spec ++++++
--- /var/tmp/diff_new_pack.tLk4xw/_old 2021-12-09 19:46:23.217155717 +0100
+++ /var/tmp/diff_new_pack.tLk4xw/_new 2021-12-09 19:46:23.217155717 +0100
@@ -22,7 +22,7 @@
%define main_requires %{assetpack_requires} perl(Carp) perl(DBD::Pg) >= 3.7.4
perl(DBI) >= 1.632 perl(DBIx::Class) >= 0.082801
perl(DBIx::Class::DynamicDefault) perl(DateTime) perl(Encode) perl(Time::Piece)
perl(Time::Seconds) perl(Time::ParseDate) perl(DateTime::Format::Pg)
perl(Exporter) perl(File::Basename) perl(LWP::UserAgent) perl(Mojo::Base)
perl(Mojo::ByteStream) perl(Mojo::IOLoop) perl(Mojo::JSON) perl(Mojo::Pg)
perl(Mojo::URL) perl(Mojo::Util) perl(Mojolicious::Commands)
perl(Mojolicious::Plugin) perl(Mojolicious::Plugin::RenderFile)
perl(Mojolicious::Static) perl(Net::OpenID::Consumer) perl(POSIX)
perl(Sort::Versions) perl(URI::Escape) perl(XML::Writer) perl(base)
perl(constant) perl(diagnostics) perl(strict) perl(warnings) shadow
rubygem(sass) perl(Net::DNS) perl(LWP::Protocol::https) perl(Digest::SHA)
%define build_requires %{assetpack_requires} rubygem(sass) tidy sysuser-shadow
sysuser-tools
Name: MirrorCache
-Version: 1.017
+Version: 1.019
Release: 0
Summary: WebApp to redirect and manage mirrors
License: GPL-2.0-or-later
++++++ MirrorCache-1.017.obscpio -> MirrorCache-1.019.obscpio ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.017/assets/javascripts/mirrorlist.js
new/MirrorCache-1.019/assets/javascripts/mirrorlist.js
--- old/MirrorCache-1.017/assets/javascripts/mirrorlist.js 2021-11-11
09:43:45.000000000 +0100
+++ new/MirrorCache-1.019/assets/javascripts/mirrorlist.js 2021-12-01
13:56:26.000000000 +0100
@@ -11,10 +11,11 @@
element.innerText = 'Error: ' + error;
}
-function loadMirrorlist(url, element1, element2, element3) {
+function loadMirrorlist(url, country, region, element1, element2, element3) {
$.ajax({
url: url,
method: 'GET',
+ data:{ COUNTRY: country, REGION: region },
success: function(response) {
var l1 = response.l1;
for(var i = 0; i < l1.length; i++) {
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.017/lib/MirrorCache/Datamodule.pm
new/MirrorCache-1.019/lib/MirrorCache/Datamodule.pm
--- old/MirrorCache-1.017/lib/MirrorCache/Datamodule.pm 2021-11-11
09:43:45.000000000 +0100
+++ new/MirrorCache-1.019/lib/MirrorCache/Datamodule.pm 2021-12-01
13:56:26.000000000 +0100
@@ -366,7 +366,7 @@
$self->must_render_from_root(1)
if !$self->mirrorlist
&& ( !$self->metalink || $self->metalink_accept )
- && $path =~
m/.*\/(repodata\/repomd.xml[^\/]*|media\.1\/media|.*\.sha256(\.asc)|Release(.key|.gpg)?|InRelease|Packages(.gz)?|Sources(.gz)?)$/;
+ && $path =~
m/.*\/(repodata\/repomd.xml[^\/]*|media\.1\/media|.*\.sha256(\.asc)|Release(.key|.gpg)?|InRelease|Packages(.gz)?|Sources(.gz)?|.*_Arch\.(files|db|key)(\.(sig|tar\.gz(\.sig)?)?))$/;
$self->_path($path);
$self->_trailing_slash($trailing_slash);
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.017/lib/MirrorCache/Schema/Result/Folder.pm
new/MirrorCache-1.019/lib/MirrorCache/Schema/Result/Folder.pm
--- old/MirrorCache-1.017/lib/MirrorCache/Schema/Result/Folder.pm
2021-11-11 09:43:45.000000000 +0100
+++ new/MirrorCache-1.019/lib/MirrorCache/Schema/Result/Folder.pm
2021-12-01 13:56:26.000000000 +0100
@@ -62,7 +62,7 @@
{ data_type => "varchar", is_nullable => 0, size => 512 },
wanted => {
data_type => 'timestamp',
- dynamic_default_on_create => 'DBIx::Class::Timestamps::now',
+ dynamic_default_on_create => 'DBIx::Class::Timestamps::now',
is_nullable => 1
},
sync_requested => {
@@ -90,6 +90,10 @@
data_type => 'timestamp',
is_nullable => 1
},
+ hash_last_import => {
+ data_type => 'timestamp',
+ is_nullable => 1
+ },
"size",
{ data_type => "bigint", is_nullable => 1 },
);
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.017/lib/MirrorCache/Schema/Result/Subsidiary.pm
new/MirrorCache-1.019/lib/MirrorCache/Schema/Result/Subsidiary.pm
--- old/MirrorCache-1.017/lib/MirrorCache/Schema/Result/Subsidiary.pm
2021-11-11 09:43:45.000000000 +0100
+++ new/MirrorCache-1.019/lib/MirrorCache/Schema/Result/Subsidiary.pm
2021-12-01 13:56:26.000000000 +0100
@@ -48,5 +48,7 @@
{ data_type => "varchar", is_nullable => 0, size => 128 },
"uri",
{ data_type => "varchar", is_nullable => 0, size => 128 },
+ "local",
+ { data_type => "boolean", is_nullable => 0 },
);
1;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.017/lib/MirrorCache/Schema/ResultSet/File.pm
new/MirrorCache-1.019/lib/MirrorCache/Schema/ResultSet/File.pm
--- old/MirrorCache-1.017/lib/MirrorCache/Schema/ResultSet/File.pm
2021-11-11 09:43:45.000000000 +0100
+++ new/MirrorCache-1.019/lib/MirrorCache/Schema/ResultSet/File.pm
2021-12-01 13:56:26.000000000 +0100
@@ -29,13 +29,21 @@
# html parser may loose seconds from file.mtime, so we allow hash.mtime
differ for up to 1 min for now
my $sql = <<'END_SQL';
-select file.*, hash.mtime as hash_mtime, hash.md5, hash.sha1, hash.sha256,
hash.piece_size, hash.pieces,
+select file.id, file.folder_id, file.name,
+case when coalesce(file.size, 0::bigint) = 0::bigint and coalesce(hash.size,
0::bigint) != 0::bigint then hash.size else file.size end,
+case when coalesce(file.mtime, 0::bigint) = 0::bigint and coalesce(hash.mtime,
0::bigint) != 0::bigint then hash.mtime else file.mtime end,
+file.mtime, file.dt, hash.mtime as hash_mtime, hash.md5, hash.sha1,
hash.sha256, hash.piece_size, hash.pieces,
(DATE_PART('day', now() - file.dt) * 24 * 3600 +
DATE_PART('hour', now() - file.dt) * 3600 +
DATE_PART('minute', now() - file.dt) * 60 +
DATE_PART('second', now() - file.dt)) as age
from file
-left join hash on file_id = id and file.size = hash.size and abs(file.mtime -
hash.mtime) < 61
+left join hash on file_id = id and
+(
+ (file.size = hash.size and abs(file.mtime - hash.mtime) < 61)
+ or
+ (coalesce(file.size, 0::bigint) = 0::bigint and coalesce(hash.size,
0::bigint) != 0::bigint and file.dt <= hash.dt)
+)
where file.folder_id = ? and name = ?
END_SQL
my $prep = $dbh->prepare($sql);
@@ -43,21 +51,49 @@
return $dbh->selectrow_hashref($prep);
}
+# returns list of file_id for which we need hash to (re)calculate
sub hash_needed {
- my ($self, $folder_id) = @_;
+ my ($self, $folder_id, $dt) = @_;
my $rsource = $self->result_source;
my $schema = $rsource->schema;
my $dbh = $schema->storage->dbh;
+ unless ($dt) {
+ my $res = $self->need_hashes($folder_id);
+ $dt = $res->{max_dt} if $res;
+ }
+
my $sql = <<'END_SQL';
select file.id, file.name, file.size
from file
-left join hash on file_id = id and file.size = hash.size
+left join hash on file_id = id
+where file.folder_id = ?
+and (hash.file_id is null or coalesce(file.dt > ?, 't'))
+END_SQL
+ return $dbh->selectall_hashref($sql, 'id', {}, $folder_id, $dt);
+}
+
+# returns pair (bool, max_dt)
+sub need_hashes {
+ my ($self, $folder_id) = @_;
+
+ my $rsource = $self->result_source;
+ my $schema = $rsource->schema;
+ my $dbh = $schema->storage->dbh;
+
+ my $sql = <<'END_SQL';
+select file.id,
+(select max(hash.dt) from hash join file on file_id = id where folder_id = ?)
as max_dt
+from file
+left join hash on file_id = id
where file.folder_id = ?
-and hash.file_id is null
+and (hash.file_id is null or coalesce(file.dt > hash.dt, 't'))
+limit 1;
END_SQL
- return $dbh->selectall_hashref($sql, 'id', {}, $folder_id);
+ my $prep = $dbh->prepare($sql);
+ $prep->execute($folder_id, $folder_id);
+ return $dbh->selectrow_hashref($prep);
}
1;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.017/lib/MirrorCache/Schema/ResultSet/Hash.pm
new/MirrorCache-1.019/lib/MirrorCache/Schema/ResultSet/Hash.pm
--- old/MirrorCache-1.017/lib/MirrorCache/Schema/ResultSet/Hash.pm
2021-11-11 09:43:45.000000000 +0100
+++ new/MirrorCache-1.019/lib/MirrorCache/Schema/ResultSet/Hash.pm
2021-12-01 13:56:26.000000000 +0100
@@ -69,19 +69,4 @@
return $dbh->selectall_arrayref($prep, {Slice => {}});
}
-sub hashes_max_dt {
- my ($self, $folder_id) = @_;
-
- my $rsource = $self->result_source;
- my $schema = $rsource->schema;
- my $dbh = $schema->storage->dbh;
-
- my $sql = <<'END_SQL';
-select max(hash.dt) from hash where file_id in ( select id from file where
folder_id = ? )
-END_SQL
- my $prep = $dbh->prepare($sql);
- $prep->execute($folder_id);
- return $dbh->selectrow_array($prep);
-}
-
1;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.017/lib/MirrorCache/Schema/ResultSet/Server.pm
new/MirrorCache-1.019/lib/MirrorCache/Schema/ResultSet/Server.pm
--- old/MirrorCache-1.017/lib/MirrorCache/Schema/ResultSet/Server.pm
2021-11-11 09:43:45.000000000 +0100
+++ new/MirrorCache-1.019/lib/MirrorCache/Schema/ResultSet/Server.pm
2021-12-01 13:56:26.000000000 +0100
@@ -37,7 +37,7 @@
my $schema = $rsource->schema;
my $dbh = $schema->storage->dbh;
- my $avoid_country;
+ my $avoid_country = $avoid_region;
my $country_condition;
my @country_params = ($country);
$avoid_region = ($region && $avoid_region) ? '!' : '';
@@ -57,7 +57,7 @@
}
}
elsif ($country) {
- $avoid_country = $avoid_region ? '!' : '';
+ $avoid_country = $avoid_country ? '!' : '';
my $region_condition = '';
if ($region) {
$region_condition = " and s.region $avoid_region= ?";
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.017/lib/MirrorCache/Schema/ResultSet/ServerCapabilityDeclaration.pm
new/MirrorCache-1.019/lib/MirrorCache/Schema/ResultSet/ServerCapabilityDeclaration.pm
---
old/MirrorCache-1.017/lib/MirrorCache/Schema/ResultSet/ServerCapabilityDeclaration.pm
2021-11-11 09:43:45.000000000 +0100
+++
new/MirrorCache-1.019/lib/MirrorCache/Schema/ResultSet/ServerCapabilityDeclaration.pm
2021-12-01 13:56:26.000000000 +0100
@@ -47,9 +47,11 @@
where 't'
AND (fhttp.server_id IS NULL or fhttps.server_id IS NULL) -- do not show
servers which have both http and https force disabled
AND (fipv4.server_id IS NULL or fipv6.server_id IS NULL) -- do not show
servers which have both ipv4 and ipv6 force disabled
- AND s.country = lower(?)
AND s.enabled
END_SQL
+ return $dbh->selectall_hashref($sql, 'id', {}) unless $country;
+
+ $sql = $sql . ' AND s.country = lower(?)';
return $dbh->selectall_hashref($sql, 'id', {}, $country);
}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.017/lib/MirrorCache/Schema/ResultSet/Stat.pm
new/MirrorCache-1.019/lib/MirrorCache/Schema/ResultSet/Stat.pm
--- old/MirrorCache-1.017/lib/MirrorCache/Schema/ResultSet/Stat.pm
2021-11-11 09:43:45.000000000 +0100
+++ new/MirrorCache-1.019/lib/MirrorCache/Schema/ResultSet/Stat.pm
2021-12-01 13:56:26.000000000 +0100
@@ -106,7 +106,7 @@
my $dbh = $self->result_source->schema->storage->dbh;
my $sql = << "END_SQL";
-select stat.id, mirror_id, stat.country,
+select stat.id, mirror_id, trim(stat.country),
concat(case when secure then 'https://' else 'http://' end, CASE WHEN
length(server.hostname_vpn)>0 THEN server.hostname_vpn ELSE server.hostname
END, server.urldir, case when metalink then regexp_replace(path,
'(.*)\.metalink', E'\\1') else path end) as url,
substring(path,'(^(/.*)+)/') as folder, folder_id
from stat join server on mirror_id = server.id
@@ -128,9 +128,10 @@
my $sql = << 'END_SQL';
select * from (
-select stat.id, stat.path, stat.folder_id, country
+select stat.id, stat.path, stat.folder_id, trim(country)
from stat left join folder on folder.id = stat.folder_id
-where mirror_id in (-1, 0)
+where mirror_id < 1
+and ( mirror_id in (0,-1) or mirrorlist )
and file_id is null
and stat.path !~
'.*\/(repodata\/repomd.xml[^\/]*|media\.1\/media|.*\.sha256(\.asc)|Release(.key|.gpg)?|InRelease|Packages(.gz)?|Sources(.gz)?)$'
and stat.agent NOT ILIKE '%bot%'
@@ -174,7 +175,8 @@
my $dbh = $schema->storage->dbh;
my $sql = << "END_SQL";
-select stat.id, stat.folder_id, country
+select * from (
+select stat.id, stat.folder_id, trim(country)
from stat join folder on folder.id = stat.folder_id
where mirror_id in (-1, 0) and file_id is not null
and stat.agent NOT ILIKE '%bot%'
@@ -184,9 +186,9 @@
)
END_SQL
$sql = "$sql and stat.id > $prev_stat_id" if $prev_stat_id;
+ $sql = "$sql order by stat.id desc limit ($limit+1) ) x";
$sql = "$sql union all select max(id), 0, null from stat"; # this is just
to get max(id) in the same query
$sql = "$sql order by id desc";
- $sql = "$sql limit ($limit+1)";
my $prep = $dbh->prepare($sql);
$prep->execute();
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.017/lib/MirrorCache/Task/FolderHashesCreate.pm
new/MirrorCache-1.019/lib/MirrorCache/Task/FolderHashesCreate.pm
--- old/MirrorCache-1.017/lib/MirrorCache/Task/FolderHashesCreate.pm
2021-11-11 09:43:45.000000000 +0100
+++ new/MirrorCache-1.019/lib/MirrorCache/Task/FolderHashesCreate.pm
2021-12-01 13:56:26.000000000 +0100
@@ -29,7 +29,7 @@
}
sub _sync {
- my ($app, $job, $path) = @_;
+ my ($app, $job, $path, $dt) = @_;
return $job->fail('Empty path is not allowed') unless $path;
return $job->fail('Trailing slash is not allowed') if '/' eq
substr($path,-1) && $path ne '/';
@@ -48,7 +48,7 @@
my $folder_id = $folder->id;
my $count = 0;
my $errcount = 0;
- my $rows = $schema->resultset('File')->hash_needed($folder_id);
+ my $rows = $schema->resultset('File')->hash_needed($folder_id, $dt);
for my $id (sort keys %$rows) {
my $file = $rows->{$id};
my $basename = $file->{name};
@@ -72,7 +72,7 @@
sub calcBlockSize {
my $fileSize = shift;
- return 0 if $fileSize <= $HASHES_PIECES_MIN_SIZE; # don't create piece
hashes
+ return 0 if !$fileSize || $fileSize <= $HASHES_PIECES_MIN_SIZE; # don't
create piece hashes
return 256*1024 if $fileSize < 8*1024*1024;
return 1024*1024 if $fileSize < 256*1024*1024;
return 4*1024*1024;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.017/lib/MirrorCache/Task/FolderHashesImport.pm
new/MirrorCache-1.019/lib/MirrorCache/Task/FolderHashesImport.pm
--- old/MirrorCache-1.017/lib/MirrorCache/Task/FolderHashesImport.pm
2021-11-11 09:43:45.000000000 +0100
+++ new/MirrorCache-1.019/lib/MirrorCache/Task/FolderHashesImport.pm
2021-12-01 13:56:26.000000000 +0100
@@ -43,8 +43,9 @@
my $count = 0;
my $errcount = 0;
- my $max_dt = $schema->resultset('Hash')->hashes_max_dt($folder_id);
+ my $max_dt = $folder->hash_last_import;
$max_dt = $max_dt ? "&since=$max_dt" : '';
+ $job->note(max_dt => $max_dt);
my $hq_url = $ENV{MIRRORCACHE_HEADQUARTER} . $path . '?hashes' . $max_dt;
$hq_url = "http://"; . $hq_url unless 'http' eq substr($hq_url, 0, 4);
@@ -55,7 +56,7 @@
if $res->code != 200;
my $res_json = $res->json;
-
+ my $last_import;
for my $hash (@$res_json) {
my $basename = $hash->{name};
next unless $basename;
@@ -64,6 +65,7 @@
eval {
$schema->resultset('Hash')->store($file->id, $hash->{mtime},
$hash->{size}, $hash->{md5},
$hash->{sha1}, $hash->{sha256}, $hash->{piece_size},
$hash->{pieces});
+ $last_import = Mojo::Date($hash->{dt}) if ($last_import &&
$hash->{dt} && $last_import->epoch < Mojo::Date->new($hash->{dt})->epoch);
$count++;
};
if ($@) {
@@ -74,6 +76,7 @@
}
}
+ $folder->hash_last_import($last_import) if $last_import && $count;
$job->note(count => $count, errors => $errcount);
}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.017/lib/MirrorCache/Task/FolderSync.pm
new/MirrorCache-1.019/lib/MirrorCache/Task/FolderSync.pm
--- old/MirrorCache-1.017/lib/MirrorCache/Task/FolderSync.pm 2021-11-11
09:43:45.000000000 +0100
+++ new/MirrorCache-1.019/lib/MirrorCache/Task/FolderSync.pm 2021-12-01
13:56:26.000000000 +0100
@@ -18,6 +18,7 @@
use MirrorCache::Utils 'datetime_now';
my $HASHES_COLLECT = $ENV{MIRRORCACHE_HASHES_COLLECT} // 0;
+my $HASHES_IMPORT = $ENV{MIRRORCACHE_HASHES_IMPORT} // 0;
my $HASHES_QUEUE = $ENV{MIRRORCACHE_HASHES_QUEUE} // 'hashes';
sub register {
@@ -118,9 +119,10 @@
if ($count) {
$schema->resultset('Folder')->request_scan($folder->id);
- $schema->resultset('Folder')->request_scan($otherFolder->id) if
$otherFolder;
$minion->enqueue('folder_hashes_create' => [$realpath] => {queue
=> $HASHES_QUEUE}) if $HASHES_COLLECT &&
!$app->backstage->inactive_jobs_exceed_limit(1000, 'folder_hashes_create',
$HASHES_QUEUE);
+ $minion->enqueue('folder_hashes_import' => [$realpath] => {queue
=> $HASHES_QUEUE}) if $HASHES_IMPORT &&
!$app->backstage->inactive_jobs_exceed_limit(1000, 'folder_hashes_import',
$HASHES_QUEUE);
}
+ $schema->resultset('Folder')->request_scan($otherFolder->id) if
$otherFolder && ($count || !$otherFolder->scan_requested);
return;
};
return $job->fail("Couldn't create folder $path in DB") unless $folder &&
$folder->id;
@@ -148,7 +150,7 @@
my $id = delete $dbfileidstodelete{$file};
if ((defined $size && defined $mtime) && ($size !=
$dbfilesizes{$file} || $mtime != $dbfilemtimes{$file})) {
$schema->storage->dbh->prepare(
- "UPDATE file set size = ?, mtime = ? where id = ?"
+ "UPDATE file set size = ?, mtime = ?, dt =
NOW()::timestamp(0) where id = ?"
)->execute($size, $mtime, $id);
$updated = $updated + 1;
}
@@ -173,11 +175,27 @@
$job->note(updated => $realpath, count => $cnt, deleted => $deleted,
updated => $updated);
if ($cnt || $updated) {
- $folder->update({sync_last => \"NOW()", scan_requested => \"NOW()",
sync_scheduled => \'coalesce(sync_scheduled, NOW())'});
- $schema->resultset('Folder')->request_scan($folder_id);
- $minion->enqueue('folder_hashes_create' => [$realpath] => {queue =>
$HASHES_QUEUE}) if $HASHES_COLLECT &&
!$app->backstage->inactive_jobs_exceed_limit(1000, 'folder_hashes_create',
$HASHES_QUEUE);
+ $folder->update( {sync_last => \"NOW()", scan_requested =>
\"NOW()", sync_scheduled => \'coalesce(sync_scheduled, NOW())'});
} else {
- $folder->update({sync_last => \"NOW()", sync_scheduled =>
\'coalesce(sync_scheduled, NOW())'});
+ $folder->update( {sync_last => \"NOW()", sync_scheduled =>
\'coalesce(sync_scheduled, NOW())'});
+ }
+ my $need_hashes = $cnt || $updated ? 1 : 0;
+ my $max_dt;
+ if ( ($HASHES_COLLECT || $HASHES_IMPORT) ) {
+ if(my $res = $schema->resultset('File')->need_hashes($folder_id)) {
+ $need_hashes = $res->{file_id} unless $need_hashes;
+ $max_dt = $res->{max_dt};
+ }
+ }
+ if ($need_hashes) {
+ $minion->enqueue('folder_hashes_create' => [$realpath, $max_dt] =>
{queue => $HASHES_QUEUE}) if $HASHES_COLLECT &&
!$app->backstage->inactive_jobs_exceed_limit(1000, 'folder_hashes_create',
$HASHES_QUEUE);
+ $minion->enqueue('folder_hashes_import' => [$realpath, $max_dt] =>
{queue => $HASHES_QUEUE}) if $HASHES_IMPORT &&
!$app->backstage->inactive_jobs_exceed_limit(1000, 'folder_hashes_import',
$HASHES_QUEUE);
+ }
+
+ if ($otherFolder && ($cnt || $updated || !$otherFolder->scan_requested)) {
+ $otherFolder->update({sync_last => \"NOW()", scan_requested =>
\"NOW()", sync_scheduled => \'coalesce(sync_scheduled, NOW())'});
+ } else {
+ $otherFolder->update({sync_last => \"NOW()", sync_scheduled =>
\'coalesce(sync_scheduled, NOW())'}) if $otherFolder;
}
}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.017/lib/MirrorCache/Task/FolderSyncScheduleFromMisses.pm
new/MirrorCache-1.019/lib/MirrorCache/Task/FolderSyncScheduleFromMisses.pm
--- old/MirrorCache-1.017/lib/MirrorCache/Task/FolderSyncScheduleFromMisses.pm
2021-11-11 09:43:45.000000000 +0100
+++ new/MirrorCache-1.019/lib/MirrorCache/Task/FolderSyncScheduleFromMisses.pm
2021-12-01 13:56:26.000000000 +0100
@@ -65,6 +65,7 @@
$rs->request_scan($folder->id);
}
}
+ @$country_list = ('') if $cnt && !@$country_list;
for my $country (@$country_list) {
next unless $minion->lock('mirror_probe_scheduled_' . $country,
60); # don't schedule if schedule happened in last 60 sec
next unless $minion->lock('mirror_probe_incomplete_for_' .
$country, 600); # don't schedule until probe job completed
@@ -73,7 +74,7 @@
}
$last_run = $last_run + $cnt;
last unless $cnt;
- $limit = 50;
+ $limit = 1000;
($stat_id, $folders, $country_list) =
$schema->resultset('Stat')->path_misses($prev_stat_id, $limit);
}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.017/lib/MirrorCache/Task/MirrorProbe.pm
new/MirrorCache-1.019/lib/MirrorCache/Task/MirrorProbe.pm
--- old/MirrorCache-1.017/lib/MirrorCache/Task/MirrorProbe.pm 2021-11-11
09:43:45.000000000 +0100
+++ new/MirrorCache-1.019/lib/MirrorCache/Task/MirrorProbe.pm 2021-12-01
13:56:26.000000000 +0100
@@ -29,7 +29,7 @@
sub _probe {
my ($app, $job, $country) = @_;
- return $job->fail('Empty country is not allowed') unless $country;
+ $country = '' unless $country;
my $minion = $app->minion;
return $job->finish("Previous mirror probe for {$country} job is still
active")
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.017/lib/MirrorCache/Task/MirrorScanScheduleFromMisses.pm
new/MirrorCache-1.019/lib/MirrorCache/Task/MirrorScanScheduleFromMisses.pm
--- old/MirrorCache-1.017/lib/MirrorCache/Task/MirrorScanScheduleFromMisses.pm
2021-11-11 09:43:45.000000000 +0100
+++ new/MirrorCache-1.019/lib/MirrorCache/Task/MirrorScanScheduleFromMisses.pm
2021-12-01 13:56:26.000000000 +0100
@@ -53,6 +53,7 @@
$cnt = $cnt + 1;
$rs->request_scan($folder_id);
}
+ @$country_list = ('') if $cnt && !@$country_list;
for my $country (@$country_list) {
next unless $minion->lock('mirror_probe_scheduled_' . $country,
60); # don't schedule if schedule happened in last 60 sec
next unless $minion->lock('mirror_probe_incomplete_for_' .
$country, 6000); # don't schedule until probe job completed
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.017/lib/MirrorCache/Task/MirrorScanScheduleFromPathErrors.pm
new/MirrorCache-1.019/lib/MirrorCache/Task/MirrorScanScheduleFromPathErrors.pm
---
old/MirrorCache-1.017/lib/MirrorCache/Task/MirrorScanScheduleFromPathErrors.pm
2021-11-11 09:43:45.000000000 +0100
+++
new/MirrorCache-1.019/lib/MirrorCache/Task/MirrorScanScheduleFromPathErrors.pm
2021-12-01 13:56:26.000000000 +0100
@@ -54,6 +54,7 @@
$rs->request_scan_array(@$rescan_ids);
$cnt += @$rescan_ids;
}
+ @$country_list = ('') if $cnt && !@$country_list;
for my $country (@$country_list) {
next unless $minion->lock('mirror_probe_scheduled_' . $country,
60); # don't schedule if schedule hapened in last 60 sec
next unless $minion->lock('mirror_probe_incomplete_for_' .
$country, 300); # don't schedule until probe job completed
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.017/lib/MirrorCache/WebAPI/Plugin/Dir.pm
new/MirrorCache-1.019/lib/MirrorCache/WebAPI/Plugin/Dir.pm
--- old/MirrorCache-1.017/lib/MirrorCache/WebAPI/Plugin/Dir.pm 2021-11-11
09:43:45.000000000 +0100
+++ new/MirrorCache-1.019/lib/MirrorCache/WebAPI/Plugin/Dir.pm 2021-12-01
13:56:26.000000000 +0100
@@ -251,31 +251,40 @@
my ($path, $trailing_slash) = $dm->path;
my $rsFolder = $schema->resultset('Folder');
- if (my $folder = $rsFolder->find_folder_or_redirect($dm->root_subtree .
$path)) {
- return $dm->redirect($folder->{pathto}) if $folder->{pathto};
- # folder must have trailing slash, otherwise it will be a challenge to
render links on webpage
- return $dm->redirect($dm->route . $path . '/') if !$trailing_slash &&
$path ne '/';
- $dm->folder_id($folder->{id});
- $dm->file_id(-1);
- return _render_dir($dm, $path, $rsFolder) if ($folder->{sync_last});
- } elsif (!$trailing_slash && $path ne '/') {
+ if (!$trailing_slash && $path ne '/') {
my $f = Mojo::File->new($path);
my $dirname = $root->realpath($f->dirname);
$dirname = $dm->root_subtree . $f->dirname unless $dirname;
if (my $parent_folder = $rsFolder->find({path => $dirname})) {
- $dm->folder_id($parent_folder->id);
+ if ($dirname eq $dm->root_subtree . $f->dirname) {
+ $dm->folder_id($parent_folder->id);
+ } else {
+ my $another_folder = $rsFolder->find({path =>
$dm->root_subtree . $f->dirname});
+ if (!$another_folder) {
+ my $res = $c->render(status => 425, text => "The file is
unknown, retry later");
+ # log miss here even thoough we haven't rendered anything
+ $c->stat->redirect_to_root($dm, 0);
+ return $res;
+ }
+ $dm->folder_id($another_folder->id);
+ }
my $file;
$file = $schema->resultset('File')->find({ name => $f->basename,
folder_id => $parent_folder->id }) if $parent_folder && !$trailing_slash;
# folders are stored with trailing slash in file table, so they
will not be selected here
if ($file) {
$dm->file_id($file->id);
- # regular file has trailing slash in db? That is probably
incorrect, so let the root handle it
- return $root->render_file($dm, $path . '/') if $trailing_slash;
# find a mirror for it
$c->mirrorcache->render_file($path, $dm);
return 1;
}
}
+ } elsif (my $folder = $rsFolder->find_folder_or_redirect($dm->root_subtree
. $path)) {
+ return $dm->redirect($folder->{pathto}) if $folder->{pathto};
+ # folder must have trailing slash, otherwise it will be a challenge to
render links on webpage
+ return $dm->redirect($dm->route . $path . '/') if !$trailing_slash &&
$path ne '/';
+ $dm->folder_id($folder->{id});
+ $dm->file_id(-1);
+ return _render_dir($dm, $path, $rsFolder) if ($folder->{sync_last});
}
return undef;
}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.017/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
new/MirrorCache-1.019/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
--- old/MirrorCache-1.017/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
2021-11-11 09:43:45.000000000 +0100
+++ new/MirrorCache-1.019/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
2021-12-01 13:56:26.000000000 +0100
@@ -195,7 +195,7 @@
sha256 => $file->{sha256},
};
- my @regions = $c->subsidiary->regions($region) if $region;
+ my @regions = $c->subsidiary->regions($region);
$c->stash('nonavbar' => 1) if ($ENV{MIRRORCACHE_BRANDING});
$c->stash('mirrorlist' => 1);
my ($lat, $lng) = $dm->coord;
@@ -207,6 +207,7 @@
mirrordata_region => \@mirrordata_region,
mirrordata_rest => \@mirrordata_rest,
country => uc($country),
+ region => $region,
ip => $dm->ip,
lat => $lat,
lng => $lng,
@@ -490,7 +491,7 @@
}
}
- if ($found_count < $limit || $mirrorlist) {
+ if (!$found_count || ($found_count < $limit && !$dm->root_country) ||
$mirrorlist) {
$m = $rs->mirrors_query(
$country, $region, $folder_id, $file_id, $scheme,
$ipv, $lat, $lng, $avoid_countries, $limit, 1,
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.017/lib/MirrorCache/WebAPI/Plugin/Subsidiary.pm
new/MirrorCache-1.019/lib/MirrorCache/WebAPI/Plugin/Subsidiary.pm
--- old/MirrorCache-1.017/lib/MirrorCache/WebAPI/Plugin/Subsidiary.pm
2021-11-11 09:43:45.000000000 +0100
+++ new/MirrorCache-1.019/lib/MirrorCache/WebAPI/Plugin/Subsidiary.pm
2021-12-01 13:56:26.000000000 +0100
@@ -21,6 +21,7 @@
my $subsidiaries_initialized = 0;
my %subsidiary_urls;
+my %subsidiary_local;
my @regions;
sub register {
@@ -44,15 +45,23 @@
push @regions, $region;
my $obj = Mojo::URL->new($url)->to_abs;
$subsidiary_urls{$region} = $obj;
+ $subsidiary_local{$region} = 1 if $s->local;
$app->routes->get("/rest/$region" => sub {
my $c = shift;
my $file = $c->param('file');
return $c->render(status => 400) unless $file;
+ my $dm = MirrorCache::Datamodule->new->app($c->app);
+ $dm->reset($c);
+
my $req = $obj->clone;
$req->scheme($c->req->url->to_abs->scheme);
$req->path($req->path . $file);
+ my $country = $dm->country;
+ my $region = $dm->region;
$req->query('mirrorlist&json');
+ $req->query->merge(COUNTRY => $country) if $country;
+ $req->query->merge(REGION => $region) if $region;
$c->proxy->get_p($req);
});
}
@@ -84,6 +93,7 @@
for my $s (@regions) {
next if $region eq $s;
+ next if $subsidiary_local{$s};
push @res, $s;
}
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.017/lib/MirrorCache/resources/migrations/pg.sql
new/MirrorCache-1.019/lib/MirrorCache/resources/migrations/pg.sql
--- old/MirrorCache-1.017/lib/MirrorCache/resources/migrations/pg.sql
2021-11-11 09:43:45.000000000 +0100
+++ new/MirrorCache-1.019/lib/MirrorCache/resources/migrations/pg.sql
2021-12-01 13:56:26.000000000 +0100
@@ -13,6 +13,7 @@
scan_requested timestamp, -- when it was determined that scan is needed
scan_scheduled timestamp, -- when scan job was created (scheduled)
scan_last timestamp, -- when scan job started
+ hash_last_import timestamp,
files int,
size bigint
);
@@ -111,7 +112,8 @@
create table if not exists subsidiary (
region varchar(2) PRIMARY KEY,
hostname varchar(128) NOT NULL,
- uri varchar(128) default ''
+ uri varchar(128) default '',
+ local boolean default 'f'
);
create table if not exists audit_event (
@@ -227,3 +229,6 @@
-- 14 up
create index if not exists folder_sync_requested_idx on folder(sync_requested,
wanted, sync_scheduled);
create index if not exists folder_scan_requested_idx on folder(scan_requested,
wanted, scan_scheduled);
+-- 15 up
+alter table subsidiary add column if not exists local boolean default 'f';
+alter table folder add column if not exists hash_last_import timestamp;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.017/t/environ/01-smoke-nogeo.sh
new/MirrorCache-1.019/t/environ/01-smoke-nogeo.sh
--- old/MirrorCache-1.017/t/environ/01-smoke-nogeo.sh 2021-11-11
09:43:45.000000000 +0100
+++ new/MirrorCache-1.019/t/environ/01-smoke-nogeo.sh 2021-12-01
13:56:26.000000000 +0100
@@ -30,13 +30,13 @@
$mc/backstage/job folder_sync_schedule
$mc/backstage/shoot
-$mc/backstage/job -e mirror_scan -a '["/folder1"]'
+$mc/backstage/job mirror_scan_schedule
$mc/backstage/shoot
-$mc/db/sql "select * from minion_jobs order by id"
-
$mc/curl /download/folder1/ | grep file1.1.dat
$mc/curl -I /download/folder1/file1.1.dat | grep -C10 302 | grep -E
"($($ap7/print_address)|$($ap8/print_address))"
+cnt=$($mc/sql "select count(*) from minion_jobs where args = '[\" \"]'")
+test $cnt == 0
###################################
# test files are removed properly
@@ -44,7 +44,6 @@
# resync the folder
$mc/backstage/job folder_sync_schedule
-$mc/backstage/job -e mirror_probe -a '["/folder1"]'
$mc/backstage/shoot
$mc/curl -s /download/folder1/ | grep file1.1.dat || :
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.017/t/environ/01-smoke-subtree.sh
new/MirrorCache-1.019/t/environ/01-smoke-subtree.sh
--- old/MirrorCache-1.017/t/environ/01-smoke-subtree.sh 2021-11-11
09:43:45.000000000 +0100
+++ new/MirrorCache-1.019/t/environ/01-smoke-subtree.sh 2021-12-01
13:56:26.000000000 +0100
@@ -4,7 +4,7 @@
mc=$(environ mc $(pwd))
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=0
-$mc/gen_env
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=$MIRRORCACHE_SCHEDULE_RETRY_INTERVAL
+$mc/gen_env
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=$MIRRORCACHE_SCHEDULE_RETRY_INTERVAL
MIRRORCACHE_TOP_FOLDERS="'f1 f2 f3'"
$mc/start
$mc/status
@@ -36,6 +36,7 @@
$mc/curl -Is /download/updates/f1/f1.1.dat
$mcsub/curl -Is /download/f1/f1.1.dat
+$mc/curl -Is /download/updates/f3/f1.1.dat
$mc/sql_test 0 == "select count(*) from stat where path like '/f1%'"
$mc/sql_test 2 == "select count(*) from stat where path like '/updates/f1%'"
@@ -65,7 +66,6 @@
# resync the folder
$mc/backstage/job folder_sync_schedule
-$mc/backstage/job -e mirror_probe -a '["/updates/f1"]'
$mc/backstage/shoot
if $mc/curl -s /download/updates/f1/ | grep f1.1.dat ; then
@@ -82,4 +82,18 @@
$mcsub/curl -s /download/f1/f2.1.dat.metalink | grep '<url type="http"
location="US" preference="100">http://127.0.0.1:1304/updates/f1/f2.1.dat</url>'
# test unknown file is rendered with render_local()
-$mcsub/curl -IL /download/f2/f2.1.dat | grep -i '200 OK'
+$mcsub/curl -I /download/f2/f2.1.dat | grep -i '200 OK'
+# test mirrorlist for unknown file
+$mcsub/curl /f3/f1.1.dat.mirrorlist | grep
http://127.0.0.1:1314/updates/f3/f1.1.dat
+
+$mc/backstage/job folder_sync_schedule_from_misses
+$mc/backstage/job folder_sync_schedule
+$mc/backstage/shoot
+$mc/backstage/job mirror_scan_schedule
+$mc/backstage/shoot
+
+# test file which was unknown is now redirected
+$mcsub/curl -I /download/f2/f2.1.dat | grep -i '302 Found'
+# test mirrorlist for file which was unknown
+$mcsub/curl /download/f3/f1.1.dat.mirrorlist | grep
'href="http://127.0.0.1:1314/updates/f3/f1.1.dat";'
+
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.017/t/environ/02-files-hashes-import.sh
new/MirrorCache-1.019/t/environ/02-files-hashes-import.sh
--- old/MirrorCache-1.017/t/environ/02-files-hashes-import.sh 2021-11-11
09:43:45.000000000 +0100
+++ new/MirrorCache-1.019/t/environ/02-files-hashes-import.sh 2021-12-01
13:56:26.000000000 +0100
@@ -71,4 +71,9 @@
echo /download shouldnt be shown when MIRRORCACHE_TOP_FOLDERS is set and
MIRRORCACHE_BRANDING==SUSE
mc9/curl -s /folder1/file1.1.dat.mirrorlist | grep /download || rc=$?
test $rc -gt 0
+
+echo Erase size in file and make sure it is taken from hash
+mc9/sql 'update file set size=0, mtime=null'
+mc9/curl -sL /folder1/file1.1.dat.metalink | grep -F -C10 '<size>11</size>'
+
echo success
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.017/t/environ/02-files.sh
new/MirrorCache-1.019/t/environ/02-files.sh
--- old/MirrorCache-1.017/t/environ/02-files.sh 2021-11-11 09:43:45.000000000
+0100
+++ new/MirrorCache-1.019/t/environ/02-files.sh 2021-12-01 13:56:26.000000000
+0100
@@ -41,7 +41,7 @@
# remove a file from one mirror
rm $ap8/dt/folder1/file2.1.dat
-# this file is different size on one mirrors
+# this file is different size on one mirror
echo 1 > $ap8/dt/folder1/file1.dat
# force scan
@@ -69,7 +69,7 @@
$mc/curl -I /download/folder1/file1.dat | grep 200
# make root the same size of folder1/file1.dat
-echo 1 > $mc/dt/folder1/file1.dat
+cp $ap7/dt/folder1/file1.dat $mc/dt/folder1/file1.dat
$mc/backstage/job mirror_scan_schedule_from_path_errors
$mc/backstage/job folder_sync_schedule
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.017/t/environ/03-geo.sh
new/MirrorCache-1.019/t/environ/03-geo.sh
--- old/MirrorCache-1.017/t/environ/03-geo.sh 2021-11-11 09:43:45.000000000
+0100
+++ new/MirrorCache-1.019/t/environ/03-geo.sh 2021-12-01 13:56:26.000000000
+0100
@@ -27,6 +27,12 @@
$mc/backstage/job mirror_scan_schedule
$mc/backstage/shoot
+# check unknown country
+$mc/curl --interface 127.0.0.4 -I /download/folder1/file1.1.dat?COUNTRY=xx
+$mc/curl --interface 127.0.0.4 -I /download/folder1/file1.1.dat?COUNTRY=xx |
grep -C10 302 | grep -E '1304|1314|1324'
+$mc/curl --interface 127.0.0.4 -i
/download/folder1/file1.1.dat.mirrorlist?COUNTRY=xx | grep -C10 1304 | grep
-C10 1314| grep 1324
+
+# check country routing
$mc/curl --interface 127.0.0.4 -I /download/folder1/file1.1.dat | grep 1324
$mc/curl --interface 127.0.0.3 -I /download/folder1/file1.1.dat | grep 1314
$mc/curl --interface 127.0.0.2 -I /download/folder1/file1.1.dat | grep 1304
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.017/t/environ/03-headquarter-subsidiaries-hashes.sh
new/MirrorCache-1.019/t/environ/03-headquarter-subsidiaries-hashes.sh
--- old/MirrorCache-1.017/t/environ/03-headquarter-subsidiaries-hashes.sh
1970-01-01 01:00:00.000000000 +0100
+++ new/MirrorCache-1.019/t/environ/03-headquarter-subsidiaries-hashes.sh
2021-12-01 13:56:26.000000000 +0100
@@ -0,0 +1,75 @@
+#!lib/test-in-container-environ.sh
+set -ex
+
+# environ by number:
+# 9 - headquarter
+# 6 - NA subsidiary
+# 7 - EU subsidiary
+# 8 - ASIA subsidiary
+
+for i in 6 7 8 9; do
+ x=$(environ mc$i $(pwd))
+ mkdir -p $x/dt/{folder1,folder2,folder3}
+ echo $x/dt/{folder1,folder2,folder3}/{file1.1,file2.1}.dat | xargs -n 1
touch
+ echo 1111111111 > $x/dt/folder1/file1.1.dat
+ eval mc$i=$x
+done
+
+hq_address=$($mc9/print_address)
+na_address=$($mc6/print_address)
+na_interface=127.0.0.2
+eu_address=$($mc7/print_address)
+eu_interface=127.0.0.3
+as_address=$($mc8/print_address)
+as_interface=127.0.0.4
+
+# deploy db
+$mc9/gen_env MIRRORCACHE_TOP_FOLDERS="'folder1 folder2 folder3'"
MIRRORCACHE_HASHES_COLLECT=1
+$mc9/backstage/shoot
+
+$mc9/db/sql "insert into subsidiary(hostname,region,uri) select
'$na_address','na','/download'"
+$mc9/db/sql "insert into subsidiary(hostname,region,uri) select
'$eu_address','eu','/download'"
+$mc9/db/sql "insert into subsidiary(hostname,region,uri) select
'$as_address','as','/download'"
+
+$mc9/start
+
+$mc6/gen_env MIRRORCACHE_REGION=na MIRRORCACHE_HASHES_IMPORT=1
MIRRORCACHE_HEADQUARTER=$($mc9/print_address)
+$mc6/start
+$mc7/gen_env MIRRORCACHE_REGION=eu MIRRORCACHE_HASHES_IMPORT=1
MIRRORCACHE_HEADQUARTER=$($mc9/print_address)
+$mc7/start
+$mc8/gen_env MIRRORCACHE_REGION=as MIRRORCACHE_HASHES_IMPORT=1
MIRRORCACHE_HEADQUARTER=$($mc9/print_address)
+$mc8/start
+
+
+$mc9/curl -I /download/folder1/file1.1.dat.mirrorlist
+$mc9/backstage/job folder_sync_schedule_from_misses
+$mc9/backstage/job folder_sync_schedule
+$mc9/backstage/shoot
+$mc9/backstage/shoot -q hashes
+
+$mc9/curl -I /download/folder1/file1.1.dat.mirrorlist
+# these calls normally called by ajax in mirrorlist UI
+$mc6/curl -I '/download/folder1/file1.1.dat?mirrorlist&json'
+$mc7/curl -I '/download/folder1/file1.1.dat?mirrorlist&json'
+$mc8/curl -I '/download/folder1/file1.1.dat?mirrorlist&json'
+
+for i in 6 7 8 9; do
+ mc$i/backstage/job folder_sync_schedule_from_misses
+ mc$i/backstage/job folder_sync_schedule
+ mc$i/backstage/shoot
+ mc$i/backstage/shoot -q hashes
+ test b2c5860a03d2c4f1f049a3b2409b39a8 == $(mc$i/db/sql 'select md5 from
hash where file_id=1')
+done
+
+echo Step 2. Add more files to folder1 and make sure only new hashes are
transfered
+
+for i in 9 6 7 8; do
+ echo 1111111112 > mc$i/dt/folder1/file1.1.dat
+ echo 1111111112 > mc$i/dt/folder1/file4.1.dat
+ mc$i/backstage/job -e folder_sync -a '["/folder1"]'
+ mc$i/backstage/shoot
+ mc$i/backstage/shoot -q hashes
+ test $(mc$i/db/sql "select md5 from hash where file_id=3") ==
$(mc$i/db/sql 'select md5 from hash where file_id=1')
+done
+
+
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.017/t/environ/03-headquarter-subsidiaries-mirrorlist.sh
new/MirrorCache-1.019/t/environ/03-headquarter-subsidiaries-mirrorlist.sh
--- old/MirrorCache-1.017/t/environ/03-headquarter-subsidiaries-mirrorlist.sh
2021-11-11 09:43:45.000000000 +0100
+++ new/MirrorCache-1.019/t/environ/03-headquarter-subsidiaries-mirrorlist.sh
2021-12-01 13:56:26.000000000 +0100
@@ -40,6 +40,7 @@
na_interface=127.0.0.2
eu_address=$($mc7/print_address)
eu_interface=127.0.0.3
+as_interface=127.0.0.4
$mc9/gen_env "MIRRORCACHE_TOP_FOLDERS='folder1 folder2 folder3'" \
MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT=0 \
@@ -71,8 +72,9 @@
done
-curl -s "http://$na_address/folder1/file1.1.dat?mirrorlist&json";
-curl -s "http://$eu_address/folder1/file1.1.dat?mirrorlist&json";
+curl -s "http://$na_address/folder1/file1.1.dat?mirrorlist&json"; |
grep -F
'{"l1":[{"location":"US","url":"http:\/\/127.0.0.1:1264\/folder1\/file1.1.dat"}],"l2":[{"location":"CA","url":"http:\/\/127.0.0.1:1274\/folder1\/file1.1.dat"}],"l3":[]}'
+curl -s "http://$na_address/folder1/file1.1.dat?mirrorlist&json&COUNTRY=ca"; |
grep -F
'{"l1":[{"location":"CA","url":"http:\/\/127.0.0.1:1274\/folder1\/file1.1.dat"}],"l2":[{"location":"US","url":"http:\/\/127.0.0.1:1264\/folder1\/file1.1.dat"}],"l3":[]}'
+curl -s "http://$eu_address/folder1/file1.1.dat?mirrorlist&json"; | grep -F
'{"l1":[],"l2":[],"l3":[{"location":"DE"'
curl -s "http://$hq_address/rest/eu?file=/folder1/file1.1.dat"; | grep -C50
$($ap5/print_address) | grep $($ap6/print_address)
curl -s "http://$hq_address/rest/na?file=/folder1/file1.1.dat"; | grep -C50
$($ap3/print_address) | grep $($ap4/print_address)
@@ -97,17 +99,17 @@
mc9/backstage/job -e folder_sync -a '["/folder2"]'
mc9/backstage/shoot
-curl -sL --interface 127.0.0.4
http://$hq_address/folder2/file1.1.dat.mirrorlist | grep 'file1.1.dat'
+curl -sL --interface $as_interface
http://$hq_address/folder2/file1.1.dat.mirrorlist | grep 'file1.1.dat'
mc9/backstage/job -e mirror_scan_schedule_from_misses
mc9/backstage/job -e mirror_scan_schedule
mc9/backstage/shoot
-curl -sL --interface 127.0.0.4
http://$hq_address/folder2/file1.1.dat.mirrorlist | grep -C10
$($ap1/print_address) | grep $($ap2/print_address)
+curl -sL --interface $as_interface
http://$hq_address/folder2/file1.1.dat.mirrorlist | grep -C10
$($ap1/print_address) | grep $($ap2/print_address)
###########################################
# test table demand_mirrorlist:
# if mirrorlist was requested for unknown file - all countries will be scanned
-curl -sL --interface 127.0.0.4
http://$hq_address/folder3/file1.1.dat.mirrorlist | grep 'unknown'
+curl -sL --interface $as_interface
http://$hq_address/folder3/file1.1.dat.mirrorlist | grep 'unknown'
mc9/backstage/job -e folder_sync_schedule_from_misses
mc9/backstage/job -e folder_sync_schedule
@@ -115,10 +117,11 @@
mc9/backstage/job -e mirror_scan_schedule
mc9/backstage/shoot
-curl -sL --interface 127.0.0.4
http://$hq_address/folder3/file1.1.dat.mirrorlist | grep -C10
$($ap1/print_address) | grep $($ap2/print_address)
+curl -sL --interface $as_interface
http://$hq_address/folder3/file1.1.dat.mirrorlist | grep -C10
$($ap1/print_address) | grep $($ap2/print_address)
###########################################
curl -Is --interface $eu_interface http://$eu_address/folder1/file1.1.dat |
grep -C10 302 | grep $($ap5/print_address)
curl -Is --interface $na_interface http://$eu_address/folder1/file1.1.dat |
grep -C10 302 | grep -E "$($ap5/print_address)|$($ap6/print_address)"
+curl -siL --interface $eu_interface
http://$hq_address/folder1/file1.1.dat?COUNTRY=jp | grep -E 'Mirrors|http'
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.017/t/environ/03-headquarter-subsidiaries-samedb.sh
new/MirrorCache-1.019/t/environ/03-headquarter-subsidiaries-samedb.sh
--- old/MirrorCache-1.017/t/environ/03-headquarter-subsidiaries-samedb.sh
2021-11-11 09:43:45.000000000 +0100
+++ new/MirrorCache-1.019/t/environ/03-headquarter-subsidiaries-samedb.sh
2021-12-01 13:56:26.000000000 +0100
@@ -85,8 +85,8 @@
mc$i/backstage/shoot
done
-# curl -s "http://$hq_address/rest/eu?file=/folder1/file1.1.dat"; | grep -C50
$($ap5/print_address) | grep $($ap6/print_address)
-# curl -s "http://$hq_address/rest/na?file=/folder1/file1.1.dat"; | grep -C50
$($ap3/print_address) | grep $($ap4/print_address)
+curl -s "http://$hq_address/rest/eu?file=/folder1/file1.1.dat"; | grep -C50
$($ap5/print_address) | grep $($ap6/print_address)
+curl -s "http://$hq_address/rest/na?file=/folder1/file1.1.dat"; | grep -C50
$($ap3/print_address) | grep $($ap4/print_address)
curl -sL http://$hq_address/folder1/file1.1.dat.metalink | grep file1.1.dat
curl -sL --interface 127.0.0.4 http://$hq_address/folder1/file1.1.dat.metalink
| grep file1.1.dat
@@ -105,10 +105,12 @@
mc9/backstage/job -e folder_sync -a '["/folder2"]'
mc9/backstage/shoot
-curl -sL --interface 127.0.0.4
http://$hq_address/folder2/file1.1.dat.mirrorlist | grep 'file1.1.dat'
+curl -sL --interface $as_interface
http://$hq_address/folder2/file1.1.dat.mirrorlist | grep 'file1.1.dat'
mc9/backstage/job -e mirror_scan_schedule_from_misses
mc9/backstage/job -e mirror_scan_schedule
mc9/backstage/shoot
-curl -sL --interface 127.0.0.4
http://$hq_address/folder2/file1.1.dat.mirrorlist | grep -C10
$($ap1/print_address) | grep $($ap2/print_address)
+curl -sL --interface $as_interface
http://$hq_address/folder2/file1.1.dat.mirrorlist | grep -C10
$($ap1/print_address) | grep $($ap2/print_address)
+
+test 2 == $(curl -sL --interface $as_interface
http://$hq_address/folder2/file1.1.dat.mirrorlist?COUNTRY=br | grep -A5
'Mirrors which handle this country' | grep '(BR)' | wc -l)
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.017/t/environ/06-remote-geo-root-country-same-region.sh
new/MirrorCache-1.019/t/environ/06-remote-geo-root-country-same-region.sh
--- old/MirrorCache-1.017/t/environ/06-remote-geo-root-country-same-region.sh
2021-11-11 09:43:45.000000000 +0100
+++ new/MirrorCache-1.019/t/environ/06-remote-geo-root-country-same-region.sh
2021-12-01 13:56:26.000000000 +0100
@@ -44,6 +44,9 @@
$mc/curl --interface 127.0.0.2
'/download/folder1/file1.1.dat.metalink?REGION=eu' | grep -A1 $DE_ADDRESS |
grep $CZ_ADDRESS
$mc/curl --interface 127.0.0.4
'/download/folder1/file1.1.dat.metalink?REGION=eu' | grep -A1 $CZ_ADDRESS |
grep $DE_ADDRESS
+# check requests from na are served from DE_ADDRESS
+$mc/curl /download/folder1/file1.1.dat.metalink?REGION=na | grep -A1
$DE_ADDRESS | grep $CZ_ADDRESS
+
#########################################
echo test scan is scheduled when metadata is missing
$mc/curl -Is --interface 127.0.0.3
'/download/folder2/file1.1.dat.metalink?COUNTRY=de'
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.017/t/environ/10-remote-redirect.sh
new/MirrorCache-1.019/t/environ/10-remote-redirect.sh
--- old/MirrorCache-1.017/t/environ/10-remote-redirect.sh 2021-11-11
09:43:45.000000000 +0100
+++ new/MirrorCache-1.019/t/environ/10-remote-redirect.sh 2021-12-01
13:56:26.000000000 +0100
@@ -26,7 +26,8 @@
MOJO_CA_FILE=$(pwd)/ca/ca.pem \
MOJO_REVERSE_PROXY=1 \
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=2 \
- MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT=0 \
+ MIRRORCACHE_COUNTRY_RESCAN_TIMEOUT=1 \
+ MIRRORCACHE_BACKSTAGE_WORKERS=8 \
MIRRORCACHE_METALINK_PUBLISHER_URL=http://metalink_publisher.net
$mc/start
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.017/t/environ/15-local-symlink-subtree.sh
new/MirrorCache-1.019/t/environ/15-local-symlink-subtree.sh
--- old/MirrorCache-1.017/t/environ/15-local-symlink-subtree.sh 2021-11-11
09:43:45.000000000 +0100
+++ new/MirrorCache-1.019/t/environ/15-local-symlink-subtree.sh 2021-12-01
13:56:26.000000000 +0100
@@ -26,6 +26,10 @@
mkdir -p $ap8/dt/updates/tool/v1
cp $mc/dt/folder1/* $ap8/dt/updates/tool/v1/
+cp -r $mc/dt/folder2/ $ap7/dt/
+mkdir -p $ap8/dt/updates/tool/v2
+cp $mc/dt/folder2/* $ap8/dt/updates/tool/v2/
+
$ap7/start
$ap7/curl /folder1/ | grep file1.1.dat
@@ -44,6 +48,7 @@
$mcsub/start
$mc/curl -Is /download/updates/tool/v1/file1.1.dat.mirrorlist
+$mc/curl -Is /download/folder2/file1.1.dat.mirrorlist
$mc/backstage/job folder_sync_schedule_from_misses
$mc/backstage/job folder_sync_schedule
@@ -57,6 +62,16 @@
$mcsub/curl /tool/v1/file1.1.dat.mirrorlist | grep -C 10
$($ap7/print_address)/folder1/file1.1.dat | grep
$($ap8/print_address)/updates/tool/v1/file1.1.dat
$mcsub/curl /tool/v1/file1.1.dat.metalink | grep -C 10
$($ap7/print_address)/folder1/file1.1.dat | grep
$($ap8/print_address)/updates/tool/v1/file1.1.dat
+$mcsub/curl -I /tool/v2/file1.1.dat.metalink | grep 425
+
+$mc/backstage/job folder_sync_schedule_from_misses
+$mc/backstage/job folder_sync_schedule
+$mc/backstage/shoot
+$mc/backstage/job mirror_scan_schedule
+$mc/backstage/shoot
+
+$mcsub/curl /tool/v2/file1.1.dat.metalink | grep -C 10
$($ap7/print_address)/folder2/file1.1.dat | grep
$($ap8/print_address)/updates/tool/v2/file1.1.dat
+
rc=0
$mcsub/curl / | grep tool1 || rc=$?
test $rc -gt 0
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.017/t/lib/environ/mc/source/backstage/start.sh.m4
new/MirrorCache-1.019/t/lib/environ/mc/source/backstage/start.sh.m4
--- old/MirrorCache-1.017/t/lib/environ/mc/source/backstage/start.sh.m4
2021-11-11 09:43:45.000000000 +0100
+++ new/MirrorCache-1.019/t/lib/environ/mc/source/backstage/start.sh.m4
2021-12-01 13:56:26.000000000 +0100
@@ -8,7 +8,7 @@
(
cd __workdir
-__srcdir/script/mirrorcache backstage run -C 1 -j
${MIRRORCACHE_BACKSTAGE_WORKERS:-1} >> __workdir/.cout 2>> __workdir/.cerr &
+__srcdir/script/mirrorcache backstage run -C 1 -j
${MIRRORCACHE_BACKSTAGE_WORKERS:-2} >> __workdir/.cout 2>> __workdir/.cerr &
pid=$!
echo $pid > __workdir/.pid
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.017/templates/mirrorlist.html.ep
new/MirrorCache-1.019/templates/mirrorlist.html.ep
--- old/MirrorCache-1.017/templates/mirrorlist.html.ep 2021-11-11
09:43:45.000000000 +0100
+++ new/MirrorCache-1.019/templates/mirrorlist.html.ep 2021-12-01
13:56:26.000000000 +0100
@@ -8,10 +8,10 @@
% if ($regions && @$regions) {
% content_for 'ready_function' => begin
% my $i = 0;
-% for my $region (@$regions) {
-% if ($region) {
-% my $url = "/rest/$region?file=$cur_path";
- loadMirrorlist("<%= $url %>", "ul1000", "ul1001", "ul1002");
+% for my $reg (@$regions) {
+% if ($reg) {
+% my $url = "/rest/$reg?file=$cur_path";
+ loadMirrorlist("<%= $url %>", "<%= $country %>", "<%= $region %>",
"ul1000", "ul1001", "ul1002");
% }
% $i = $i+1;
% }
++++++ MirrorCache.obsinfo ++++++
--- /var/tmp/diff_new_pack.tLk4xw/_old 2021-12-09 19:46:23.645155923 +0100
+++ /var/tmp/diff_new_pack.tLk4xw/_new 2021-12-09 19:46:23.649155925 +0100
@@ -1,5 +1,5 @@
name: MirrorCache
-version: 1.017
-mtime: 1636620225
-commit: c6b91b1a3892d5972354173af00be3cdb8254bdc
+version: 1.019
+mtime: 1638363386
+commit: 3ddb8020f3772d4c49953e24232946d874c85e90
