Script 'mail_helper' called by obssrc
Hello community,
here is the log from the commit of package MirrorCache for openSUSE:Factory
checked in at 2024-08-01 22:05:36
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/MirrorCache (Old)
and /work/SRC/openSUSE:Factory/.MirrorCache.new.7232 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "MirrorCache"
Thu Aug 1 22:05:36 2024 rev:43 rq:1190907 version:1.079
Changes:
--------
--- /work/SRC/openSUSE:Factory/MirrorCache/MirrorCache.changes 2024-06-27
16:05:08.234656119 +0200
+++ /work/SRC/openSUSE:Factory/.MirrorCache.new.7232/MirrorCache.changes
2024-08-01 22:06:25.245394943 +0200
@@ -1,0 +2,15 @@
+Thu Jul 25 07:40:38 UTC 2024 - Andrii Nikitin <andrii.niki...@suse.com>
+
+- Update to version 1.079:
+ * Show size of projects (#502)
+
+-------------------------------------------------------------------
+Thu Jul 18 08:23:40 UTC 2024 - Andrii Nikitin <andrii.niki...@suse.com>
+
+- Update to version 1.078:
+ * Respect If-Modified-Since header (#501)
+ * Add content-disposition header in torrent responses (#497)
+ * Reuse file extention instead of url parameter in redirects (#497)
+ * Allow caches to serve files while re-checking (#492)
+
+-------------------------------------------------------------------
Old:
----
MirrorCache-1.077.obscpio
New:
----
MirrorCache-1.079.obscpio
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ MirrorCache.spec ++++++
--- /var/tmp/diff_new_pack.Zlvxh3/_old 2024-08-01 22:06:25.873420858 +0200
+++ /var/tmp/diff_new_pack.Zlvxh3/_new 2024-08-01 22:06:25.873420858 +0200
@@ -22,7 +22,7 @@
%define main_requires %{assetpack_requires} perl(Carp) perl(DBD::Pg) >= 3.7.4
perl(DBI) >= 1.632 perl(DBIx::Class) >= 0.082801
perl(DBIx::Class::DynamicDefault) perl(DateTime) perl(Encode) perl(Time::Piece)
perl(Time::Seconds) perl(Time::ParseDate) perl(DateTime::Format::Pg)
perl(Exporter) perl(File::Basename) perl(LWP::UserAgent) perl(Mojo::Base)
perl(Mojo::ByteStream) perl(Mojo::IOLoop) perl(Mojo::JSON) perl(Mojo::Pg)
perl(Mojo::URL) perl(Mojo::Util) perl(Mojolicious::Commands)
perl(Mojolicious::Plugin) perl(Mojolicious::Plugin::RenderFile)
perl(Mojolicious::Static) perl(Net::OpenID::Consumer) perl(POSIX)
perl(Sort::Versions) perl(URI::Escape) perl(XML::Writer) perl(base)
perl(constant) perl(diagnostics) perl(strict) perl(warnings) shadow
rubygem(sass) perl(Net::DNS) perl(LWP::Protocol::https) perl(Digest::SHA)
perl(Config::IniFiles)
%define build_requires %{assetpack_requires} rubygem(sass) tidy sysuser-shadow
sysuser-tools
Name: MirrorCache
-Version: 1.077
+Version: 1.079
Release: 0
Summary: WebApp to redirect and manage mirrors
License: GPL-2.0-or-later
++++++ MirrorCache-1.077.obscpio -> MirrorCache-1.079.obscpio ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.077/assets/javascripts/admintable.js
new/MirrorCache-1.079/assets/javascripts/admintable.js
--- old/MirrorCache-1.077/assets/javascripts/admintable.js 2024-06-20
07:43:31.000000000 +0200
+++ new/MirrorCache-1.079/assets/javascripts/admintable.js 2024-07-24
16:14:18.000000000 +0200
@@ -257,6 +257,14 @@
return htmlEscape(data);
}
+function renderAdminUnixtime(data, type, row, meta) {
+ return data ? new Date(data*1000).toISOString() : '';
+}
+
+function renderAdminReadonly(data, type, row, meta) {
+ return data ? data : '';
+}
+
function renderAdminTableSettingsList(data, type, row, meta) {
var plainText = type !== 'display';
var edit = isEditingAdminTableRow(meta);
@@ -409,6 +417,11 @@
} else {
// columnName = th.text().trim().toLowerCase().replace(/ /g, '_');
columnName = th.text().trim().toLowerCase();
+ if (columnName == 'file count') {
+ columnName = 'file cnt';
+ } else if (columnName == 'last modified') {
+ columnName = 'lm';
+ }
}
columns.push({ data: columnName });
@@ -428,6 +441,12 @@
columnDef.render = renderAdminTableValue;
}
emptyRow[columnName] = "";
+ } else if (th.hasClass('col_ro')) {
+ columnDef.render = renderAdminReadonly;
+ emptyRow.settings = {};
+ } else if (th.hasClass('col_unixtime')) {
+ columnDef.render = renderAdminUnixtime;
+ emptyRow.settings = {};
} else if (th.hasClass('col_settings')) {
columnDef.render = renderAdminTableSettings;
emptyRow.settings = {};
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.077/lib/MirrorCache/Datamodule.pm
new/MirrorCache-1.079/lib/MirrorCache/Datamodule.pm
--- old/MirrorCache-1.077/lib/MirrorCache/Datamodule.pm 2024-06-20
07:43:31.000000000 +0200
+++ new/MirrorCache-1.079/lib/MirrorCache/Datamodule.pm 2024-07-24
16:14:18.000000000 +0200
@@ -356,8 +356,7 @@
my $param = $c->req->params;
if (!$skip_xtra && $self->_original_path =~
m/(\.metalink|\.meta4|\.zsync|\.mirrorlist|\.torrent|\.magnet|\.btih)$/) {
$xtra = $1;
- $xtra = substr($xtra, 1);
- $param->append($xtra => 1);
+ $url = $url . $xtra;
}
if (my $version =
Directory::Scanner::OBSMediaVersion::parse_version($url)) {
$c->res->headers->add('X-MEDIA-VERSION' => $version);
@@ -616,8 +615,9 @@
) {
$self->must_render_from_root(1);
my $time = ~time() & 0xff;
+ my $time2 = 60*60 + $time; # allow caches to serve content for 1h
while they re-check
- $self->c->res->headers->cache_control("public, max-age=$time");
+ $self->c->res->headers->cache_control("public, max-age=$time
stale-while-revalidate=$time2 stale-if-error=86400");
}
my ($ext) = $path =~ /([^.]+)$/;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.077/lib/MirrorCache/Schema/Result/Project.pm
new/MirrorCache-1.079/lib/MirrorCache/Schema/Result/Project.pm
--- old/MirrorCache-1.077/lib/MirrorCache/Schema/Result/Project.pm
2024-06-20 07:43:31.000000000 +0200
+++ new/MirrorCache-1.079/lib/MirrorCache/Schema/Result/Project.pm
2024-07-24 16:14:18.000000000 +0200
@@ -35,6 +35,9 @@
is_nullable => 1
},
db_sync_full_every => { data_type => "integer", is_nullable => 1 },
+ size => { data_type => "bigint", is_nullable => 1 },
+ file_cnt => { data_type => "bigint", is_nullable => 1 },
+ lm => { data_type => "bigint", is_nullable => 1 },
);
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.077/lib/MirrorCache/Schema/ResultSet/Folder.pm
new/MirrorCache-1.079/lib/MirrorCache/Schema/ResultSet/Folder.pm
--- old/MirrorCache-1.077/lib/MirrorCache/Schema/ResultSet/Folder.pm
2024-06-20 07:43:31.000000000 +0200
+++ new/MirrorCache-1.079/lib/MirrorCache/Schema/ResultSet/Folder.pm
2024-07-24 16:14:18.000000000 +0200
@@ -404,4 +404,21 @@
$prep->execute($pathfrom, $pathto, $pathto);
}
+sub calculate_disk_usage {
+ my ($self, $path) = @_;
+
+ my $rsource = $self->result_source;
+ my $schema = $rsource->schema;
+ my $dbh = $schema->storage->dbh;
+
+ $path = $path . "%";
+
+ my $sql = "select sum(file.size) size, count(*) file_cnt, max(mtime) lm
from file join folder on folder_id = folder.id where path like ?";
+ my $prep = $dbh->prepare($sql);
+ $prep->execute($path);
+ my $res = $dbh->selectrow_hashref($prep);
+ return ($res->{size}, $res->{file_cnt}, $res->{lm});
+}
+
+
1;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.077/lib/MirrorCache/Schema/ResultSet/Project.pm
new/MirrorCache-1.079/lib/MirrorCache/Schema/ResultSet/Project.pm
--- old/MirrorCache-1.077/lib/MirrorCache/Schema/ResultSet/Project.pm
2024-06-20 07:43:31.000000000 +0200
+++ new/MirrorCache-1.079/lib/MirrorCache/Schema/ResultSet/Project.pm
2024-07-24 16:14:18.000000000 +0200
@@ -22,5 +22,22 @@
$prep->execute($project_id);
}
+sub update_disk_usage {
+ my ($self, $path, $size, $file_cnt, $lm) = @_;
+
+ my $rsource = $self->result_source;
+ my $schema = $rsource->schema;
+ my $dbh = $schema->storage->dbh;
+
+ my $sql = << "END_SQL";
+update project set
+size = ?,
+file_cnt = ?,
+lm = (case when lm > ? then lm else ? end)
+where path = ?
+END_SQL
+ my $prep = $dbh->prepare($sql);
+ $prep->execute($size, $file_cnt, $lm, $lm, $path);
+}
1;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.077/lib/MirrorCache/Task/ReportProjectSize.pm
new/MirrorCache-1.079/lib/MirrorCache/Task/ReportProjectSize.pm
--- old/MirrorCache-1.077/lib/MirrorCache/Task/ReportProjectSize.pm
1970-01-01 01:00:00.000000000 +0100
+++ new/MirrorCache-1.079/lib/MirrorCache/Task/ReportProjectSize.pm
2024-07-24 16:14:18.000000000 +0200
@@ -0,0 +1,45 @@
+# Copyright (C) 2024 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package MirrorCache::Task::ReportProjectSize;
+use Mojo::Base 'Mojolicious::Plugin';
+
+
+sub register {
+ my ($self, $app) = @_;
+ $app->minion->add_task(report_project_size => sub { _run($app, @_) });
+}
+
+sub _run {
+ my ($app, $job, $path) = @_;
+ return $job->fail('Empty path is not allowed') unless $path;
+
+ my $minion = $app->minion;
+ return $job->finish('Previous report job is still active for ' . $path)
+ unless my $guard = $minion->guard('report_project_size_' . $path,
30*60);
+
+ my $realpath = $app->mc->root->rootpath($path);
+ return $job->fail('Path not found: ' . $path) unless $realpath;
+
+ my ($size, $file_cnt, $lm) =
$app->schema->resultset('Folder')->calculate_disk_usage($path);
+
+ $job->note("total size" => $size, "file count" => $file_cnt, "last
modified" => $lm);
+
+ $app->schema->resultset('Project')->update_disk_usage($path, $size,
$file_cnt, $lm);
+
+ return $job->finish;
+}
+
+1;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.077/lib/MirrorCache/Task/ReportProjectSizeSchedule.pm
new/MirrorCache-1.079/lib/MirrorCache/Task/ReportProjectSizeSchedule.pm
--- old/MirrorCache-1.077/lib/MirrorCache/Task/ReportProjectSizeSchedule.pm
1970-01-01 01:00:00.000000000 +0100
+++ new/MirrorCache-1.079/lib/MirrorCache/Task/ReportProjectSizeSchedule.pm
2024-07-24 16:14:18.000000000 +0200
@@ -0,0 +1,49 @@
+# Copyright (C) 2024 SUSE LLC
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 2 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, see <http://www.gnu.org/licenses/>.
+
+package MirrorCache::Task::ReportProjectSizeSchedule;
+use Mojo::Base 'Mojolicious::Plugin';
+use DateTime;
+
+sub register {
+ my ($self, $app) = @_;
+ $app->minion->add_task(report_project_size_schedule => sub { _run($app,
@_) });
+}
+
+my $HASHES_QUEUE = $ENV{MIRRORCACHE_HASHES_QUEUE} // 'hashes';
+my $DELAY =
int($ENV{MIRRORCACHE_SCHEDULE_REPORT_PROJECT_SIZE_SCHEDULE_RETRY_INTERVAL} //
12 * 60 * 60);
+
+sub _run {
+ my ($app, $job, $once) = @_;
+ my $minion = $app->minion;
+ return $job->finish('Previous report job is still active')
+ unless my $guard = $minion->guard('report_project_size_schedule',
2*60*60);
+
+ my $schema = $app->schema;
+ my @projects = $schema->resultset('Project')->all;
+ my $sunday = DateTime->today->day_of_week;
+ $sunday = 0 unless $sunday == 7;
+ for my $proj (@projects) {
+ my $path = $proj->path;
+ my $prio = $proj->prio // 1;
+ next if !$sunday && $prio < 1;
+ $minion->enqueue('report_project_size' => [ $proj->path ] => {priority
=> 15} => {queue => $HASHES_QUEUE});
+ }
+
+ return $job->finish if $once;
+ return $job->retry({delay => $DELAY});
+}
+
+1;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.077/lib/MirrorCache/WebAPI/Controller/Rest/Table.pm
new/MirrorCache-1.079/lib/MirrorCache/WebAPI/Controller/Rest/Table.pm
--- old/MirrorCache-1.077/lib/MirrorCache/WebAPI/Controller/Rest/Table.pm
2024-06-20 07:43:31.000000000 +0200
+++ new/MirrorCache-1.079/lib/MirrorCache/WebAPI/Controller/Rest/Table.pm
2024-07-24 16:14:18.000000000 +0200
@@ -40,7 +40,7 @@
},
Project => {
keys => [['id'], ['name'],],
- cols => ['id', 'name', 'path', 'prio'],
+ cols => ['id', 'name', 'path', 'prio', 'size', 'file cnt', 'lm'],
},
);
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.077/lib/MirrorCache/WebAPI/Plugin/Backstage.pm
new/MirrorCache-1.079/lib/MirrorCache/WebAPI/Plugin/Backstage.pm
--- old/MirrorCache-1.077/lib/MirrorCache/WebAPI/Plugin/Backstage.pm
2024-06-20 07:43:31.000000000 +0200
+++ new/MirrorCache-1.079/lib/MirrorCache/WebAPI/Plugin/Backstage.pm
2024-07-24 16:14:18.000000000 +0200
@@ -46,6 +46,10 @@
if ($app->mcconfig->mirror_provider) {
push @permanent_jobs, 'mirror_provider_sync';
}
+ eval {
+ my $projects = $app->mcproject->list;
+ push @permanent_jobs, 'report_project_size_schedule' if @$projects;
+ };
$app->plugin($_)
for (
@@ -67,6 +71,8 @@
qw(MirrorCache::Task::ProjectSyncSchedule),
qw(MirrorCache::Task::Cleanup),
qw(MirrorCache::Task::Report),
+ qw(MirrorCache::Task::ReportProjectSize),
+ qw(MirrorCache::Task::ReportProjectSizeSchedule),
qw(MirrorCache::Task::StatAggSchedule),
);
}
@@ -113,7 +119,7 @@
$app->hook(
before_server_start => sub {
- my $every = $ENV{MIRRORCACHE_PERMANENT_JOBS_CHECK_INTERVAL} // 15
* 60;
+ my $every = $ENV{MIRRORCACHE_PERMANENT_JOBS_CHECK_INTERVAL} // 5 *
60;
$self->check_permanent_jobs;
Mojo::IOLoop->next_tick(
sub {
@@ -132,6 +138,9 @@
eval {
my $app = shift->app;
my $minion = $app->minion;
+ my $guard = $minion->guard('check_permanent_jobs', 60);
+ return undef unless $guard;
+
my $jobs = $minion->jobs(
{
tasks => \@permanent_jobs,
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.077/lib/MirrorCache/WebAPI/Plugin/Dir.pm
new/MirrorCache-1.079/lib/MirrorCache/WebAPI/Plugin/Dir.pm
--- old/MirrorCache-1.077/lib/MirrorCache/WebAPI/Plugin/Dir.pm 2024-06-20
07:43:31.000000000 +0200
+++ new/MirrorCache-1.079/lib/MirrorCache/WebAPI/Plugin/Dir.pm 2024-07-24
16:14:18.000000000 +0200
@@ -23,6 +23,8 @@
use Time::Piece;
use Time::Seconds;
use Time::ParseDate;
+use Mojo::Date;
+
use Digest::Metalink;
use Digest::Meta4;
use MirrorCache::Utils;
@@ -82,6 +84,7 @@
eval {
$success =
_render_hashes($dm)
+ || _render_if_not_modified($dm)
|| _render_small($dm)
|| _set_cache_control($dm)
|| _redirect_project_ln_geo($dm)
@@ -745,6 +748,30 @@
return $c->render( 'dir', files => \@items, route => $dm->route, cur_path
=> $dir, folder_id => $id );
}
+sub _render_if_not_modified {
+ my $dm = shift;
+ return undef if $root->is_remote;
+ my $c = $dm->c;
+ my $x = $c->req->headers->if_modified_since;
+ $c->log->error($c->dumper('DIR::render_if_not_modified1', $x, ref $x,
scalar($x), ref scalar($x))) if $MCDEBUG;
+ return undef unless $x;
+ $x = scalar($x);
+ my $lms;
+ eval {
+ $lms = Mojo::Date->new($x)->epoch;
+ };
+ $c->log->error($c->dumper('DIR::render_if_not_modified2', $lms)) if
$MCDEBUG;
+ return undef unless int($lms // 0);
+ $dm->_init_path;
+ my ($path, undef) = $dm->path;
+ my $mtime = $root->is_file_mtime($path);
+ $c->log->error($c->dumper('DIR::render_if_not_modified3', $mtime)) if
$MCDEBUG;
+ return undef unless int($mtime // 0);
+ return $c->render(text => '', status => 304) if $mtime <= $lms;
+ $c->log->error($c->dumper('DIR::render_if_not_modified4', $mtime, $lms))
if $MCDEBUG;
+ return undef;
+}
+
sub _render_small {
my $dm = shift;
my $root_nfs = $mc_config->root_nfs;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.077/lib/MirrorCache/WebAPI/Plugin/Project.pm
new/MirrorCache-1.079/lib/MirrorCache/WebAPI/Plugin/Project.pm
--- old/MirrorCache-1.077/lib/MirrorCache/WebAPI/Plugin/Project.pm
2024-06-20 07:43:31.000000000 +0200
+++ new/MirrorCache-1.079/lib/MirrorCache/WebAPI/Plugin/Project.pm
2024-07-24 16:14:18.000000000 +0200
@@ -54,11 +54,13 @@
eval { #the table may be missing - no big deal (only reports might be
inaccurate if some other error occurred).
eval {
my @rows = $c->schema->resultset('Project')->search(undef, {
order_by => { -desc => [qw/prio name/] } });
+ my @projs;
# we want to cache it, so move to simpler structure
for my $r (@rows) {
my %proj = ( id => $r->id, name => $r->name, path => $r->path,
redirect => $r->redirect, prio => $r->prio );
- push @projects, \%proj;
+ push @projs, \%proj;
}
+ @projects = @projs;
$wasdberror = 0;
$initialized = 1;
};
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.077/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
new/MirrorCache-1.079/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
--- old/MirrorCache-1.077/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
2024-06-20 07:43:31.000000000 +0200
+++ new/MirrorCache-1.079/lib/MirrorCache/WebAPI/Plugin/RenderFileFromMirror.pm
2024-07-24 16:14:18.000000000 +0200
@@ -119,7 +119,7 @@
$c->log->error($c->dumper('RENDER FOLDER REAL', $realfolder_id ?
$realfolder_id : 'NULL')) if $MCDEBUG;
my $fileoriginpath = $filepath;
$fileoriginpath = $realdirname . '/' . $basename if $realdirname ne
$dirname;
- return $root->render_file($dm, $fileoriginpath, 1) if
$dm->must_render_from_root; # && $root->is_reachable;
+ return $root->render_file($dm, $fileoriginpath, 1) if
$dm->must_render_from_root && !$c->req->headers->if_modified_since; # &&
$root->is_reachable;
if ($folder || $realfolder_id) {
my $fldid = ($realfolder_id? $realfolder_id : $folder_id);
@@ -152,6 +152,22 @@
$c->res->headers->vary('Accept, COUNTRY, X-COUNTRY, Fastly-SSL');
$c->res->headers->etag($dm->etag) if defined $dm->file_size;
$c->res->headers->add('X-MEDIA-VERSION' => $dm->media_version) if
$dm->media_version;
+
+ my $mtime = $file->{mtime};
+ if ($mtime) { # Check Last Modified Since header
+ my $lms;
+ eval {
+ if (my $x = $c->req->headers->if_modified_since) {
+ $c->log->error($c->dumper('RENDER IF MODIFIED SINCE1',
$x)) if $MCDEBUG;
+ $lms = Mojo::Date->new($x)->epoch;
+ }
+ };
+ $c->log->error($c->dumper('RENDER IF MODIFIED SINCE', $lms,
$mtime)) if $MCDEBUG;
+
+ # Not Modified
+ return $c->render(status => 304, text => '') if int($lms // 0) &&
int($mtime // 0) && $mtime <= $lms;
+ }
+
my $baseurl; # just hostname + eventual urldir (without folder and
file)
my $fullurl; # baseurl with path and filename
if ($dm->metalink || $dm->meta4 || $dm->torrent || $dm->zsync ||
$dm->magnet) {
@@ -844,6 +860,7 @@
return 1;
}
+ $c->res->headers->content_disposition('attachment; filename="' .$filename.
'.btih"');
$c->render(text => "$filename " . _calc_btih($filename, $file));
return 1;
}
@@ -862,6 +879,7 @@
$filename = Mojo::Util::url_escape($filename);
$url = Mojo::Util::url_escape($url);
+ $c->res->headers->content_disposition('attachment; filename="' .$filename.
'.magnet"');
$c->render(text =>
"magnet:?xt=urn:btih:$btih&xt=urn:md5:$md5&xl=$size&dn=$filename&as=$url");
return 1;
}
@@ -911,6 +929,7 @@
$footer = $footer . 'ee';
$c->res->headers->content_length(length($header) + length($pieces) +
length($footer));
+ $c->res->headers->content_disposition('attachment; filename="' .$filename.
'.torrent"');
$c->write($header => sub () {
$c->write($pieces => sub () {
$c->write($footer => sub () {$c->finish});
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.077/lib/MirrorCache/WebAPI/Plugin/RootLocal.pm
new/MirrorCache-1.079/lib/MirrorCache/WebAPI/Plugin/RootLocal.pm
--- old/MirrorCache-1.077/lib/MirrorCache/WebAPI/Plugin/RootLocal.pm
2024-06-20 07:43:31.000000000 +0200
+++ new/MirrorCache-1.079/lib/MirrorCache/WebAPI/Plugin/RootLocal.pm
2024-07-24 16:14:18.000000000 +0200
@@ -78,6 +78,16 @@
return 0;
}
+sub is_file_mtime {
+ return 1 unless $_[1];
+ for my $root (@roots) {
+ my $f = $root->[dir] . $root_subtree . $_[1];
+ next unless -f $root->[dir] . $root_subtree . $_[1];
+ return Mojo::File->new($f)->stat->mtime;
+ }
+ return 0;
+}
+
sub is_dir {
my ($self, $path) = @_;
return 1 if !$path || $path eq '/';
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.077/lib/MirrorCache/WebAPI/Plugin/Subsidiary.pm
new/MirrorCache-1.079/lib/MirrorCache/WebAPI/Plugin/Subsidiary.pm
--- old/MirrorCache-1.077/lib/MirrorCache/WebAPI/Plugin/Subsidiary.pm
2024-06-20 07:43:31.000000000 +0200
+++ new/MirrorCache-1.079/lib/MirrorCache/WebAPI/Plugin/Subsidiary.pm
2024-07-24 16:14:18.000000000 +0200
@@ -67,7 +67,7 @@
my $c = shift;
my $file = $c->param('file');
my $https = $c->param('https');
- return $c->render(status => 400) unless $file;
+ return $c->render(text => '', status => 400) unless $file;
my $dm = MirrorCache::Datamodule->new->app($c->app);
$dm->reset($c);
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.077/lib/MirrorCache/resources/migrations/Pg.sql
new/MirrorCache-1.079/lib/MirrorCache/resources/migrations/Pg.sql
--- old/MirrorCache-1.077/lib/MirrorCache/resources/migrations/Pg.sql
2024-06-20 07:43:31.000000000 +0200
+++ new/MirrorCache-1.079/lib/MirrorCache/resources/migrations/Pg.sql
2024-07-24 16:14:18.000000000 +0200
@@ -414,3 +414,8 @@
alter table folder_diff add column if not exists realfolder_id bigint;
-- 36 up
alter table project drop column etalon;
+-- 37 up
+alter table project
+ add column if not exists size bigint,
+ add column if not exists file_cnt bigint,
+ add column if not exists lm bigint;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.077/lib/MirrorCache/resources/migrations/mysql.sql
new/MirrorCache-1.079/lib/MirrorCache/resources/migrations/mysql.sql
--- old/MirrorCache-1.077/lib/MirrorCache/resources/migrations/mysql.sql
2024-06-20 07:43:31.000000000 +0200
+++ new/MirrorCache-1.079/lib/MirrorCache/resources/migrations/mysql.sql
2024-07-24 16:14:18.000000000 +0200
@@ -426,3 +426,8 @@
alter table folder_diff add column if not exists realfolder_id bigint;
-- 36 up
alter table project drop constraint fk_project_etalon, drop column etalon;
+-- 37 up
+alter table project
+ add column if not exists size bigint,
+ add column if not exists file_cnt bigint,
+ add column if not exists lm bigint;
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.077/t/environ/02-files-hashes.sh
new/MirrorCache-1.079/t/environ/02-files-hashes.sh
--- old/MirrorCache-1.077/t/environ/02-files-hashes.sh 2024-06-20
07:43:31.000000000 +0200
+++ new/MirrorCache-1.079/t/environ/02-files-hashes.sh 2024-07-24
16:14:18.000000000 +0200
@@ -74,11 +74,11 @@
$mc/curl /download/folder1/file1.1.dat.meta4 | grep -o
'<hash>5179db3d4263c9cb4ecf0edbc653ca460e3678b7</hash>'
$mc/curl /download/folder1/file1.1.dat.metalink | grep -o '<hash
piece="0">5179db3d4263c9cb4ecf0edbc653ca460e3678b7</hash>'
-$mc/curl -I /download/folder1/file1.1.dat.btih | grep '200 OK'
+$mc/curl -I /download/folder1/file1.1.dat.btih | grep -C20 '200 OK' | grep
'Content-Disposition: attachment; filename="file1.1.dat.btih"'
$mc/curl /download/folder1/file1.1.dat.btih
-$mc/curl -I /download/folder1/file1.1.dat.magnet | grep '200 OK'
+$mc/curl -I /download/folder1/file1.1.dat.magnet | grep -C20 '200 OK' | grep
'Content-Disposition: attachment; filename="file1.1.dat.magnet"'
$mc/curl /download/folder1/file1.1.dat.magnet
-$mc/curl -I /download/folder1/file1.1.dat.torrent | grep '200 OK'
+$mc/curl -I /download/folder1/file1.1.dat.torrent | grep -C20 '200 OK' | grep
'Content-Disposition: attachment; filename="file1.1.dat.torrent"'
$mc/curl /download/folder1/file1.1.dat.torrent
$mc/curl /download/folder1/file1.1.dat.metalink | xmllint --noout --format -
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.077/t/environ/02-files.sh
new/MirrorCache-1.079/t/environ/02-files.sh
--- old/MirrorCache-1.077/t/environ/02-files.sh 2024-06-20 07:43:31.000000000
+0200
+++ new/MirrorCache-1.079/t/environ/02-files.sh 2024-07-24 16:14:18.000000000
+0200
@@ -6,9 +6,6 @@
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=0
$mc/gen_env
MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=$MIRRORCACHE_SCHEDULE_RETRY_INTERVAL
-$mc/start
-$mc/status
-
ap8=$(environ ap8)
ap7=$(environ ap7)
@@ -35,10 +32,17 @@
done
done
+$mc/start
+$mc/status
+
$ap7/start
$ap8/start
$mc/curl -I -H "Accept: */*, application/metalink+xml"
/download/Folder1/repodata/repomd.xml | grep '200 OK'
+$mc/curl -I -H "Accept: */*, application/metalink+xml" -H "If-Modified-Since:
$(date -u --rfc-3339=seconds --date='1 second ago')"
/download/Folder1/repodata/repomd.xml | grep '304 Not Modified'
+$mc/curl -I -H "Accept: */*, application/metalink+xml" -H "If-Modified-Since:
$(date -u --rfc-3339=seconds --date='1 hour ago')"
/download/Folder1/repodata/repomd.xml | grep '200 OK'
+$mc/curl -I -H "Accept: */*, application/metalink+xml" -H "If-Modified-Since:
Sun, 06 Nov 1994 08:49:37 GMT" /download/Folder1/repodata/repomd.xml | grep
'200 OK'
+$mc/curl -I -H "Accept: */*, application/metalink+xml" -H "If-Modified-Since:
Smoe 10 Garbage 10:53:46 UTC 2024x" /download/Folder1/repodata/repomd.xml |
grep '200 OK'
$mc/sql "insert into server(hostname,urldir,enabled,country,region) select
'$($ap7/print_address)','','t','us','na'"
$mc/sql "insert into server(hostname,urldir,enabled,country,region) select
'$($ap8/print_address)','','t','ca','na'"
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.077/t/environ/03-headquarter-subsidiaries-hashes.sh
new/MirrorCache-1.079/t/environ/03-headquarter-subsidiaries-hashes.sh
--- old/MirrorCache-1.077/t/environ/03-headquarter-subsidiaries-hashes.sh
2024-06-20 07:43:31.000000000 +0200
+++ new/MirrorCache-1.079/t/environ/03-headquarter-subsidiaries-hashes.sh
2024-07-24 16:14:18.000000000 +0200
@@ -85,10 +85,8 @@
mc$i/sql_test file4.1.dat == "select hash.target from hash join file on id
= file_id where name='file-Media.iso'"
for x in '' metalink mirrorlist; do
ext=""
- par=""
[ -z "$x" ] || ext=.$x
- [ -z "$x" ] || par=?$x
- mc$i/curl -I /folder1/file-Media.iso$ext | grep -C 10 302 | grep
/folder1/file4.1.dat$par | grep -v /download/folder1/file4.1.dat
+ mc$i/curl -I /folder1/file-Media.iso$ext | grep -C 10 302 | grep
/folder1/file4.1.dat$ext | grep -v /download/folder1/file4.1.dat
done
done
@@ -115,24 +113,26 @@
sleep $DELAY
mc9/curl -I /download/folder1/file-Media.iso | grep 'Location:
/download/folder1/file4.1.dat'
-mc9/curl -I /download/folder1/file-Media.iso.metalink | grep 'Location:
/download/folder1/file4.1.dat?metalink=1'
-mc9/curl -I /download/folder1/file-Media.iso.mirrorlist | grep 'Location:
/download/folder1/file4.1.dat?mirrorlist=1'
+mc9/curl -I /download/folder1/file-Media.iso.metalink | grep 'Location:
/download/folder1/file4.1.dat\.metalink'
+mc9/curl -I /download/folder1/file-Media.iso?metalink | grep 'Location:
/download/folder1/file4.1.dat?metalink='
+mc9/curl -I /download/folder1/file-Media.iso.mirrorlist | grep 'Location:
/download/folder1/file4.1.dat\.mirrorlist'
+mc9/curl -I /download/folder1/file-Media.iso?mirrorlist | grep 'Location:
/download/folder1/file4.1.dat?mirrorlist='
mc9/curl -I /download/folder1/xcurr.dat | grep "Location:
http://$na_address/download/folder1/xcurr.dat";
mc9/curl -I /download/folder1/xcurr.dat.metalink | grep "Location:
http://$na_address/download/folder1/xcurr.dat.metalink";
-mc9/curl -I /download/folder1/xcurr.dat?meta4 | grep "Location:
http://$na_address/download/folder1/xcurr.dat.meta4";
-mc9/curl -I /download/folder1/xcurr.dat.mirrorlist | grep
'/folder1/file4.1.dat?mirrorlist='
+mc9/curl -I /download/folder1/xcurr.dat?meta4 | grep "Location:
http://$na_address/download/folder1/xcurr.dat?meta4";
+mc9/curl -I /download/folder1/xcurr.dat.mirrorlist | grep
'/folder1/file4.1.dat.mirrorlist'
-mc9/curl -I /download/folder1/xcurr.dat.zsync.mirrorlist | grep 'Location:
/download/folder1/file4.1.dat.zsync?mirrorlist='
mc9/curl -I /download/folder1/xcurr.dat.zsync?mirrorlist | grep 'Location:
/download/folder1/file4.1.dat.zsync?mirrorlist='
+mc9/curl -I /download/folder1/xcurr.dat.zsync.mirrorlist | grep 'Location:
/download/folder1/file4.1.dat.zsync\.mirrorlist'
mc6/sql_test file4.1.dat == "select target from file where name = 'xcurr.dat'"
mc6/curl -IL /download/folder1/xcurr.dat | grep '200 OK'
-mc6/curl -I /download/folder1/xcurr.dat.metalink | grep
'/folder1/file4.1.dat?metalink='
+mc6/curl -I /download/folder1/xcurr.dat.metalink | grep
'/folder1/file4.1.dat\.metalink'
mc6/curl -I /download/folder1/xcurr.dat?meta4 | grep
'/folder1/file4.1.dat?meta4='
-mc6/curl -I /download/folder1/xcurr.dat.mirrorlist | grep
'/folder1/file4.1.dat?mirrorlist='
+mc6/curl -I /download/folder1/xcurr.dat.mirrorlist | grep
'/folder1/file4.1.dat\.mirrorlist'
# now the hashes on subsidiaries should be retried and match the headquarter
for i in 6 7 8; do
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.077/t/environ/04-remote-mc.sh
new/MirrorCache-1.079/t/environ/04-remote-mc.sh
--- old/MirrorCache-1.077/t/environ/04-remote-mc.sh 2024-06-20
07:43:31.000000000 +0200
+++ new/MirrorCache-1.079/t/environ/04-remote-mc.sh 2024-07-24
16:14:18.000000000 +0200
@@ -48,6 +48,7 @@
test 1 == $($mc/db/sql "select count(*) from folder_diff_file")
$mc/curl -I /download/folder1/file2.1.dat | grep $($ng7/print_address)
+$mc/curl -I -H "If-Modified-Since: $(date -u --rfc-3339=seconds)"
/download/folder1/file2.1.dat | grep '304 Not Modified'
mv $ng7/dt/folder1/file2.1.dat $ng8/dt/folder1/
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore' old/MirrorCache-1.077/t/environ/14-project-report.sh
new/MirrorCache-1.079/t/environ/14-project-report.sh
--- old/MirrorCache-1.077/t/environ/14-project-report.sh 2024-06-20
07:43:31.000000000 +0200
+++ new/MirrorCache-1.079/t/environ/14-project-report.sh 2024-07-24
16:14:18.000000000 +0200
@@ -21,6 +21,8 @@
echo $x/dt/{folder1,folder2,folder3}/{file1.1,file2.1}.dat | xargs -n 1
touch
echo $x/dt/project1/{folder1,folder2,folder3}/{file1.1,file2.1}.dat |
xargs -n 1 touch
echo $x/dt/project2/{folder1,folder2,folder3}/{file1.1,file2.1}.dat |
xargs -n 1 touch
+ echo 1234567 > $x/dt/project2/folder2/file1.1.dat
+ echo 1234 > $x/dt/project2/folder2/file2.1.dat
done
$ap3/start
@@ -74,8 +76,13 @@
$mc/backstage/job mirror_probe_projects
$mc/backstage/job -e report -a '["once"]'
+$mc/backstage/job -e report_project_size_schedule -a '["once"]'
$mc/backstage/shoot
+$mc/sql_test 4 == "select file_cnt from project where path = '/project1'"
+$mc/sql_test 2 == "select file_cnt from project where path =
'/project2/folder1'"
+$mc/sql_test 13 == "select size from project where path = '/project2/folder2'"
+
$mc/curl /report/mirrors | tidy --drop-empty-elements no | \
grep -A4 -F '<div class="repo">' | \
grep -A2 -F '"http://127.0.0.1:1304/project2/folder2";>' | \
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn'
'--exclude=.svnignore'
old/MirrorCache-1.077/templates/app/project/index.html.ep
new/MirrorCache-1.079/templates/app/project/index.html.ep
--- old/MirrorCache-1.077/templates/app/project/index.html.ep 2024-06-20
07:43:31.000000000 +0200
+++ new/MirrorCache-1.079/templates/app/project/index.html.ep 2024-07-24
16:14:18.000000000 +0200
@@ -18,6 +18,9 @@
<th class="col_value">Name</th>
<th class="col_value">Path</th>
<th class="col_value">Prio</th>
+ <th class="col_ro">Size</th>
+ <th class="col_ro">File Count</th>
+ <th class="col_unixtime">Last Modified</th>
<th class="col_action">Actions</th>
</tr>
</thead>
++++++ MirrorCache.obsinfo ++++++
--- /var/tmp/diff_new_pack.Zlvxh3/_old 2024-08-01 22:06:26.361440995 +0200
+++ /var/tmp/diff_new_pack.Zlvxh3/_new 2024-08-01 22:06:26.397442481 +0200
@@ -1,5 +1,5 @@
name: MirrorCache
-version: 1.077
-mtime: 1718862211
-commit: 8e27b8285ca98c3110c0cc8c1438fe4a88157fce
+version: 1.079
+mtime: 1721830458
+commit: 1fd848f467a79f3c2697593716894e3ee3ea7238
