Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package MirrorCache for openSUSE:Factory checked in at 2025-05-26 18:33:56 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/MirrorCache (Old) and /work/SRC/openSUSE:Factory/.MirrorCache.new.2732 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "MirrorCache" Mon May 26 18:33:56 2025 rev:51 rq:1279511 version:1.094 Changes: -------- --- /work/SRC/openSUSE:Factory/MirrorCache/MirrorCache.changes 2025-04-02 18:25:28.678517589 +0200 +++ /work/SRC/openSUSE:Factory/.MirrorCache.new.2732/MirrorCache.changes 2025-05-26 18:35:42.971874432 +0200 @@ -1,0 +2,10 @@ +Fri May 16 09:24:29 UTC 2025 - Andrii Nikitin <[email protected]> + +- Update to version 1.094: + * Cleanup agg_download_pkg (#589) + * Add BACKSTAGE_QUEUE parameter to backstage startup scripts (#583) + * Shard background jobs (#582) + * Fix workflow test-salt-package-from-obs (#578) + * Show package download statistics for month and week (#577) + +------------------------------------------------------------------- Old: ---- MirrorCache-1.093.obscpio New: ---- MirrorCache-1.094.obscpio ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ MirrorCache.spec ++++++ --- /var/tmp/diff_new_pack.rmWrQw/_old 2025-05-26 18:35:43.503896860 +0200 +++ /var/tmp/diff_new_pack.rmWrQw/_new 2025-05-26 18:35:43.503896860 +0200 @@ -22,7 +22,7 @@ %define main_requires %{assetpack_requires} perl(Carp) perl(DBD::Pg) >= 3.7.4 perl(DBI) >= 1.632 perl(DBIx::Class) >= 0.082801 perl(DBIx::Class::DynamicDefault) perl(DateTime) perl(Encode) perl(Time::Piece) perl(Time::Seconds) perl(Time::ParseDate) perl(DateTime::Format::Pg) perl(Exporter) perl(File::Basename) perl(LWP::UserAgent) perl(Mojo::Base) perl(Mojo::ByteStream) perl(Mojo::IOLoop) perl(Mojo::JSON) perl(Mojo::Pg) perl(Mojo::URL) perl(Mojo::Util) perl(Mojolicious::Commands) perl(Mojolicious::Plugin) perl(Mojolicious::Plugin::RenderFile) perl(Mojolicious::Static) perl(Net::OpenID::Consumer) perl(POSIX) perl(Sort::Versions) perl(URI::Escape) perl(XML::Writer) perl(base) perl(constant) perl(diagnostics) perl(strict) perl(warnings) shadow rubygem(sass) perl(Net::DNS) perl(LWP::Protocol::https) perl(Digest::SHA) perl(Config::IniFiles) %define build_requires %{assetpack_requires} rubygem(sass) tidy sysuser-shadow sysuser-tools Name: MirrorCache -Version: 1.093 +Version: 1.094 Release: 0 Summary: WebApp to redirect and manage mirrors License: GPL-2.0-or-later ++++++ MirrorCache-1.093.obscpio -> MirrorCache-1.094.obscpio ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/assets/javascripts/package.js new/MirrorCache-1.094/assets/javascripts/package.js --- old/MirrorCache-1.093/assets/javascripts/package.js 2025-03-26 13:06:12.000000000 +0100 +++ new/MirrorCache-1.094/assets/javascripts/package.js 2025-05-12 23:43:54.000000000 +0200 @@ -11,6 +11,8 @@ var pkg_stat_first_seen; var pkg_stat_dl_todate = ''; // total excluding today +var pkg_stat_dl_month = ''; // last 30 days excluding today +var pkg_stat_dl_week = ''; // last 7 days excluding today var pkg_stat_dl_today = ''; // today excluding last hour var pkg_stat_dl_curr = ''; // last hour @@ -247,13 +249,28 @@ } }], lengthMenu: [ - [100, 1000, 10, -1], - [100, 1000, 10, 'All'], + [30, 100, 1000, 10, -1], + [30, 100, 1000, 10, 'All'], ], }); } +function setupDownloadStatUIElement(elementName, v, v1) { + var dot = '...'; + var res; + if (typeof v != 'undefined') { + res = v; + if ( +parseInt(v1) > 0) { + res = + res + +parseInt(v1); + } + if ( +parseInt(pkg_stat_dl_curr) > 0) { + res = +res + +parseInt(pkg_stat_dl_curr); + dot = ''; + } + } + document.getElementById(elementName).textContent = String(res).concat(dot); +} function setupDownloadStatUI() { @@ -266,23 +283,12 @@ } document.getElementById("download-stat-first-seen").textContent = res; } - if (typeof pkg_stat_dl_todate != 'undefined') { - var res = pkg_stat_dl_todate; - if ( +parseInt(pkg_stat_dl_today) > 0) { - res = +res + +parseInt(pkg_stat_dl_today); - } - if ( +parseInt(pkg_stat_dl_curr) > 0) { - res = +res + +parseInt(pkg_stat_dl_curr); - } - document.getElementById("download-stat-total").textContent = res; - } - if (typeof pkg_stat_dl_today != 'undefined') { - var res = pkg_stat_dl_today; - if ( +parseInt(pkg_stat_dl_curr) > 0) { - res = +res + +parseInt(pkg_stat_dl_curr); - } - document.getElementById("download-stat-today").textContent = res; - } + + setupDownloadStatUIElement("download-stat-total", pkg_stat_dl_todate, pkg_stat_dl_today); + setupDownloadStatUIElement("download-stat-month", pkg_stat_dl_month, pkg_stat_dl_today); + setupDownloadStatUIElement("download-stat-week", pkg_stat_dl_week, pkg_stat_dl_today); + setupDownloadStatUIElement("download-stat-today", pkg_stat_dl_today); + if (typeof pkg_stat_dl_today != 'undefined') { var res = pkg_stat_dl_curr; document.getElementById("download-stat-curr").textContent = res; @@ -302,6 +308,14 @@ if (typeof c !== 'undefined' && c > 0) { pkg_stat_dl_todate = c; } + c = data.cnt_30d; + if (typeof c !== 'undefined' && c > 0) { + pkg_stat_dl_month = c; + } + c = data.cnt_7d; + if (typeof c !== 'undefined' && c > 0) { + pkg_stat_dl_week = c; + } c = data.cnt_today; if (typeof c !== 'undefined' && c > 0) { pkg_stat_dl_today = c; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/dist/salt/test/Dockerfile new/MirrorCache-1.094/dist/salt/test/Dockerfile --- old/MirrorCache-1.093/dist/salt/test/Dockerfile 2025-03-26 13:06:12.000000000 +0100 +++ new/MirrorCache-1.094/dist/salt/test/Dockerfile 2025-05-12 23:43:54.000000000 +0200 @@ -1,17 +1,26 @@ -FROM opensuse/leap -ENV container docker +FROM registry.opensuse.org/opensuse/leap +ENV container podman ENV LANG en_US.UTF-8 # packages needed for test RUN zypper -vvv -n install vim curl sudo salt-minion git -RUN mkdir -p /srv/salt/ -RUN sed -i 's^\#*\s*file_client: .*$^file_client: local\nsystemd.scope: False\nenable_fqdns_grains: False^' /etc/salt/minion +RUN mkdir -p /srv/salt/ && \ + sed -i 's^\#*\s*file_client: .*$^file_client: local\nsystemd.scope: False\nenable_fqdns_grains: False^' /etc/salt/minion && \ + sed -i '/pam_systemd.so/d' /etc/pam.d/common-session-pc # delete pam_systemd , otherwise sudo will hang + +RUN mkdir -p /srv/pillar/ + +RUN echo "{{ saltenv }}:" > /srv/pillar/top.sls +RUN echo ' "*":' >> /srv/pillar/top.sls +RUN echo " - repo" >> /srv/pillar/top.sls + +RUN echo mirrorcache_formula_enable_repository: True > /srv/pillar/repo.sls WORKDIR /opt ADD mirrors-eu.sql /opt RUN git clone https://github.com/andrii-suse/mirrorcache-formula -RUN ln -s /opt/mirrorcache-formula/mirrorcache /srv/salt/mirrorcache +RUN cp -r /opt/mirrorcache-formula/mirrorcache /srv/salt/mirrorcache ENV MIRRORCACHE_DB_PROVIDER postgresql diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/lib/MirrorCache/Config.pm new/MirrorCache-1.094/lib/MirrorCache/Config.pm --- old/MirrorCache-1.093/lib/MirrorCache/Config.pm 2025-03-26 13:06:12.000000000 +0100 +++ new/MirrorCache-1.094/lib/MirrorCache/Config.pm 2025-05-12 23:43:54.000000000 +0200 @@ -46,6 +46,8 @@ has ip2location => $ENV{MIRRORCACHE_IP2LOCATION}; has top_folders => $ENV{MIRRORCACHE_TOP_FOLDERS}; +has workers => $ENV{MIRRORCACHE_WORKERS}; + has plugin_status => $ENV{MIRRORCACHE_PLUGIN_STATUS}; has regions => $ENV{MIRRORCACHE_REGIONS}; @@ -65,7 +67,7 @@ my $cfg; $cfg = Config::IniFiles->new(-file => $cfgfile, -fallback => 'default') if $cfgfile; if ($cfg) { - for my $k (qw/root root_nfs redirect redirect_huge huge_file_size small_file_size city_mmdb ip2location top_folders regions mirror_provider browser_agent_mask custom_footer_message country_image_dir vpn_prefix/) { + for my $k (qw/root root_nfs redirect redirect_huge huge_file_size small_file_size city_mmdb ip2location top_folders workers regions mirror_provider browser_agent_mask custom_footer_message country_image_dir vpn_prefix/) { if (my $v = $cfg->val('default', $k)) { $self->$k($v); } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/lib/MirrorCache/Schema/Result/Project.pm new/MirrorCache-1.094/lib/MirrorCache/Schema/Result/Project.pm --- old/MirrorCache-1.093/lib/MirrorCache/Schema/Result/Project.pm 2025-03-26 13:06:12.000000000 +0100 +++ new/MirrorCache-1.094/lib/MirrorCache/Schema/Result/Project.pm 2025-05-12 23:43:54.000000000 +0200 @@ -25,6 +25,8 @@ { data_type => "varchar", is_nullable => 0, size => 512 }, "redirect", { data_type => "varchar", is_nullable => 0, size => 512 }, + "shard", + { data_type => "varchar", is_nullable => 0, size => 32 }, db_sync_last => { data_type => 'timestamp', is_nullable => 1 diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/lib/MirrorCache/Schema/ResultSet/Stat.pm new/MirrorCache-1.094/lib/MirrorCache/Schema/ResultSet/Stat.pm --- old/MirrorCache-1.093/lib/MirrorCache/Schema/ResultSet/Stat.pm 2025-03-26 13:06:12.000000000 +0100 +++ new/MirrorCache-1.094/lib/MirrorCache/Schema/ResultSet/Stat.pm 2025-05-12 23:43:54.000000000 +0200 @@ -222,6 +222,52 @@ return ($id, \@folders, \@country_list); } + +# this should return recent requests for unknown yet files or folders +sub path_misses_shard { + my ($self, $min_stat_id, $max_stat_id, $shard) = @_; + + my $rsource = $self->result_source; + my $schema = $rsource->schema; + my $dbh = $schema->storage->dbh; + + my $sql = << 'END_SQL'; +select stat.id, stat.path, stat.folder_id, trim(country) +from stat left join folder on folder.id = stat.folder_id +where +( + (( mirror_id in (0,-1) or mirrorlist ) and ( file_id is null )) -- unknown file +or + ( folder_id is null and mirror_id > -2 ) -- file may be known, but requested folder is unknown - happens when realpath shows to a different folder +) +and stat.path !~ '\/(repodata\/repomd\.xml[^\/]*|media\.1\/(media|products)|content|.*\.sha\d\d\d(\.asc)?|Release(\.key|\.gpg)?|InRelease|Packages(\.gz|\.zst)?|Sources(\.gz|\.zst)?|.*_Arch\.(files|db|key)(\.(sig|tar\.gz(\.sig)?|tar\.zst(\.sig)?))?|(files|primary|other)\.xml\.(gz|zck|zst)|[Pp]ackages(\.[A-Z][A-Z])?\.(xz|gz|zst)|gpg-pubkey.*\.asc|CHECKSUMS(\.asc)?|APKINDEX\.tar\.gz)$' +and lower(stat.agent) NOT LIKE '%bot%' +and lower(stat.agent) NOT LIKE '%rclone%' +and stat.id between ? and ? +and stat.path like ? +and ( + stat.folder_id is null or + folder.sync_requested < folder.sync_scheduled + ) +END_SQL + + $sql =~ s/\!\~/not regexp/g unless $dbh->{Driver}->{Name} eq 'Pg'; + + my $prep = $dbh->prepare($sql); + $prep->execute($min_stat_id, $max_stat_id, "/$shard"); + my $arrayref = $dbh->selectall_arrayref($prep, { Slice => {} }); + my $id; + my %folders = (); + foreach my $miss ( @$arrayref ) { + my $path = $miss->{path}; + next unless $path; + $path = path($path)->dirname; + $folders{$path} = 1; + } + my @folders = (sort keys %folders); + return (\@folders); +} + sub mirror_misses { my ($self, $prev_stat_id, $limit) = @_; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/lib/MirrorCache/Task/Cleanup.pm new/MirrorCache-1.094/lib/MirrorCache/Task/Cleanup.pm --- old/MirrorCache-1.093/lib/MirrorCache/Task/Cleanup.pm 2025-03-26 13:06:12.000000000 +0100 +++ new/MirrorCache-1.094/lib/MirrorCache/Task/Cleanup.pm 2025-05-12 23:43:54.000000000 +0200 @@ -20,6 +20,7 @@ sub register { my ($self, $app) = @_; $app->minion->add_task(cleanup => sub { _run($app, @_) }); + $app->minion->add_task(cleanup_agg_download_pkg => sub { _run_agg_download_pkg($app, @_) }); } my $DELAY = int($ENV{MIRRORCACHE_SCHEDULE_CLEANUP_RETRY_INTERVAL} // 2 * 60); @@ -33,6 +34,7 @@ my $schema = $app->schema; $minion->enqueue('mirror_probe_projects'); + $minion->enqueue('cleanup_agg_download_pkg'); # detect stalled backstage jobs my $sql; @@ -139,4 +141,51 @@ return $job->retry({delay => $DELAY}); } +sub _run_agg_download_pkg { + my ($app, $job) = @_; + my $minion = $app->minion; + return $job->finish('Previous cleanup job is still active') + unless my $guard = $minion->guard('cleanup_agg_download_pkg', 120); + + my $schema = $app->schema; + + # detect stalled backstage jobs + my $sql; + if ($schema->pg) { +$sql = <<'END_SQL'; +delete from agg_download_pkg +where ctid in ( + select ctid from agg_download_pkg + where period = 'hour' and dt < (current_timestamp - interval '14 day') + LIMIT 100000 +) +END_SQL + } else { + $sql = "delete from agg_download_pkg where period = 'hour' and dt < date_sub(current_timestamp, interval 14 day) LIMIT 100000"; + } + eval { + $schema->storage->dbh->prepare($sql)->execute(); + 1; + } or $job->note(sql_error => $@, at => datetime_now()); + + if ($schema->pg) { +$sql = <<'END_SQL'; +delete from agg_download_pkg +where ctid in ( + select ctid from agg_download_pkg + where period = 'day' and dt < (current_timestamp - interval '6 month') + LIMIT 100000 +) +END_SQL + } else { + $sql = "delete from agg_download_pkg where period = 'day' and dt < date_sub(current_timestamp, interval 6 month) LIMIT 100000"; + } + eval { + $schema->storage->dbh->prepare($sql)->execute(); + 1; + } or $job->note(sql_error_2 => $@, at_2 => datetime_now()); + + return $job->finish; +} + 1; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/lib/MirrorCache/Task/FolderHashesCreate.pm new/MirrorCache-1.094/lib/MirrorCache/Task/FolderHashesCreate.pm --- old/MirrorCache-1.093/lib/MirrorCache/Task/FolderHashesCreate.pm 2025-03-26 13:06:12.000000000 +0100 +++ new/MirrorCache-1.094/lib/MirrorCache/Task/FolderHashesCreate.pm 2025-05-12 23:43:54.000000000 +0200 @@ -57,6 +57,7 @@ my $block_size = calcBlockSize($file->{size}); eval { my $indir = $root->rootpath($path); + die "Not found: $path" unless $indir; calcMetalink($indir, $path, $basename, $block_size, $schema, $file->{id}); $count++; }; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/lib/MirrorCache/Task/FolderSync.pm new/MirrorCache-1.094/lib/MirrorCache/Task/FolderSync.pm --- old/MirrorCache-1.093/lib/MirrorCache/Task/FolderSync.pm 2025-03-26 13:06:12.000000000 +0100 +++ new/MirrorCache-1.094/lib/MirrorCache/Task/FolderSync.pm 2025-05-12 23:43:54.000000000 +0200 @@ -150,13 +150,13 @@ $schema->resultset('Folder')->request_scan($folder->id); $minion->enqueue('folder_hashes_create' => [$realpath] => {queue => $HASHES_QUEUE}) if $HASHES_COLLECT && !$app->backstage->inactive_jobs_exceed_limit(1000, 'folder_hashes_create', $HASHES_QUEUE); $minion->enqueue('folder_hashes_import' => [$realpath] => {queue => $HASHES_QUEUE}) if $HASHES_IMPORT && !$app->backstage->inactive_jobs_exceed_limit(1000, 'folder_hashes_import', $HASHES_QUEUE); - $minion->enqueue('folder_pkg_sync' => [$realpath]) if $has_pkg && !$app->backstage->inactive_jobs_exceed_limit(1000, 'folder_pkg_sync'); + $minion->enqueue('folder_pkg_sync' => [$realpath] => { queue => $job->info->{queue} }) if $has_pkg && !$app->backstage->inactive_jobs_exceed_limit(1000, 'folder_pkg_sync'); } $schema->resultset('Folder')->request_scan($otherFolder->id) if $otherFolder && ($count || !$otherFolder->scan_requested); $schema->resultset('Rollout')->add_rollout($proj->{project_id}, $obsrelease->versionmtime, $obsrelease->version, $obsrelease->versionfilename, $proj_prefix) if $obsrelease && $obsrelease->versionfilename; for my $subfolder (@subfolders) { - $minion->enqueue('folder_sync' => ["$path/$subfolder"]); + $app->backstage->enqueue('folder_sync', "$path/$subfolder"); } return; }; @@ -252,7 +252,7 @@ $minion->enqueue('folder_hashes_create' => [$realpath, $max_dt] => {queue => $HASHES_QUEUE}) if $HASHES_COLLECT && !$app->backstage->inactive_jobs_exceed_limit(1000, 'folder_hashes_create', $HASHES_QUEUE); $minion->enqueue('folder_hashes_import' => [$realpath, $max_dt] => {queue => $HASHES_QUEUE}) if $HASHES_IMPORT && !$app->backstage->inactive_jobs_exceed_limit(1000, 'folder_hashes_import', $HASHES_QUEUE); } - $minion->enqueue('folder_pkg_sync' => [$realpath]) if $has_pkg && !$app->backstage->inactive_jobs_exceed_limit(1000, 'folder_pkg_sync'); + $minion->enqueue('folder_pkg_sync' => [$realpath] => { queue => $job->info->{queue} }) if $has_pkg && !$app->backstage->inactive_jobs_exceed_limit(1000, 'folder_pkg_sync', $job->info->{queue}); if ($otherFolder && ($cnt || $updated || !$otherFolder->scan_requested)) { $otherFolder->update({sync_last => \"CURRENT_TIMESTAMP(3)", scan_requested => \"CURRENT_TIMESTAMP(3)", sync_scheduled => \'coalesce(sync_scheduled, CURRENT_TIMESTAMP(3))'}); @@ -263,7 +263,7 @@ for my $subfolder (@subfolders) { - $minion->enqueue('folder_sync' => ["$path/$subfolder"]); + $app->backstage->enqueue('folder_sync', "$path/$subfolder"); } } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/lib/MirrorCache/Task/FolderSyncSchedule.pm new/MirrorCache-1.094/lib/MirrorCache/Task/FolderSyncSchedule.pm --- old/MirrorCache-1.093/lib/MirrorCache/Task/FolderSyncSchedule.pm 2025-03-26 13:06:12.000000000 +0100 +++ new/MirrorCache-1.094/lib/MirrorCache/Task/FolderSyncSchedule.pm 2025-05-12 23:43:54.000000000 +0200 @@ -99,7 +99,10 @@ for my $folder (@folders) { $folder->update({sync_scheduled => \'CURRENT_TIMESTAMP(3)'}); # , sync_requested => \'if(sync_requested > sync_scheduled, sync_scheduled, sync_requested)'}); - $minion->enqueue('folder_sync' => [$folder->path] => {priority => 2} => {notes => {$folder->path => 1}} ); + my $queue = "default"; + my $shard = $app->mcproject->shard_for_path($folder->path); + $queue = $shard if $shard; + $app->backstage->enqueue('folder_sync', $folder->path); $cnt = $cnt + 1; } $job->note(count => $cnt); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/lib/MirrorCache/Task/FolderSyncScheduleFromMisses.pm new/MirrorCache-1.094/lib/MirrorCache/Task/FolderSyncScheduleFromMisses.pm --- old/MirrorCache-1.093/lib/MirrorCache/Task/FolderSyncScheduleFromMisses.pm 2025-03-26 13:06:12.000000000 +0100 +++ new/MirrorCache-1.094/lib/MirrorCache/Task/FolderSyncScheduleFromMisses.pm 2025-05-12 23:43:54.000000000 +0200 @@ -20,6 +20,7 @@ sub register { my ($self, $app) = @_; $app->minion->add_task(folder_sync_schedule_from_misses => sub { _run($app, @_) }); + $app->minion->add_task(folder_sync_schedule_from_misses_shard => sub { _run_shard($app, @_) }); } my $DELAY = int($ENV{MIRRORCACHE_SCHEDULE_RETRY_INTERVAL} // 10); @@ -54,15 +55,21 @@ my $rs = $schema->resultset('Folder'); my $last_run = 0; print STDERR $app->dumper($pref, $folders, $stat_id, $prev_stat_id) if $MCDEBUG; + + # we will schedule folder_sync_schedule_from_misses_shard for each affected shard with range ($prev_stat_id, $next_stat_id) + my %affected_shards; + while (scalar(@$folders)) { my $cnt = 0; - $prev_stat_id = $stat_id; print(STDERR "$pref read id from stat up to: $stat_id\n"); for my $path (@$folders) { print STDERR $pref . $path . "\n" if $MCDEBUG; my $folder = $rs->find({ path => $path }); if (!$folder) { - if (!$app->mc->root->is_dir($path)) { + my $shard = $app->mcproject->shard_for_path($path); + if ($shard) { + $affected_shards{$shard} = 1; + } elsif (!$app->mc->root->is_dir($path)) { $path = Mojo::File->new($path)->dirname; next unless $app->mc->root->is_dir($path); } @@ -86,7 +93,7 @@ $last_run = $last_run + $cnt; last unless $cnt; $limit = 10000; - ($stat_id, $folders, $country_list) = $schema->resultset('Stat')->path_misses($prev_stat_id, $limit); + ($stat_id, $folders, $country_list) = $schema->resultset('Stat')->path_misses($stat_id, $limit); } if ($minion->lock('mirror_force_done', 9000)) { @@ -103,10 +110,58 @@ print(STDERR "$pref will retry with id: $prev_stat_id\n") if $MCDEBUG; my $total = $job->info->{notes}{total}; $total = 0 unless $total; - $job->note(stat_id => $prev_stat_id, total => $total, last_run => $last_run); + $total += $last_run if $last_run; + $job->note(stat_id => $stat_id, total => $total, last_run => $last_run); + + for my $shard (keys %affected_shards) { + $minion->enqueue('folder_sync_schedule_from_misses_shard' => [$prev_stat_id, $stat_id, $shard] => {queue => $shard}); + } return $job->finish unless $DELAY; return $job->retry({delay => $DELAY}); } + +# files from shards are available only special workeds, thus a dedicated job to run in a queue of those workers +sub _run_shard { + my ($app, $job, $min_stat_id, $max_stat_id, $shard) = @_; + my $job_id = $job->id; + my $pref = "[schedule_from_misses$shard $job_id]"; + + return $job->finish("something is missing: $min_stat_id, $max_stat_id, $shard") + unless defined $min_stat_id && $max_stat_id && $shard; + + my $minion = $app->minion; + # prevent multiple scheduling tasks to run in parallel + return $job->finish("Previous schedule_from_misses_$shard job is still active") + unless my $guard = $minion->guard("folder_sync_schedule_from_misses_$shard", 180); + + my $schema = $app->schema; + + my $folders = $schema->resultset('Stat')->path_misses_shard($min_stat_id, $max_stat_id, $shard); + my $rs = $schema->resultset('Folder'); + + while (scalar(@$folders)) { + my $cnt = 0; + for my $path (@$folders) { + my $folder = $rs->find({ path => $path }); + if (!$folder) { + if (!$app->mc->root->is_dir($path)) { + $path = Mojo::File->new($path)->dirname; + next unless $app->mc->root->is_dir($path); + } + } + $folder = $rs->find({ path => $path }) unless $folder; + + $cnt = $cnt + 1; + $rs->request_sync($path); + if ($folder && $folder->id) { + $rs->request_scan($folder->id); + } + } + } + + return $job->finish("Done $shard for range $min_stat_id, $max_stat_id"); +} + 1; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/lib/MirrorCache/Task/ProjectScanSchedule.pm new/MirrorCache-1.094/lib/MirrorCache/Task/ProjectScanSchedule.pm --- old/MirrorCache-1.093/lib/MirrorCache/Task/ProjectScanSchedule.pm 2025-03-26 13:06:12.000000000 +0100 +++ new/MirrorCache-1.094/lib/MirrorCache/Task/ProjectScanSchedule.pm 2025-05-12 23:43:54.000000000 +0200 @@ -61,7 +61,7 @@ my $needsync = $project->get_column('needsync'); next unless $needsync; $rs->mark_scheduled($project->id); - $minion->enqueue('folder_sync' => [$project->path, 1] => {priority => 2} => {notes => {$project->path => 1}} ); + $app->backstage->enqueue('folder_sync', $project->path, 1); $cnt++; } $job->note(count => $cnt); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/lib/MirrorCache/Task/ProjectSyncSchedule.pm new/MirrorCache-1.094/lib/MirrorCache/Task/ProjectSyncSchedule.pm --- old/MirrorCache-1.093/lib/MirrorCache/Task/ProjectSyncSchedule.pm 2025-03-26 13:06:12.000000000 +0100 +++ new/MirrorCache-1.094/lib/MirrorCache/Task/ProjectSyncSchedule.pm 2025-05-12 23:43:54.000000000 +0200 @@ -61,9 +61,10 @@ my $needsync = $project->get_column('needsync'); next unless $needsync; $rs->mark_scheduled($project->id); - $minion->enqueue('folder_sync' => [$project->path, 1] => {priority => 2} => {notes => {$project->path => 1}} ); + $app->backstage->enqueue('folder_sync', $project->path, 1); $cnt++; } + $job->note(count => $cnt); return $job->finish unless $DELAY; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/lib/MirrorCache/Task/Report.pm new/MirrorCache-1.094/lib/MirrorCache/Task/Report.pm --- old/MirrorCache-1.093/lib/MirrorCache/Task/Report.pm 2025-03-26 13:06:12.000000000 +0100 +++ new/MirrorCache-1.094/lib/MirrorCache/Task/Report.pm 2025-05-12 23:43:54.000000000 +0200 @@ -94,7 +94,7 @@ bytes) select 'hour'::stat_period_t cperiod, date_trunc('hour', stat.dt) cdt, coalesce(p.id, 0) cpid, coalesce(stat.country, '') ccountry, stat.mirror_id cmirror_id, coalesce(ft.id, 0) cft_id, - coalesce(os.id, 0) cos_id, coalesce(regexp_replace(stat.path, os.mask, os.version), '') cos_version, + coalesce(os.id, 0) cos_id, lower(coalesce(regexp_replace(stat.path, os.mask, os.version), '')) cos_version, coalesce(arch.id, 0) carch_id, 0 cmeta_id, count(*) cnt, @@ -112,7 +112,7 @@ and d.project_id = coalesce(p.id, 0) and d.file_type = coalesce(ft.id, 0) and d.os_id = coalesce(os.id, 0) - and d.os_version = coalesce(regexp_replace(stat.path, os.mask, os.version), '') + and d.os_version = lower(coalesce(regexp_replace(stat.path, os.mask, os.version), '')) and d.arch_id = coalesce(arch.id, 0) and d.dt = date_trunc('hour', stat.dt) and d.dt > now() - interval '7 hour' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/lib/MirrorCache/WebAPI/Controller/Rest/FolderJobs.pm new/MirrorCache-1.094/lib/MirrorCache/WebAPI/Controller/Rest/FolderJobs.pm --- old/MirrorCache-1.093/lib/MirrorCache/WebAPI/Controller/Rest/FolderJobs.pm 2025-03-26 13:06:12.000000000 +0100 +++ new/MirrorCache-1.094/lib/MirrorCache/WebAPI/Controller/Rest/FolderJobs.pm 2025-05-12 23:43:54.000000000 +0200 @@ -67,7 +67,7 @@ my $job_id; eval { - $job_id = $self->minion->enqueue('folder_tree' => [$path] => {priority => 10}); + $job_id = $self->backstage->enqueue('folder_tree', $path); }; return $self->render(status => 500, text => Dumper($@)) unless $job_id; @@ -85,7 +85,7 @@ my $job_id; eval { - $job_id = $self->minion->enqueue('folder_sync' => [$path] => {priority => 10, notes => {$path => 1}} ); + $job_id = $self->backstage->enqueue('folder_sync', $path); }; return $self->render(status => 500, text => Dumper($@)) unless $job_id; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/lib/MirrorCache/WebAPI/Controller/Rest/Metapkg.pm new/MirrorCache-1.094/lib/MirrorCache/WebAPI/Controller/Rest/Metapkg.pm --- old/MirrorCache-1.093/lib/MirrorCache/WebAPI/Controller/Rest/Metapkg.pm 2025-03-26 13:06:12.000000000 +0100 +++ new/MirrorCache-1.094/lib/MirrorCache/WebAPI/Controller/Rest/Metapkg.pm 2025-05-12 23:43:54.000000000 +0200 @@ -179,7 +179,7 @@ my $sql; $sql = <<'END_SQL'; select - extract(epoch from ( select min(dt) from agg_download_pkg where metapkg_id = ? and period = 'hour' ))::int as first_seen, + extract(epoch from ( select min(dt) from agg_download_pkg where metapkg_id = ? and period = 'day' ))::int as first_seen, coalesce( (select sum(cnt) as cnt from agg_download_pkg where metapkg_id = ? and period = 'total' and dt = (select max(dt) from agg_download_pkg where period = 'total')), 0 ) as cnt_total, coalesce( (select sum(cnt) as cnt from agg_download_pkg where metapkg_id = ? and period = 'hour' and dt > (select max(dt) from agg_download_pkg where period = 'total')), 0 ) as cnt_today, sum(cnt) as cnt_30d, @@ -203,20 +203,15 @@ my $name = $self->param('name'); my $sql; $sql = <<'END_SQL'; -select - count(*) as cnt_curr -from - stat - left join (select max(dt) as dt from agg_download_pkg where period = 'hour') last_agg_hour on 1 = 1 -where - stat.dt > coalesce(last_agg_hour.dt, now() - interval '1 hour') and path like concat('%',?::text,'%rpm') and ? = regexp_replace(path, '^(.*\/)+(.*)-[^-]+-[^-]+\.(x86_64|noarch|ppc64le|(a|loong)arch64.*|s390x|i[3-6]86|armv.*|src|riscv64|ppc.*|nosrc|ia64)(\.d?rpm)$', '\2') +select count(*) as cnt_curr +from stat +where stat.dt > coalesce((select max(dt) as dt from agg_download_pkg where period = 'hour') , now() - interval '1 hour') and pkg = ?::text; END_SQL unless ($self->schema->pg) { - $sql =~ s/\\2/\\\\2/g; $sql =~ s/::text//g; $sql =~ s/interval '(\d+) (day|hour)'/interval $1 $2/g; } - my $res = $self->schema->storage->dbh->selectall_arrayref($sql, {Columns => {}}, $name, $name); + my $res = $self->schema->storage->dbh->selectall_arrayref($sql, {Columns => {}}, $name); return $self->render(json => { data => $res }); } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/lib/MirrorCache/WebAPI/Plugin/Backstage.pm new/MirrorCache-1.094/lib/MirrorCache/WebAPI/Plugin/Backstage.pm --- old/MirrorCache-1.093/lib/MirrorCache/WebAPI/Plugin/Backstage.pm 2025-03-26 13:06:12.000000000 +0100 +++ new/MirrorCache-1.094/lib/MirrorCache/WebAPI/Plugin/Backstage.pm 2025-05-12 23:43:54.000000000 +0200 @@ -186,16 +186,39 @@ return $db->query($sql, $limit, $queue, $task, $limit1)->expand->hash->{exceed}; } +sub enqueue { + my $self = shift; + my $task = shift; + my @args = @_; + my $app = $self->app; + + my %params; + if ($task eq 'folder_sync') { + $params{"notes"} = { $args[0] => 1 }; + if (my $queue = $app->mcproject->shard_for_path($args[0])) { + $params{"queue"} = $queue; + } + $params{"priority"} = 2; + } elsif ($task eq 'folder_tree') { + if (my $queue = $app->mcproject->shard_for_path($args[0])) { + $params{"queue"} = $queue; + } + $params{"priority"} = 10; + } + return $app->minion->enqueue($task => [@args] => \%params); +} + # raсe condition here souldn't be big issue sub enqueue_unless_scheduled_with_parameter_or_limit { - my ( $self, $task, $arg1, $arg2 ) = @_; + my ( $self, $task, $arg, $queue ) = @_; + $queue = 'default' unless $queue; - return 0 if $self->inactive_jobs_exceed_limit(300); + return 0 if $self->inactive_jobs_exceed_limit(300, $task, $queue); my $minion = $self->app->minion; - my $res = $minion->backend->list_jobs(0, 1, {tasks => [$task], states => ['inactive','active'], notes => [$arg1] }); + my $res = $minion->backend->list_jobs(0, 1, {tasks => [$task], states => ['inactive','active'], notes => [$arg]}); return -1 unless ( $res || !exists $res->{total} || $res->{total} > 0 ); - return $minion->enqueue($task => [($arg1, $arg2)] => {priority => 10} => {notes => { $arg1 => 1 }} ); + return $minion->enqueue($task => [($arg)] => {notes => { $arg => 1 }, queue => $queue, priority => 10}); } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/lib/MirrorCache/WebAPI/Plugin/Project.pm new/MirrorCache-1.094/lib/MirrorCache/WebAPI/Plugin/Project.pm --- old/MirrorCache-1.093/lib/MirrorCache/WebAPI/Plugin/Project.pm 2025-03-26 13:06:12.000000000 +0100 +++ new/MirrorCache-1.094/lib/MirrorCache/WebAPI/Plugin/Project.pm 2025-05-12 23:43:54.000000000 +0200 @@ -25,6 +25,8 @@ my %projects_alias; my %projects_redirect; my %projects_region_redirect; +my %projects_shard; + my $last_init_warning; my $caching = 1; @@ -40,6 +42,7 @@ $app->helper('mcproject.redirect' => \&_redirect); $app->helper('mcproject.caching' => \&_caching); $app->helper('mcproject.cache_dir' => \&_cache_dir); + $app->helper('mcproject.shard_for_path' => \&_shard_for_path); $caching = 0 unless -w $cache_dir; @@ -57,7 +60,7 @@ my @projs; # we want to cache it, so move to simpler structure for my $r (@rows) { - my %proj = ( id => $r->id, name => $r->name, path => $r->path, redirect => $r->redirect, prio => $r->prio ); + my %proj = ( id => $r->id, name => $r->name, path => $r->path, redirect => $r->redirect, prio => $r->prio, shard => $r->shard ); push @projs, \%proj; } @projects = @projs; @@ -90,7 +93,7 @@ if ($caching && @projects && !$wasdberror) { eval { my $f = Mojo::File->new( "$cache_dir/$cache_filename.json" ); - $f->spurt(encode_json([ @projects ])); + $f->spew(encode_json([ @projects ])); }; } @@ -105,6 +108,9 @@ $projects_path{$name} = $p->{path}; $projects_path{$id} = $p->{path}; $projects_alias{$name} = $alias; + if (my $shard = $p->{shard}) { + $projects_shard{$p->{path}} = $shard; + } my $redirect = $p->{redirect}; next unless $redirect; my @parts = split ';', $redirect; @@ -183,4 +189,13 @@ $cache_dir; } +sub _shard_for_path { + my ($c, $path) = @_; + _init_if_needed($c); + return '' unless keys %projects_shard; + + $path = substr($path, 0, index($path, '/', 1)); + return $projects_shard{$path}; +} + 1; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/lib/MirrorCache/WebAPI.pm new/MirrorCache-1.094/lib/MirrorCache/WebAPI.pm --- old/MirrorCache-1.093/lib/MirrorCache/WebAPI.pm 2025-03-26 13:06:12.000000000 +0100 +++ new/MirrorCache-1.094/lib/MirrorCache/WebAPI.pm 2025-05-12 23:43:54.000000000 +0200 @@ -88,7 +88,7 @@ my $secret = random_string(16); $self->config->{hypnotoad}{listen} = [$ENV{MOJO_LISTEN} // 'http://*:8080']; $self->config->{hypnotoad}{proxy} = $ENV{MOJO_REVERSE_PROXY} // 0, - $self->config->{hypnotoad}{workers} = $ENV{MIRRORCACHE_WORKERS}, + $self->config->{hypnotoad}{workers} = $mcconfig->workers, # $self->config->{hypnotoad}{pid_file} = $ENV{MIRRORCACHE_HYPNOTOAD_PID}, - already set in constructor $self->config->{_openid_secret} = $secret; $self->secrets([$secret]); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/lib/MirrorCache/resources/migrations/Pg.sql new/MirrorCache-1.094/lib/MirrorCache/resources/migrations/Pg.sql --- old/MirrorCache-1.093/lib/MirrorCache/resources/migrations/Pg.sql 2025-03-26 13:06:12.000000000 +0100 +++ new/MirrorCache-1.094/lib/MirrorCache/resources/migrations/Pg.sql 2025-05-12 23:43:54.000000000 +0200 @@ -459,3 +459,7 @@ -- 43 up alter table stat add column if not exists pkg varchar(512); create index if not exists stat_dt_pkg_folder_id_country_idx on stat(dt, pkg, folder_id, country); +-- 44 up +create index if not exists agg_download_pkg_period_metapkg_id_dt_idx on agg_download_pkg(period, metapkg_id, dt); +-- 45 up +alter table project add column if not exists shard varchar(32); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/lib/MirrorCache/resources/migrations/mysql.sql new/MirrorCache-1.094/lib/MirrorCache/resources/migrations/mysql.sql --- old/MirrorCache-1.093/lib/MirrorCache/resources/migrations/mysql.sql 2025-03-26 13:06:12.000000000 +0100 +++ new/MirrorCache-1.094/lib/MirrorCache/resources/migrations/mysql.sql 2025-05-12 23:43:54.000000000 +0200 @@ -472,3 +472,7 @@ -- 43 up alter table stat add column if not exists pkg varchar(512); create index if not exists i_stat_dt_pkg_folder_id_country on stat(dt, pkg, folder_id, country); +-- 44 up +create index if not exists i_agg_download_pkg_period_metapkg_id_dt on agg_download_pkg(period, metapkg_id, dt); +-- 45 up +alter table project add column if not exists shard varchar(32); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/script/mirrorcache-backstage new/MirrorCache-1.094/script/mirrorcache-backstage --- old/MirrorCache-1.093/script/mirrorcache-backstage 2025-03-26 13:06:12.000000000 +0100 +++ new/MirrorCache-1.094/script/mirrorcache-backstage 2025-05-12 23:43:54.000000000 +0200 @@ -1,2 +1,2 @@ #!/bin/sh -e -exec "$(dirname "$0")"/mirrorcache backstage run -j ${MIRRORCACHE_BACKSTAGE_WORKERS:-12} -q default +exec "$(dirname "$0")"/mirrorcache backstage run -j ${MIRRORCACHE_BACKSTAGE_WORKERS:-12} -q ${MIRRORCACHE_BACKSTAGE_QUEUE:-default} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/script/mirrorcache-backstage-hashes new/MirrorCache-1.094/script/mirrorcache-backstage-hashes --- old/MirrorCache-1.093/script/mirrorcache-backstage-hashes 2025-03-26 13:06:12.000000000 +0100 +++ new/MirrorCache-1.094/script/mirrorcache-backstage-hashes 2025-05-12 23:43:54.000000000 +0200 @@ -1,2 +1,2 @@ #!/bin/sh -e -exec "$(dirname "$0")"/mirrorcache backstage run -j ${MIRRORCACHE_BACKSTAGE_WORKERS:-12} -q hashes +exec "$(dirname "$0")"/mirrorcache backstage run -j ${MIRRORCACHE_BACKSTAGE_WORKERS:-12} -q ${MIRRORCACHE_BACKSTAGE_QUEUE:-hashes} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/t/environ/14-project-shard.sh new/MirrorCache-1.094/t/environ/14-project-shard.sh --- old/MirrorCache-1.093/t/environ/14-project-shard.sh 1970-01-01 01:00:00.000000000 +0100 +++ new/MirrorCache-1.094/t/environ/14-project-shard.sh 2025-05-12 23:43:54.000000000 +0200 @@ -0,0 +1,61 @@ +#!lib/test-in-container-environ.sh +set -ex + +# contains everything, just place holder and will not be really used in this test +mcall=$(environ mc2 $(pwd)) + +mkdir -p $mcall/dt/repositories/home1 +mkdir -p $mcall/dt/repositories/home2 +mkdir -p $mcall/dt/folder1/dir1 +mkdir -p $mcall/dt/folder1/dir2 +mkdir -p $mcall/dt/folder2/dir1 +mkdir -p $mcall/dt/folder2/dir2 + +# contains /repositories +mcrepo=$(environ mc3 $(pwd)) +# contains the rest +mcmain=$(environ mc4 $(pwd)) + + +# deploy DB +$mcall/backstage/shoot + +$mcall/sql "insert into project(name,path) select 'proj1','/folder1'" +$mcall/sql "insert into project(name,path) select 'proj 2','/folder2'" +$mcall/sql "insert into project(name,path,shard) select 'repositories','/repositories','repositories'" + +# gen config and link DB +$mcrepo/gen_env MIRRORCACHE_TOP_FOLDERS="'repositories'" +rm -r $mcrepo/db +ln -s $mcall/db $mcrepo/db +$mcrepo/start + +$mcmain/gen_env MIRRORCACHE_TOP_FOLDERS="'folder1 folder2'" +rm -r $mcmain/db +ln -s $mcall/db $mcmain/db +$mcmain/start + +( cd $mcrepo/dt ; ln -s $mcall/dt/repositories repositories ) +( cd $mcmain/dt ; ln -s $mcall/dt/folder1 folder1; ln -s $mcall/dt/folder2 folder2 ) + +echo $mcall/dt/{folder1,folder2}/{dir1,dir2}/{file1.1,file2.1}.dat | xargs -n 1 touch +echo $mcall/dt/repositories/{home1,home2}/{file1.1,file2.1}.dat | xargs -n 1 touch + +echo smoke check files exist +$mcrepo/curl -I /repositories/home1/file1.1.dat | grep '200 OK' +$mcmain/curl -I /folder1/dir1/file1.1.dat | grep '200 OK' + +$mcmain/backstage/job folder_sync_schedule_from_misses +$mcmain/backstage/job folder_sync_schedule + +$mcmain/backstage/shoot +$mcrepo/backstage/shoot -q repositories + +$mcmain/sql_test 2 == 'select count(*) from folder' +$mcmain/sql_test 2 == 'select count(*) from folder where sync_scheduled > sync_requested' +$mcmain/sql_test 2 == "select count(*) from minion_jobs where task = 'folder_sync'" +$mcmain/sql_test 2 == "select count(*) from minion_jobs where task = 'folder_sync' and state = 'finished'" +$mcmain/sql_test 1 == "select count(*) from minion_jobs where task = 'folder_sync' and queue = 'default' " +$mcmain/sql_test 1 == "select count(*) from minion_jobs where task = 'folder_sync' and queue = 'repositories'" + +echo success diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/t/environ/20-report-download-case.sh new/MirrorCache-1.094/t/environ/20-report-download-case.sh --- old/MirrorCache-1.093/t/environ/20-report-download-case.sh 1970-01-01 01:00:00.000000000 +0100 +++ new/MirrorCache-1.094/t/environ/20-report-download-case.sh 2025-05-12 23:43:54.000000000 +0200 @@ -0,0 +1,63 @@ +#!lib/test-in-container-environ.sh +set -ex + +mc=$(environ mc $(pwd)) +MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=0 + +$mc/gen_env MIRRORCACHE_SCHEDULE_RETRY_INTERVAL=$MIRRORCACHE_SCHEDULE_RETRY_INTERVAL + +$mc/start +$mc/status + +ap8=$(environ ap8) +ap7=$(environ ap7) + +files=( + /repositories/test1/debian_testing/arm64/libethercat_1.5.2-33_arm64.deb + /repositories/test2/Debian_Testing/arm64/libethercat_1.5.2-33_arm64.deb + ) + + +for f in ${files[@]}; do + for x in $mc $ap7 $ap8; do + mkdir -p $x/dt${f%/*} + echo 1111111111 > $x/dt$f + done +done + +$ap7/start +$ap8/start + +$mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap7/print_address)','','t','us','na'" +$mc/sql "insert into server(hostname,urldir,enabled,country,region) select '$($ap8/print_address)','','t','de','eu'" + + +for f in ${files[@]}; do + $mc/curl -Is /download$f +done + +$mc/backstage/job folder_sync_schedule_from_misses +$mc/backstage/job folder_sync_schedule +$mc/backstage/shoot +$mc/backstage/job mirror_scan_schedule +$mc/backstage/shoot + +for f in ${files[@]}; do + $mc/curl -Is /download$f | grep 302 + $mc/curl -Is /download$f?COUNTRY=de | grep 302 + $mc/curl -Is /download$f?COUNTRY=cn | grep 302 +done + +$mc/sql "update stat set dt = dt - interval '1 hour'" + +$mc/sql "insert into stat(ip_sha1, agent, path, country, dt, mirror_id, folder_id, file_id, secure, ipv4, metalink, head, mirrorlist, pid, execution_time, pkg) select ip_sha1, agent, path, country, dt - interval '1 hour', mirror_id, folder_id, file_id, secure, ipv4, metalink, head, mirrorlist, pid, execution_time, pkg from stat" +$mc/sql "insert into stat(ip_sha1, agent, path, country, dt, mirror_id, folder_id, file_id, secure, ipv4, metalink, head, mirrorlist, pid, execution_time, pkg) select ip_sha1, agent, path, country, dt - interval '2 hour', mirror_id, folder_id, file_id, secure, ipv4, metalink, head, mirrorlist, pid, execution_time, pkg from stat" +$mc/sql "insert into stat(ip_sha1, agent, path, country, dt, mirror_id, folder_id, file_id, secure, ipv4, metalink, head, mirrorlist, pid, execution_time, pkg) select ip_sha1, agent, path, country, dt - interval '1 day', mirror_id, folder_id, file_id, secure, ipv4, metalink, head, mirrorlist, pid, execution_time, pkg from stat" +$mc/sql "insert into stat(ip_sha1, agent, path, country, dt, mirror_id, folder_id, file_id, secure, ipv4, metalink, head, mirrorlist, pid, execution_time, pkg) select ip_sha1, agent, path, country, dt - interval '1 minute', mirror_id, folder_id, file_id, secure, ipv4, metalink, head, mirrorlist, pid, execution_time, pkg from stat" + +$mc/backstage/job -e report -a '["once"]' +$mc/backstage/shoot + +$mc/curl /rest/repdownload | grep '"known_files_no_mirrors":"4","known_files_redirected":"12","known_files_requested":"12"' | grep '"bytes_redirected":"132","bytes_served":"0","bytes_total":"132"' + +echo success diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/t/environ/20-report-download.sh new/MirrorCache-1.094/t/environ/20-report-download.sh --- old/MirrorCache-1.093/t/environ/20-report-download.sh 2025-03-26 13:06:12.000000000 +0100 +++ new/MirrorCache-1.094/t/environ/20-report-download.sh 2025-05-12 23:43:54.000000000 +0200 @@ -136,4 +136,13 @@ $mc/curl /rest/package/cargo1.64/stat_download_curr | grep '{"cnt_curr":3}' + +echo testing cleanup, for it populate old data +$mc/sql "update agg_download_pkg set dt = dt - interval '1 month'" +$mc/sql_test 72 == "select count(*) from agg_download_pkg" + +$mc/backstage/job cleanup +$mc/backstage/shoot +$mc/sql_test 48 == "select count(*) from agg_download_pkg" + echo success diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/t/lib/Dockerfile.environ.mariadb new/MirrorCache-1.094/t/lib/Dockerfile.environ.mariadb --- old/MirrorCache-1.093/t/lib/Dockerfile.environ.mariadb 2025-03-26 13:06:12.000000000 +0100 +++ new/MirrorCache-1.094/t/lib/Dockerfile.environ.mariadb 2025-05-12 23:43:54.000000000 +0200 @@ -11,7 +11,7 @@ RUN zypper -vvv -n install MirrorCache perl-MaxMind-DB-Reader perl-Mojolicious-Plugin-ClientIP \ vim mariadb mariadb-server curl sudo git-core wget tar m4 make rsync \ apache2 perl-Digest-MD4 tidy nginx bbe perl-DBD-mysql perl-Mojo-mysql perl-Minion-Backend-mysql perl-DateTime-HiRes \ - perl-Config-IniFiles environ + perl-Config-IniFiles environ eatmydata # optional dependencies used in testing RUN zypper -vvv -n install perl-Geo-IP2Location perl-Digest-Zsync perl-DateTime-Format-MySQL libxml2-tools diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/t/lib/Dockerfile.environ.mariadb.experimental new/MirrorCache-1.094/t/lib/Dockerfile.environ.mariadb.experimental --- old/MirrorCache-1.093/t/lib/Dockerfile.environ.mariadb.experimental 2025-03-26 13:06:12.000000000 +0100 +++ new/MirrorCache-1.094/t/lib/Dockerfile.environ.mariadb.experimental 2025-05-12 23:43:54.000000000 +0200 @@ -13,7 +13,7 @@ RUN zypper -vvv -n install MirrorCache perl-MaxMind-DB-Reader perl-Mojolicious-Plugin-ClientIP \ vim MariaDB-server MariaDB-client curl sudo git-core wget tar m4 make rsync \ apache2 perl-Digest-MD4 tidy nginx bbe perl-DBD-mysql perl-Mojo-mysql perl-Minion-Backend-mysql perl-DateTime-HiRes \ - perl-Config-IniFiles environ + perl-Config-IniFiles environ eatmydata # optional dependencies used in testing RUN zypper -vvv -n install perl-Geo-IP2Location perl-Digest-Zsync perl-DateTime-Format-MySQL libxml2-tools diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/t/lib/environ/mc/source/gen_env.sh.m4 new/MirrorCache-1.094/t/lib/environ/mc/source/gen_env.sh.m4 --- old/MirrorCache-1.093/t/lib/environ/mc/source/gen_env.sh.m4 2025-03-26 13:06:12.000000000 +0100 +++ new/MirrorCache-1.094/t/lib/environ/mc/source/gen_env.sh.m4 2025-05-12 23:43:54.000000000 +0200 @@ -17,6 +17,7 @@ export MIRRORCACHE_STAT_FLUSH_COUNT=1 export MIRRORCACHE_RECKLESS=1 export MIRRORCACHE_SCAN_MTIME_DIFF=0 +export MARIADB_TLS_DISABLE_PEER_VERIFICATION=1 " for i in "$@"; do diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/MirrorCache-1.093/templates/app/package/show.html.ep new/MirrorCache-1.094/templates/app/package/show.html.ep --- old/MirrorCache-1.093/templates/app/package/show.html.ep 2025-03-26 13:06:12.000000000 +0100 +++ new/MirrorCache-1.094/templates/app/package/show.html.ep 2025-05-12 23:43:54.000000000 +0200 @@ -196,16 +196,22 @@ <div id="download-statistics" class="card"> <div class="card-header">Download Statistics</div> <div class="card-body row"> - <div class="col-sm-3"> + <div class="col-sm"> First seen: <span class="badge text-bg-success" id="download-stat-first-seen"></span> </div> - <div class="col-sm-3"> + <div class="col-sm"> Total: <span class="badge text-bg-primary" id="download-stat-total"></span> </div> - <div class="col-sm-3"> + <div class="col-sm"> + Month: <span class="badge text-bg-secondary" id="download-stat-month"></span> + </div> + <div class="col-sm"> + Week: <span class="badge text-bg-secondary" id="download-stat-week"></span> + </div> + <div class="col-sm"> Today: <span class="badge text-bg-secondary" id="download-stat-today"></span> </div> - <div class="col-sm-3"> + <div class="col-sm"> Recent: <span class="badge text-bg-dark" id="download-stat-curr"></span> </div> </div> ++++++ MirrorCache.obsinfo ++++++ --- /var/tmp/diff_new_pack.rmWrQw/_old 2025-05-26 18:35:43.911914060 +0200 +++ /var/tmp/diff_new_pack.rmWrQw/_new 2025-05-26 18:35:43.915914229 +0200 @@ -1,5 +1,5 @@ name: MirrorCache -version: 1.093 -mtime: 1742990772 -commit: 110a7a829c8bf15b9ea7e951166a719e1f0b86e7 +version: 1.094 +mtime: 1747086234 +commit: ee1d47c0eab73867dd275d2c0ccf00d95f02647f
