Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package openQA for openSUSE:Factory checked in at 2026-01-14 16:22:09 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/openQA (Old) and /work/SRC/openSUSE:Factory/.openQA.new.1928 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "openQA" Wed Jan 14 16:22:09 2026 rev:793 rq:1327095 version:5.1768323619.9a70ab91 Changes: -------- --- /work/SRC/openSUSE:Factory/openQA/openQA.changes 2026-01-13 21:26:45.600615757 +0100 +++ /work/SRC/openSUSE:Factory/.openQA.new.1928/openQA.changes 2026-01-14 16:22:26.395339789 +0100 @@ -1,0 +2,14 @@ +Tue Jan 13 23:23:36 UTC 2026 - [email protected] + +- Update to version 5.1768323619.9a70ab91: + * refactor: Extend tests of df-based cleanup + * fix: Avoid wrong deletion of archived jobs in df-based cleanup + * refactor: Move logic for validating percentage into helper + * refactor: Clarify wording in comment regarding job cleanup + * Use template literals in certain JavaScript code + * Retry delete_needles job on server restart + * Add test for _delete_needles + * feat(OpenQA::Git): Cleanup git dir in commit() on shutdown + * feat: Improve rendering results on the scheduled product page + +------------------------------------------------------------------- Old: ---- openQA-5.1768209690.f34c2973.obscpio New: ---- openQA-5.1768323619.9a70ab91.obscpio ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ openQA-client-test.spec ++++++ --- /var/tmp/diff_new_pack.xYbLWT/_old 2026-01-14 16:22:30.963529280 +0100 +++ /var/tmp/diff_new_pack.xYbLWT/_new 2026-01-14 16:22:30.967529445 +0100 @@ -18,7 +18,7 @@ %define short_name openQA-client Name: %{short_name}-test -Version: 5.1768209690.f34c2973 +Version: 5.1768323619.9a70ab91 Release: 0 Summary: Test package for %{short_name} License: GPL-2.0-or-later ++++++ openQA-devel-test.spec ++++++ --- /var/tmp/diff_new_pack.xYbLWT/_old 2026-01-14 16:22:31.119535751 +0100 +++ /var/tmp/diff_new_pack.xYbLWT/_new 2026-01-14 16:22:31.127536083 +0100 @@ -18,7 +18,7 @@ %define short_name openQA-devel Name: %{short_name}-test -Version: 5.1768209690.f34c2973 +Version: 5.1768323619.9a70ab91 Release: 0 Summary: Test package for %{short_name} License: GPL-2.0-or-later ++++++ openQA-test.spec ++++++ --- /var/tmp/diff_new_pack.xYbLWT/_old 2026-01-14 16:22:31.307543549 +0100 +++ /var/tmp/diff_new_pack.xYbLWT/_new 2026-01-14 16:22:31.311543715 +0100 @@ -18,7 +18,7 @@ %define short_name openQA Name: %{short_name}-test -Version: 5.1768209690.f34c2973 +Version: 5.1768323619.9a70ab91 Release: 0 Summary: Test package for openQA License: GPL-2.0-or-later ++++++ openQA-worker-test.spec ++++++ --- /var/tmp/diff_new_pack.xYbLWT/_old 2026-01-14 16:22:31.487551016 +0100 +++ /var/tmp/diff_new_pack.xYbLWT/_new 2026-01-14 16:22:31.507551846 +0100 @@ -18,7 +18,7 @@ %define short_name openQA-worker Name: %{short_name}-test -Version: 5.1768209690.f34c2973 +Version: 5.1768323619.9a70ab91 Release: 0 Summary: Test package for %{short_name} License: GPL-2.0-or-later ++++++ openQA.spec ++++++ --- /var/tmp/diff_new_pack.xYbLWT/_old 2026-01-14 16:22:31.855566282 +0100 +++ /var/tmp/diff_new_pack.xYbLWT/_new 2026-01-14 16:22:31.863566613 +0100 @@ -99,7 +99,7 @@ %define devel_requires %devel_no_selenium_requires chromedriver Name: openQA -Version: 5.1768209690.f34c2973 +Version: 5.1768323619.9a70ab91 Release: 0 Summary: The openQA web-frontend, scheduler and tools License: GPL-2.0-or-later ++++++ openQA-5.1768209690.f34c2973.obscpio -> openQA-5.1768323619.9a70ab91.obscpio ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openQA-5.1768209690.f34c2973/assets/javascripts/audit_log.js new/openQA-5.1768323619.9a70ab91/assets/javascripts/audit_log.js --- old/openQA-5.1768209690.f34c2973/assets/javascripts/audit_log.js 2026-01-12 10:21:30.000000000 +0100 +++ new/openQA-5.1768323619.9a70ab91/assets/javascripts/audit_log.js 2026-01-13 18:00:19.000000000 +0100 @@ -209,16 +209,68 @@ } } +function renderFailedJobInfo(results) { + const failedJobInfo = results.failed_job_info; + delete results.failed_job_info; + if (!Array.isArray(failedJobInfo) || failedJobInfo.length === 0) { + return []; + } + const heading = createElement('h4', 'Errors occurred when scheduling this product'); + const failedElements = createElement( + 'ul', + failedJobInfo.map(info => { + return createElement('li', [renderJobLink(info.job_id), ': ', renderMessages(info.error_messages)]); + }) + ); + return [createElement('div', [heading, failedElements], {class: 'alert alert-danger'})]; +} + +function renderSuccessfullyScheduledJobs(results) { + const successfulJobIDs = results.successful_job_ids; + delete results.successful_job_ids; + if (!Array.isArray(successfulJobIDs) || successfulJobIDs.length === 0) { + return []; + } + const heading = createElement('h4', `Successfully scheduled ${successfulJobIDs.length} job(s)`); + const jobLinks = successfulJobIDs.flatMap(id => [renderJobLink(id), ', ']); + if (jobLinks.length > 1) { + jobLinks.pop(); + } + return [createElement('div', [heading, ...jobLinks], {class: 'alert alert-success'})]; +} + +function renderNotes(results, property, type = 'info') { + const note = results[property]; + delete results[property]; + return note !== undefined ? [createElement('div', [renderMessages(note)], {class: `alert alert-${type}`})] : []; +} + +function renderUnknownResults(results) { + return Object.keys(results).length > 0 ? [createElement('pre', JSON.stringify(results, undefined, 4))] : []; +} + function renderScheduledProductResults(results) { - const element = document.createElement(results ? 'pre' : 'p'); - element.textContent = results ? JSON.stringify(results, undefined, 4) : 'No results available.'; - return element; + if (results === null || typeof results !== 'object') { + return createElement('p', 'No results available.'); + } + delete results.error_code; // not meaningful to users and results.error is enough + return createElement('div', [ + ...renderNotes(results, 'note'), // notes added by external tools via API + ...renderNotes(results, 'error', 'danger'), // error message in case scheduling was not possible at all + ...renderFailedJobInfo(results), // error messages about failed jobs in case the overall scheduling succeeded + ...renderSuccessfullyScheduledJobs(results), // IDs of successfully scheduled jobs + ...renderNotes(results, 'notes'), // notes added by openQA itself under certain conditions + ...renderUnknownResults(results) // everything not covered by the above + ]); } function showScheduledProductResults(link) { const rowData = dataForLink(link); if (rowData !== undefined) { - showScheduledProductModalDialog('Scheduled product results', renderScheduledProductResults(rowData.results)); + showScheduledProductModalDialog( + 'Scheduled product results', + renderScheduledProductResults(Object.create(rowData.results)) + ); } } @@ -248,10 +300,10 @@ function showSettingsAndResults(rowData) { const scheduledProductsDiv = $('#scheduled-products'); - scheduledProductsDiv.append($('<h3>Settings</h3>')); - scheduledProductsDiv.append(renderScheduledProductSettings(rowData.settings)); scheduledProductsDiv.append($('<h3>Results</h3>')); scheduledProductsDiv.append(renderScheduledProductResults(rowData.results)); + scheduledProductsDiv.append($('<h3>Settings</h3>')); + scheduledProductsDiv.append(renderScheduledProductSettings(rowData.settings)); } function loadProductLogTable(dataTableUrl, rescheduleUrlTemplate, showActions) { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openQA-5.1768209690.f34c2973/assets/javascripts/render.js new/openQA-5.1768323619.9a70ab91/assets/javascripts/render.js --- old/openQA-5.1768209690.f34c2973/assets/javascripts/render.js 2026-01-12 10:21:30.000000000 +0100 +++ new/openQA-5.1768323619.9a70ab91/assets/javascripts/render.js 2026-01-13 18:00:19.000000000 +0100 @@ -1,6 +1,6 @@ // jshint esversion: 9 -function createElement(tag, content = [], attrs = {}) { +function createElement(tag, content = [], attrs = {}, options = {}) { const elem = document.createElement(tag); for (const [key, value] of Object.entries(attrs)) { @@ -10,6 +10,9 @@ } elem.append(...content); + if (options.preWrap) { + elem.style.whiteSpace = 'pre-wrap'; + } return elem; } @@ -245,3 +248,16 @@ tbody.appendChild(renderModuleRow(module, response.snippets)); } } + +function renderJobLink(jobId) { + return createElement('a', [jobId], {href: `/tests/${jobId}`}); +} + +function renderMessages(messages) { + return Array.isArray(messages) && messages.length > 1 + ? createElement( + 'ul', + messages.map(m => createElement('li', m, {}, {preWrap: true})) + ) + : createElement('span', [messages], {}, {preWrap: true}); +} diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openQA-5.1768209690.f34c2973/lib/OpenQA/Error/Cmd.pm new/openQA-5.1768323619.9a70ab91/lib/OpenQA/Error/Cmd.pm --- old/openQA-5.1768209690.f34c2973/lib/OpenQA/Error/Cmd.pm 2026-01-12 10:21:30.000000000 +0100 +++ new/openQA-5.1768323619.9a70ab91/lib/OpenQA/Error/Cmd.pm 2026-01-13 18:00:19.000000000 +0100 @@ -3,14 +3,8 @@ package OpenQA::Error::Cmd; -use Mojo::Base -base, -signatures; +use Mojo::Base 'OpenQA::Error', -signatures; -has [qw(status return_code stdout stderr signal msg)]; - -# Perl::Critic::Policy::Community::OverloadOptions -# Automatically render error message in string context -use overload '""' => \&to_string, bool => sub { 1 }, fallback => 1; - -sub to_string ($self, @) { (ref $self) . ': ' . $self->msg } +has [qw(status return_code stdout stderr)]; 1; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openQA-5.1768209690.f34c2973/lib/OpenQA/Error.pm new/openQA-5.1768323619.9a70ab91/lib/OpenQA/Error.pm --- old/openQA-5.1768209690.f34c2973/lib/OpenQA/Error.pm 1970-01-01 01:00:00.000000000 +0100 +++ new/openQA-5.1768323619.9a70ab91/lib/OpenQA/Error.pm 2026-01-13 18:00:19.000000000 +0100 @@ -0,0 +1,21 @@ +# Copyright SUSE LLC +# SPDX-License-Identifier: GPL-2.0-or-later + +package OpenQA::Error; + +use Mojo::Base -base, -signatures; + +has signal => 0; +has 'msg'; + +# Automatically render error message in string context +# Perl::Critic::Policy::Community::OverloadOptions +use overload '""' => \&to_string, bool => sub { 1 }, fallback => 1; + +sub to_string ($self, @) { (ref $self) . ': ' . $self->msg } + +sub shutting_down ($self) { + grep { ($self->signal // 0) eq $_ } qw(INT TERM); +} + +1; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openQA-5.1768209690.f34c2973/lib/OpenQA/Git.pm new/openQA-5.1768323619.9a70ab91/lib/OpenQA/Git.pm --- old/openQA-5.1768209690.f34c2973/lib/OpenQA/Git.pm 2026-01-12 10:21:30.000000000 +0100 +++ new/openQA-5.1768323619.9a70ab91/lib/OpenQA/Git.pm 2026-01-13 18:00:19.000000000 +0100 @@ -11,6 +11,7 @@ use OpenQA::App; use Feature::Compat::Try; use OpenQA::Error::Cmd; +use OpenQA::Task::SignalGuard; use Carp qw(croak); has 'app'; @@ -36,6 +37,11 @@ } sub _run_cmd ($self, $args, $options = {}) { + if (my $signal = OpenQA::Task::SignalGuard->signaled and !$options->{force}) { + my $error + = OpenQA::Error->new(signal => $signal, msg => "Not running git command (@$args) because of signal $signal"); + croak $error; + } my $include_git_path = $options->{include_git_path} // 1; my $batchmode = $options->{batchmode} // 0; my @cmd; @@ -106,7 +112,14 @@ # commit changes my $message = $args->{message}; my $author = sprintf('--author=%s <%s>', $self->user->fullname, $self->user->email); - $self->_run_cmd(['commit', '-q', '-m', $message, $author, @files], {croak => 'Unable to commit via Git'}); + try { + $self->_run_cmd(['commit', '-q', '-m', $message, $author, @files], {croak => 'Unable to commit via Git'}); + } + catch ($e) { + $self->_run_cmd(['restore', '--staged', @files], {force => 1, croak => 'Unable to restore'}) + if ref $e && $e->shutting_down; + croak $e; + } # push changes if (($self->config->{do_push} || '') eq 'yes') { @@ -116,10 +129,17 @@ return undef; } catch ($e) { - if ($e->return_code == 128 and $e->stderr =~ m/Authentication failed for .http/) { - $msg - .= '. See https://open.qa/docs/#_setting_up_git_support on how to setup git support and possibly push via ssh.'; - $e->msg($msg); + if (ref $e) { + $self->_run_cmd(['reset', '--hard', 'HEAD^1'], {force => 1, croak => 'Unable to reset'}) + if $e->shutting_down; + if ( ref $e eq 'OpenQA::Error::Cmd' + and $e->return_code == 128 + and $e->stderr =~ m/Authentication failed for .http/) + { + $msg + .= '. See https://open.qa/docs/#_setting_up_git_support on how to setup git support and possibly push via ssh.'; + $e->msg($msg); + } } croak $e; } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openQA-5.1768209690.f34c2973/lib/OpenQA/Task/Job/Limit.pm new/openQA-5.1768323619.9a70ab91/lib/OpenQA/Task/Job/Limit.pm --- old/openQA-5.1768209690.f34c2973/lib/OpenQA/Task/Job/Limit.pm 2026-01-12 10:21:30.000000000 +0100 +++ new/openQA-5.1768323619.9a70ab91/lib/OpenQA/Task/Job/Limit.pm 2026-01-13 18:00:19.000000000 +0100 @@ -153,6 +153,52 @@ return $margin_bytes; } +sub _is_valid_percentage ($value) { looks_like_number($value) && $value >= 0 && $value <= 100 } + +sub _delete_results ($jobs, $max_job_id, $not_important_cond, $important_cond, $margin_bytes, $archived) { + # caveat: The subsequent cleanup simply deletes stuff from old jobs first. It does not take the retention periods + # configured on job group level into account anymore. + # caveat: We're considering possibly lots of jobs at once here. Maybe we need to select a range here when dealing + # with a huge number of jobs. + + my $from = $archived ? 'archive' : 'results dir'; + log_debug "Deleting videos from non-important jobs starting from oldest job (balance is $margin_bytes)"; + my @job_id_args = (id => {'<=' => $max_job_id}, archived => $archived); + my %jobs_params = (order_by => {-asc => 'id'}); + my $relevant_jobs = $jobs->search({@job_id_args, @$not_important_cond, logs_present => 1}, \%jobs_params); + while (my $openqa_job = $relevant_jobs->next) { + log_debug 'Deleting video of job ' . $openqa_job->id; + return (1, "Done with $from after deleting videos from non-important jobs") + if ($margin_bytes += $openqa_job->delete_videos) >= 0; + } + + log_debug "Deleting results from non-important jobs starting from oldest job (balance is $margin_bytes)"; + $relevant_jobs = $jobs->search({@job_id_args, @$not_important_cond}, \%jobs_params); + while (my $openqa_job = $relevant_jobs->next) { + log_debug 'Deleting results of job ' . $openqa_job->id; + return (1, "Done with $from after deleting results from non-important jobs") + if ($margin_bytes += $openqa_job->delete_results) >= 0; + } + + log_debug "Deleting videos from important jobs starting from oldest job (balance is $margin_bytes)"; + $relevant_jobs = $jobs->search({@job_id_args, @$important_cond, logs_present => 1}, \%jobs_params); + while (my $openqa_job = $relevant_jobs->next) { + log_debug 'Deleting video of important job ' . $openqa_job->id; + return (1, "Done with $from after deleting videos from important jobs") + if ($margin_bytes += $openqa_job->delete_videos) >= 0; + } + + log_debug "Deleting results from important jobs starting from oldest job (balance is $margin_bytes)"; + $relevant_jobs = $jobs->search({@job_id_args, @$important_cond}, \%jobs_params); + while (my $openqa_job = $relevant_jobs->next) { + log_debug 'Deleting results of important job ' . $openqa_job->id; + return (1, "Done with $from after deleting results from important jobs") + if ($margin_bytes += $openqa_job->delete_results) >= 0; + } + + return (0, "Unable to cleanup enough results from $from"); +} + sub _ensure_results_below_threshold ($job, @) { my $ensure_task_retry_on_termination_signal_guard = OpenQA::Task::SignalGuard->new($job); # prevent multiple limit_* tasks to run in parallel @@ -164,7 +210,7 @@ my $min_free_percentage = $job->app->config->{misc_limits}->{results_min_free_disk_space_percentage}; return $job->finish('No minimum free disk space percentage configured') unless defined $min_free_percentage; return $job->fail('Configured minimum free disk space is not a number between 0 and 100') - unless looks_like_number($min_free_percentage) && $min_free_percentage >= 0 && $min_free_percentage <= 100; + unless _is_valid_percentage($min_free_percentage); # check free percentage # caveat: We're using `df` here which might not be appropriate for any filesystem, e.g. one might want @@ -183,11 +229,10 @@ # note: If a new important build is scheduled while the cleanup is ongoing we must not accidentally clean these # jobs up because our list of important builds is outdated. It would be possible to use a transaction # to avoid this. However, this would make things more complicated because the actual screenshot deletion - # must *not* run within that transaction so we needed to determine non-important jobs upfront. This + # must *not* run within such a transaction. So we needed to determine non-important jobs upfront. This # would eliminate the possibility to query jobs in ranges for better scalability. (The screenshot - # deletion must not run within the transaction because we rely on getting a foreign key violation to - # prevent deleting a screenshot which has in the meantime been linked to a new job. This conflict must - # not only occur at the end of the transaction.) + # deletion must not run within a transaction because we rely on getting a foreign key violation to + # prevent deleting a screenshot which has in the meantime been linked to a new job.) my $schema = $app->schema; my ($max_job_id) = $schema->storage->dbh->selectrow_array('select max(id) from jobs'); return $job->finish('Done, no jobs present') unless $max_job_id; @@ -213,47 +258,10 @@ $job->note(important_builds_with_version => \@important_builds_with_version); $job->note(important_builds_without_version => \@important_builds_without_version); - # caveat: The subsequent cleanup simply deletes stuff from old jobs first. It does not take the retention periods - # configured on job group level into account anymore. - # caveat: We're considering possibly lots of jobs at once here. Maybe we need to select a range here when dealing - # with a huge number of jobs. - - log_debug "Deleting videos from non-important jobs startinng from oldest job (balance is $margin_bytes)"; my $jobs = $schema->resultset('Jobs'); - my @job_id_args = (id => {'<=' => $max_job_id}); - my %jobs_params = (order_by => {-asc => 'id'}); - my $relevant_jobs = $jobs->search({@job_id_args, @not_important_cond, logs_present => 1}, \%jobs_params); - while (my $openqa_job = $relevant_jobs->next) { - log_debug 'Deleting video of job ' . $openqa_job->id; - return $job->finish('Done after deleting videos from non-important jobs') - if ($margin_bytes += $openqa_job->delete_videos) >= 0; - } - - log_debug "Deleting results from non-important jobs startinng from oldest job (balance is $margin_bytes)"; - $relevant_jobs = $jobs->search({@job_id_args, @not_important_cond}, \%jobs_params); - while (my $openqa_job = $relevant_jobs->next) { - log_debug 'Deleting results of job ' . $openqa_job->id; - return $job->finish('Done after deleting results from non-important jobs') - if ($margin_bytes += $openqa_job->delete_results) >= 0; - } - - log_debug "Deleting videos from important jobs startinng from oldest job (balance is $margin_bytes)"; - $relevant_jobs = $jobs->search({@job_id_args, @important_cond, logs_present => 1}, \%jobs_params); - while (my $openqa_job = $relevant_jobs->next) { - log_debug 'Deleting video of important job ' . $openqa_job->id; - return $job->finish('Done after deleting videos from important jobs') - if ($margin_bytes += $openqa_job->delete_videos) >= 0; - } - - log_debug "Deleting results from important jobs startinng from oldest job (balance is $margin_bytes)"; - $relevant_jobs = $jobs->search({@job_id_args, @important_cond}, \%jobs_params); - while (my $openqa_job = $relevant_jobs->next) { - log_debug 'Deleting results of important job ' . $openqa_job->id; - return $job->finish('Done after deleting results from important jobs') - if ($margin_bytes += $openqa_job->delete_results) >= 0; - } - - return $job->fail('Unable to cleanup enough results'); + my ($ok, $message) = _delete_results($jobs, $max_job_id, \@not_important_cond, \@important_cond, $margin_bytes, 0); + my $method = $ok ? 'finish' : 'fail'; + $job->$method($message); } 1; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openQA-5.1768209690.f34c2973/lib/OpenQA/Task/Needle/Delete.pm new/openQA-5.1768323619.9a70ab91/lib/OpenQA/Task/Needle/Delete.pm --- old/openQA-5.1768209690.f34c2973/lib/OpenQA/Task/Needle/Delete.pm 2026-01-12 10:21:30.000000000 +0100 +++ new/openQA-5.1768323619.9a70ab91/lib/OpenQA/Task/Needle/Delete.pm 2026-01-13 18:00:19.000000000 +0100 @@ -6,16 +6,19 @@ use OpenQA::Utils; use Scalar::Util 'looks_like_number'; -use Time::Seconds 'ONE_HOUR'; +use Time::Seconds qw(ONE_HOUR ONE_DAY); use OpenQA::Task::SignalGuard; use Feature::Compat::Try; +use Carp qw(croak); sub register { my ($self, $app) = @_; - $app->minion->add_task(delete_needles => sub { _delete_needles($app, @_) }); + $app->minion->add_task(delete_needles => sub { _task_delete_needles($app, @_) }); } -sub _delete_needles ($app, $minion_job, $args) { +sub restart_delay { $ENV{OPENQA_GRU_SERVER_RESTART_DELAY} // 5 } + +sub _task_delete_needles ($app, $minion_job, $args) { # SignalGuard will prevent the delete task to interrupt with no recovery, # instead will retry once the gru server returned up and running. The popup # on the frontend will wait until the retried job finished. @@ -25,10 +28,11 @@ my $user = $schema->resultset('Users')->find($args->{user_id}); my $needle_ids = $args->{needle_ids}; - my (@removed_ids, @errors); - - my %to_remove; + my (@errors, %to_remove); + my @removed_ids = @{$minion_job->info->{notes}->{removed_ids} || []}; + my %removed = map { $_ => 1 } @removed_ids; for my $needle_id (@$needle_ids) { + next if $removed{$needle_id}; my $needle = looks_like_number($needle_id) ? $needles->find($needle_id) : undef; if (!$needle) { push @errors, @@ -42,13 +46,32 @@ } $signal_guard->retry(0); + try { + _delete_needles($app, $user, \%to_remove, \@removed_ids, \@errors); + } + catch ($e) { + if (ref $e && $e->shutting_down) { + $minion_job->note(removed_ids => \@removed_ids); + # Explicitly set high value for expire, otherwise it would be only 60s + return $minion_job->retry({delay => restart_delay, expire => ONE_DAY}); + } + croak $e; + } - for my $dir (sort keys %to_remove) { - my $needles = $to_remove{$dir}; + return $minion_job->finish( + { + removed_ids => \@removed_ids, + errors => \@errors + }); +} + +sub _delete_needles ($app, $user, $to_remove, $removed_ids, $errors) { + DIR: for my $dir (sort keys %$to_remove) { + my $needles = $to_remove->{$dir}; # prevent multiple git tasks to run in parallel my $guard; unless ($guard = $app->minion->guard("git_clone_${dir}_task", 2 * ONE_HOUR)) { - push @errors, + push @$errors, { id => $_, message => "Another git task for $dir is ongoing. Try again later.", @@ -62,23 +85,18 @@ $needle->remove($user); } catch ($e) { - push @errors, + croak $e if ref $e && $e->shutting_down; + push @$errors, { id => $needle_id, display_name => $needle->filename, - message => ref $e ? $e->msg : $e, + message => "$e", }; next; } - push @removed_ids, $needle_id; + push @$removed_ids, $needle_id; } } - - return $minion_job->finish( - { - removed_ids => \@removed_ids, - errors => \@errors - }); } 1; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openQA-5.1768209690.f34c2973/lib/OpenQA/Task/SignalGuard.pm new/openQA-5.1768323619.9a70ab91/lib/OpenQA/Task/SignalGuard.pm --- old/openQA-5.1768209690.f34c2973/lib/OpenQA/Task/SignalGuard.pm 2026-01-12 10:21:30.000000000 +0100 +++ new/openQA-5.1768323619.9a70ab91/lib/OpenQA/Task/SignalGuard.pm 2026-01-13 18:00:19.000000000 +0100 @@ -17,6 +17,8 @@ # action twice. has abort => 0; +my $SIGNALED = 0; + # retries the specified Minion job when receiving SIGTERM/SIGINT as long as the returned object exists # note: Prevents the job to fail with "Job terminated unexpectedly". sub new ($class, $job, @attributes) { @@ -33,6 +35,7 @@ } sub _handle_signal ($self_weak, $signal) { + $SIGNALED = $signal; # abort job if the corresponding flag is set return undef unless $self_weak; my $job = $self_weak->{_job}; @@ -51,6 +54,8 @@ exit; } +sub signaled ($class) { $SIGNALED } + sub DESTROY ($self) { $SIG{TERM} = $self->{_old_term_handler}; $SIG{INT} = $self->{_old_int_handler}; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openQA-5.1768209690.f34c2973/t/14-grutasks-git.t new/openQA-5.1768323619.9a70ab91/t/14-grutasks-git.t --- old/openQA-5.1768209690.f34c2973/t/14-grutasks-git.t 2026-01-12 10:21:30.000000000 +0100 +++ new/openQA-5.1768323619.9a70ab91/t/14-grutasks-git.t 2026-01-13 18:00:19.000000000 +0100 @@ -21,6 +21,9 @@ use Mojo::File qw(path tempdir); use Time::Seconds; use File::Copy::Recursive qw(dircopy); +use Test::MockModule; +use Test::MockObject; +use OpenQA::Error; # Avoid using tester's ~/.gitconfig delete $ENV{HOME}; @@ -481,6 +484,47 @@ $error = $res->{result}->{errors}->[0]; like $error->{message}, qr{Another git task for.*fedora.*is ongoing}, 'expected error message'; }; + + subtest 'error handling' => sub { + my $user = Test::MockObject->new; + my @needles = map { $schema->resultset('Needles')->new({id => $_}) } (23, 42); + my $mock_needle = Test::MockModule->new('OpenQA::Schema::Result::Needles'); + $mock_needle->redefine( + remove => sub ($self, $user) { + return if $self->id == 23; + die OpenQA::Error->new(signal => 'INT', msg => "Not running git command (...) because of signal INT"); + }); + my %to_remove = ('/foo' => \@needles); + my @removed_ids; + my @errors; + throws_ok { + OpenQA::Task::Needle::Delete::_delete_needles($t->app, $user, \%to_remove, \@removed_ids, \@errors) + } + qr{OpenQA::Error: Not running git command}, 'got error for second needle'; + is $removed_ids[0], 23, 'Successfully removed first needle'; + }; + + subtest 'retry on server restart' => sub { + my $mock_needle = Test::MockModule->new('OpenQA::Task::Needle::Delete'); + $mock_needle->redefine( + _delete_needles => sub ($, $, $, $removed_ids, $) { + push @$removed_ids, 23; + die OpenQA::Error->new(signal => 'INT', msg => "Not running git command (...) because of signal INT"); + }); + my $res = run_gru_job(@gru_args); + is $res->{state}, 'inactive', 'job inactive'; + is $res->{notes}->{removed_ids}->[0], 23, 'removed ids are recorded'; + + subtest 'Unexpected error' => sub { + $mock_needle->redefine(_delete_needles => sub (@) { die "Something else" }); + combined_like { + $res = run_gru_job(@gru_args); + } + qr{Gru job error: Something else}, 'error message logged'; + is $res->{state}, 'failed', 'job failed'; + like $res->{result}, qr{Something else}, 'error message recorded'; + }; + }; }; subtest ServerAvailability => sub { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openQA-5.1768209690.f34c2973/t/16-utils-runcmd.t new/openQA-5.1768323619.9a70ab91/t/16-utils-runcmd.t --- old/openQA-5.1768209690.f34c2973/t/16-utils-runcmd.t 2026-01-12 10:21:30.000000000 +0100 +++ new/openQA-5.1768323619.9a70ab91/t/16-utils-runcmd.t 2026-01-13 18:00:19.000000000 +0100 @@ -333,6 +333,65 @@ ok $signal_guard->abort, 'signal guard is set to abort'; }; +subtest 'OpenQA::Git signal handling' => sub { + my $signal_guard_mock = Test::MockModule->new('OpenQA::Task::SignalGuard'); + my $git_mock = Test::MockModule->new('OpenQA::Git'); + my $git = OpenQA::Git->new({app => $t->app, dir => $empty_tmp_dir, user => $first_user}); + + $git_mock->redefine( + run_cmd_with_log_return_error => sub { {status => 0, stdout => '', stderr => '', signal => 'INT'} }); + throws_ok { $git->commit({add => ['file.txt']}) } qr{OpenQA::Error::Cmd: Unable to add via Git}, + 'Aborts git command when signal was received'; + + $signal_guard_mock->redefine(signaled => 'INT'); + throws_ok { $git->commit({add => ['file.txt']}) } qr{OpenQA::Error: Not running git command.*signal INT}, + 'Does not execute command when signal was received'; + + $signal_guard_mock->unmock('signaled'); + + subtest 'rollback commit' => sub { + + my @results = ( + {status => 1, return_code => 0, stdout => 'add ok', stderr => '', signal => 0}, + {status => 0, return_code => 0, stdout => '', stderr => '', signal => 'INT'}, + {status => 1, return_code => 0, stdout => 'restore ok', stderr => '', signal => 0}, + ); + my @executed; + $git_mock->redefine( + run_cmd_with_log_return_error => sub ($cmd, %args) { + push @executed, $cmd; + return shift @results; + }); + subtest 'git commit' => sub { + my @expected = (qr{git .* add file.txt}, qr{git .* commit}, qr{git .* restore.*file.txt}); + + throws_ok { $git->commit({add => ['file.txt'], message => 'test'}) } + qr{OpenQA::Error::Cmd: Unable to commit}, 'git commit fails'; + for my $i (0 .. $#expected) { + like "@{$executed[$i]}", $expected[$i], "executed git command $i like expected"; + } + }; + + subtest 'git push' => sub { + $git->config->{do_push} = 'yes'; + @results = ( + {status => 1, return_code => 0, stdout => 'add ok', stderr => '', signal => 0}, + {status => 1, return_code => 0, stdout => 'commit ok', stderr => '', signal => 0}, + {status => 0, return_code => 0, stdout => '', stderr => '', signal => 'INT'}, + {status => 1, return_code => 0, stdout => 'restore ok', stderr => '', signal => 0}, + ); + my @expected = (qr{git .* add file.txt}, qr{git .* commit}, qr{git .* push}, qr{git .* reset .* HEAD\^1}); + + @executed = (); + throws_ok { $git->commit({add => ['file.txt'], message => 'test'}) } qr{OpenQA::Error::Cmd: Unable to push}, + 'git push failed'; + for my $i (0 .. $#expected) { + like "@{$executed[$i]}", $expected[$i], "executed git command $i like expected"; + } + }; + }; +}; + subtest 'save_needle returns and logs error when set_to_latest_master fails' => sub { package Test::FailingMinionJob { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openQA-5.1768209690.f34c2973/t/42-df-based-cleanup.t new/openQA-5.1768323619.9a70ab91/t/42-df-based-cleanup.t --- old/openQA-5.1768209690.f34c2973/t/42-df-based-cleanup.t 2026-01-12 10:21:30.000000000 +0100 +++ new/openQA-5.1768323619.9a70ab91/t/42-df-based-cleanup.t 2026-01-13 18:00:19.000000000 +0100 @@ -3,6 +3,7 @@ # SPDX-License-Identifier: GPL-2.0-or-later use Test::Most; +use Mojo::Base -signatures; use FindBin; use lib "$FindBin::Bin/lib", "$FindBin::Bin/../external/os-autoinst-common/lib"; @@ -28,8 +29,7 @@ $app->log(Mojo::Log->new(level => 'debug')); -sub job_log_like { - my ($regex, $test_name) = @_; +sub job_log_like ($regex, $test_name) { my $job; combined_like { $job = run_gru_job($app, ensure_results_below_threshold => []) } $regex, $test_name; return $job; @@ -161,7 +161,7 @@ Deleting\svideo\sof\simportant\sjob\s$important_job_id.*Deleting\sresults\sof\simportant\sjob\s$important_job_id /xs, 'cleanup steps in right order'; is $job->{state}, 'failed', 'job considered failed'; - is $job->{result}, 'Unable to cleanup enough results', 'unable to make enough room'; + is $job->{result}, 'Unable to cleanup enough results from results dir', 'unable to make enough room'; }; my $new_job_id = $jobs->search({}, {rows => 1, order_by => {-desc => 'id'}})->first->id; @@ -174,7 +174,8 @@ my $job = job_log_like qr/Deleting\svideo\sof\sjob\s$unimportant_job_id/s, 'cleanup steps in right order'; is $job->{state}, 'finished', 'job considered successful'; - is $job->{result}, 'Done after deleting videos from non-important jobs', 'finished within expected step'; + is $job->{result}, 'Done with results dir after deleting videos from non-important jobs', + 'finished within expected step'; }; subtest 'job done triggers cleanup' => sub { @@ -204,7 +205,8 @@ my $job; my $output = combined_from { $job = run_gru_job($app, ensure_results_below_threshold => []) }; is $job->{state}, 'finished', 'job considered successful'; - is $job->{result}, 'Done after deleting videos from important jobs', 'finished within expected step'; + is $job->{result}, 'Done with results dir after deleting videos from important jobs', + 'finished within expected step'; like $output, qr/Deleting video of important job $important_job_id/, 'video of "old" important job deleted'; unlike $output, qr/Deleting video.*$new_job_id/, 'video of job more recent important job not considered'; }; @@ -226,7 +228,8 @@ Deleting\sresults\sof\sjob\s$unimportant_job_id /xs, 'cleanup steps in right order'; is $job->{state}, 'finished', 'job considered successful'; - is $job->{result}, 'Done after deleting results from non-important jobs', 'finished within expected step'; + is $job->{result}, 'Done with results dir after deleting results from non-important jobs', + 'finished within expected step'; }; $available_bytes_mock = 19; @@ -244,7 +247,24 @@ Deleting\sresults\sof\simportant\sjob\s$important_job_id /xs, 'cleanup steps in right order'; is $job->{state}, 'finished', 'job considered successful'; - is $job->{result}, 'Done after deleting results from important jobs', 'finished within expected step'; + is $job->{result}, 'Done with results dir after deleting results from important jobs', + 'finished within expected step'; +}; + +subtest 'jobs in archive not considered' => sub { + # setup: as in the previous subtest but this time the important job is flagged as archived and is therefore + # not supposed to be considered + %gained_disk_space_by_deleting_results_of_job = ($important_job_id => 1); + $jobs->find($important_job_id)->update({archived => 1}); + + my $job = job_log_like qr/ + Deleting\svideo\sof\sjob\s$unimportant_job_id.* + Deleting\sresults\sof\sjob\s$unimportant_job_id.* + Deleting\svideo\sof\simportant\sjob\s$new_job_id.* + /xs, 'cleanup steps in right order without archived job mentioned'; + is $job->{state}, 'failed', 'job considered failed'; + is $job->{result}, 'Unable to cleanup enough results from results dir', + 'not finished as job that could gain disk space is in the archive'; }; done_testing(); diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openQA-5.1768209690.f34c2973/t/ui/17-product-log.t new/openQA-5.1768323619.9a70ab91/t/ui/17-product-log.t --- old/openQA-5.1768209690.f34c2973/t/ui/17-product-log.t 2026-01-12 10:21:30.000000000 +0100 +++ new/openQA-5.1768323619.9a70ab91/t/ui/17-product-log.t 2026-01-13 18:00:19.000000000 +0100 @@ -99,20 +99,10 @@ # show results $action_links[1]->click(); wait_for_ajax; - my $results = decode_json($driver->find_element('.modal-body')->get_text()); - my $failed_job_info = $results->{failed_job_info}; - is(scalar @{$results->{successful_job_ids}}, 9, '9 jobs successful'); - is(scalar @{$failed_job_info}, 2, '2 errors present'); - is_deeply( - $failed_job_info->[0]->{error_messages}, - ['START_AFTER_TEST=kda@64bit not found - check for dependency typos and dependency cycles'], - 'error message' - ); - is_deeply( - $failed_job_info->[1]->{error_messages}, - ['textmode@32bit has no child, check its machine placed or dependency setting typos'], - 'error message' - ); + my $results = $driver->find_element('.modal-body')->get_text; + like $results, qr/Successfully scheduled 9 job\(s\)/i, '9 jobs successful'; + like $results, qr/99987: START_AFTER_TEST=kda\@64bit not found - check for/i, 'error message for 99987'; + like $results, qr/99982: textmode\@32bit has no child, check its machine/i, 'error message for 99982'; $driver->find_element('.modal-footer button')->click(); # trigger rescheduling @@ -164,7 +154,7 @@ is(scalar @rows, 1, 'only one row shown'); like($rows[0]->get_text, qr/perci.*whatever\.iso/, 'row data'); like($driver->find_element('#scheduled-products h3 + table')->get_text, qr/FOO.*bar/, 'settings'); - like($driver->find_element('#scheduled-products h3 + pre')->get_text, + like($driver->find_element('#scheduled-products .alert-danger')->get_text, qr/check for dependency typos and dependency cycles/, 'results'); }; ++++++ openQA.obsinfo ++++++ --- /var/tmp/diff_new_pack.xYbLWT/_old 2026-01-14 16:22:50.140315121 +0100 +++ /var/tmp/diff_new_pack.xYbLWT/_new 2026-01-14 16:22:50.144315283 +0100 @@ -1,5 +1,5 @@ name: openQA -version: 5.1768209690.f34c2973 -mtime: 1768209690 -commit: f34c29732fde5f2386315ab8129298675d9863a4 +version: 5.1768323619.9a70ab91 +mtime: 1768323619 +commit: 9a70ab916b9b0b0f04ea18643f48a56f7dff2409
