Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package openQA for openSUSE:Factory checked in at 2026-11-09 16:05:06 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/openQA (Old) and /work/SRC/openSUSE:Factory/.openQA.new.3759290 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "openQA" Mon Nov 9 16:05:06 2026 rev:906 rq:1396212 version:5.1794183862.c3440c51 Changes: -------- --- /work/SRC/openSUSE:Factory/openQA/openQA.changes 2026-11-07 23:34:57.614366127 +0100 +++ /work/SRC/openSUSE:Factory/.openQA.new.3759290/openQA.changes 2026-11-09 16:05:51.100851226 +0100 @@ -1,0 +2,6 @@ +Mon Nov 09 00:24:35 UTC 2026 - [email protected] + +- Update to version 5.1794183862.c3440c51: + * feat(investigation): List recent commits to files of failed modules + +------------------------------------------------------------------- Old: ---- openQA-5.1794057672.cbb4e678.obscpio New: ---- openQA-5.1794183862.c3440c51.obscpio ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ openQA-client-test.spec ++++++ --- /var/tmp/diff_new_pack.Di3F2J/_old 2026-11-09 16:05:52.778921575 +0100 +++ /var/tmp/diff_new_pack.Di3F2J/_new 2026-11-09 16:05:52.780921659 +0100 @@ -18,7 +18,7 @@ %define short_name openQA-client Name: %{short_name}-test -Version: 5.1794057672.cbb4e678 +Version: 5.1794183862.c3440c51 Release: 0 Summary: Test package for %{short_name} License: GPL-2.0-or-later ++++++ openQA-test.spec ++++++ --- /var/tmp/diff_new_pack.Di3F2J/_old 2026-11-09 16:05:52.808922833 +0100 +++ /var/tmp/diff_new_pack.Di3F2J/_new 2026-11-09 16:05:52.809922875 +0100 @@ -18,7 +18,7 @@ %define short_name openQA Name: %{short_name}-test -Version: 5.1794057672.cbb4e678 +Version: 5.1794183862.c3440c51 Release: 0 Summary: Test package for openQA License: GPL-2.0-or-later ++++++ openQA-worker-test.spec ++++++ --- /var/tmp/diff_new_pack.Di3F2J/_old 2026-11-09 16:05:52.837924048 +0100 +++ /var/tmp/diff_new_pack.Di3F2J/_new 2026-11-09 16:05:52.839924132 +0100 @@ -18,7 +18,7 @@ %define short_name openQA-worker Name: %{short_name}-test -Version: 5.1794057672.cbb4e678 +Version: 5.1794183862.c3440c51 Release: 0 Summary: Test package for %{short_name} License: GPL-2.0-or-later ++++++ openQA.spec ++++++ --- /var/tmp/diff_new_pack.Di3F2J/_old 2026-11-09 16:05:52.871925474 +0100 +++ /var/tmp/diff_new_pack.Di3F2J/_new 2026-11-09 16:05:52.872925516 +0100 @@ -104,7 +104,7 @@ %define devel_requires %devel_no_selenium_requires chromedriver Name: openQA -Version: 5.1794057672.cbb4e678 +Version: 5.1794183862.c3440c51 Release: 0 Summary: Framework for automated system-level testing (web-frontend, scheduler and tools) Group: Development/Tools/Other ++++++ openQA-5.1794057672.cbb4e678.obscpio -> openQA-5.1794183862.c3440c51.obscpio ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openQA-5.1794057672.cbb4e678/assets/javascripts/test_result.js new/openQA-5.1794183862.c3440c51/assets/javascripts/test_result.js --- old/openQA-5.1794057672.cbb4e678/assets/javascripts/test_result.js 2026-11-07 14:21:12.000000000 +0100 +++ new/openQA-5.1794183862.c3440c51/assets/javascripts/test_result.js 2026-11-09 01:24:22.000000000 +0100 @@ -983,6 +983,38 @@ return commits; } +// "(#<number>)" is only known to mean a pull request on GitHub and Gitea/Forgejo hosts +const PULL_REQUEST_PATHS = [ + [/^github\.com$/, 'pull/'], + [/^(?:src\.opensuse\.org|codeberg\.org)$|gitea|forgejo/, 'pulls/'] +]; + +function pullRequestUrl(commitUrl, number) { + let url; + try { + url = new URL(commitUrl); + } catch { + return null; + } + const path = PULL_REQUEST_PATHS.find(([host]) => host.test(url.hostname)); + return path && commitUrl.endsWith('/commit/') ? commitUrl.replace(/commit\/$/, path[1]) + number : null; +} + +function createCommitMessageLink(msg, repoUrl) { + const fragment = document.createDocumentFragment(); + const match = msg.match(/^(.*) \(#(\d+)\)$/); + const href = match && pullRequestUrl(repoUrl, match[2]); + if (!href) { + fragment.append(msg); + return fragment; + } + const link = document.createElement('a'); + link.href = href; + link.textContent = '#' + match[2]; + fragment.append(match[1] + ' (', link, ')'); + return fragment; +} + function setupTestDetailsFilter(tabConfig) { if (tabConfig._hasFilterHandlers) return; @@ -1206,8 +1238,9 @@ const needlegiturl = response.needlegiturl; delete response.testgiturl; delete response.needlegiturl; - document.getElementById('investigation').setAttribute('data-testgiturl', testgiturl); - document.getElementById('investigation').setAttribute('data-needlegiturl', needlegiturl); + const investigation = document.getElementById('investigation'); + if (testgiturl) investigation.setAttribute('data-testgiturl', testgiturl); + if (needlegiturl) investigation.setAttribute('data-needlegiturl', needlegiturl); const theadElement = document.createElement('thead'); const headTrElement = document.createElement('tr'); @@ -1281,7 +1314,8 @@ const logDetailsDiv = document.createElement('div'); logDetailsDiv.id = 'collapse' + key + i; logDetailsDiv.className = 'collapse'; - spanElem.innerHTML = gitstats[i].link + ' ' + gitstats[i].msg; + spanElem.innerHTML = gitstats[i].link + ' '; + spanElem.appendChild(createCommitMessageLink(gitstats[i].msg, repoUrl)); logDetailsDiv.innerHTML = gitstats[i].stat.join('\n'); statItem.append(collapseSign, spanElem, logDetailsDiv); @@ -1350,7 +1384,12 @@ } function getInvestigationDataAttr(key) { - const attrs = {test_log: 'data-testgiturl', needles_log: 'data-needlegiturl'}; + const attrs = { + test_log: 'data-testgiturl', + needles_log: 'data-needlegiturl', + failure_test_log: 'data-testgiturl', + failure_needles_log: 'data-needlegiturl' + }; const investigation = document.getElementById('investigation'); return investigation ? investigation.getAttribute(attrs[key]) : null; } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openQA-5.1794057672.cbb4e678/docs/UsersGuide.md new/openQA-5.1794183862.c3440c51/docs/UsersGuide.md --- old/openQA-5.1794057672.cbb4e678/docs/UsersGuide.md 2026-11-07 14:21:12.000000000 +0100 +++ new/openQA-5.1794183862.c3440c51/docs/UsersGuide.md 2026-11-09 01:24:22.000000000 +0100 @@ -720,6 +720,32 @@ autologin-timeout=0 ``` +### Investigation of failed jobs + +The "Investigation" tab of a failed job, also available as JSON from +`/tests/<job id>/investigation_ajax`, lists what changed since the last good +job in the same scenario. For jobs with failed modules the keys +`failure_test_log` and `failure_needles_log` only list the commits to the files +that are involved in the failure: + +- test code files named in the stack traces of the failed modules, or else the + module's own file +- needles considered by failed screen checks and needles with the tags of those + checks + +The files are looked up in the commit the job was tested with +(`TEST_GIT_HASH`, `NEEDLES_GIT_HASH`) within the checkouts on the web UI host, +so later renames or removals do not change the result. Without a last good job +the commits of the 5 days before the job started are listed. Each line is +`<10-char sha> <title> (#<pull request>)`; the pull request is added if the +commit was merged by a "Merge pull request" commit. The tab links it for +GitHub and Gitea/Forgejo repositories. + +Instead of a log the keys contain a message if the tested commit is not +available on the web UI host (e.g. for a `CASEDIR` that is a URL or absolute +path) or if git took longer than `job_investigate_git_timeout` in total or +failed. Pull request numbers are skipped if time runs out. + <a id="carry-over"></a> ### Carry over of bug references from previous jobs in same scenario diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openQA-5.1794057672.cbb4e678/lib/OpenQA/JobInvestigation.pm new/openQA-5.1794183862.c3440c51/lib/OpenQA/JobInvestigation.pm --- old/openQA-5.1794057672.cbb4e678/lib/OpenQA/JobInvestigation.pm 1970-01-01 01:00:00.000000000 +0100 +++ new/openQA-5.1794183862.c3440c51/lib/OpenQA/JobInvestigation.pm 2026-11-09 01:24:22.000000000 +0100 @@ -0,0 +1,117 @@ +# Copyright SUSE LLC +# SPDX-License-Identifier: GPL-2.0-or-later + +package OpenQA::JobInvestigation; + +use Mojo::Base -strict, -signatures; + +use Exporter 'import'; +use List::Util 'uniq'; +use OpenQA::Utils 'run_cmd_with_log_return_error'; + +our @EXPORT_OK = qw( + RETURN_CODE_TIMEOUT_EXCEEDED tree_file git_has_commit git_timed git_failure_message git_tree tagged_needles + annotate_pull_requests failed_test_files failed_needle_files +); + +use constant JOB_INVESTIGATE_PULL_REQUESTS_MAX => 200; +use constant RETURN_CODE_TIMEOUT_EXCEEDED => 124; # see `man timeout` + +# returns the longest suffix of $path that is a file in the git tree (a hash reference of paths) +sub tree_file ($tree, $path) { + my @parts = grep { length } split m{/}, $path // ''; + for (; @parts; shift @parts) { + my $rel = join '/', @parts; + return $rel if $tree->{$rel}; + } + return undef; +} + +sub git_has_commit ($deadline, $dir, $hash) { + return 0 unless defined $hash && $hash =~ m/^[0-9a-f]{7,40}$/i; + return git_timed($deadline, $dir, ['cat-file', '-e', "$hash^{commit}"])->{status}; +} + +# runs git within the time left until $deadline (epoch seconds), which is shared by all commands of a request +sub git_timed ($deadline, $dir, $args, %options) { + my $left = $deadline - time; + return {status => 0, return_code => RETURN_CODE_TIMEOUT_EXCEEDED, stdout => '', stderr => ''} if $left <= 0; + return run_cmd_with_log_return_error(['timeout', $left, 'git', '-C', $dir, @$args], stdout => 'trace', %options); +} + +sub git_failure_message ($return_code) { + return $return_code == RETURN_CODE_TIMEOUT_EXCEEDED + ? 'Computing changes timed out' + : 'Cannot list changes because of a git problem'; +} + +# returns the files of the commit $hash in $dir as hash reference or, as second value, why that is not possible +sub git_tree ($deadline, $dir, $hash) { + my $unknown = 'Tested git commit is unknown or not available on this host, cannot list changes'; + return (undef, $unknown) unless defined $hash && $hash =~ m/^[0-9a-f]{7,40}$/i; + my $res = git_timed($deadline, $dir, ['ls-tree', '-r', '-z', $hash]); + unless ($res->{status}) { + return (undef, + $res->{return_code} == RETURN_CODE_TIMEOUT_EXCEEDED + ? git_failure_message(RETURN_CODE_TIMEOUT_EXCEEDED) + : $unknown); + } + return (+{map { $_ => 1 } map { /^100\d{3} blob \S+\t(.*)$/s ? $1 : () } split /\0/, $res->{stdout}}); +} + +# returns the paths of the needles in the commit $hash that have one of the $tags, or the git error message +sub tagged_needles ($deadline, $dir, $tree, $hash, $tags) { + my $pattern = join '|', map { s/([.+])/\\$1/gr } @$tags; + my @cmd = ('grep', '-l', '-z', '-E', '-e', qq{"($pattern)"}, $hash, '--', '*.json'); + my $res = git_timed($deadline, $dir, \@cmd, expected_return_codes => {0 => 1, 1 => 1}); + my $return_code = $res->{return_code} // 255; + return (undef, git_failure_message($return_code)) if $return_code > 1; + return ([grep { $tree->{$_} } map { s/^\Q$hash\E://r } split /\0/, $res->{stdout}]); +} + +# adds " (#<number>)" to commits that were merged via a "Merge pull request #<number>" merge commit, as far as +# the time until $options{deadline} allows +sub annotate_pull_requests ($dir, $log, $range, %options) { + my %pending = map { $_ => 1 } $log =~ /^([0-9a-f]{10}) (?!.*\(#\d+\)$)/mg; + return $log unless %pending; + my $deadline = $options{deadline}; + my @since = $options{since} ? "--since=$options{since}" : (); + # commits on the first-parent line were not merged via pull request and would make us search all merges + my $direct = git_timed($deadline, $dir, ['rev-list', '--first-parent', '--no-merges', @since, $range]); + return $log unless $direct->{status}; + delete $pending{$_} for $direct->{stdout} =~ /^([0-9a-f]{10})/mg; + return $log unless %pending; + my @merges = ('log', '--first-parent', '--merges', '--pretty=%H %s', '-' . JOB_INVESTIGATE_PULL_REQUESTS_MAX); + my $merges = git_timed($deadline, $dir, [@merges, @since, $range]); + return $log unless $merges->{status}; + my %pull_request; + for my $merge (split /\n/, $merges->{stdout}) { + next unless $merge =~ m/^([0-9a-f]{40}) Merge pull request #(\d+) /; + my ($merge_hash, $number) = ($1, $2); + my $commits = git_timed($deadline, $dir, ['rev-list', '--no-merges', "$merge_hash^1..$merge_hash"]); + last unless $commits->{status}; + $pull_request{$_} = $number for grep { delete $pending{$_} } $commits->{stdout} =~ /^([0-9a-f]{10})/mg; + last unless %pending; + } + return $log =~ s/^([0-9a-f]{10}) (.*)$/exists $pull_request{$1} ? "$1 $2 (#$pull_request{$1})" : "$1 $2"/mger; +} + +# the tested files of the failed modules, i.e. those of their stack traces, else the module's own file +sub failed_test_files ($tree, $modules) { + my @files; + for my $module (@$modules) { + my ($traced, $script) = @$module; + my @found = map { tree_file($tree, $_) // () } @$traced; + push @files, @found ? @found : (tree_file($tree, $script) // ()); + } + return [uniq @files]; +} + +# the needles considered by failed screen checks, their pngs and the needles with the tags of those checks +sub failed_needle_files ($tree, $needles, $tagged) { + my @files = map { tree_file($tree, $_) // () } @$needles; + @files = uniq @files, @$tagged; + return [map { my $png = s/\.json$/.png/r; ($_, $png ne $_ && $tree->{$png} ? $png : ()) } @files]; +} + +1; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openQA-5.1794057672.cbb4e678/lib/OpenQA/Schema/Result/Jobs.pm new/openQA-5.1794183862.c3440c51/lib/OpenQA/Schema/Result/Jobs.pm --- old/openQA-5.1794057672.cbb4e678/lib/OpenQA/Schema/Result/Jobs.pm 2026-11-07 14:21:12.000000000 +0100 +++ new/openQA-5.1794183862.c3440c51/lib/OpenQA/Schema/Result/Jobs.pm 2026-11-09 01:24:22.000000000 +0100 @@ -16,6 +16,9 @@ use OpenQA::App; use OpenQA::Constants qw(DOWNLOAD_PERMISSION); use OpenQA::Jobs::Constants; +use OpenQA::JobInvestigation + qw(RETURN_CODE_TIMEOUT_EXCEEDED git_has_commit git_failure_message git_tree tagged_needles), + qw(annotate_pull_requests failed_test_files failed_needle_files); use OpenQA::JobDependencies::Constants; use OpenQA::Markdown 'markdown_to_html'; use OpenQA::Setup; @@ -37,8 +40,9 @@ use OpenQA::File; use OpenQA::Parser 'parser'; use OpenQA::WebSockets::Client; -use List::Util qw(any all); +use List::Util qw(any all uniq); use Scalar::Util qw(looks_like_number); +use Time::Piece (); # The state and results constants are duplicated in the Python client: # if you change them or add any, please also update const.py. @@ -53,7 +57,9 @@ use constant MAX_LENGTH_REASON => 300; use constant JOB_INVESTIGATE_GIT_TIMEOUT => 20; -use constant RETURN_CODE_TIMEOUT_EXCEEDED => 124; # see `man timeout` +use constant JOB_INVESTIGATE_FAILURE_WINDOW_DAYS => 5; +use constant JOB_INVESTIGATE_FAILURE_MODULES_MAX => 3; +use constant JOB_INVESTIGATE_FAILURE_FILES_MAX => 50; __PACKAGE__->table('jobs'); __PACKAGE__->load_components(qw(InflateColumn::DateTime FilterColumn Timestamps)); @@ -1938,14 +1944,20 @@ return "Invalid range $refspec_range" if $refspec_range =~ m/UNKNOWN|unreadable git hash/; my $limit = $options{limit}; my $stat = $options{stat} // '300,200'; + my @abbrev = $options{abbrev} ? "--abbrev=$options{abbrev}" : (); + my @since = $options{since} ? "--since=$options{since}" : (); + my @pickaxe = $options{pickaxe} ? "-G$options{pickaxe}" : (); + my @paths = $options{paths} ? ('--', @{$options{paths}}) : (); + my @timeout = $options{timeout} ? ('timeout', $options{timeout}) : (); my $res = run_cmd_with_log_return_error( [ - 'git', '-C', $dir, 'log', - ($limit ? "-$limit" : ()), "--stat=$stat", '--pretty=oneline', '--abbrev-commit', - '--no-merges', $refspec_range + @timeout, 'git', '-C', $dir, 'log', ($limit ? "-$limit" : ()), + "--stat=$stat", '--pretty=oneline', + '--abbrev-commit', @abbrev, '--no-merges', @since, @pickaxe, $refspec_range, @paths ], stdout => 'trace' ); + ${$options{failed}} = $res->{return_code} // 255 if $options{failed} && !$res->{status}; # regardless of success or not the output contains the information we need return $res->{stdout} . $res->{stderr}; } @@ -1982,6 +1994,96 @@ return $out; } +# returns the test code paths, needle paths and tags involved in the failed modules, found via their stack +# traces and failed screen steps, as array references; for the test code as per module [trace paths, script] +sub _failed_module_inputs ($self) { + my (@modules, @needles, @tags); + my %attrs = (order_by => 't_updated', rows => JOB_INVESTIGATE_FAILURE_MODULES_MAX); + my $modules = $self->modules->search({result => 'failed'}, \%attrs); + while (my $module = $modules->next) { + my @steps = grep { ($_->{result} // '') eq 'fail' } @{$module->results->{details} // []}; + my @traced = map { /(\S+\.(?:pm|py)) line \d+/g } map { $_->{text_data} // () } @steps; + push @modules, [\@traced, $module->script]; + for my $step (@steps) { + push @needles, map { $_->{json} // () } @{$step->{needles} // []}; + my @missing = ($step->{text_data} // '') =~ /no candidate needle with tag\(s\) '([^']+)'/g; + push @tags, grep { /^[A-Za-z0-9_.:+-]+$/ } @{$step->{tags} // []}, map { split /[\s,]+/ } @missing; + } + } + return \@modules, [uniq @needles], [uniq @tags]; +} + +sub _failure_window_start ($self) { + my $started = $self->get_column('t_started'); + my $epoch = $started ? Time::Piece->strptime($started =~ s/\..*//r, '%Y-%m-%d %H:%M:%S')->epoch : time; + return $epoch - JOB_INVESTIGATE_FAILURE_WINDOW_DAYS * 24 * 60 * 60; +} + +# lists commits to the files of the $selections (git log options) between the last good and this job, or else +# within the window before this job started; returns a message instead if git failed +sub _failure_log ($self, $dir, $self_hash, $prev_hash, $selections, %options) { + my $deadline = $options{deadline}; + my %git = (limit => $options{limit}, abbrev => 10); + my $has_range = git_has_commit($deadline, $dir, $prev_hash); + my $range = $has_range ? "$prev_hash..$self_hash" : $self_hash; + $git{since} = $self->_failure_window_start unless $has_range; + my $log = ''; + for my $selection (@$selections) { + my $left = $deadline - time; + return git_failure_message(RETURN_CODE_TIMEOUT_EXCEEDED) if $left <= 0; + my $failed; + $log .= $self->git_log_diff($dir, $range, %git, timeout => $left, failed => \$failed, %$selection); + return git_failure_message($failed) if $failed; + } + my %seen; + $log = join '', grep { !$seen{/^(\S+)/ ? $1 : ''}++ } split /(?=^[0-9a-f]{10} )/m, $log; + return $log ? annotate_pull_requests($dir, $log, $range, since => $git{since}, deadline => $deadline) : ''; +} + +# commits to the files named in the stack traces and to the needles of the failed screen steps +sub _failure_investigation ($self, $inv, $self_vars, $prev_vars, %args) { + return unless $self->has_failed_modules && $self->result_dir && %$self_vars; + my $timeout = OpenQA::App->singleton->config->{global}->{job_investigate_git_timeout} + // JOB_INVESTIGATE_GIT_TIMEOUT; + my $deadline = time + $timeout; + my $effective_distri = effective_distri($self_vars); + my $casedir = testcasedir($effective_distri, $self->VERSION); + my $needles_dir = needledir($effective_distri, $self->VERSION); + my ($modules, $needles, $tags) = $self->_failed_module_inputs; + my ($test_hash, $needles_hash) = map { $self_vars->{$_} } qw(TEST_GIT_HASH NEEDLES_GIT_HASH); + my ($prev_test, $prev_needles) = map { $prev_vars->{$_} } qw(TEST_GIT_HASH NEEDLES_GIT_HASH); + my %log = (limit => $args{git_limit}, deadline => $deadline); + if (@$modules) { + my ($tree, $error) = git_tree($deadline, $casedir, $test_hash); + my $files = $tree ? failed_test_files($tree, $modules) : []; + splice @$files, JOB_INVESTIGATE_FAILURE_FILES_MAX; + my @selections = ({paths => [map { ":(literal)$_" } @$files]}); + $inv->{failure_test_log} + = $error + || (@$files ? $self->_failure_log($casedir, $test_hash, $prev_test, \@selections, %log) : '') + || 'No recent changes to the test files involved in the failure' + if $error + || @$files; + } + if (@$needles || @$tags) { + my ($tree, $error) = git_tree($deadline, $needles_dir, $needles_hash); + my $tagged = []; + ($tagged, $error) = tagged_needles($deadline, $needles_dir, $tree, $needles_hash, $tags) if $tree && @$tags; + my $files = $tree && !$error ? failed_needle_files($tree, $needles, $tagged) : []; + splice @$files, JOB_INVESTIGATE_FAILURE_FILES_MAX; + my @selections; + push @selections, {paths => [map { ":(literal)$_" } @$files]} if @$files; + my $pattern = join '|', map { s/([.+])/\\$1/gr } @$tags; + push @selections, {pickaxe => qq{"($pattern)"}, paths => ['*.json']} if @$tags; + $inv->{failure_needles_log} + = $error + || $self->_failure_log($needles_dir, $needles_hash, $prev_needles, \@selections, %log) + || 'No recent changes to the needles involved in the failure' + if $error + || @selections; + } +} + =head2 investigate Find pointers for investigation on failures, e.g. what changed vs. a "last @@ -1994,8 +2096,18 @@ # this job's own isolated history (e.g. the same PR), not an unrelated one my $isolation_keys = exists $args{isolation_keys} ? $args{isolation_keys} : undef; my @previous = $self->_previous_scenario_jobs(undef, {}, $isolation_keys); - return {error => 'No previous job in this scenario, cannot provide hints'} unless @previous; + unless (@previous) { + my $vars_file = $self->result_dir ? eval { path($self->result_dir, 'vars.json')->slurp } : undef; + my $vars = decode_json($vars_file // '{}'); + my %inv = ( + error => 'No previous job in this scenario, cannot provide hints', + testgiturl => git_commit_url($vars->{TEST_GIT_URL}), + needlegiturl => git_commit_url($vars->{NEEDLES_GIT_URL})); + $self->_failure_investigation(\%inv, $vars, {}, %args); + return \%inv; + } my %inv; + my $prev_vars = {}; return {error => 'No result directory available for current job'} unless $self->result_dir(); my $ignore = OpenQA::App->singleton->config->{global}->{job_investigate_ignore}; my $self_file = eval { Mojo::File->new($self->result_dir(), 'vars.json')->slurp }; @@ -2010,6 +2122,7 @@ $inv{last_good} = {type => 'link', link => '/tests/' . $prev->id, text => $prev->id}; last unless $prev->result_dir; my $prev_file = eval { Mojo::File->new($prev->result_dir(), 'vars.json')->slurp }; + $prev_vars = decode_json($prev_file) if $prev_file; $inv{diff_packages_to_last_good} = $self->packages_diff($prev, $ignore, 'worker_packages.txt'); $inv{diff_sut_packages_to_last_good} = $self->packages_diff($prev, $ignore, 'sut_packages.txt'); last unless $self_file && $prev_file; @@ -2017,7 +2130,6 @@ # files missing. This is a best-effort approach. my $diff = eval { diff(\$prev_file, \$self_file, {CONTEXT => 0}) }; $inv{diff_to_last_good} = join "\n", grep { !/(?:^@@|$ignore)/ } split /\n/, $diff; - my $prev_vars = decode_json($prev_file // '{}'); my $effective_distri = effective_distri($self_vars); my $casedir = testcasedir($effective_distri, $self->VERSION); my $refspec_range = "$prev_vars->{TEST_GIT_HASH}..$self_vars->{TEST_GIT_HASH}"; @@ -2038,6 +2150,7 @@ last; } $inv{last_good} //= 'not found'; + $self->_failure_investigation(\%inv, $self_vars, $prev_vars, %args); $inv{testgiturl} = git_commit_url($self_vars->{TEST_GIT_URL}); $inv{needlegiturl} = git_commit_url($self_vars->{NEEDLES_GIT_URL}); return \%inv; diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openQA-5.1794057672.cbb4e678/t/10-jobs.t new/openQA-5.1794183862.c3440c51/t/10-jobs.t --- old/openQA-5.1794057672.cbb4e678/t/10-jobs.t 2026-11-07 14:21:12.000000000 +0100 +++ new/openQA-5.1794183862.c3440c51/t/10-jobs.t 2026-11-09 01:24:22.000000000 +0100 @@ -16,6 +16,7 @@ use autodie ':all'; use File::Copy; use OpenQA::Jobs::Constants; +use OpenQA::JobInvestigation 'annotate_pull_requests'; use OpenQA::Test::Case; use Test::MockModule 'strict'; use Test::Mojo; @@ -538,6 +539,175 @@ like $inv->{diff_packages_to_last_good}, qr/^\+python/m, 'diff packages for job is shown'; }; + subtest 'investigation lists recent changes to files involved in the failure' => sub { + my $repo = tempdir; + my $git = sub (@args) { system 'git', '-C', "$repo", '-c', 'commit.gpgsign=false', @args }; + local @ENV{qw(GIT_AUTHOR_NAME GIT_AUTHOR_EMAIL GIT_COMMITTER_NAME GIT_COMMITTER_EMAIL)} + = qw(t [email protected] t [email protected]); + my $commit = sub ($file, $message, $content = $message, $date = time) { + local @ENV{qw(GIT_AUTHOR_DATE GIT_COMMITTER_DATE)} = ("$date +0000") x 2; + $repo->child($file)->dirname->make_path; + $repo->child($file)->spew($content); + $git->(qw(add -A)); + $git->(qw(commit -q -m), $message); + chomp(my $hash = qx{git -C "$repo" rev-parse HEAD}); + return $hash; + }; + $git->(qw(init -q -b main)); + $commit->('tests/x.pm', 'Ancient change', 'Ancient change', 946_684_800); + $git->(qw(switch -q -c feature)); + $commit->('tests/x.pm', 'Break x'); + $commit->('tests/unrelated.pm', 'Unrelated'); + $git->(qw(switch -q main)); + $git->(qw(merge -q --no-ff -m), 'Merge pull request #42 from fork/feature', 'feature'); + chomp(my $merge = qx{git -C "$repo" rev-parse HEAD}); + $commit->('tests/x.pm', 'Squash fix (#43)'); + $commit->('tests/old.pm', 'Add old'); + $commit->('lib/helper.pm', 'Change helper'); + $commit->('needles/foo-1.png', 'Add foo needle png'); + $commit->('needles/foo-1.json', 'Add foo needle', '{"tags": ["foo-tag"]}'); + $commit->('needles/bar.json', 'Add bar needle', '{"tags": ["bar"]}'); + $commit->('needles/foo-2.json', 'Retag needle', '{"tags": ["foo-tag", "other"]}'); + $commit->('needles/area.json', 'Add area needle', '{"tags": ["area-tag"], "area": [{"xpos": 1}]}'); + $commit->('needles/gone.json', 'Add gone needle', '{"tags": ["removed-tag"]}'); + $commit->('needles/foo-1.png', 'Update foo png only'); + $commit->('needles/area.json', 'Move area', '{"tags": ["area-tag"], "area": [{"xpos": 2}]}'); + my $tested = $commit->('needles/gone.json', 'Drop tag from gone needle', '{"tags": ["other"]}'); + $git->(qw(rm -q tests/old.pm)); + $git->(qw(commit -q -m), 'Remove old'); + my $head = $commit->('tests/later.pm', 'Later'); + + my $trace = join "\n", '# Test died: oops', '--- # stack trace', + "testapi::assert_script_run('x') called at tests/x.pm line 19", + 'prepare::y at /var/lib/openqa/pool/1/tests/old.pm line 7', 'at ../outside.pm line 3', + 'at .git/hook.pm line 1'; + my $needle_step = { + result => 'fail', + tags => ['foo-tag', 'v1.2'], + needles => [{json => '/var/lib/openqa/needles/foo-1.json'}]}; + my $text_step = {result => 'fail', text_data => "no candidate needle with tag(s) 'bar, removed-tag'"}; + my $area_step = {result => 'fail', tags => ['area-tag']}; + $job->update_module( + 'b', + { + result => 'fail', + details => [{result => 'fail', text_data => $trace}, $needle_step, $text_step, $area_step]}); + $job->insert_module({name => 'c', category => 'c', script => 'tests/x.pm', flags => {}}); + $job->update_module('c', {result => 'fail', details => []}); + my $needle_trace = join "\n", "# Test died: no candidate needle with tag(s) 'foo-tag' matched", + '--- # stack trace', + 'testapi::assert_screen("foo-tag") called at lib/helper.pm line 94', + 'tests::other::run() called at tests/other.pm line 5'; + $job->insert_module({name => 'd', category => 'd', script => 'tests/other.pm', flags => {}}); + $job->update_module('d', {result => 'fail', details => [{result => 'fail', text_data => $needle_trace}]}); + + my $job_mock = Test::MockModule->new('OpenQA::Schema::Result::Jobs', no_auto => 1); + $job_mock->redefine(testcasedir => sub (@) { "$repo" }); + $job_mock->redefine(needledir => sub (@) { $repo->child('needles')->to_string }); + my $set_vars = sub ($j, $hash, %extra) { + path($j->result_dir, 'vars.json') + ->spew(encode_json {TEST_GIT_HASH => $hash, NEEDLES_GIT_HASH => $hash, %extra}); + }; + my $last_good = ($job->_previous_scenario_jobs)[1]; + $set_vars->($job, $tested); + + subtest 'window before the job started without last good vars.json' => sub { + path($last_good->result_dir, 'vars.json')->remove; + my $inv = $job->investigate; + my $log = $inv->{failure_test_log}; + like $log, qr/^[0-9a-f]{10} Break x \(#42\)$/m, 'commit merged via merge commit links its pull request'; + like $log, qr/^[0-9a-f]{10} Squash fix \(#43\)$/m, 'pull request of squashed commit not added twice'; + like $log, qr/tests\/x\.pm \|/, 'diff stat of involved file shown'; + like $log, qr/^[0-9a-f]{10} Add old$/m, 'file removed after the tested commit still considered'; + like $log, qr/^[0-9a-f]{10} Change helper$/m, 'shared library of a failed needle check considered'; + unlike $log, qr/Ancient change/, 'commit older than the window not listed'; + unlike $log, qr/Unrelated|Later|Remove old|outside|hook/, 'uninvolved or later changes ignored'; + $log = $inv->{failure_needles_log}; + like $log, qr/^[0-9a-f]{10} Add foo needle$/m, 'commit of candidate needle listed'; + like $log, qr/^[0-9a-f]{10} Update foo png only$/m, 'commit changing only the png of a candidate listed'; + like $log, qr/^[0-9a-f]{10} Retag needle$/m, 'commit changing a needle with the failed tag listed'; + like $log, qr/^[0-9a-f]{10} Add bar needle$/m, 'tag from text of failed step considered'; + like $log, qr/^[0-9a-f]{10} Move area$/m, 'commit not touching the tag lines of a needle with it listed'; + like $log, qr/^[0-9a-f]{10} Drop tag from gone needle$/m, 'commit removing the failed tag listed'; + is scalar(() = $log =~ /Add foo needle$/mg), 1, 'commits found twice only listed once'; + }; + + subtest 'range to the last good job' => sub { + $set_vars->($last_good, $merge); + my $inv = $job->investigate(git_limit => 200); + unlike $inv->{failure_test_log}, qr/Break x/, 'commits before the last good job not listed'; + like $inv->{failure_test_log}, qr/Squash fix \(#43\)/, 'commits after the last good job listed'; + $set_vars->($last_good, $tested); + $inv = $job->investigate(git_limit => 200); + is $inv->{failure_test_log}, 'No recent changes to the test files involved in the failure', 'no changes'; + like $inv->{failure_needles_log}, qr/^No recent changes to the needles/, 'no needle changes'; + }; + + subtest 'tested commit not available' => sub { + $set_vars->($job, 'a' x 40); + my $inv = $job->investigate(git_limit => 200); + like $inv->{failure_test_log}, qr/unknown or not available/, 'message about missing commit'; + like $inv->{failure_needles_log}, qr/unknown or not available/, 'message for needles as well'; + $set_vars->($job, $tested); + }; + + subtest 'git problems are not reported as missing changes' => sub { + my $global = $t->app->config->{global}; + { + local $global->{job_investigate_git_timeout} = 0; + my $inv = $job->investigate(git_limit => 200); + is $inv->{failure_test_log}, 'Computing changes timed out', 'timeout of test log reported'; + is $inv->{failure_needles_log}, 'Computing changes timed out', 'timeout of needles log reported'; + } + my $run_cmd = \&OpenQA::Schema::Result::Jobs::run_cmd_with_log_return_error; + $job_mock->redefine( + run_cmd_with_log_return_error => sub ($cmd, %args) { + return {status => 0, return_code => 128, stdout => '', stderr => 'fatal'} + if grep { $_ eq 'log' } @$cmd; + return $run_cmd->($cmd, %args); + }); + my $inv = $job->investigate(git_limit => 200); + is $inv->{failure_test_log}, 'Cannot list changes because of a git problem', 'failure of test log reported'; + is $inv->{failure_needles_log}, 'Cannot list changes because of a git problem', 'needles failure reported'; + $job_mock->redefine(run_cmd_with_log_return_error => $run_cmd); + }; + + subtest 'pull requests are looked up as far as time allows' => sub { + chomp(my $short = qx{git -C "$repo" rev-parse --short=10 $head}); + my $log = "$short Later\n"; + my $run_cmd = \&OpenQA::JobInvestigation::run_cmd_with_log_return_error; + my @commands; + my $investigation_mock = Test::MockModule->new('OpenQA::JobInvestigation', no_auto => 1); + $investigation_mock->redefine( + run_cmd_with_log_return_error => sub ($cmd, %args) { push @commands, $cmd; $run_cmd->($cmd, %args) }); + is annotate_pull_requests("$repo", $log, $head, deadline => time + 20), $log, 'direct commit as is'; + is scalar @commands, 1, 'merge commits not searched for a commit on the first-parent line'; + @commands = (); + is annotate_pull_requests("$repo", $log, $head, deadline => time - 1), $log, 'log kept if too late'; + is scalar @commands, 0, 'no git command run after the deadline'; + }; + + subtest 'no previous job in the scenario' => sub { + $job_mock->redefine(_previous_scenario_jobs => sub (@) { () }); + $set_vars->( + $job, $tested, + TEST_GIT_URL => 'https://github.com/o/r.git', + NEEDLES_GIT_URL => '[email protected]:o/n.git' + ); + my $inv = $job->investigate(git_limit => 200); + like $inv->{error}, qr/No previous job/, 'error reported'; + like $inv->{failure_test_log}, qr/^[0-9a-f]{10} Break x \(#42\)$/m, 'test changes of the window listed'; + like $inv->{failure_needles_log}, qr/^[0-9a-f]{10} Move area$/m, 'needle changes of the window listed'; + is $inv->{testgiturl}, 'https://github.com/o/r/commit/', 'repository URL of tests provided for links'; + is $inv->{needlegiturl}, 'https://gitea.example.com/o/n/commit/', 'repository URL of needles provided'; + path($job->result_dir, 'vars.json')->spew('{'); + throws_ok { $job->investigate(git_limit => 200) } qr/expected, at character offset/, + 'invalid vars.json is not ignored'; + $job_mock->unmock('_previous_scenario_jobs'); + }; + $job_mock->unmock_all; + }; + subtest 'external hook is called on done job if specified' => sub { my $task_mock = Test::MockModule->new('OpenQA::Task::Job::HookScript', no_auto => 1); $task_mock->redefine( diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openQA-5.1794057672.cbb4e678/t/16-utils-runcmd.t new/openQA-5.1794183862.c3440c51/t/16-utils-runcmd.t --- old/openQA-5.1794057672.cbb4e678/t/16-utils-runcmd.t 2026-11-07 14:21:12.000000000 +0100 +++ new/openQA-5.1794183862.c3440c51/t/16-utils-runcmd.t 2026-11-09 01:24:22.000000000 +0100 @@ -122,6 +122,19 @@ 'git_diff - full filename'; like OpenQA::Schema::Result::Jobs::git_diff(undef, $empty_tmp_dir, 'HEAD', limit => 3, stat => '15,10'), qr/\.\.\.ile.txt/, 'git_diff - filename shortened'; + my $failed; + like OpenQA::Schema::Result::Jobs::git_log_diff( + undef, $empty_tmp_dir, 'HEAD', + timeout => 10, + failed => \$failed + ), + qr/$msg/, 'git_log_diff - commit mentioned with timeout'; + ok !$failed, 'no failure reported on success'; + combined_like { + OpenQA::Schema::Result::Jobs::git_log_diff(undef, $empty_tmp_dir, 'unknown-ref', failed => \$failed) + } + qr/cmd returned 128/, 'failure logged'; + is $failed, 128, 'return code of failed git reported'; }; subtest 'cache_ref' => sub { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openQA-5.1794057672.cbb4e678/t/ui/18-tests-details.t new/openQA-5.1794183862.c3440c51/t/ui/18-tests-details.t --- old/openQA-5.1794057672.cbb4e678/t/ui/18-tests-details.t 2026-11-07 14:21:12.000000000 +0100 +++ new/openQA-5.1794183862.c3440c51/t/ui/18-tests-details.t 2026-11-09 01:24:22.000000000 +0100 @@ -1087,6 +1087,28 @@ $value = '9b8aaf060 Containers: Inc…\n stat1\n stat2\na54a4bb34Remove select…\n stat3\n stat4\n'; $links = $driver->execute_script(qq{return githashToLink("$value", 'test/')}); is_deeply $links, undef, 'null returned as second line is invalid' or always_explain $links; + + my $html = sub ($msg, $repo) { + $driver->execute_script( +'const d = document.createElement("div"); d.appendChild(createCommitMessageLink(arguments[0], arguments[1]));' + . 'return d.innerHTML;', + $msg, $repo + ); + }; + is $html->('Fix <b>foo</b> (#42)', 'https://github.com/o/r/commit/'), + 'Fix <b>foo</b> (<a href="https://github.com/o/r/pull/42";>#42</a>)', + 'pull request linked and message escaped for GitHub'; + like $html->('Fix foo (#42)', 'https://gitea.example.com/o/r/commit/'), + qr{href="https://gitea.example.com/o/r/pulls/42"}, + 'pull request linked for Gitea'; + like $html->('Fix foo (#42)', 'https://src.opensuse.org/o/r/commit/'), + qr{href="https://src.opensuse.org/o/r/pulls/42"}, + 'pull request linked for src.opensuse.org'; + is $html->('Fix foo (#42)', 'https://gitlab.com/o/r/commit/'), 'Fix foo (#42)', 'no link for GitLab'; + is $html->('Fix foo (#42)', 'https://github.example.com/o/r/commit/'), 'Fix foo (#42)', 'no link for unknown host'; + is $html->('Fix foo (#42)', 'https://github.com/o/r/'), 'Fix foo (#42)', 'no link for unexpected URL'; + is $html->('Fix foo (#42)', 'undefined'), 'Fix foo (#42)', 'no link without repository URL'; + is $html->('Fix foo', 'https://github.com/o/r/commit/'), 'Fix foo', 'message without pull request unchanged'; }; subtest 'additional investigation notes provided on new failed' => sub { @@ -1114,6 +1136,8 @@ $driver->find_element('#investigation_status_entry') ->text_like(qr/error\nNo previous job in this scenario, cannot provide hints/, 'linked to investigation tab directly'); + is $driver->find_element_by_id('investigation')->get_attribute('data-testgiturl'), undef, + 'no repository URL set if unknown'; $driver->find_element_by_xpath(q{//div[@class='tab-content']}) ->text_unlike(qr/Investigate the first bad test/, 'no alert shown for first bad itself'); }; ++++++ openQA.obsinfo ++++++ --- /var/tmp/diff_new_pack.Di3F2J/_old 2026-11-09 16:06:01.537288749 +0100 +++ /var/tmp/diff_new_pack.Di3F2J/_new 2026-11-09 16:06:01.542288959 +0100 @@ -1,5 +1,5 @@ name: openQA -version: 5.1794057672.cbb4e678 -mtime: 1794057672 -commit: cbb4e67823bfcd5b54655489ec40dde9d3c82c44 +version: 5.1794183862.c3440c51 +mtime: 1794183862 +commit: c3440c5101e56a2ff28383650d26026c414618db
