Script 'mail_helper' called by obssrc Hello community, here is the log from the commit of package openqa-mon for openSUSE:Factory checked in at 2023-04-01 19:32:35 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/openqa-mon (Old) and /work/SRC/openSUSE:Factory/.openqa-mon.new.9019 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "openqa-mon" Sat Apr 1 19:32:35 2023 rev:7 rq:1076567 version:1.0.1 Changes: -------- --- /work/SRC/openSUSE:Factory/openqa-mon/openqa-mon.changes 2023-03-19 16:17:03.935464221 +0100 +++ /work/SRC/openSUSE:Factory/.openqa-mon.new.9019/openqa-mon.changes 2023-04-01 19:32:50.457536097 +0200 @@ -1,0 +2,18 @@ +Thu Mar 30 13:47:25 UTC 2023 - Felix Niederwanger <[email protected]> + +- Update to version 1.0.1 +* Upgrade to gopenqa 0.7.1 and align version number + +------------------------------------------------------------------- +Thu Mar 30 13:04:00 UTC 2023 - Felix Niederwanger <[email protected]> + +- Update to version 1.0.0 +* Switch to single request for multiple jobs (openqa-mon and openqa-revtui) +* Improve review templates +* Consider the parallel_failed status (openqa-revtui) +* Add progress when loading jobs (openqa-revtui) +* Introduce colors in status field (openqa-revtui) +* Rate limit continuous refresh on OSD and O3 +* Show job progress in Refresh line (openqa-revtui) + +------------------------------------------------------------------- Old: ---- openqa-mon-0.27.tar.gz New: ---- openqa-mon-1.0.1.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ openqa-mon.spec ++++++ --- /var/tmp/diff_new_pack.35bpAt/_old 2023-04-01 19:32:51.081539373 +0200 +++ /var/tmp/diff_new_pack.35bpAt/_new 2023-04-01 19:32:51.085539394 +0200 @@ -17,7 +17,7 @@ Name: openqa-mon -Version: 0.27 +Version: 1.0.1 Release: 0 Summary: CLI monitoring utility for openQA License: GPL-3.0-or-later ++++++ _service ++++++ --- /var/tmp/diff_new_pack.35bpAt/_old 2023-04-01 19:32:51.125539605 +0200 +++ /var/tmp/diff_new_pack.35bpAt/_new 2023-04-01 19:32:51.137539668 +0200 @@ -3,7 +3,7 @@ <param name="url">https://github.com/grisu48/openqa-mon.git</param> <param name="scm">git</param> <param name="revision">main</param> - <param name="version">v0.27</param> + <param name="version">v1.0.1</param> <param name="versionrewrite-pattern">v(.*)</param> </service> <service name="tar" mode="localonly"/> ++++++ openqa-mon-0.27.tar.gz -> openqa-mon-1.0.1.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openqa-mon-0.27/_review/opensuse-microos.toml new/openqa-mon-1.0.1/_review/opensuse-microos.toml --- old/openqa-mon-0.27/_review/opensuse-microos.toml 2023-03-13 10:29:03.000000000 +0100 +++ new/openqa-mon-1.0.1/_review/opensuse-microos.toml 2023-03-30 15:43:35.000000000 +0200 @@ -1,13 +1,13 @@ ## Review template file for MicroOS test runs on O3 -Instance = "https://openqa.opensuse.org"; # openQA instance to query -RabbitMQ = "amqps://opensuse:[email protected]" # RabbitMQ instance to query -RabbitMQTopic = "opensuse.openqa.job.done" # RabbitMQ topic to query -HideStatus = [ "scheduled","passed","assigned","running","softfailed","reviewed" ] # Hide scheduled, passed, assigned and reviewed jobs -RefreshInterval = 60 # Refresh from API once every minute -MaxJobs = 20 # Max. job per group to display -GroupBy = "groups" # Group by defined groups ("none" or "groups") -DefaultParams = { distri="microos", version = "Tumbleweed" } # Set of default parameters +Instance = "https://openqa.opensuse.org"; # openQA instance to query +RabbitMQ = "amqps://opensuse:[email protected]" # RabbitMQ instance to query +RabbitMQTopic = "opensuse.openqa.job.done" # RabbitMQ topic to query +HideStatus = ["scheduled", "passed", "softfailed", "cancelled", "skipped", "running", "uploading", "parallel_failed", "reviewed" ] # Hide scheduled and passed jobs +RefreshInterval = 60 # Refresh from API once every minute +MaxJobs = 20 # Max. job per group to display +GroupBy = "groups" # Group by defined groups ("none" or "groups") +DefaultParams = { distri="microos", version = "Tumbleweed" } # Set of default parameters ## Define job groups [[Groups]] diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openqa-mon-0.27/_review/opensuse-tumbleweed.toml new/openqa-mon-1.0.1/_review/opensuse-tumbleweed.toml --- old/openqa-mon-0.27/_review/opensuse-tumbleweed.toml 2023-03-13 10:29:03.000000000 +0100 +++ new/openqa-mon-1.0.1/_review/opensuse-tumbleweed.toml 2023-03-30 15:43:35.000000000 +0200 @@ -1,13 +1,13 @@ ## Review template file for Tumbleweed test runs on O3 -Instance = "https://openqa.opensuse.org"; # openQA instance to query -RabbitMQ = "amqps://opensuse:[email protected]" # RabbitMQ instance to query -RabbitMQTopic = "opensuse.openqa.job.done" # RabbitMQ topic to query -HideStatus = [ "scheduled","passed","assigned","running","softfailed","reviewed" ] # Hide scheduled, passed, assigned and reviewed jobs -RefreshInterval = 60 # Refresh from API once every minute -MaxJobs = 20 # Max. job per group to display -GroupBy = "groups" # Group by defined groups ("none" or "groups") -DefaultParams = { distri="opensuse", version = "Tumbleweed" } # Set of default parameters +Instance = "https://openqa.opensuse.org"; # openQA instance to query +RabbitMQ = "amqps://opensuse:[email protected]" # RabbitMQ instance to query +RabbitMQTopic = "opensuse.openqa.job.done" # RabbitMQ topic to query +HideStatus = ["scheduled", "passed", "softfailed", "cancelled", "skipped", "running", "uploading", "parallel_failed", "reviewed" ] # Hide scheduled and passed jobs +RefreshInterval = 60 # Refresh from API once every minute +MaxJobs = 20 # Max. job per group to display +GroupBy = "groups" # Group by defined groups ("none" or "groups") +DefaultParams = { distri="opensuse", version = "Tumbleweed" } # Set of default parameters ## Define job groups diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openqa-mon-0.27/_review/qe-core-O3.toml new/openqa-mon-1.0.1/_review/qe-core-O3.toml --- old/openqa-mon-0.27/_review/qe-core-O3.toml 2023-03-13 10:29:03.000000000 +0100 +++ new/openqa-mon-1.0.1/_review/qe-core-O3.toml 2023-03-30 15:43:35.000000000 +0200 @@ -1,13 +1,13 @@ ## Review template file for QE-Core job groups in O3 # See: https://confluence.suse.com/display/qasle/Bugbusters+and+Review+Shifts#BugbustersandReviewShifts-Links -Instance = "https://openqa.opensuse.org"; # openQA instance to query -RabbitMQ = "amqps://opensuse:[email protected]" # RabbitMQ instance to query -RabbitMQTopic = "opensuse.openqa.job.done" # RabbitMQ topic to query -HideStatus = [ "scheduled", "passed", "softfailed", "running", "reviewed" ] # Hide scheduled and passed jobs -RefreshInterval = 60 # Refresh from API once every minute -MaxJobs = 20 # Max. job per group to display -GroupBy = "groups" # Group by defined groups ("none" or "groups") +Instance = "https://openqa.opensuse.org"; # openQA instance to query +RabbitMQ = "amqps://opensuse:[email protected]" # RabbitMQ instance to query +RabbitMQTopic = "opensuse.openqa.job.done" # RabbitMQ topic to query +HideStatus = ["scheduled", "passed", "softfailed", "cancelled", "skipped", "running", "uploading", "parallel_failed", "reviewed" ] # Hide scheduled and passed jobs +RefreshInterval = 60 # Refresh from API once every minute +MaxJobs = 20 # Max. job per group to display +GroupBy = "groups" # Group by defined groups ("none" or "groups") # Tumbleweed diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openqa-mon-0.27/_review/qe-core-OSD.toml new/openqa-mon-1.0.1/_review/qe-core-OSD.toml --- old/openqa-mon-0.27/_review/qe-core-OSD.toml 2023-03-13 10:29:03.000000000 +0100 +++ new/openqa-mon-1.0.1/_review/qe-core-OSD.toml 2023-03-30 15:43:35.000000000 +0200 @@ -4,7 +4,7 @@ Instance = "https://openqa.suse.de"; # openQA instance to query RabbitMQ = "amqps://suse:[email protected]" # RabbitMQ instance to query RabbitMQTopic = "suse.openqa.job.done" # RabbitMQ topic to query -HideStatus = [ "scheduled", "passed", "softfailed", "running", "reviewed" ] # Hide scheduled and passed jobs +HideStatus = ["scheduled", "passed", "softfailed", "cancelled", "skipped", "running", "uploading", "parallel_failed", "reviewed" ] # Hide scheduled and passed jobs RefreshInterval = 60 # Refresh from API once every minute MaxJobs = 20 # Max. job per group to display GroupBy = "groups" # Group by defined groups ("none" or "groups") @@ -63,6 +63,18 @@ Params = { groupid = "53", build = "" } MaxLifetime = 86400 +# Quarterly Refresh + +[[Groups]] +Name = "Maintenance: QR SLE 15-SP4" +Params = { groupid = "458", build = "" } +MaxLifetime = 86400 + +[[Groups]] +Name = "Maintenance: QR SLE 15-SP3" +Params = { groupid = "373", build = "" } +MaxLifetime = 86400 + # Functional [[Groups]] diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openqa-mon-0.27/_review/qec-containers.toml new/openqa-mon-1.0.1/_review/qec-containers.toml --- old/openqa-mon-0.27/_review/qec-containers.toml 2023-03-13 10:29:03.000000000 +0100 +++ new/openqa-mon-1.0.1/_review/qec-containers.toml 2023-03-30 15:43:35.000000000 +0200 @@ -3,7 +3,7 @@ Instance = "https://openqa.suse.de"; # openQA instance to query RabbitMQ = "amqps://suse:[email protected]" # RabbitMQ instance to query RabbitMQTopic = "suse.openqa.job.done" # RabbitMQ topic to query -HideStatus = [ "scheduled", "passed", "softfailed", "running", "reviewed" ] # Hide scheduled and passed jobs +HideStatus = ["scheduled", "passed", "softfailed", "cancelled", "skipped", "running", "uploading", "parallel_failed", "reviewed" ] # Hide scheduled and passed jobs RefreshInterval = 60 # Refresh from API once every minute MaxJobs = 20 # Max. job per group to display GroupBy = "groups" # Group by defined groups ("none" or "groups") diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openqa-mon-0.27/_review/qec-jeos.toml new/openqa-mon-1.0.1/_review/qec-jeos.toml --- old/openqa-mon-0.27/_review/qec-jeos.toml 2023-03-13 10:29:03.000000000 +0100 +++ new/openqa-mon-1.0.1/_review/qec-jeos.toml 2023-03-30 15:43:35.000000000 +0200 @@ -3,7 +3,7 @@ Instance = "https://openqa.suse.de"; # openQA instance to query RabbitMQ = "amqps://suse:[email protected]" # RabbitMQ instance to query RabbitMQTopic = "suse.openqa.job.done" # RabbitMQ topic to query -HideStatus = [ "scheduled", "passed", "softfailed", "running", "reviewed" ] # Hide scheduled and passed jobs +HideStatus = ["scheduled", "passed", "softfailed", "cancelled", "skipped", "running", "uploading", "parallel_failed", "reviewed" ] # Hide scheduled and passed jobs RefreshInterval = 60 # Refresh from API once every minute MaxJobs = 20 # Max. job per group to display GroupBy = "groups" # Group by defined groups ("none" or "groups") diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openqa-mon-0.27/_review/qec-publiccloud.toml new/openqa-mon-1.0.1/_review/qec-publiccloud.toml --- old/openqa-mon-0.27/_review/qec-publiccloud.toml 2023-03-13 10:29:03.000000000 +0100 +++ new/openqa-mon-1.0.1/_review/qec-publiccloud.toml 2023-03-30 15:43:35.000000000 +0200 @@ -3,7 +3,7 @@ Instance = "https://openqa.suse.de"; # openQA instance to query RabbitMQ = "amqps://suse:[email protected]" # RabbitMQ instance to query RabbitMQTopic = "suse.openqa.job.done" # RabbitMQ topic to query -HideStatus = [ "scheduled", "passed", "softfailed", "cancelled", "skipped", "running", "reviewed" ] # Hide scheduled and passed jobs +HideStatus = ["scheduled", "passed", "softfailed", "cancelled", "skipped", "running", "uploading", "parallel_failed", "reviewed" ] # Hide scheduled and passed jobs Notify = false # No notifications by default RefreshInterval = 60 # Refresh from API once every minute MaxJobs = 20 # Max. job per group to display diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openqa-mon-0.27/cmd/openqa-mon/openqa-mon.go new/openqa-mon-1.0.1/cmd/openqa-mon/openqa-mon.go --- old/openqa-mon-0.27/cmd/openqa-mon/openqa-mon.go 2023-03-13 10:29:03.000000000 +0100 +++ new/openqa-mon-1.0.1/cmd/openqa-mon/openqa-mon.go 2023-03-30 15:43:35.000000000 +0200 @@ -17,7 +17,7 @@ "github.com/grisu48/gopenqa" ) -const VERSION = "0.8.0" +const VERSION = "1.0.1" var config Config var tui TUI @@ -677,6 +677,17 @@ singleCall(remotes) os.Exit(0) } + + // Refresh rates below 30 seconds are not allowed on public instances + if config.Continuous < 30 { + for _, remote := range remotes { + if strings.Contains(remote.URI, "://openqa.suse.de") || strings.Contains(remote.URI, "://openqa.opensuse.org") { + config.Continuous = 30 + break + } + } + } + tui = CreateTUI() tui.EnterAltScreen() tui.Clear() @@ -700,7 +711,7 @@ for i, remote := range remotes { instance := gopenqa.CreateInstance(ensureHTTP(remote.URI)) instance.SetUserAgent("openqa-mon") - instance.SetMaxRecursionDepth(100) // Certain jobs (e.g. verification runs) can have a lot of clones + instance.SetMaxRecursionDepth(20) // Certain jobs (e.g. verification runs) can have a lot of clones // If no jobs are defined, fetch overview if len(remote.Jobs) == 0 { overview, err := instance.GetOverview("", gopenqa.EmptyParams()) @@ -713,23 +724,21 @@ } else { // Fetch individual jobs jobsModified := false // If remote.Jobs has been modified (e.g. id changes when detecting a restarted job) - for i, id := range remote.Jobs { - var job gopenqa.Job - var err error - if config.Follow { - job, err = instance.GetJobFollow(id) - } else { - job, err = instance.GetJob(id) - } - if err != nil { - // It's better to ignore a single failure than to suppress following jobs as well - continue - } - if job.ID != id { + jobs, err := instance.GetJobs(remote.Jobs) + if err != nil { + return remotes, err + } + for i, job := range jobs { + if config.Follow && (job.IsCloned()) { + job, err = instance.GetJobFollow(job.ID) + if err != nil { + // It's better to ignore a single failure than to suppress following jobs as well + continue + } remote.Jobs[i] = job.ID jobsModified = true } - callback(id, job) + callback(job.ID, job) // Fetch children if config.Hierarchy { diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openqa-mon-0.27/cmd/openqa-mq/openqa-mq.go new/openqa-mon-1.0.1/cmd/openqa-mq/openqa-mq.go --- old/openqa-mon-0.27/cmd/openqa-mq/openqa-mq.go 2023-03-13 10:29:03.000000000 +0100 +++ new/openqa-mon-1.0.1/cmd/openqa-mq/openqa-mq.go 2023-03-30 15:43:35.000000000 +0200 @@ -10,7 +10,7 @@ "github.com/streadway/amqp" ) -const VERSION = "1.2" +const VERSION = "1.0.1" type Config struct { Remote string // Remote address diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openqa-mon-0.27/cmd/openqa-revtui/openqa-revtui.go new/openqa-mon-1.0.1/cmd/openqa-revtui/openqa-revtui.go --- old/openqa-mon-0.27/cmd/openqa-revtui/openqa-revtui.go 2023-03-13 10:29:03.000000000 +0100 +++ new/openqa-mon-1.0.1/cmd/openqa-revtui/openqa-revtui.go 2023-03-30 15:43:35.000000000 +0200 @@ -11,7 +11,7 @@ "github.com/grisu48/gopenqa" ) -const VERSION = "0.4.0" +const VERSION = "1.0.1" /* Group is a single configurable monitoring unit. A group contains all parameters that will be queried from openQA */ type Group struct { @@ -36,6 +36,7 @@ var cf Config var knownJobs []gopenqa.Job +var updatedRefresh bool func (cf *Config) LoadToml(filename string) error { if _, err := toml.DecodeFile(filename, cf); err != nil { @@ -126,8 +127,29 @@ return deltaT > maxlifetime } -func isReviewed(job gopenqa.Job, instance gopenqa.Instance) (bool, error) { - comments, err := instance.GetComments(job.ID) +func isReviewed(job gopenqa.Job, instance gopenqa.Instance, checkParallel bool) (bool, error) { + reviewed, err := checkReviewed(job.ID, instance) + if err != nil || reviewed { + return reviewed, err + } + + // If not reviewed but "parallel_failed", check parallel jobs if they are reviewed + if checkParallel { + for _, childID := range job.Children.Parallel { + reviewed, err := checkReviewed(childID, instance) + if err != nil { + return reviewed, err + } + if reviewed { + return true, nil + } + } + } + return false, nil +} + +func checkReviewed(job int64, instance gopenqa.Instance) (bool, error) { + comments, err := instance.GetComments(job) if err != nil { return false, nil } @@ -136,11 +158,11 @@ return true, nil } // Manually check for poo or bsc reference - if strings.Contains(c.Text, "poo#") || strings.Contains(c.Text, "bsc#") { + if strings.Contains(c.Text, "poo#") || strings.Contains(c.Text, "bsc#") || strings.Contains(c.Text, "boo#") { return true, nil } // Or for link to progress/bugzilla ticket - if strings.Contains(c.Text, "://progress.opensuse.org/issues/") || strings.Contains(c.Text, "://bugzilla.suse.com/show_bug.cgi?id=") { + if strings.Contains(c.Text, "://progress.opensuse.org/issues/") || strings.Contains(c.Text, "://bugzilla.suse.com/show_bug.cgi?id=") || strings.Contains(c.Text, "://bugzilla.opensuse.org/show_bug.cgi?id=") { return true, nil } } @@ -168,7 +190,7 @@ if err != nil { return job, err } - if job.CloneID != 0 && job.CloneID != job.ID { + if job.IsCloned() { id = job.CloneID time.Sleep(100 * time.Millisecond) // Don't spam the instance continue @@ -178,9 +200,33 @@ return job, fmt.Errorf("max recursion depth reached") } -func FetchJobs(instance gopenqa.Instance) ([]gopenqa.Job, error) { +/* Fetch the given jobs from the instance at once */ +func fetchJobs(ids []int64, instance gopenqa.Instance) ([]gopenqa.Job, error) { + jobs := make([]gopenqa.Job, 0) + + jobs, err := instance.GetJobs(ids) + if err != nil { + return jobs, err + } + + // Get cloned jobs, if present + for i, job := range jobs { + if job.IsCloned() { + job, err = FetchJob(job.ID, instance) + if err != nil { + return jobs, err + } + jobs[i] = job + } + } + return jobs, nil +} + +type FetchJobsCallback func(int, int, int, int) + +func FetchJobs(instance gopenqa.Instance, callback FetchJobsCallback) ([]gopenqa.Job, error) { ret := make([]gopenqa.Job, 0) - for _, group := range cf.Groups { + for i, group := range cf.Groups { params := group.Params jobs, err := instance.GetOverview("", params) if err != nil { @@ -190,15 +236,24 @@ if len(jobs) > cf.MaxJobs { jobs = jobs[:cf.MaxJobs] } - // Get detailed job instances + + // Get detailed job instances. Fetch them at once + ids := make([]int64, 0) for _, job := range jobs { - if job, err = FetchJob(job.ID, instance); err != nil { - return ret, err - } else { - // Filter too old jobs - if !isJobTooOld(job, group.MaxLifetime) { - ret = append(ret, job) - } + ids = append(ids, job.ID) + } + if callback != nil { + // Add one to the counter to indicate the progress to humans (0/16 looks weird) + callback(i+1, len(cf.Groups), 0, len(jobs)) + } + jobs, err = fetchJobs(ids, instance) + if err != nil { + return jobs, err + } + for _, job := range jobs { + // Filter too old jobs + if !isJobTooOld(job, group.MaxLifetime) { + ret = append(ret, job) } } } @@ -214,21 +269,26 @@ return remote } -/** Try to update the given job, if it exists and if not the same. Returns the found job and true, if an update was successful */ -func updateJob(orig_id int64, job gopenqa.Job, instance gopenqa.Instance) (gopenqa.Job, bool) { +/** Updates the known job based on the Job ID. Returns true if it has been updated and the updated instance and false if and the job if the job id hasn't been found */ +func updateJob(job gopenqa.Job) (gopenqa.Job, bool) { for i, j := range knownJobs { - if j.ID == orig_id { - if j.ID != job.ID || j.State != job.State || j.Result != job.Result { - knownJobs[i] = job - return knownJobs[i], true - } else { - return job, false - } + if j.ID == job.ID { + knownJobs[i] = job + return knownJobs[i], true } } return job, false } +func getKnownJob(id int64) (gopenqa.Job, bool) { + for _, j := range knownJobs { + if j.ID == id { + return j, true + } + } + return gopenqa.Job{}, false +} + /** Try to update the job with the given status, if present. Returns the found job and true if the job was present */ func updateJobStatus(status gopenqa.JobStatus) (gopenqa.Job, bool) { var job gopenqa.Job @@ -422,6 +482,15 @@ instance := gopenqa.CreateInstance(cf.Instance) instance.SetUserAgent("openqa-mon/revtui") + // Refresh rates below 5 minutes are not allowed on public instances due to the rather large load it puts on them + updatedRefresh = false + if cf.RefreshInterval < 300 { + if strings.Contains(cf.Instance, "://openqa.suse.de") || strings.Contains(cf.Instance, "://openqa.opensuse.org") { + cf.RefreshInterval = 300 + updatedRefresh = true + } + } + // Run TUI and use the return code tui := CreateTUI() switch cf.GroupBy { @@ -445,37 +514,49 @@ func refreshJobs(tui *TUI, instance gopenqa.Instance) error { // Get fresh jobs status := tui.Status() - tui.SetStatus("Refreshing jobs ... ") + oldJobs := tui.Model.Jobs() + tui.SetStatus(fmt.Sprintf("Refreshing %d jobs ... ", len(oldJobs))) tui.Update() - jobs := tui.Model.Jobs() - for i, job := range jobs { - orig_id := job.ID - job, err := FetchJob(job.ID, instance) - if err != nil { - return err + // Refresh all jobs at once in one request + ids := make([]int64, 0) + for _, job := range oldJobs { + ids = append(ids, job.ID) + } + jobs, err := instance.GetJobs(ids) + if err != nil { + return err + } + for _, job := range jobs { + updated := false + if j, found := getKnownJob(job.ID); found { + updated = j.ID != job.ID || j.State != job.State || j.Result != job.Result + } else { + updated = true } - job, found := updateJob(orig_id, job, instance) - if found { + + if updated { status = fmt.Sprintf("Last update: [%s] Job %d-%s %s", time.Now().Format("15:04:05"), job.ID, job.Name, job.JobState()) tui.SetStatus(status) - jobs[i] = job tui.Model.Apply(jobs) tui.Update() if cf.Notify && !hideJob(job) { NotifySend(fmt.Sprintf("%s: %s %s", job.JobState(), job.Name, job.Test)) } } - // Failed jobs will be also scanned for comments + tui.Update() + // Scan failed jobs for comments state := job.JobState() - if state == "failed" || state == "incomplete" { - reviewed, err := isReviewed(job, instance) + if state == "failed" || state == "incomplete" || state == "parallel_failed" { + reviewed, err := isReviewed(job, instance, state == "parallel_failed") if err != nil { return err } + tui.Model.SetReviewed(job.ID, reviewed) tui.Update() } } + knownJobs = jobs tui.Model.Apply(jobs) tui.SetStatus(status) tui.Update() @@ -530,6 +611,9 @@ fmt.Println(title) fmt.Println("") + if updatedRefresh { + fmt.Printf(ANSI_YELLOW + "WARNING: For OSD and O3 a rate limit of 5 minutes between polling has been applied." + ANSI_RESET + "\n\n") + } fmt.Printf("Initial querying instance %s ... \n", cf.Instance) fmt.Println("\tGet job groups ... ") jobgroups, err := FetchJobGroups(instance) @@ -540,16 +624,27 @@ fmt.Fprintf(os.Stderr, "Warn: No job groups\n") } tui.Model.SetJobGroups(jobgroups) - fmt.Printf("\tGet jobs for %d groups ... \n", len(cf.Groups)) - jobs, err := FetchJobs(instance) + fmt.Print("\033[s") // Save cursor position + fmt.Printf("\tGet jobs for %d groups ...", len(cf.Groups)) + jobs, err := FetchJobs(instance, func(group int, groups int, job int, jobs int) { + fmt.Print("\033[u") // Restore cursor position + fmt.Print("\033[K") // Erase till end of line + fmt.Printf("\tGet jobs for %d groups ... %d/%d", len(cf.Groups), group, groups) + if job == 0 { + fmt.Printf(" (%d jobs)", jobs) + } else { + fmt.Printf(" (%d/%d jobs)", job, jobs) + } + }) + fmt.Println() if err != nil { return fmt.Errorf("Error fetching jobs: %s", err) } // Failed jobs will be also scanned for comments for _, job := range jobs { state := job.JobState() - if state == "failed" || state == "incomplete" { - reviewed, err := isReviewed(job, instance) + if state == "failed" || state == "incomplete" || state == "parallel_failed" { + reviewed, err := isReviewed(job, instance, state == "parallel_failed") if err != nil { return fmt.Errorf("Error fetching job comment: %s", err) } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openqa-mon-0.27/cmd/openqa-revtui/tui.go new/openqa-mon-1.0.1/cmd/openqa-revtui/tui.go --- old/openqa-mon-0.27/cmd/openqa-revtui/tui.go 2023-03-13 10:29:03.000000000 +0100 +++ new/openqa-mon-1.0.1/cmd/openqa-revtui/tui.go 2023-03-30 15:43:35.000000000 +0200 @@ -273,8 +273,9 @@ if state == s { return true } + // Special reviewed keyword - if s == "reviewed" && state == "failed" { + if s == "reviewed" && (state == "failed" || state == "parallel_failed" || state == "incomplete") { if reviewed, found := tui.Model.reviewed[job.ID]; found && reviewed { return true } @@ -306,6 +307,21 @@ return ret } +func jobGroupHeader(group gopenqa.JobGroup, width int) string { + if width <= 0 { + return "" + } + line := fmt.Sprintf("===== %s =====", group.Name) + for len(line) < width { + line += "=" + } + // Crop if necessary + if len(line) > width { + line = line[:width] + } + return line +} + func (tui *TUI) buildJobsScreenByGroup(width int) []string { lines := make([]string, 0) @@ -337,7 +353,8 @@ } else { lines = append(lines, "") } - lines = append(lines, fmt.Sprintf("===== %s ====================", grp.Name)) + lines = append(lines, jobGroupHeader(grp, width)) + for _, job := range jobs { if !tui.hideJob(job) { lines = append(lines, tui.formatJobLine(job, width)) @@ -356,7 +373,23 @@ stats := sortedKeys(statC) for _, s := range stats { c := statC[s] - line += fmt.Sprintf(", %s: %d", s, c) + line += ", " + // Add some color + if s == "passed" { + line += ANSI_GREEN + } else if s == "cancelled" { + line += ANSI_MAGENTA + } else if s == "failed" || s == "parallel_failed" || s == "incomplete" { + line += ANSI_RED + } else if s == "softfailed" { + line += ANSI_YELLOW + } else if s == "uploading" || s == "scheduled" || s == "running" { + line += ANSI_BLUE + } else if s == "skipped" { + line += ANSI_WHITE + } + line += fmt.Sprintf("%s: %d", s, c) + line += ANSI_RESET // Clear color } if hidden > 0 { line += fmt.Sprintf(" (hidden: %d)", hidden) @@ -458,10 +491,15 @@ tui.Model.mutex.Lock() defer tui.Model.mutex.Unlock() width, height := terminalSize() - if width < 0 || height < 0 { + if width <= 0 || height <= 0 { return } + // Check for unreasonable values + if width > 1000 { + width = 1000 + } + // Header and footer are separate. We only scroll through the "screen" screen := tui.buildScreen(width) header := tui.buildHeader(width) @@ -487,7 +525,7 @@ if len(footer) > remainingLines { footer = make([]string, 0) } else { - remainingLines -= (len(footer) + 1) + remainingLines -= len(footer) } // Print screen @@ -506,9 +544,9 @@ // Print footer if len(footer) > 0 { - fmt.Println("") for _, line := range footer { - fmt.Println(line) + fmt.Println("") + fmt.Print(line) } } } @@ -529,7 +567,7 @@ return ANSI_GREEN } else if state == "softfail" || state == "softfailed" { return ANSI_YELLOW - } else if state == "fail" || state == "failed" || state == "incomplete" { + } else if state == "fail" || state == "failed" || state == "incomplete" || state == "parallel_failed" { return ANSI_RED } else if state == "cancelled" || state == "user_cancelled" { return ANSI_MAGENTA @@ -571,7 +609,7 @@ tStr = timestamp.Format("2006-01-02-15:04:05") } // For failed jobs check if they are reviewed - if job.Result == "failed" || job.Result == "incomplete" { + if state == "failed" || state == "incomplete" || state == "parallel_failed" { if reviewed, found := tui.Model.reviewed[job.ID]; found && reviewed { c2 = ANSI_MAGENTA } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openqa-mon-0.27/go.mod new/openqa-mon-1.0.1/go.mod --- old/openqa-mon-0.27/go.mod 2023-03-13 10:29:03.000000000 +0100 +++ new/openqa-mon-1.0.1/go.mod 2023-03-30 15:43:35.000000000 +0200 @@ -4,7 +4,7 @@ require ( github.com/BurntSushi/toml v1.2.1 - github.com/grisu48/gopenqa v0.5.2 + github.com/grisu48/gopenqa v0.7.1 github.com/streadway/amqp v1.0.0 golang.org/x/crypto v0.4.0 ) diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/openqa-mon-0.27/go.sum new/openqa-mon-1.0.1/go.sum --- old/openqa-mon-0.27/go.sum 2023-03-13 10:29:03.000000000 +0100 +++ new/openqa-mon-1.0.1/go.sum 2023-03-30 15:43:35.000000000 +0200 @@ -2,8 +2,8 @@ github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/grisu48/gopenqa v0.5.2 h1:ofiN5Q9KIpeI+8Eq+Mh+Smk+Ia3q70nHz7EKKpJtcZU= -github.com/grisu48/gopenqa v0.5.2/go.mod h1:D7EFTPhtzNvnHnDol9UoPCFmnzOiLBVa1tOOYqJDgGo= +github.com/grisu48/gopenqa v0.7.1 h1:XhmsWY93WjqdnXmdDjjHTFdfLjQwz7kApviDM9lX27U= +github.com/grisu48/gopenqa v0.7.1/go.mod h1:D7EFTPhtzNvnHnDol9UoPCFmnzOiLBVa1tOOYqJDgGo= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/streadway/amqp v1.0.0 h1:kuuDrUJFZL1QYL9hUNuCxNObNzB0bV/ZG5jV3RWAQgo= ++++++ openqa-mon.obsinfo ++++++ --- /var/tmp/diff_new_pack.35bpAt/_old 2023-04-01 19:32:51.293540486 +0200 +++ /var/tmp/diff_new_pack.35bpAt/_new 2023-04-01 19:32:51.297540507 +0200 @@ -1,5 +1,5 @@ name: openqa-mon -version: 0.27 -mtime: 1678699743 -commit: f3c259de42cfcbff0afd4285313b48f8d9475a84 +version: 1.0.1 +mtime: 1680183815 +commit: 53cb17dd5a792df874a72a88248f3ef87556255a ++++++ vendor.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/github.com/grisu48/gopenqa/gopenqa.go new/vendor/github.com/grisu48/gopenqa/gopenqa.go --- old/vendor/github.com/grisu48/gopenqa/gopenqa.go 2023-03-16 10:37:09.000000000 +0100 +++ new/vendor/github.com/grisu48/gopenqa/gopenqa.go 2023-03-30 15:45:47.000000000 +0200 @@ -11,6 +11,7 @@ "net/url" "os" "strings" + "sync" "time" ) @@ -20,8 +21,10 @@ apikey string apisecret string verbose bool - maxRecursions int // Maximum number of recursions - userAgent string // Useragent sent with the request + maxRecursions int // Maximum number of recursions + userAgent string // Useragent sent with the request + allowParallel bool // Allow parallel requests (default: No) + mutFetching sync.Mutex // Mutex to ensure only one request at the time is performed } // the settings are given as dict: @@ -132,13 +135,21 @@ return params.Encode() } +func ExtractJobIDS(jobs []Job) []int64 { + ret := make([]int64, 0) + for _, job := range jobs { + ret = append(ret, job.ID) + } + return ret +} + func EmptyParams() map[string]string { return make(map[string]string, 0) } /* Create a openQA instance module */ func CreateInstance(url string) Instance { - inst := Instance{URL: url, maxRecursions: 10, verbose: false, userAgent: "gopenqa"} + inst := Instance{URL: url, maxRecursions: 10, verbose: false, userAgent: "gopenqa", allowParallel: false} return inst } @@ -168,6 +179,11 @@ i.userAgent = userAgent } +// Set to allow or disallow parallel requests to the instance +func (i *Instance) SetAllowParallel(allow bool) { + i.allowParallel = allow +} + func assignInstance(jobs []Job, instance *Instance) []Job { for i, j := range jobs { j.instance = instance @@ -226,6 +242,12 @@ * Add the APIKEY and APISECRET credentials, if given */ func (i *Instance) request(method string, url string, data []byte) ([]byte, error) { + // Request mutex to ensure, only one request at the time + if !i.allowParallel { + i.mutFetching.Lock() + defer i.mutFetching.Unlock() + } + contentType := "" if data == nil { data = make([]byte, 0) @@ -337,6 +359,9 @@ return jobs.Jobs, err } err = json.Unmarshal(resp, &jobs) + if err != nil { + return jobs.Jobs, err + } // Now, get only the latest job per group_id mapped := make(map[int]Job) @@ -360,19 +385,41 @@ ret = append(ret, v) } return ret, nil +} +func (job *Job) applyInstance(i *Instance) { + job.Link = fmt.Sprintf("%s/tests/%d", i.URL, job.ID) + job.instance = i + job.Remote = i.URL } // GetJob fetches detailled job information func (i *Instance) GetJob(id int64) (Job, error) { url := fmt.Sprintf("%s/api/v1/jobs/%d", i.URL, id) job, err := i.fetchJob(url) - job.Link = fmt.Sprintf("%s/tests/%d", i.URL, id) - job.instance = i - job.Remote = i.URL return job, err } +// GetJob fetches detailled information about a list of jobs +func (i *Instance) GetJobs(ids []int64) ([]Job, error) { + if len(ids) == 0 { + return make([]Job, 0), nil + } + url := fmt.Sprintf("%s/api/v1/jobs", i.URL) + // Add job ids to URL + // Note: I'm not using strings.Join because that requires me to first convert ids to a []string and I believe the following approach is not worse + first := true + for _, id := range ids { + if first { + first = false + url = fmt.Sprintf("%s?ids=%d", url, id) + } else { + url = fmt.Sprintf("%s&ids=%d", url, id) + } + } + return i.fetchJobsArray(url) +} + func (i *Instance) DeleteJob(id int64) error { url := fmt.Sprintf("%s/api/v1/jobs/%d", i.URL, id) buf, err := i.delete(url, nil) @@ -383,23 +430,26 @@ } // GetJob fetches detailled job information and follows the job, if it contains a CloneID -func (i *Instance) GetJobFollow(id int64) (Job, error) { - recursions := 0 // keep track of the number of recursions -fetch: - url := fmt.Sprintf("%s/api/v1/jobs/%d", i.URL, id) - job, err := i.fetchJob(url) - if job.CloneID != 0 && job.CloneID != job.ID { - recursions++ - if i.maxRecursions != 0 && recursions >= i.maxRecursions { - return job, fmt.Errorf("maximum recusion depth reached") +func (inst *Instance) GetJobFollow(id int64) (Job, error) { + for recursion := 0; recursion < inst.maxRecursions; recursion++ { + url := fmt.Sprintf("%s/api/v1/jobs/%d", inst.URL, id) + job, err := inst.fetchJob(url) + if err != nil { + return job, err + } + if job.IsCloned() { + id = job.CloneID + continue } - id = job.CloneID - goto fetch + return job, nil } - job.Link = fmt.Sprintf("%s/tests/%d", i.URL, id) - job.instance = i - job.Remote = i.URL - return job, err + return Job{}, fmt.Errorf("maximum recusion depth reached") +} + +// GetJobState uses the (currently experimental) API call to quickly fetch a job state +func (i *Instance) GetJobState(id int64) (JobState, error) { + url := fmt.Sprintf("%s/api/v1//experimental/jobs/%d/status", i.URL, id) + return i.fetchJobState(url) } func (i *Instance) GetJobGroups() ([]JobGroup, error) { @@ -467,21 +517,44 @@ return i.fetchWorkers(url) } -func (i *Instance) fetchJobs(url string) ([]Job, error) { +// fetchJobs fetches the given url and returns all jobs returned by it (as direct array) +func (inst *Instance) fetchJobs(url string) ([]Job, error) { jobs := make([]Job, 0) - resp, err := i.get(url, nil) + resp, err := inst.get(url, nil) if err != nil { return jobs, err } err = json.Unmarshal(resp, &jobs) + for i, job := range jobs { + job.applyInstance(inst) + jobs[i] = job + } return jobs, err } -func (i *Instance) fetchJobGroups(url string) ([]JobGroup, error) { +// fetchJobs fetches the given url and returns all jobs, It expects the jobs to be within the "jobs" dict of the result +func (inst *Instance) fetchJobsArray(url string) ([]Job, error) { + type ResultJob struct { // Expected result structure + Jobs []Job `json:"jobs"` + } + var ret ResultJob + resp, err := inst.get(url, nil) + if err != nil { + return make([]Job, 0), err + } + err = json.Unmarshal(resp, &ret) + for i, job := range ret.Jobs { + job.applyInstance(inst) + ret.Jobs[i] = job + } + return ret.Jobs, err +} + +func (inst *Instance) fetchJobGroups(url string) ([]JobGroup, error) { jobs := make([]JobGroup, 0) - resp, err := i.get(url, nil) + resp, err := inst.get(url, nil) if err != nil { return jobs, err } @@ -539,20 +612,31 @@ return make([]Machine, 0), nil } -func (i *Instance) fetchJob(url string) (Job, error) { +func (inst *Instance) fetchJob(url string) (Job, error) { type ResultJob struct { // Expected result structure Job Job `json:"job"` } var job ResultJob - resp, err := i.get(url, nil) + resp, err := inst.get(url, nil) if err != nil { return job.Job, err } // TODO: Sometimes SizeLimit is returned as string but it should be an int. Fix this. err = json.Unmarshal(resp, &job) + job.Job.applyInstance(inst) return job.Job, err } +func (i *Instance) fetchJobState(url string) (JobState, error) { + var state JobState + resp, err := i.get(url, nil) + if err != nil { + return state, err + } + err = json.Unmarshal(resp, &state) + return state, err +} + /* merge given parameter string to URL parameters */ func mergeParams(params map[string]string) string { if len(params) == 0 { @@ -569,17 +653,24 @@ * Fetch the given child jobs. Use with j.Children.Chained, j.Children.DirectlyChained and j.Children.Parallel * if follow is set to true, the method will return the cloned job instead of the original one, if present */ -func (j *Job) FetchChildren(children []int64, follow bool) ([]Job, error) { - jobs := make([]Job, 0) - for _, id := range children { - job, err := j.instance.GetJobFollow(id) - if err != nil { - return jobs, err +func (j *Job) FetchChildren(ids []int64, follow bool) ([]Job, error) { + children, err := j.instance.GetJobs(ids) + if err != nil { + return children, err + } + if follow { + for i, job := range children { + // Fetch cloned job, if present + if job.CloneID != 0 && job.CloneID != job.ID { + job, err := j.instance.GetJobFollow(job.ID) + if err != nil { + return children, err + } + children[i] = job + } } - jobs = append(jobs, job) } - - return jobs, nil + return children, nil } /* Fetch all child jobs diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/github.com/grisu48/gopenqa/job.go new/vendor/github.com/grisu48/gopenqa/job.go --- old/vendor/github.com/grisu48/gopenqa/job.go 2023-03-16 10:37:09.000000000 +0100 +++ new/vendor/github.com/grisu48/gopenqa/job.go 2023-03-30 15:45:47.000000000 +0200 @@ -42,10 +42,19 @@ Machine string `json:"MACHINE"` } +/* Special struct for getting quick job status */ +type JobState struct { + BlockedBy int64 `json:"blocked_by_id"` + Result string `json:"result"` + State string `json:"state"` +} + /* Format job as a string */ func (j *Job) String() string { return fmt.Sprintf("%d %s (%s)", j.ID, j.Name, j.Test) } + +/* JobState returns or the job state or the result, if the job is done */ func (j *Job) JobState() string { if j.State == "done" { return j.Result @@ -53,6 +62,11 @@ return j.State } +/* IsCloned returns true, if the job has been cloned/restarted */ +func (j *Job) IsCloned() bool { + return j.CloneID != 0 && j.CloneID != j.ID +} + /* Compares two jobs according to their unique parameters (ID, GroupID, Test) */ func (j1 *Job) Equals(j2 Job) bool { // Compare only relevant parameters diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/github.com/grisu48/gopenqa/rabbitmq.go new/vendor/github.com/grisu48/gopenqa/rabbitmq.go --- old/vendor/github.com/grisu48/gopenqa/rabbitmq.go 2023-03-16 10:37:09.000000000 +0100 +++ new/vendor/github.com/grisu48/gopenqa/rabbitmq.go 2023-03-30 15:45:47.000000000 +0200 @@ -41,29 +41,80 @@ type RabbitMQ struct { remote string con *amqp.Connection + closed bool } +// Callback when the connection was closed +type RabbitMQCloseCallback func(error) + // Close connection func (mq *RabbitMQ) Close() { + mq.closed = true mq.con.Close() } +// Connected returns true if RabbitMQ is connected +func (mq *RabbitMQ) Connected() bool { + return !mq.closed && !mq.con.IsClosed() +} + +// Connected returns true if RabbitMQ is closing or if it is closed. +func (mq *RabbitMQ) Closed() bool { + if mq.closed { + return true + } + if mq.con.IsClosed() { + mq.closed = true + return true + } + return false +} + +// Reconnect to the RabbitMQ server. This will close any previous connections and channels +func (mq *RabbitMQ) Reconnect() error { + var err error + mq.con.Close() + mq.closed = false + mq.con, err = amqp.Dial(mq.remote) + return err +} + +// NotifyClose registeres a defined callback function for when the RabbitMQ connection is closed +func (mq *RabbitMQ) NotifyClose(callback RabbitMQCloseCallback) { + go func() { + recvChannel := make(chan *amqp.Error, 1) + mq.con.NotifyClose(recvChannel) + for err := range recvChannel { + callback(fmt.Errorf(err.Error())) + } + }() +} + // RabbitMQSubscription handles a single subscription type RabbitMQSubscription struct { channel *amqp.Channel key string obs <-chan amqp.Delivery mq *RabbitMQ + con *amqp.Connection // Keep a reference to the connection to check if it is still connected. This is necessary because mq can reconnect and therefore have another new mq.con instance +} + +// Connected returns true if RabbitMQ is connected +func (sub *RabbitMQSubscription) Connected() bool { + return !sub.con.IsClosed() } // Receive receives a raw non-empty RabbitMQ messages func (sub *RabbitMQSubscription) Receive() (amqp.Delivery, error) { - for { - msg := <-sub.obs + for msg, ok := <-sub.obs; ok; { if len(msg.Body) > 0 { return msg, nil } } + if sub.mq == nil || sub.mq.closed || sub.con == nil || sub.con.IsClosed() { + return amqp.Delivery{}, fmt.Errorf("EOF") + } + return amqp.Delivery{}, fmt.Errorf("channel unexpectedly closed") } // ReceiveJob receives the next message and try to parse it as job @@ -174,7 +225,7 @@ // ConnectRabbitMQ connects to a RabbitMQ instance and returns the RabbitMQ object func ConnectRabbitMQ(remote string) (RabbitMQ, error) { var err error - rmq := RabbitMQ{remote: remote} + rmq := RabbitMQ{remote: remote, closed: false} rmq.con, err = amqp.Dial(remote) if err != nil { @@ -213,5 +264,6 @@ sub.channel = ch sub.key = key sub.obs = obs + sub.con = mq.con return sub, nil } diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/vendor/modules.txt new/vendor/modules.txt --- old/vendor/modules.txt 2023-03-16 10:37:10.000000000 +0100 +++ new/vendor/modules.txt 2023-03-30 15:45:47.000000000 +0200 @@ -2,7 +2,7 @@ ## explicit; go 1.16 github.com/BurntSushi/toml github.com/BurntSushi/toml/internal -# github.com/grisu48/gopenqa v0.5.2 +# github.com/grisu48/gopenqa v0.7.1 ## explicit; go 1.14 github.com/grisu48/gopenqa # github.com/streadway/amqp v1.0.0
