This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 8b6a1bae7 fix(sonarqube): issue collector failed when issue status
updated (#5544) (#5548)
8b6a1bae7 is described below
commit 8b6a1bae76535cad7014506ee0ba301b16e1eaef
Author: Leric Zhang <[email protected]>
AuthorDate: Mon Jun 26 09:50:55 2023 +0800
fix(sonarqube): issue collector failed when issue status updated (#5544)
(#5548)
* fix(sonarqube): issue collector failed when issue status updated (#5544)
cherry-pick
* fix: fix golangci-lint error
---
.../pluginhelper/subtaskmeta_sorter/dependency_test.go | 8 ++++----
backend/plugins/dora/impl/impl_test.go | 6 +++---
backend/plugins/sonarqube/e2e/account_test.go | 3 +++
backend/plugins/sonarqube/e2e/filemetrics_test.go | 3 +++
backend/plugins/sonarqube/e2e/hotspot_test.go | 3 +++
backend/plugins/sonarqube/e2e/issue_test.go | 3 +++
backend/plugins/sonarqube/e2e/project_test.go | 2 ++
backend/plugins/sonarqube/impl/impl.go | 6 ++++--
backend/plugins/sonarqube/tasks/hotspots_collector.go | 16 ++++++++++++++++
backend/plugins/sonarqube/tasks/issues_collector.go | 11 ++++++-----
backend/plugins/sonarqube/tasks/task_data.go | 1 +
11 files changed, 48 insertions(+), 14 deletions(-)
diff --git a/backend/helpers/pluginhelper/subtaskmeta_sorter/dependency_test.go
b/backend/helpers/pluginhelper/subtaskmeta_sorter/dependency_test.go
index 924b3be79..451837ff2 100644
--- a/backend/helpers/pluginhelper/subtaskmeta_sorter/dependency_test.go
+++ b/backend/helpers/pluginhelper/subtaskmeta_sorter/dependency_test.go
@@ -66,15 +66,15 @@ func Test_topologicalSort(t *testing.T) {
{
name: "cycle error",
args: args{[]*plugin.SubTaskMeta{
- &plugin.SubTaskMeta{
+ {
Name: "D",
- Dependencies:
[]*plugin.SubTaskMeta{&plugin.SubTaskMeta{
+ Dependencies: []*plugin.SubTaskMeta{{
Name: "E",
}},
},
- &plugin.SubTaskMeta{
+ {
Name: "E",
- Dependencies:
[]*plugin.SubTaskMeta{&plugin.SubTaskMeta{
+ Dependencies: []*plugin.SubTaskMeta{{
Name: "D",
}},
},
diff --git a/backend/plugins/dora/impl/impl_test.go
b/backend/plugins/dora/impl/impl_test.go
index 7c5d8715a..c34679d1f 100644
--- a/backend/plugins/dora/impl/impl_test.go
+++ b/backend/plugins/dora/impl/impl_test.go
@@ -39,12 +39,12 @@ func TestMakeMetricPluginPipelinePlanV200(t *testing.T) {
doraOutputPlan := plugin.PipelinePlan{
plugin.PipelineStage{
{
- Plugin: "dora",
+ Plugin: "dora",
Subtasks: []string{
"generateDeploymentCommits",
"enrichPrevSuccessDeploymentCommits",
},
- Options: map[string]interface{}{"projectName":
projectName},
+ Options: map[string]interface{}{"projectName":
projectName},
},
},
plugin.PipelineStage{
@@ -56,7 +56,7 @@ func TestMakeMetricPluginPipelinePlanV200(t *testing.T) {
},
plugin.PipelineStage{
{
- Plugin: "dora",
+ Plugin: "dora",
Subtasks: []string{
"calculateChangeLeadTime",
"ConnectIncidentToDeployment",
diff --git a/backend/plugins/sonarqube/e2e/account_test.go
b/backend/plugins/sonarqube/e2e/account_test.go
index 0a98d9f99..add755e87 100644
--- a/backend/plugins/sonarqube/e2e/account_test.go
+++ b/backend/plugins/sonarqube/e2e/account_test.go
@@ -19,6 +19,7 @@ package e2e
import (
"testing"
+ "time"
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
@@ -37,6 +38,7 @@ func TestSonarqubeAccountDataFlow(t *testing.T) {
Options: &tasks.SonarqubeOptions{
ConnectionId: 1,
},
+ TaskStartTime: time.Now(),
}
// import raw data table
@@ -51,6 +53,7 @@ func TestSonarqubeAccountDataFlow(t *testing.T) {
Options: &tasks.SonarqubeOptions{
ConnectionId: 2,
},
+ TaskStartTime: time.Now(),
}
dataflowTester.Subtask(tasks.ExtractAccountsMeta, taskData2)
diff --git a/backend/plugins/sonarqube/e2e/filemetrics_test.go
b/backend/plugins/sonarqube/e2e/filemetrics_test.go
index 9c34ef99a..488ec5f5e 100644
--- a/backend/plugins/sonarqube/e2e/filemetrics_test.go
+++ b/backend/plugins/sonarqube/e2e/filemetrics_test.go
@@ -17,6 +17,7 @@ package e2e
import (
"testing"
+ "time"
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/models/domainlayer/codequality"
@@ -43,6 +44,7 @@ func TestSonarqubeFileMetricsDataFlow(t *testing.T) {
ConnectionId: 2,
ProjectKey: "testDevLake",
},
+ TaskStartTime: time.Now(),
}
// Interfered data
taskData2 := &tasks.SonarqubeTaskData{
@@ -50,6 +52,7 @@ func TestSonarqubeFileMetricsDataFlow(t *testing.T) {
ConnectionId: 1,
ProjectKey: "testNone",
},
+ TaskStartTime: time.Now(),
}
// verify extraction
diff --git a/backend/plugins/sonarqube/e2e/hotspot_test.go
b/backend/plugins/sonarqube/e2e/hotspot_test.go
index 8be9024ef..8a73fd202 100644
--- a/backend/plugins/sonarqube/e2e/hotspot_test.go
+++ b/backend/plugins/sonarqube/e2e/hotspot_test.go
@@ -17,6 +17,7 @@ package e2e
import (
"testing"
+ "time"
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/models/domainlayer/codequality"
@@ -41,6 +42,7 @@ func TestSonarqubeHotspotDataFlow(t *testing.T) {
ConnectionId: 1,
ProjectKey: "f5a50c63-2e8f-4107-9014-853f6f467757",
},
+ TaskStartTime: time.Now(),
}
// Interfered data
taskData2 := &tasks.SonarqubeTaskData{
@@ -48,6 +50,7 @@ func TestSonarqubeHotspotDataFlow(t *testing.T) {
ConnectionId: 2,
ProjectKey: "testWarrenEtcd",
},
+ TaskStartTime: time.Now(),
}
// verify extraction
diff --git a/backend/plugins/sonarqube/e2e/issue_test.go
b/backend/plugins/sonarqube/e2e/issue_test.go
index c425224ca..66875de14 100644
--- a/backend/plugins/sonarqube/e2e/issue_test.go
+++ b/backend/plugins/sonarqube/e2e/issue_test.go
@@ -17,6 +17,7 @@ package e2e
import (
"testing"
+ "time"
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/models/domainlayer/codequality"
@@ -41,6 +42,7 @@ func TestSonarqubeIssueDataFlow(t *testing.T) {
ConnectionId: 1,
ProjectKey: "f5a50c63-2e8f-4107-9014-853f6f467757",
},
+ TaskStartTime: time.Now(),
}
// Interfered data
taskData2 := &tasks.SonarqubeTaskData{
@@ -48,6 +50,7 @@ func TestSonarqubeIssueDataFlow(t *testing.T) {
ConnectionId: 2,
ProjectKey: "testWarrenEtcd",
},
+ TaskStartTime: time.Now(),
}
// verify extraction
diff --git a/backend/plugins/sonarqube/e2e/project_test.go
b/backend/plugins/sonarqube/e2e/project_test.go
index 930b9a835..e3dc8fc67 100644
--- a/backend/plugins/sonarqube/e2e/project_test.go
+++ b/backend/plugins/sonarqube/e2e/project_test.go
@@ -19,6 +19,7 @@ package e2e
import (
"testing"
+ "time"
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/models/domainlayer/codequality"
@@ -42,6 +43,7 @@ func TestSonarqubeProjectDataFlow(t *testing.T) {
ConnectionId: 2,
ProjectKey: "e2c6d5e9-a321-4e8c-b322-03d9599ef962",
},
+ TaskStartTime: time.Now(),
}
dataflowTester.FlushTabler(&codequality.CqProject{})
diff --git a/backend/plugins/sonarqube/impl/impl.go
b/backend/plugins/sonarqube/impl/impl.go
index da728a13c..514554339 100644
--- a/backend/plugins/sonarqube/impl/impl.go
+++ b/backend/plugins/sonarqube/impl/impl.go
@@ -19,6 +19,7 @@ package impl
import (
"fmt"
+ "time"
"github.com/apache/incubator-devlake/core/dal"
@@ -127,8 +128,9 @@ func (p Sonarqube) PrepareTaskData(taskCtx
plugin.TaskContext, options map[strin
return nil, errors.Default.Wrap(err, "unable to get Sonarqube
API client instance")
}
taskData := &tasks.SonarqubeTaskData{
- Options: op,
- ApiClient: apiClient,
+ Options: op,
+ ApiClient: apiClient,
+ TaskStartTime: time.Now(),
}
// even we have project in _tool_sonaqube_projects, we still need to
collect project to update LastAnalysisDate
var scope models.SonarqubeProject
diff --git a/backend/plugins/sonarqube/tasks/hotspots_collector.go
b/backend/plugins/sonarqube/tasks/hotspots_collector.go
index e6ccf547c..b725e4367 100644
--- a/backend/plugins/sonarqube/tasks/hotspots_collector.go
+++ b/backend/plugins/sonarqube/tasks/hotspots_collector.go
@@ -57,6 +57,22 @@ func CollectHotspots(taskCtx plugin.SubTaskContext)
errors.Error {
Data []json.RawMessage `json:"hotspots"`
}
err := helper.UnmarshalResponse(res, &resData)
+
+ // check if sonar report updated during collecting
+ var issue struct {
+ UpdateDate *helper.Iso8601Time
`json:"updateDate"`
+ }
+ for _, v := range resData.Data {
+ err = errors.Convert(json.Unmarshal(v, &issue))
+ if err != nil {
+ return nil, err
+ }
+ if
issue.UpdateDate.ToTime().After(data.TaskStartTime) {
+ return nil,
errors.Default.New(fmt.Sprintf(`Your data is affected by the latest analysis\n
+ Please recollect this project:
%s`, data.Options.ProjectKey))
+ }
+ }
+
return resData.Data, err
},
})
diff --git a/backend/plugins/sonarqube/tasks/issues_collector.go
b/backend/plugins/sonarqube/tasks/issues_collector.go
index 11a0d9514..117199311 100644
--- a/backend/plugins/sonarqube/tasks/issues_collector.go
+++ b/backend/plugins/sonarqube/tasks/issues_collector.go
@@ -41,7 +41,6 @@ type SonarqubeIssueTimeIteratorNode struct {
func CollectIssues(taskCtx plugin.SubTaskContext) (err errors.Error) {
logger := taskCtx.GetLogger()
logger.Info("collect issues")
-
iterator := helper.NewQueueIterator()
iterator.Push(
&SonarqubeIssueTimeIteratorNode{
@@ -154,19 +153,21 @@ func CollectIssues(taskCtx plugin.SubTaskContext) (err
errors.Error) {
var resData struct {
Data []json.RawMessage `json:"issues"`
}
- var issue struct {
- UpdateDate *helper.Iso8601Time
`json:"updateDate"`
- }
err = helper.UnmarshalResponse(res, &resData)
if err != nil {
return nil, err
}
+
+ // check if sonar report updated during collecting
+ var issue struct {
+ UpdateDate *helper.Iso8601Time
`json:"updateDate"`
+ }
for _, v := range resData.Data {
err = errors.Convert(json.Unmarshal(v, &issue))
if err != nil {
return nil, err
}
- if
issue.UpdateDate.ToTime().After(*data.LastAnalysisDate) {
+ if
issue.UpdateDate.ToTime().After(data.TaskStartTime) {
return nil,
errors.Default.New(fmt.Sprintf(`Your data is affected by the latest analysis\n
Please recollect this project:
%s`, data.Options.ProjectKey))
}
diff --git a/backend/plugins/sonarqube/tasks/task_data.go
b/backend/plugins/sonarqube/tasks/task_data.go
index b366b28ea..998054a69 100644
--- a/backend/plugins/sonarqube/tasks/task_data.go
+++ b/backend/plugins/sonarqube/tasks/task_data.go
@@ -37,6 +37,7 @@ type SonarqubeTaskData struct {
Options *SonarqubeOptions
ApiClient *api.ApiAsyncClient
LastAnalysisDate *time.Time
+ TaskStartTime time.Time
}
func DecodeAndValidateTaskOptions(options map[string]interface{})
(*SonarqubeOptions, errors.Error) {
