This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new cb461b5fe refactor: sonarqube adopts timeAfter (#4489)
cb461b5fe is described below
commit cb461b5feda15e564d34be0c425c4b15e08c2ce6
Author: abeizn <[email protected]>
AuthorDate: Thu Feb 23 16:50:17 2023 +0800
refactor: sonarqube adopts timeAfter (#4489)
* refactor: sonarqube adopts timeAfter
* refactor: remove sonarqube timeAfter
* fix: sonarqube shapshot e2e
---
backend/plugins/sonarqube/api/blueprint_v200.go | 9 +++----
.../e2e/snapshot_tables/issue_hotspots.csv | 6 ++---
backend/plugins/sonarqube/impl/impl.go | 12 ---------
backend/plugins/sonarqube/sonarqube.go | 6 ++---
.../plugins/sonarqube/tasks/accounts_collector.go | 19 ++++++--------
.../sonarqube/tasks/filemetrics_collector.go | 29 +++++++++++-----------
.../plugins/sonarqube/tasks/hotspots_collector.go | 28 ++++++++++-----------
.../plugins/sonarqube/tasks/hotspots_convertor.go | 5 ++--
.../plugins/sonarqube/tasks/issues_collector.go | 23 ++++++++---------
backend/plugins/sonarqube/tasks/task_data.go | 11 ++++----
10 files changed, 63 insertions(+), 85 deletions(-)
diff --git a/backend/plugins/sonarqube/api/blueprint_v200.go
b/backend/plugins/sonarqube/api/blueprint_v200.go
index 611cec815..dd6f9f99c 100644
--- a/backend/plugins/sonarqube/api/blueprint_v200.go
+++ b/backend/plugins/sonarqube/api/blueprint_v200.go
@@ -19,6 +19,9 @@ package api
import (
"fmt"
+ "net/http"
+ "net/url"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
@@ -30,9 +33,6 @@ import (
aha
"github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
"github.com/apache/incubator-devlake/plugins/sonarqube/models"
"github.com/apache/incubator-devlake/plugins/sonarqube/tasks"
- "net/http"
- "net/url"
- "time"
)
func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta,
connectionId uint64, bpScopes []*plugin.BlueprintScopeV200, syncPolicy
*plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope,
errors.Error) {
@@ -65,9 +65,6 @@ func makeDataSourcePipelinePlanV200(
options := make(map[string]interface{})
options["connectionId"] = connectionId
options["projectKey"] = bpScope.Id
- if syncPolicy.CreatedDateAfter != nil {
- options["createdDateAfter"] =
syncPolicy.CreatedDateAfter.Format(time.RFC3339)
- }
subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas,
bpScope.Entities)
if err != nil {
diff --git a/backend/plugins/sonarqube/e2e/snapshot_tables/issue_hotspots.csv
b/backend/plugins/sonarqube/e2e/snapshot_tables/issue_hotspots.csv
index 1bf95e693..261d166ec 100644
--- a/backend/plugins/sonarqube/e2e/snapshot_tables/issue_hotspots.csv
+++ b/backend/plugins/sonarqube/e2e/snapshot_tables/issue_hotspots.csv
@@ -1,4 +1,4 @@
id,rule,severity,component,project_key,line,status,message,debt,effort,commit_author_email,assignee,hash,tags,type,scope,start_line,end_line,start_offset,end_offset,vulnerability_probability,security_category,creation_date,update_date
-sonarqube:SonarqubeHotspot:1:AYUwBajH46XwcL-YZONY,,weak-cryptography,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/models/MlpModel.java,sonarqube:SonarqubeProject:1:f5a50c63-2e8f-4107-9014-853f6f467757,143,TO_REVIEW,Make
sure that using this pseudorandom number generator is safe
here.,0,0,[email protected],,,,HOTSPOTS,,143,0,0,0,MEDIUM,,2016-02-26T22:41:48.000+00:00,2022-12-20T14:50:30.000+00:00
-sonarqube:SonarqubeHotspot:1:AYUwBajH46XwcL-YZONZ,,weak-cryptography,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/models/MlpModel.java,sonarqube:SonarqubeProject:1:f5a50c63-2e8f-4107-9014-853f6f467757,172,TO_REVIEW,Make
sure that using this pseudorandom number generator is safe
here.,0,0,[email protected],,,,HOTSPOTS,,172,0,0,0,MEDIUM,,2016-02-26T22:41:48.000+00:00,2022-12-20T14:50:30.000+00:00
-sonarqube:SonarqubeHotspot:1:AYUwBamj46XwcL-YZOPh,,weak-cryptography,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/function/MultiDimensionSpline.java,sonarqube:SonarqubeProject:1:f5a50c63-2e8f-4107-9014-853f6f467757,295,REVIEWED,Make
sure that using this pseudorandom number generator is safe
here.,0,0,[email protected],,,,HOTSPOTS,,295,0,0,0,MEDIUM,,2016-04-25T19:02:16.000+00:00,2022-12-21T08:30:43.000+00:00
+sonarqube:SonarqubeHotspot:1:AYUwBajH46XwcL-YZONY,,,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/models/MlpModel.java,sonarqube:SonarqubeProject:1:f5a50c63-2e8f-4107-9014-853f6f467757,143,TO_REVIEW,Make
sure that using this pseudorandom number generator is safe
here.,0,0,[email protected],,,,HOTSPOTS,,143,0,0,0,MEDIUM,weak-cryptography,2016-02-26T22:41:48.000+00:00,2022-12-20T14:50:30.000+00:00
+sonarqube:SonarqubeHotspot:1:AYUwBajH46XwcL-YZONZ,,,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/models/MlpModel.java,sonarqube:SonarqubeProject:1:f5a50c63-2e8f-4107-9014-853f6f467757,172,TO_REVIEW,Make
sure that using this pseudorandom number generator is safe
here.,0,0,[email protected],,,,HOTSPOTS,,172,0,0,0,MEDIUM,weak-cryptography,2016-02-26T22:41:48.000+00:00,2022-12-20T14:50:30.000+00:00
+sonarqube:SonarqubeHotspot:1:AYUwBamj46XwcL-YZOPh,,,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/function/MultiDimensionSpline.java,sonarqube:SonarqubeProject:1:f5a50c63-2e8f-4107-9014-853f6f467757,295,REVIEWED,Make
sure that using this pseudorandom number generator is safe
here.,0,0,[email protected],,,,HOTSPOTS,,295,0,0,0,MEDIUM,weak-cryptography,2016-04-25T19:02:16.000+00:00,2022-12-21T08:30:43.000+00:00
diff --git a/backend/plugins/sonarqube/impl/impl.go
b/backend/plugins/sonarqube/impl/impl.go
index c3be0ba08..a92953bdf 100644
--- a/backend/plugins/sonarqube/impl/impl.go
+++ b/backend/plugins/sonarqube/impl/impl.go
@@ -19,7 +19,6 @@ package impl
import (
"fmt"
- "time"
"github.com/apache/incubator-devlake/core/dal"
@@ -139,17 +138,6 @@ func (p Sonarqube) PrepareTaskData(taskCtx
plugin.TaskContext, options map[strin
}
taskData.LastAnalysisDate = scope.LastAnalysisDate.ToNullableTime()
- var createdDateAfter time.Time
- if op.CreatedDateAfter != "" {
- createdDateAfter, err =
errors.Convert01(time.Parse(time.RFC3339, op.CreatedDateAfter))
- if err != nil {
- return nil, errors.BadInput.Wrap(err, "invalid value
for `createdDateAfter`")
- }
- }
- if !createdDateAfter.IsZero() {
- taskData.CreatedDateAfter = &createdDateAfter
- logger.Debug("collect data updated createdDateAfter %s",
createdDateAfter)
- }
return taskData, nil
}
diff --git a/backend/plugins/sonarqube/sonarqube.go
b/backend/plugins/sonarqube/sonarqube.go
index 64a683db0..64b9363f3 100644
--- a/backend/plugins/sonarqube/sonarqube.go
+++ b/backend/plugins/sonarqube/sonarqube.go
@@ -30,15 +30,13 @@ func main() {
cmd := &cobra.Command{Use: "sonarqube"}
connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "sonarqube
connection id")
projectKey := cmd.Flags().StringP("projectKey", "p", "", "sonarqube
projectKey")
- createdDateAfter := cmd.Flags().StringP("createdDateAfter", "a", "",
"collect data that are created after specified time, ie
2006-05-06T07:08:09+08:00")
_ = cmd.MarkFlagRequired("connectionId")
//_ = cmd.MarkFlagRequired("projectKey")
cmd.Run = func(cmd *cobra.Command, args []string) {
runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
- "connectionId": *connectionId,
- "projectKey": *projectKey,
- "createdDateAfter": *createdDateAfter,
+ "connectionId": *connectionId,
+ "projectKey": *projectKey,
})
}
runner.RunCmd(cmd)
diff --git a/backend/plugins/sonarqube/tasks/accounts_collector.go
b/backend/plugins/sonarqube/tasks/accounts_collector.go
index 29842ac7c..d762abe3d 100644
--- a/backend/plugins/sonarqube/tasks/accounts_collector.go
+++ b/backend/plugins/sonarqube/tasks/accounts_collector.go
@@ -33,18 +33,15 @@ const RAW_ACCOUNTS_TABLE = "sonarqube_accounts"
var _ plugin.SubTaskEntryPoint = CollectAccounts
func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_ACCOUNTS_TABLE)
logger := taskCtx.GetLogger()
logger.Info("collect accounts")
- collectorWithState, err :=
helper.NewApiCollectorWithState(*rawDataSubTaskArgs, data.CreatedDateAfter)
- if err != nil {
- return err
- }
- err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
- ApiClient: data.ApiClient,
- PageSize: 100,
- UrlTemplate: "users/search",
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_ACCOUNTS_TABLE)
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ UrlTemplate: "users/search",
Query: func(reqData *helper.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
query.Set("p", fmt.Sprintf("%v", reqData.Pager.Page))
@@ -56,7 +53,7 @@ func CollectAccounts(taskCtx plugin.SubTaskContext)
errors.Error {
var resData struct {
Data []json.RawMessage `json:"users"`
}
- err = helper.UnmarshalResponse(res, &resData)
+ err := helper.UnmarshalResponse(res, &resData)
return resData.Data, err
},
})
@@ -64,7 +61,7 @@ func CollectAccounts(taskCtx plugin.SubTaskContext)
errors.Error {
return err
}
- return collectorWithState.Execute()
+ return collector.Execute()
}
var CollectAccountsMeta = plugin.SubTaskMeta{
diff --git a/backend/plugins/sonarqube/tasks/filemetrics_collector.go
b/backend/plugins/sonarqube/tasks/filemetrics_collector.go
index 5ef1791f1..5d3aacd1a 100644
--- a/backend/plugins/sonarqube/tasks/filemetrics_collector.go
+++ b/backend/plugins/sonarqube/tasks/filemetrics_collector.go
@@ -20,11 +20,12 @@ package tasks
import (
"encoding/json"
"fmt"
+ "net/http"
+ "net/url"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "net/http"
- "net/url"
)
const RAW_FILEMETRICS_TABLE = "sonarqube_filemetrics"
@@ -32,18 +33,16 @@ const RAW_FILEMETRICS_TABLE = "sonarqube_filemetrics"
var _ plugin.SubTaskEntryPoint = CollectFilemetrics
func CollectFilemetrics(taskCtx plugin.SubTaskContext) errors.Error {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_FILEMETRICS_TABLE)
- collectorWithState, err :=
helper.NewApiCollectorWithState(*rawDataSubTaskArgs, data.CreatedDateAfter)
- if err != nil {
- return err
- }
- incremental := collectorWithState.IsIncremental()
+ logger := taskCtx.GetLogger()
+ logger.Info("collect filemetrics")
- err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
- Incremental: incremental,
- ApiClient: data.ApiClient,
- PageSize: 100,
- UrlTemplate: "measures/component_tree",
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_FILEMETRICS_TABLE)
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ Incremental: false,
+ UrlTemplate: "measures/component_tree",
Query: func(reqData *helper.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
query.Set("component", data.Options.ProjectKey)
@@ -57,14 +56,14 @@ func CollectFilemetrics(taskCtx plugin.SubTaskContext)
errors.Error {
var resData struct {
Data []json.RawMessage `json:"components"`
}
- err = helper.UnmarshalResponse(res, &resData)
+ err := helper.UnmarshalResponse(res, &resData)
return resData.Data, err
},
})
if err != nil {
return err
}
- return collectorWithState.Execute()
+ return collector.Execute()
}
var CollectFilemetricsMeta = plugin.SubTaskMeta{
diff --git a/backend/plugins/sonarqube/tasks/hotspots_collector.go
b/backend/plugins/sonarqube/tasks/hotspots_collector.go
index dcf064ead..b31e17c7d 100644
--- a/backend/plugins/sonarqube/tasks/hotspots_collector.go
+++ b/backend/plugins/sonarqube/tasks/hotspots_collector.go
@@ -20,11 +20,12 @@ package tasks
import (
"encoding/json"
"fmt"
+ "net/http"
+ "net/url"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "net/http"
- "net/url"
)
const RAW_HOTSPOTS_TABLE = "sonarqube_hotspots"
@@ -32,18 +33,17 @@ const RAW_HOTSPOTS_TABLE = "sonarqube_hotspots"
var _ plugin.SubTaskEntryPoint = CollectHotspots
func CollectHotspots(taskCtx plugin.SubTaskContext) errors.Error {
+ logger := taskCtx.GetLogger()
+ logger.Info("collect hotspots")
+
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_HOTSPOTS_TABLE)
- collectorWithState, err :=
helper.NewApiCollectorWithState(*rawDataSubTaskArgs, data.CreatedDateAfter)
- if err != nil {
- return err
- }
- incremental := collectorWithState.IsIncremental()
- err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
- Incremental: incremental,
- ApiClient: data.ApiClient,
- PageSize: 100,
- UrlTemplate: "hotspots/search",
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ Incremental: false,
+ UrlTemplate: "hotspots/search",
Query: func(reqData *helper.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
// no time range
@@ -56,14 +56,14 @@ func CollectHotspots(taskCtx plugin.SubTaskContext)
errors.Error {
var resData struct {
Data []json.RawMessage `json:"hotspots"`
}
- err = helper.UnmarshalResponse(res, &resData)
+ err := helper.UnmarshalResponse(res, &resData)
return resData.Data, err
},
})
if err != nil {
return err
}
- return collectorWithState.Execute()
+ return collector.Execute()
}
var CollectHotspotsMeta = plugin.SubTaskMeta{
diff --git a/backend/plugins/sonarqube/tasks/hotspots_convertor.go
b/backend/plugins/sonarqube/tasks/hotspots_convertor.go
index c04172f2e..8c198e013 100644
--- a/backend/plugins/sonarqube/tasks/hotspots_convertor.go
+++ b/backend/plugins/sonarqube/tasks/hotspots_convertor.go
@@ -18,6 +18,8 @@ limitations under the License.
package tasks
import (
+ "reflect"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
@@ -26,7 +28,6 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
sonarqubeModels
"github.com/apache/incubator-devlake/plugins/sonarqube/models"
- "reflect"
)
var ConvertHotspotsMeta = plugin.SubTaskMeta{
@@ -70,7 +71,7 @@ func ConvertHotspots(taskCtx plugin.SubTaskContext)
errors.Error {
UpdateDate:
sonarqubeHotspot.UpdateDate,
Type: "HOTSPOTS",
VulnerabilityProbability:
sonarqubeHotspot.VulnerabilityProbability,
- Severity:
sonarqubeHotspot.SecurityCategory,
+ SecurityCategory:
sonarqubeHotspot.SecurityCategory,
}
return []interface{}{
domainHotspot,
diff --git a/backend/plugins/sonarqube/tasks/issues_collector.go
b/backend/plugins/sonarqube/tasks/issues_collector.go
index 8e937e7d3..860dc0506 100644
--- a/backend/plugins/sonarqube/tasks/issues_collector.go
+++ b/backend/plugins/sonarqube/tasks/issues_collector.go
@@ -20,11 +20,12 @@ package tasks
import (
"encoding/json"
"fmt"
+ "net/http"
+ "net/url"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "net/http"
- "net/url"
)
const RAW_ISSUES_TABLE = "sonarqube_issues"
@@ -32,17 +33,15 @@ const RAW_ISSUES_TABLE = "sonarqube_issues"
var _ plugin.SubTaskEntryPoint = CollectIssues
func CollectIssues(taskCtx plugin.SubTaskContext) (err errors.Error) {
- rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_ISSUES_TABLE)
logger := taskCtx.GetLogger()
logger.Info("collect issues")
- collectorWithState, err :=
helper.NewApiCollectorWithState(*rawDataSubTaskArgs, data.CreatedDateAfter)
- if err != nil {
- return err
- }
- err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
- ApiClient: data.ApiClient,
- PageSize: 100,
- UrlTemplate: "issues/search",
+
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_ISSUES_TABLE)
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ UrlTemplate: "issues/search",
Query: func(reqData *helper.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
query.Set("componentKeys", fmt.Sprintf("%v",
data.Options.ProjectKey))
@@ -78,7 +77,7 @@ func CollectIssues(taskCtx plugin.SubTaskContext) (err
errors.Error) {
if err != nil {
return err
}
- return collectorWithState.Execute()
+ return collector.Execute()
}
var CollectIssuesMeta = plugin.SubTaskMeta{
diff --git a/backend/plugins/sonarqube/tasks/task_data.go
b/backend/plugins/sonarqube/tasks/task_data.go
index 7e191885d..c4aee2cca 100644
--- a/backend/plugins/sonarqube/tasks/task_data.go
+++ b/backend/plugins/sonarqube/tasks/task_data.go
@@ -18,9 +18,10 @@ limitations under the License.
package tasks
import (
+ "time"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "time"
)
type SonarqubeApiParams struct {
@@ -32,17 +33,15 @@ type SonarqubeOptions struct {
// options means some custom params required by plugin running.
// Such As How many rows do your want
// You can use it in subtasks, and you need to pass it to main.go and
pipelines.
- ConnectionId uint64 `json:"connectionId"`
- ProjectKey string `json:"projectKey"`
- CreatedDateAfter string `json:"createdDateAfter"
mapstructure:"createdDateAfter,omitempty"`
- Tasks []string `json:"tasks,omitempty"`
+ ConnectionId uint64 `json:"connectionId"`
+ ProjectKey string `json:"projectKey"`
+ Tasks []string `json:"tasks,omitempty"`
}
type SonarqubeTaskData struct {
Options *SonarqubeOptions
ApiClient *api.ApiAsyncClient
LastAnalysisDate *time.Time
- CreatedDateAfter *time.Time
}
func DecodeAndValidateTaskOptions(options map[string]interface{})
(*SonarqubeOptions, errors.Error) {