This is an automated email from the ASF dual-hosted git repository.

klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git


The following commit(s) were added to refs/heads/main by this push:
     new cb461b5fe refactor: sonarqube adopts timeAfter (#4489)
cb461b5fe is described below

commit cb461b5feda15e564d34be0c425c4b15e08c2ce6
Author: abeizn <[email protected]>
AuthorDate: Thu Feb 23 16:50:17 2023 +0800

    refactor: sonarqube adopts timeAfter (#4489)
    
    * refactor: sonarqube adopts timeAfter
    
    * refactor: remove sonarqube timeAfter
    
    * fix: sonarqube shapshot e2e
---
 backend/plugins/sonarqube/api/blueprint_v200.go    |  9 +++----
 .../e2e/snapshot_tables/issue_hotspots.csv         |  6 ++---
 backend/plugins/sonarqube/impl/impl.go             | 12 ---------
 backend/plugins/sonarqube/sonarqube.go             |  6 ++---
 .../plugins/sonarqube/tasks/accounts_collector.go  | 19 ++++++--------
 .../sonarqube/tasks/filemetrics_collector.go       | 29 +++++++++++-----------
 .../plugins/sonarqube/tasks/hotspots_collector.go  | 28 ++++++++++-----------
 .../plugins/sonarqube/tasks/hotspots_convertor.go  |  5 ++--
 .../plugins/sonarqube/tasks/issues_collector.go    | 23 ++++++++---------
 backend/plugins/sonarqube/tasks/task_data.go       | 11 ++++----
 10 files changed, 63 insertions(+), 85 deletions(-)

diff --git a/backend/plugins/sonarqube/api/blueprint_v200.go 
b/backend/plugins/sonarqube/api/blueprint_v200.go
index 611cec815..dd6f9f99c 100644
--- a/backend/plugins/sonarqube/api/blueprint_v200.go
+++ b/backend/plugins/sonarqube/api/blueprint_v200.go
@@ -19,6 +19,9 @@ package api
 
 import (
        "fmt"
+       "net/http"
+       "net/url"
+
        "github.com/apache/incubator-devlake/core/dal"
        "github.com/apache/incubator-devlake/core/errors"
        "github.com/apache/incubator-devlake/core/models/domainlayer"
@@ -30,9 +33,6 @@ import (
        aha 
"github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
        "github.com/apache/incubator-devlake/plugins/sonarqube/models"
        "github.com/apache/incubator-devlake/plugins/sonarqube/tasks"
-       "net/http"
-       "net/url"
-       "time"
 )
 
 func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, 
connectionId uint64, bpScopes []*plugin.BlueprintScopeV200, syncPolicy 
*plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope, 
errors.Error) {
@@ -65,9 +65,6 @@ func makeDataSourcePipelinePlanV200(
                options := make(map[string]interface{})
                options["connectionId"] = connectionId
                options["projectKey"] = bpScope.Id
-               if syncPolicy.CreatedDateAfter != nil {
-                       options["createdDateAfter"] = 
syncPolicy.CreatedDateAfter.Format(time.RFC3339)
-               }
 
                subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, 
bpScope.Entities)
                if err != nil {
diff --git a/backend/plugins/sonarqube/e2e/snapshot_tables/issue_hotspots.csv 
b/backend/plugins/sonarqube/e2e/snapshot_tables/issue_hotspots.csv
index 1bf95e693..261d166ec 100644
--- a/backend/plugins/sonarqube/e2e/snapshot_tables/issue_hotspots.csv
+++ b/backend/plugins/sonarqube/e2e/snapshot_tables/issue_hotspots.csv
@@ -1,4 +1,4 @@
 
id,rule,severity,component,project_key,line,status,message,debt,effort,commit_author_email,assignee,hash,tags,type,scope,start_line,end_line,start_offset,end_offset,vulnerability_probability,security_category,creation_date,update_date
-sonarqube:SonarqubeHotspot:1:AYUwBajH46XwcL-YZONY,,weak-cryptography,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/models/MlpModel.java,sonarqube:SonarqubeProject:1:f5a50c63-2e8f-4107-9014-853f6f467757,143,TO_REVIEW,Make
 sure that using this pseudorandom number generator is safe 
here.,0,0,[email protected],,,,HOTSPOTS,,143,0,0,0,MEDIUM,,2016-02-26T22:41:48.000+00:00,2022-12-20T14:50:30.000+00:00
-sonarqube:SonarqubeHotspot:1:AYUwBajH46XwcL-YZONZ,,weak-cryptography,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/models/MlpModel.java,sonarqube:SonarqubeProject:1:f5a50c63-2e8f-4107-9014-853f6f467757,172,TO_REVIEW,Make
 sure that using this pseudorandom number generator is safe 
here.,0,0,[email protected],,,,HOTSPOTS,,172,0,0,0,MEDIUM,,2016-02-26T22:41:48.000+00:00,2022-12-20T14:50:30.000+00:00
-sonarqube:SonarqubeHotspot:1:AYUwBamj46XwcL-YZOPh,,weak-cryptography,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/function/MultiDimensionSpline.java,sonarqube:SonarqubeProject:1:f5a50c63-2e8f-4107-9014-853f6f467757,295,REVIEWED,Make
 sure that using this pseudorandom number generator is safe 
here.,0,0,[email protected],,,,HOTSPOTS,,295,0,0,0,MEDIUM,,2016-04-25T19:02:16.000+00:00,2022-12-21T08:30:43.000+00:00
+sonarqube:SonarqubeHotspot:1:AYUwBajH46XwcL-YZONY,,,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/models/MlpModel.java,sonarqube:SonarqubeProject:1:f5a50c63-2e8f-4107-9014-853f6f467757,143,TO_REVIEW,Make
 sure that using this pseudorandom number generator is safe 
here.,0,0,[email protected],,,,HOTSPOTS,,143,0,0,0,MEDIUM,weak-cryptography,2016-02-26T22:41:48.000+00:00,2022-12-20T14:50:30.000+00:00
+sonarqube:SonarqubeHotspot:1:AYUwBajH46XwcL-YZONZ,,,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/models/MlpModel.java,sonarqube:SonarqubeProject:1:f5a50c63-2e8f-4107-9014-853f6f467757,172,TO_REVIEW,Make
 sure that using this pseudorandom number generator is safe 
here.,0,0,[email protected],,,,HOTSPOTS,,172,0,0,0,MEDIUM,weak-cryptography,2016-02-26T22:41:48.000+00:00,2022-12-20T14:50:30.000+00:00
+sonarqube:SonarqubeHotspot:1:AYUwBamj46XwcL-YZOPh,,,f5a50c63-2e8f-4107-9014-853f6f467757:core/src/main/java/com/airbnb/aerosolve/core/function/MultiDimensionSpline.java,sonarqube:SonarqubeProject:1:f5a50c63-2e8f-4107-9014-853f6f467757,295,REVIEWED,Make
 sure that using this pseudorandom number generator is safe 
here.,0,0,[email protected],,,,HOTSPOTS,,295,0,0,0,MEDIUM,weak-cryptography,2016-04-25T19:02:16.000+00:00,2022-12-21T08:30:43.000+00:00
diff --git a/backend/plugins/sonarqube/impl/impl.go 
b/backend/plugins/sonarqube/impl/impl.go
index c3be0ba08..a92953bdf 100644
--- a/backend/plugins/sonarqube/impl/impl.go
+++ b/backend/plugins/sonarqube/impl/impl.go
@@ -19,7 +19,6 @@ package impl
 
 import (
        "fmt"
-       "time"
 
        "github.com/apache/incubator-devlake/core/dal"
 
@@ -139,17 +138,6 @@ func (p Sonarqube) PrepareTaskData(taskCtx 
plugin.TaskContext, options map[strin
        }
        taskData.LastAnalysisDate = scope.LastAnalysisDate.ToNullableTime()
 
-       var createdDateAfter time.Time
-       if op.CreatedDateAfter != "" {
-               createdDateAfter, err = 
errors.Convert01(time.Parse(time.RFC3339, op.CreatedDateAfter))
-               if err != nil {
-                       return nil, errors.BadInput.Wrap(err, "invalid value 
for `createdDateAfter`")
-               }
-       }
-       if !createdDateAfter.IsZero() {
-               taskData.CreatedDateAfter = &createdDateAfter
-               logger.Debug("collect data updated createdDateAfter %s", 
createdDateAfter)
-       }
        return taskData, nil
 }
 
diff --git a/backend/plugins/sonarqube/sonarqube.go 
b/backend/plugins/sonarqube/sonarqube.go
index 64a683db0..64b9363f3 100644
--- a/backend/plugins/sonarqube/sonarqube.go
+++ b/backend/plugins/sonarqube/sonarqube.go
@@ -30,15 +30,13 @@ func main() {
        cmd := &cobra.Command{Use: "sonarqube"}
        connectionId := cmd.Flags().Uint64P("connectionId", "c", 0, "sonarqube 
connection id")
        projectKey := cmd.Flags().StringP("projectKey", "p", "", "sonarqube 
projectKey")
-       createdDateAfter := cmd.Flags().StringP("createdDateAfter", "a", "", 
"collect data that are created after specified time, ie 
2006-05-06T07:08:09+08:00")
        _ = cmd.MarkFlagRequired("connectionId")
        //_ = cmd.MarkFlagRequired("projectKey")
 
        cmd.Run = func(cmd *cobra.Command, args []string) {
                runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
-                       "connectionId":     *connectionId,
-                       "projectKey":       *projectKey,
-                       "createdDateAfter": *createdDateAfter,
+                       "connectionId": *connectionId,
+                       "projectKey":   *projectKey,
                })
        }
        runner.RunCmd(cmd)
diff --git a/backend/plugins/sonarqube/tasks/accounts_collector.go 
b/backend/plugins/sonarqube/tasks/accounts_collector.go
index 29842ac7c..d762abe3d 100644
--- a/backend/plugins/sonarqube/tasks/accounts_collector.go
+++ b/backend/plugins/sonarqube/tasks/accounts_collector.go
@@ -33,18 +33,15 @@ const RAW_ACCOUNTS_TABLE = "sonarqube_accounts"
 var _ plugin.SubTaskEntryPoint = CollectAccounts
 
 func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error {
-       rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, 
RAW_ACCOUNTS_TABLE)
        logger := taskCtx.GetLogger()
        logger.Info("collect accounts")
 
-       collectorWithState, err := 
helper.NewApiCollectorWithState(*rawDataSubTaskArgs, data.CreatedDateAfter)
-       if err != nil {
-               return err
-       }
-       err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
-               ApiClient:   data.ApiClient,
-               PageSize:    100,
-               UrlTemplate: "users/search",
+       rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, 
RAW_ACCOUNTS_TABLE)
+       collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+               RawDataSubTaskArgs: *rawDataSubTaskArgs,
+               ApiClient:          data.ApiClient,
+               PageSize:           100,
+               UrlTemplate:        "users/search",
                Query: func(reqData *helper.RequestData) (url.Values, 
errors.Error) {
                        query := url.Values{}
                        query.Set("p", fmt.Sprintf("%v", reqData.Pager.Page))
@@ -56,7 +53,7 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) 
errors.Error {
                        var resData struct {
                                Data []json.RawMessage `json:"users"`
                        }
-                       err = helper.UnmarshalResponse(res, &resData)
+                       err := helper.UnmarshalResponse(res, &resData)
                        return resData.Data, err
                },
        })
@@ -64,7 +61,7 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) 
errors.Error {
                return err
        }
 
-       return collectorWithState.Execute()
+       return collector.Execute()
 }
 
 var CollectAccountsMeta = plugin.SubTaskMeta{
diff --git a/backend/plugins/sonarqube/tasks/filemetrics_collector.go 
b/backend/plugins/sonarqube/tasks/filemetrics_collector.go
index 5ef1791f1..5d3aacd1a 100644
--- a/backend/plugins/sonarqube/tasks/filemetrics_collector.go
+++ b/backend/plugins/sonarqube/tasks/filemetrics_collector.go
@@ -20,11 +20,12 @@ package tasks
 import (
        "encoding/json"
        "fmt"
+       "net/http"
+       "net/url"
+
        "github.com/apache/incubator-devlake/core/errors"
        "github.com/apache/incubator-devlake/core/plugin"
        helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
-       "net/http"
-       "net/url"
 )
 
 const RAW_FILEMETRICS_TABLE = "sonarqube_filemetrics"
@@ -32,18 +33,16 @@ const RAW_FILEMETRICS_TABLE = "sonarqube_filemetrics"
 var _ plugin.SubTaskEntryPoint = CollectFilemetrics
 
 func CollectFilemetrics(taskCtx plugin.SubTaskContext) errors.Error {
-       rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, 
RAW_FILEMETRICS_TABLE)
-       collectorWithState, err := 
helper.NewApiCollectorWithState(*rawDataSubTaskArgs, data.CreatedDateAfter)
-       if err != nil {
-               return err
-       }
-       incremental := collectorWithState.IsIncremental()
+       logger := taskCtx.GetLogger()
+       logger.Info("collect filemetrics")
 
-       err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
-               Incremental: incremental,
-               ApiClient:   data.ApiClient,
-               PageSize:    100,
-               UrlTemplate: "measures/component_tree",
+       rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, 
RAW_FILEMETRICS_TABLE)
+       collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+               RawDataSubTaskArgs: *rawDataSubTaskArgs,
+               ApiClient:          data.ApiClient,
+               PageSize:           100,
+               Incremental:        false,
+               UrlTemplate:        "measures/component_tree",
                Query: func(reqData *helper.RequestData) (url.Values, 
errors.Error) {
                        query := url.Values{}
                        query.Set("component", data.Options.ProjectKey)
@@ -57,14 +56,14 @@ func CollectFilemetrics(taskCtx plugin.SubTaskContext) 
errors.Error {
                        var resData struct {
                                Data []json.RawMessage `json:"components"`
                        }
-                       err = helper.UnmarshalResponse(res, &resData)
+                       err := helper.UnmarshalResponse(res, &resData)
                        return resData.Data, err
                },
        })
        if err != nil {
                return err
        }
-       return collectorWithState.Execute()
+       return collector.Execute()
 }
 
 var CollectFilemetricsMeta = plugin.SubTaskMeta{
diff --git a/backend/plugins/sonarqube/tasks/hotspots_collector.go 
b/backend/plugins/sonarqube/tasks/hotspots_collector.go
index dcf064ead..b31e17c7d 100644
--- a/backend/plugins/sonarqube/tasks/hotspots_collector.go
+++ b/backend/plugins/sonarqube/tasks/hotspots_collector.go
@@ -20,11 +20,12 @@ package tasks
 import (
        "encoding/json"
        "fmt"
+       "net/http"
+       "net/url"
+
        "github.com/apache/incubator-devlake/core/errors"
        "github.com/apache/incubator-devlake/core/plugin"
        helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
-       "net/http"
-       "net/url"
 )
 
 const RAW_HOTSPOTS_TABLE = "sonarqube_hotspots"
@@ -32,18 +33,17 @@ const RAW_HOTSPOTS_TABLE = "sonarqube_hotspots"
 var _ plugin.SubTaskEntryPoint = CollectHotspots
 
 func CollectHotspots(taskCtx plugin.SubTaskContext) errors.Error {
+       logger := taskCtx.GetLogger()
+       logger.Info("collect hotspots")
+
        rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, 
RAW_HOTSPOTS_TABLE)
-       collectorWithState, err := 
helper.NewApiCollectorWithState(*rawDataSubTaskArgs, data.CreatedDateAfter)
-       if err != nil {
-               return err
-       }
-       incremental := collectorWithState.IsIncremental()
 
-       err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
-               Incremental: incremental,
-               ApiClient:   data.ApiClient,
-               PageSize:    100,
-               UrlTemplate: "hotspots/search",
+       collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+               RawDataSubTaskArgs: *rawDataSubTaskArgs,
+               ApiClient:          data.ApiClient,
+               PageSize:           100,
+               Incremental:        false,
+               UrlTemplate:        "hotspots/search",
                Query: func(reqData *helper.RequestData) (url.Values, 
errors.Error) {
                        query := url.Values{}
                        // no time range
@@ -56,14 +56,14 @@ func CollectHotspots(taskCtx plugin.SubTaskContext) 
errors.Error {
                        var resData struct {
                                Data []json.RawMessage `json:"hotspots"`
                        }
-                       err = helper.UnmarshalResponse(res, &resData)
+                       err := helper.UnmarshalResponse(res, &resData)
                        return resData.Data, err
                },
        })
        if err != nil {
                return err
        }
-       return collectorWithState.Execute()
+       return collector.Execute()
 }
 
 var CollectHotspotsMeta = plugin.SubTaskMeta{
diff --git a/backend/plugins/sonarqube/tasks/hotspots_convertor.go 
b/backend/plugins/sonarqube/tasks/hotspots_convertor.go
index c04172f2e..8c198e013 100644
--- a/backend/plugins/sonarqube/tasks/hotspots_convertor.go
+++ b/backend/plugins/sonarqube/tasks/hotspots_convertor.go
@@ -18,6 +18,8 @@ limitations under the License.
 package tasks
 
 import (
+       "reflect"
+
        "github.com/apache/incubator-devlake/core/dal"
        "github.com/apache/incubator-devlake/core/errors"
        "github.com/apache/incubator-devlake/core/models/domainlayer"
@@ -26,7 +28,6 @@ import (
        "github.com/apache/incubator-devlake/core/plugin"
        "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
        sonarqubeModels 
"github.com/apache/incubator-devlake/plugins/sonarqube/models"
-       "reflect"
 )
 
 var ConvertHotspotsMeta = plugin.SubTaskMeta{
@@ -70,7 +71,7 @@ func ConvertHotspots(taskCtx plugin.SubTaskContext) 
errors.Error {
                                UpdateDate:               
sonarqubeHotspot.UpdateDate,
                                Type:                     "HOTSPOTS",
                                VulnerabilityProbability: 
sonarqubeHotspot.VulnerabilityProbability,
-                               Severity:                 
sonarqubeHotspot.SecurityCategory,
+                               SecurityCategory:         
sonarqubeHotspot.SecurityCategory,
                        }
                        return []interface{}{
                                domainHotspot,
diff --git a/backend/plugins/sonarqube/tasks/issues_collector.go 
b/backend/plugins/sonarqube/tasks/issues_collector.go
index 8e937e7d3..860dc0506 100644
--- a/backend/plugins/sonarqube/tasks/issues_collector.go
+++ b/backend/plugins/sonarqube/tasks/issues_collector.go
@@ -20,11 +20,12 @@ package tasks
 import (
        "encoding/json"
        "fmt"
+       "net/http"
+       "net/url"
+
        "github.com/apache/incubator-devlake/core/errors"
        "github.com/apache/incubator-devlake/core/plugin"
        helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
-       "net/http"
-       "net/url"
 )
 
 const RAW_ISSUES_TABLE = "sonarqube_issues"
@@ -32,17 +33,15 @@ const RAW_ISSUES_TABLE = "sonarqube_issues"
 var _ plugin.SubTaskEntryPoint = CollectIssues
 
 func CollectIssues(taskCtx plugin.SubTaskContext) (err errors.Error) {
-       rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, 
RAW_ISSUES_TABLE)
        logger := taskCtx.GetLogger()
        logger.Info("collect issues")
-       collectorWithState, err := 
helper.NewApiCollectorWithState(*rawDataSubTaskArgs, data.CreatedDateAfter)
-       if err != nil {
-               return err
-       }
-       err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
-               ApiClient:   data.ApiClient,
-               PageSize:    100,
-               UrlTemplate: "issues/search",
+
+       rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, 
RAW_ISSUES_TABLE)
+       collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+               RawDataSubTaskArgs: *rawDataSubTaskArgs,
+               ApiClient:          data.ApiClient,
+               PageSize:           100,
+               UrlTemplate:        "issues/search",
                Query: func(reqData *helper.RequestData) (url.Values, 
errors.Error) {
                        query := url.Values{}
                        query.Set("componentKeys", fmt.Sprintf("%v", 
data.Options.ProjectKey))
@@ -78,7 +77,7 @@ func CollectIssues(taskCtx plugin.SubTaskContext) (err 
errors.Error) {
        if err != nil {
                return err
        }
-       return collectorWithState.Execute()
+       return collector.Execute()
 }
 
 var CollectIssuesMeta = plugin.SubTaskMeta{
diff --git a/backend/plugins/sonarqube/tasks/task_data.go 
b/backend/plugins/sonarqube/tasks/task_data.go
index 7e191885d..c4aee2cca 100644
--- a/backend/plugins/sonarqube/tasks/task_data.go
+++ b/backend/plugins/sonarqube/tasks/task_data.go
@@ -18,9 +18,10 @@ limitations under the License.
 package tasks
 
 import (
+       "time"
+
        "github.com/apache/incubator-devlake/core/errors"
        "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
-       "time"
 )
 
 type SonarqubeApiParams struct {
@@ -32,17 +33,15 @@ type SonarqubeOptions struct {
        // options means some custom params required by plugin running.
        // Such As How many rows do your want
        // You can use it in subtasks, and you need to pass it to main.go and 
pipelines.
-       ConnectionId     uint64   `json:"connectionId"`
-       ProjectKey       string   `json:"projectKey"`
-       CreatedDateAfter string   `json:"createdDateAfter" 
mapstructure:"createdDateAfter,omitempty"`
-       Tasks            []string `json:"tasks,omitempty"`
+       ConnectionId uint64   `json:"connectionId"`
+       ProjectKey   string   `json:"projectKey"`
+       Tasks        []string `json:"tasks,omitempty"`
 }
 
 type SonarqubeTaskData struct {
        Options          *SonarqubeOptions
        ApiClient        *api.ApiAsyncClient
        LastAnalysisDate *time.Time
-       CreatedDateAfter *time.Time
 }
 
 func DecodeAndValidateTaskOptions(options map[string]interface{}) 
(*SonarqubeOptions, errors.Error) {

Reply via email to