This is an automated email from the ASF dual-hosted git repository.
likyh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new f04a51267 refactor(sonarqube): refactor search scopes (#4707)
f04a51267 is described below
commit f04a5126747ee308bab69e0a6226ee30108b7eea
Author: Warren Chen <[email protected]>
AuthorDate: Tue Mar 21 10:46:47 2023 +0800
refactor(sonarqube): refactor search scopes (#4707)
---
backend/plugins/sonarqube/api/blueprint_v200.go | 5 +-
backend/plugins/sonarqube/api/init.go | 6 +
backend/plugins/sonarqube/api/remote.go | 306 ++++-----------------
backend/plugins/sonarqube/impl/impl.go | 6 +-
.../plugins/sonarqube/models/sonarqube_project.go | 34 +++
backend/plugins/sonarqube/tasks/shared.go | 22 --
6 files changed, 104 insertions(+), 275 deletions(-)
diff --git a/backend/plugins/sonarqube/api/blueprint_v200.go
b/backend/plugins/sonarqube/api/blueprint_v200.go
index dd6f9f99c..a60e50324 100644
--- a/backend/plugins/sonarqube/api/blueprint_v200.go
+++ b/backend/plugins/sonarqube/api/blueprint_v200.go
@@ -32,7 +32,6 @@ import (
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
aha
"github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
"github.com/apache/incubator-devlake/plugins/sonarqube/models"
- "github.com/apache/incubator-devlake/plugins/sonarqube/tasks"
)
func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta,
connectionId uint64, bpScopes []*plugin.BlueprintScopeV200, syncPolicy
*plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope,
errors.Error) {
@@ -109,9 +108,9 @@ func makeScopesV200(bpScopes []*plugin.BlueprintScopeV200,
connectionId uint64)
func GetApiProject(
projectKey string,
apiClient aha.ApiClientAbstract,
-) (*tasks.SonarqubeApiProject, errors.Error) {
+) (*models.SonarqubeApiProject, errors.Error) {
var resData struct {
- Data []tasks.SonarqubeApiProject `json:"components"`
+ Data []models.SonarqubeApiProject `json:"components"`
}
query := url.Values{}
query.Set("q", projectKey)
diff --git a/backend/plugins/sonarqube/api/init.go
b/backend/plugins/sonarqube/api/init.go
index 54c9c2bff..37bf1fcfb 100644
--- a/backend/plugins/sonarqube/api/init.go
+++ b/backend/plugins/sonarqube/api/init.go
@@ -27,6 +27,7 @@ import (
var vld *validator.Validate
var connectionHelper *api.ConnectionApiHelper
var scopeHelper *api.ScopeApiHelper[models.SonarqubeConnection,
models.SonarqubeProject, interface{}]
+var remoteHelper *api.RemoteApiHelper[models.SonarqubeConnection,
models.SonarqubeProject, models.SonarqubeApiProject, api.NoRemoteGroupResponse]
var basicRes context.BasicRes
func Init(br context.BasicRes) {
@@ -41,4 +42,9 @@ func Init(br context.BasicRes) {
vld,
connectionHelper,
)
+ remoteHelper = api.NewRemoteHelper[models.SonarqubeConnection,
models.SonarqubeProject, models.SonarqubeApiProject, api.NoRemoteGroupResponse](
+ basicRes,
+ vld,
+ connectionHelper,
+ )
}
diff --git a/backend/plugins/sonarqube/api/remote.go
b/backend/plugins/sonarqube/api/remote.go
index ba5e18c08..8f0be7030 100644
--- a/backend/plugins/sonarqube/api/remote.go
+++ b/backend/plugins/sonarqube/api/remote.go
@@ -19,48 +19,15 @@ package api
import (
"context"
- "encoding/base64"
- "encoding/json"
"fmt"
- "net/http"
- "net/url"
- "strconv"
-
+ context2 "github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/sonarqube/models"
- "github.com/apache/incubator-devlake/plugins/sonarqube/tasks"
+ "net/url"
)
-type RemoteScopesChild struct {
- Type string `json:"type"`
- ParentId *string `json:"parentId"`
- Id string `json:"id"`
- Name string `json:"name"`
- Data interface{} `json:"data"`
-}
-
-type RemoteScopesOutput struct {
- Children []RemoteScopesChild `json:"children"`
- NextPageToken string `json:"nextPageToken"`
-}
-
-type SearchRemoteScopesOutput struct {
- Children []RemoteScopesChild `json:"children"`
- Page int `json:"page"`
- PageSize int `json:"pageSize"`
-}
-
-type PageData struct {
- Page int `json:"page"`
- PerPage int `json:"per_page"`
- Tag string `json:"tag"`
-}
-
-const SonarqubeRemoteScopesPerPage int = 100
-const TypeProject string = "scope"
-
// RemoteScopes list all available scope for users
// @Summary list all available scope for users
// @Description list all available scope for users
@@ -68,90 +35,36 @@ const TypeProject string = "scope"
// @Accept application/json
// @Param connectionId path int false "connection ID"
// @Param pageToken query string false "page Token"
-// @Success 200 {object} RemoteScopesOutput
+// @Success 200 {object} api.RemoteScopesOutput
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/sonarqube/connections/{connectionId}/remote-scopes [GET]
func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
- connectionId, _ := extractParam(input.Params)
- if connectionId == 0 {
- return nil, errors.BadInput.New("invalid connectionId")
- }
-
- connection := &models.SonarqubeConnection{}
- err := connectionHelper.First(connection, input.Params)
- if err != nil {
- return nil, err
- }
-
- pageToken, ok := input.Query["pageToken"]
- if !ok || len(pageToken) == 0 {
- pageToken = []string{""}
- }
-
- // get pageData
- pageData, err := GetPageDataFromPageToken(pageToken[0])
- if err != nil {
- return nil, errors.BadInput.New("fail to get page token")
- }
-
- // create api client
- apiClient, err := api.NewApiClientFromConnection(context.TODO(),
basicRes, connection)
- if err != nil {
- return nil, err
- }
-
- var res *http.Response
- outputBody := &RemoteScopesOutput{}
-
- // list projects part
- query, err := GetQueryFromPageData(pageData)
- if err != nil {
- return nil, err
- }
- res, err = apiClient.Get("projects/search", query, nil)
-
- if err != nil {
- return nil, err
- }
-
- var resBody struct {
- Data []tasks.SonarqubeApiProject `json:"components"`
- }
- err = api.UnmarshalResponse(res, &resBody)
-
- // append project to output
- for _, project := range resBody.Data {
- child := RemoteScopesChild{
- Type: TypeProject,
- Id: project.ProjectKey,
- Name: project.Name,
- Data: tasks.ConvertProject(&project),
- }
- outputBody.Children = append(outputBody.Children, child)
- }
-
- // check project count
- if err != nil {
- return nil, err
- }
- if len(resBody.Data) < pageData.PerPage {
- pageData = nil
- }
-
- // get the next page token
- outputBody.NextPageToken = ""
- if pageData != nil {
- pageData.Page += 1
- pageData.PerPage = SonarqubeRemoteScopesPerPage
-
- outputBody.NextPageToken, err =
GetPageTokenFromPageData(pageData)
- if err != nil {
- return nil, err
- }
- }
-
- return &plugin.ApiResourceOutput{Body: outputBody, Status:
http.StatusOK}, nil
+ return remoteHelper.GetScopesFromRemote(input,
+ nil,
+ func(basicRes context2.BasicRes, gid string, queryData
*plugin.QueryData, connection models.SonarqubeConnection)
([]models.SonarqubeApiProject, errors.Error) {
+ query := initialQuery(queryData)
+ // create api client
+ apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ if err != nil {
+ return nil, err
+ }
+
+ res, err := apiClient.Get("projects/search", query, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ var resBody struct {
+ Data []models.SonarqubeApiProject
`json:"components"`
+ }
+ err = api.UnmarshalResponse(res, &resBody)
+ if err != nil {
+ return nil, err
+ }
+
+ return resBody.Data, nil
+ })
}
// SearchRemoteScopes use the Search API and only return project
@@ -162,143 +75,42 @@ func RemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, er
// @Param connectionId path int false "connection ID"
// @Param page query int false "page number"
// @Param pageSize query int false "page size per page"
-// @Success 200 {object} SearchRemoteScopesOutput
+// @Success 200 {object} api.SearchRemoteScopesOutput
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/sonarqube/connections/{connectionId}/search-remote-scopes
[GET]
func SearchRemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- connectionId, _ := extractParam(input.Params)
- if connectionId == 0 {
- return nil, errors.BadInput.New("invalid connectionId")
- }
-
- connection := &models.SonarqubeConnection{}
- err := connectionHelper.First(connection, input.Params)
- if err != nil {
- return nil, err
- }
-
- search, ok := input.Query["search"]
- if !ok || len(search) == 0 {
- search = []string{""}
- }
- var p int
- var err1 error
- page, ok := input.Query["page"]
- if !ok || len(page) == 0 {
- p = 1
- } else {
- p, err1 = strconv.Atoi(page[0])
- if err != nil {
- return nil, errors.BadInput.Wrap(err1,
fmt.Sprintf("failed to Atoi page:%s", page[0]))
- }
- }
- var ps int
- pageSize, ok := input.Query["pageSize"]
- if !ok || len(pageSize) == 0 {
- ps = SonarqubeRemoteScopesPerPage
- } else {
- ps, err1 = strconv.Atoi(pageSize[0])
- if err1 != nil {
- return nil, errors.BadInput.Wrap(err1,
fmt.Sprintf("failed to Atoi pageSize:%s", pageSize[0]))
- }
- }
-
- // create api client
- apiClient, err := api.NewApiClientFromConnection(context.TODO(),
basicRes, connection)
- if err != nil {
- return nil, err
- }
-
- // set query
- query, err := GetQueryForSearchProject(search[0], p, ps)
- if err != nil {
- return nil, err
- }
-
- // request search
- res, err := apiClient.Get("projects/search", query, nil)
- if err != nil {
- return nil, err
- }
- var resBody struct {
- Data []tasks.SonarqubeApiProject `json:"components"`
- }
- err = api.UnmarshalResponse(res, &resBody)
- if err != nil {
- return nil, err
- }
-
- // set projects return
- outputBody := &SearchRemoteScopesOutput{}
- for _, project := range resBody.Data {
- child := RemoteScopesChild{
- Type: TypeProject,
- Id: project.ProjectKey,
- ParentId: nil,
- Name: project.Name,
- Data: tasks.ConvertProject(&project),
- }
- outputBody.Children = append(outputBody.Children, child)
- }
- outputBody.Page = p
- outputBody.PageSize = ps
-
- return &plugin.ApiResourceOutput{Body: outputBody, Status:
http.StatusOK}, nil
+ return remoteHelper.GetScopesFromRemote(input,
+ nil,
+ func(basicRes context2.BasicRes, gid string, queryData
*plugin.QueryData, connection models.SonarqubeConnection)
([]models.SonarqubeApiProject, errors.Error) {
+ query := initialQuery(queryData)
+ query.Set("q", queryData.Search[0])
+ // create api client
+ apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ if err != nil {
+ return nil, err
+ }
+
+ // request search
+ res, err := apiClient.Get("projects/search", query, nil)
+ if err != nil {
+ return nil, err
+ }
+ var resBody struct {
+ Data []models.SonarqubeApiProject
`json:"components"`
+ }
+ err = api.UnmarshalResponse(res, &resBody)
+ if err != nil {
+ return nil, err
+ }
+
+ return resBody.Data, nil
+ })
}
-func GetPageTokenFromPageData(pageData *PageData) (string, errors.Error) {
- // Marshal json
- pageTokenDecode, err := json.Marshal(pageData)
- if err != nil {
- return "", errors.Default.Wrap(err, fmt.Sprintf("Marshal
pageToken failed %+v", pageData))
- }
-
- // Encode pageToken Base64
- return base64.StdEncoding.EncodeToString(pageTokenDecode), nil
-}
-
-func GetPageDataFromPageToken(pageToken string) (*PageData, errors.Error) {
- if pageToken == "" {
- return &PageData{
- Page: 1,
- PerPage: SonarqubeRemoteScopesPerPage,
- }, nil
- }
-
- // Decode pageToken Base64
- pageTokenDecode, err := base64.StdEncoding.DecodeString(pageToken)
- if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("decode
pageToken failed %s", pageToken))
- }
- // Unmarshal json
- pt := &PageData{}
- err = json.Unmarshal(pageTokenDecode, pt)
- if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("json
Unmarshal pageTokenDecode failed %s", pageTokenDecode))
- }
-
- return pt, nil
-}
-
-func GetQueryFromPageData(pageData *PageData) (url.Values, errors.Error) {
+func initialQuery(queryData *plugin.QueryData) url.Values {
query := url.Values{}
- query.Set("p", fmt.Sprintf("%v", pageData.Page))
- query.Set("ps", fmt.Sprintf("%v", pageData.PerPage))
- return query, nil
-}
-
-func GetQueryForSearchProject(search string, page int, perPage int)
(url.Values, errors.Error) {
- query, err := GetQueryFromPageData(&PageData{Page: page, PerPage:
perPage})
- if err != nil {
- return nil, err
- }
- query.Set("q", search)
- return query, nil
-}
-
-func extractParam(params map[string]string) (uint64, string) {
- connectionId, _ := strconv.ParseUint(params["connectionId"], 10, 64)
- projectKey := params["projectKey"]
- return connectionId, projectKey
+ query.Set("p", fmt.Sprintf("%v", queryData.Page))
+ query.Set("ps", fmt.Sprintf("%v", queryData.PerPage))
+ return query
}
diff --git a/backend/plugins/sonarqube/impl/impl.go
b/backend/plugins/sonarqube/impl/impl.go
index 779cffc68..92359d6dd 100644
--- a/backend/plugins/sonarqube/impl/impl.go
+++ b/backend/plugins/sonarqube/impl/impl.go
@@ -125,14 +125,14 @@ func (p Sonarqube) PrepareTaskData(taskCtx
plugin.TaskContext, options map[strin
ApiClient: apiClient,
}
// even we have project in _tool_sonaqube_projects, we still need to
collect project to update LastAnalysisDate
- var scope *models.SonarqubeProject
- var apiProject *tasks.SonarqubeApiProject
+ var scope models.SonarqubeProject
+ var apiProject *models.SonarqubeApiProject
apiProject, err = api.GetApiProject(op.ProjectKey, apiClient)
if err != nil {
return nil, err
}
logger.Debug(fmt.Sprintf("Current project: %s", apiProject.ProjectKey))
- scope = tasks.ConvertProject(apiProject)
+ scope = apiProject.ConvertApiScope().(models.SonarqubeProject)
scope.ConnectionId = op.ConnectionId
err = taskCtx.GetDal().CreateOrUpdate(&scope)
if err != nil {
diff --git a/backend/plugins/sonarqube/models/sonarqube_project.go
b/backend/plugins/sonarqube/models/sonarqube_project.go
index 47fc6037d..b590df1f1 100644
--- a/backend/plugins/sonarqube/models/sonarqube_project.go
+++ b/backend/plugins/sonarqube/models/sonarqube_project.go
@@ -19,9 +19,13 @@ package models
import (
"github.com/apache/incubator-devlake/core/models/common"
+ "github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
+var _ plugin.ToolLayerScope = (*SonarqubeProject)(nil)
+var _ plugin.ApiScope = (*SonarqubeApiProject)(nil)
+
type SonarqubeProject struct {
common.NoPKModel `json:"-" mapstructure:"-"`
ConnectionId uint64 `json:"connectionId"
validate:"required" gorm:"primaryKey"`
@@ -36,3 +40,33 @@ type SonarqubeProject struct {
func (SonarqubeProject) TableName() string {
return "_tool_sonarqube_projects"
}
+
+func (p SonarqubeProject) ScopeId() string {
+ return p.ProjectKey
+}
+
+func (p SonarqubeProject) ScopeName() string {
+ return p.Name
+}
+
+type SonarqubeApiProject struct {
+ ProjectKey string `json:"key"`
+ Name string `json:"name"`
+ Qualifier string `json:"qualifier"`
+ Visibility string `json:"visibility"`
+ LastAnalysisDate *api.Iso8601Time `json:"lastAnalysisDate"`
+ Revision string `json:"revision"`
+}
+
+// Convert the API response to our DB model instance
+func (sonarqubeApiProject SonarqubeApiProject) ConvertApiScope()
plugin.ToolLayerScope {
+ sonarqubeProject := SonarqubeProject{
+ ProjectKey: sonarqubeApiProject.ProjectKey,
+ Name: sonarqubeApiProject.Name,
+ Qualifier: sonarqubeApiProject.Qualifier,
+ Visibility: sonarqubeApiProject.Visibility,
+ LastAnalysisDate: sonarqubeApiProject.LastAnalysisDate,
+ Revision: sonarqubeApiProject.Revision,
+ }
+ return sonarqubeProject
+}
diff --git a/backend/plugins/sonarqube/tasks/shared.go
b/backend/plugins/sonarqube/tasks/shared.go
index c30eba684..bd4ced97c 100644
--- a/backend/plugins/sonarqube/tasks/shared.go
+++ b/backend/plugins/sonarqube/tasks/shared.go
@@ -72,28 +72,6 @@ type Paging struct {
Total int `json:"total"`
}
-type SonarqubeApiProject struct {
- ProjectKey string `json:"key"`
- Name string `json:"name"`
- Qualifier string `json:"qualifier"`
- Visibility string `json:"visibility"`
- LastAnalysisDate *api.Iso8601Time `json:"lastAnalysisDate"`
- Revision string `json:"revision"`
-}
-
-// Convert the API response to our DB model instance
-func ConvertProject(sonarqubeApiProject *SonarqubeApiProject)
*models.SonarqubeProject {
- sonarqubeProject := &models.SonarqubeProject{
- ProjectKey: sonarqubeApiProject.ProjectKey,
- Name: sonarqubeApiProject.Name,
- Qualifier: sonarqubeApiProject.Qualifier,
- Visibility: sonarqubeApiProject.Visibility,
- LastAnalysisDate: sonarqubeApiProject.LastAnalysisDate,
- Revision: sonarqubeApiProject.Revision,
- }
- return sonarqubeProject
-}
-
func generateId(hashCodeBlock hash.Hash, entity
*models.SonarqubeIssueCodeBlock) {
hashCodeBlock.Write([]byte(fmt.Sprintf("%s-%s-%d-%d-%d-%d-%s",
entity.IssueKey, entity.Component, entity.StartLine, entity.EndLine,
entity.StartOffset, entity.EndOffset, entity.Msg)))
entity.Id = hex.EncodeToString(hashCodeBlock.Sum(nil))
