This is an automated email from the ASF dual-hosted git repository.
likyh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 8c52c986a refactor(framework): refactor search scopes (#4692)
8c52c986a is described below
commit 8c52c986abaefdc5cd3dd1db4145d5dd60fdcf22
Author: Warren Chen <[email protected]>
AuthorDate: Fri Mar 17 15:52:41 2023 +0800
refactor(framework): refactor search scopes (#4692)
---
backend/core/plugin/plugin_connection_abstract.go | 16 +-
.../helpers/pluginhelper/api/remote_api_helper.go | 197 +++++++++++-------
backend/helpers/pluginhelper/api/scope_helper.go | 10 +-
backend/plugins/bamboo/api/blueprint_v200.go | 23 ---
backend/plugins/bamboo/api/remote.go | 207 +++++++------------
backend/plugins/bamboo/impl/impl.go | 1 -
backend/plugins/bamboo/models/connection.go | 30 +--
backend/plugins/bamboo/models/project.go | 3 +-
backend/plugins/bitbucket/api/remote.go | 217 +++++++++-----------
backend/plugins/bitbucket/models/connection.go | 57 +-----
backend/plugins/gitlab/api/remote.go | 224 ++++++++-------------
backend/plugins/gitlab/models/connection.go | 63 +-----
backend/plugins/gitlab/models/project.go | 4 +
13 files changed, 400 insertions(+), 652 deletions(-)
diff --git a/backend/core/plugin/plugin_connection_abstract.go
b/backend/core/plugin/plugin_connection_abstract.go
index 92e4a5a00..157c63df0 100644
--- a/backend/core/plugin/plugin_connection_abstract.go
+++ b/backend/core/plugin/plugin_connection_abstract.go
@@ -18,13 +18,10 @@ limitations under the License.
package plugin
import (
- context2 "github.com/apache/incubator-devlake/core/context"
-
"github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
- "net/http"
- "net/url"
-
"github.com/apache/incubator-devlake/core/errors"
+
"github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
"github.com/go-playground/validator/v10"
+ "net/http"
)
// ApiConnection represents a API Connection
@@ -34,10 +31,11 @@ type ApiConnection interface {
GetRateLimitPerHour() int
}
-type ApiConnectionForRemote[T ApiGroup, S ApiScope] interface {
- ApiConnection
- GetGroup(basicRes context2.BasicRes, gid string, query url.Values)
([]T, errors.Error)
- GetScope(basicRes context2.BasicRes, gid string, query url.Values)
([]S, errors.Error)
+type QueryData struct {
+ Page int `json:"page"`
+ PerPage int `json:"per_page"`
+ Tag string `json:"tag"`
+ Search []string
}
// ApiAuthenticator is to be implemented by a Concreate Connection if
Authorization is required
diff --git a/backend/helpers/pluginhelper/api/remote_api_helper.go
b/backend/helpers/pluginhelper/api/remote_api_helper.go
index 931ccdbae..93be9e04a 100644
--- a/backend/helpers/pluginhelper/api/remote_api_helper.go
+++ b/backend/helpers/pluginhelper/api/remote_api_helper.go
@@ -26,18 +26,37 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
"github.com/go-playground/validator/v10"
"net/http"
- "net/url"
+ "strconv"
)
+type RemoteScopesChild struct {
+ Type string `json:"type"`
+ ParentId *string `json:"parentId"`
+ Id string `json:"id"`
+ Name string `json:"name"`
+ Data interface{} `json:"data"`
+}
+
+type RemoteScopesOutput struct {
+ Children []RemoteScopesChild `json:"children"`
+ NextPageToken string `json:"nextPageToken"`
+}
+
+type SearchRemoteScopesOutput struct {
+ Children []RemoteScopesChild `json:"children"`
+ Page int `json:"page"`
+ PageSize int `json:"pageSize"`
+}
+
// RemoteApiHelper is used to write the CURD of connection
-type RemoteApiHelper[Conn plugin.ApiConnectionForRemote[Group, ApiScope],
Scope plugin.ToolLayerScope, ApiScope plugin.ApiScope, Group plugin.ApiGroup]
struct {
+type RemoteApiHelper[Conn plugin.ApiConnection, Scope plugin.ToolLayerScope,
ApiScope plugin.ApiScope, Group plugin.ApiGroup] struct {
basicRes coreContext.BasicRes
validator *validator.Validate
connHelper *ConnectionApiHelper
}
// NewRemoteHelper creates a ScopeHelper for connection management
-func NewRemoteHelper[Conn plugin.ApiConnectionForRemote[Group, ApiScope],
Scope plugin.ToolLayerScope, ApiScope plugin.ApiScope, Group plugin.ApiGroup](
+func NewRemoteHelper[Conn plugin.ApiConnection, Scope plugin.ToolLayerScope,
ApiScope plugin.ApiScope, Group plugin.ApiGroup](
basicRes coreContext.BasicRes,
vld *validator.Validate,
connHelper *ConnectionApiHelper,
@@ -55,37 +74,15 @@ func NewRemoteHelper[Conn
plugin.ApiConnectionForRemote[Group, ApiScope], Scope
}
}
-type RemoteScopesChild struct {
- Type string `json:"type"`
- ParentId *string `json:"parentId"`
- Id string `json:"id"`
- Name string `json:"name"`
- Data interface{} `json:"data"`
-}
-
-type RemoteScopesOutput struct {
- Children []RemoteScopesChild `json:"children"`
- NextPageToken string `json:"nextPageToken"`
-}
-
-type SearchRemoteScopesOutput struct {
- Children []RemoteScopesChild `json:"children"`
- Page int `json:"page"`
- PageSize int `json:"pageSize"`
-}
-
-type PageData struct {
- Page int `json:"page"`
- PerPage int `json:"per_page"`
- Tag string `json:"tag"`
-}
-
const remoteScopesPerPage int = 100
const TypeProject string = "scope"
const TypeGroup string = "group"
-func (r *RemoteApiHelper[Conn, Scope, ApiScope, Group])
GetScopesFromRemote(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
- connectionId, _ := ExtractFromReqParam(input.Params)
+func (r *RemoteApiHelper[Conn, Scope, ApiScope, Group])
GetScopesFromRemote(input *plugin.ApiResourceInput,
+ getGroup func(basicRes coreContext.BasicRes, gid string, queryData
*plugin.QueryData, connection Conn) ([]Group, errors.Error),
+ getScope func(basicRes coreContext.BasicRes, gid string, queryData
*plugin.QueryData, connection Conn) ([]ApiScope, errors.Error),
+) (*plugin.ApiResourceOutput, errors.Error) {
+ connectionId, _ := extractFromReqParam(input.Params)
if connectionId == 0 {
return nil, errors.BadInput.New("invalid connectionId")
}
@@ -108,7 +105,7 @@ func (r *RemoteApiHelper[Conn, Scope, ApiScope, Group])
GetScopesFromRemote(inpu
// get gid and pageData
gid := groupId[0]
- pageData, err := getPageDataFromPageToken(pageToken[0])
+ queryData, err := getPageDataFromPageToken(pageToken[0])
if err != nil {
return nil, errors.BadInput.New("failed to get paget token")
}
@@ -116,17 +113,13 @@ func (r *RemoteApiHelper[Conn, Scope, ApiScope, Group])
GetScopesFromRemote(inpu
outputBody := &RemoteScopesOutput{}
// list groups part
- if pageData.Tag == TypeGroup {
- query, err := getQueryFromPageData(pageData)
- if err != nil {
- return nil, err
- }
+ if queryData.Tag == TypeGroup {
var resBody []Group
- resBody, err = connection.GetGroup(r.basicRes, gid, query)
+ resBody, err = getGroup(r.basicRes, gid, queryData, connection)
if err != nil {
return nil, err
}
-
+ // if len(resBody) == 0, will skip the following steps, this
will happen in some plugins which don't have group
// append group to output
for _, group := range resBody {
child := RemoteScopesChild{
@@ -143,22 +136,17 @@ func (r *RemoteApiHelper[Conn, Scope, ApiScope, Group])
GetScopesFromRemote(inpu
outputBody.Children = append(outputBody.Children, child)
}
// check groups count
- if len(resBody) < pageData.PerPage {
- pageData.Tag = TypeProject
- pageData.Page = 1
- pageData.PerPage = pageData.PerPage - len(resBody)
+ if len(resBody) < queryData.PerPage {
+ queryData.Tag = TypeProject
+ queryData.Page = 1
+ queryData.PerPage = queryData.PerPage - len(resBody)
}
}
// list projects part
- if pageData.Tag == TypeProject {
- query, err := getQueryFromPageData(pageData)
- if err != nil {
- return nil, err
- }
+ if queryData.Tag == TypeProject {
var resBody []ApiScope
-
- resBody, err = connection.GetScope(r.basicRes, gid, query)
+ resBody, err = getScope(r.basicRes, gid, queryData, connection)
if err != nil {
return nil, err
}
@@ -181,18 +169,18 @@ func (r *RemoteApiHelper[Conn, Scope, ApiScope, Group])
GetScopesFromRemote(inpu
}
// check project count
- if len(resBody) < pageData.PerPage {
- pageData = nil
+ if len(resBody) < queryData.PerPage {
+ queryData = nil
}
}
// get the next page token
outputBody.NextPageToken = ""
- if pageData != nil {
- pageData.Page += 1
- pageData.PerPage = remoteScopesPerPage
+ if queryData != nil {
+ queryData.Page += 1
+ queryData.PerPage = remoteScopesPerPage
- outputBody.NextPageToken, err =
getPageTokenFromPageData(pageData)
+ outputBody.NextPageToken, err =
getPageTokenFromPageData(queryData)
if err != nil {
return nil, err
}
@@ -201,7 +189,80 @@ func (r *RemoteApiHelper[Conn, Scope, ApiScope, Group])
GetScopesFromRemote(inpu
return &plugin.ApiResourceOutput{Body: outputBody, Status:
http.StatusOK}, nil
}
-func getPageTokenFromPageData(pageData *PageData) (string, errors.Error) {
+func (r *RemoteApiHelper[Conn, Scope, ApiScope, Group])
SearchRemoteScopes(input *plugin.ApiResourceInput, searchScope func(basicRes
coreContext.BasicRes, queryData *plugin.QueryData, connection Conn)
([]ApiScope, errors.Error)) (*plugin.ApiResourceOutput, errors.Error) {
+ connectionId, _ := extractFromReqParam(input.Params)
+ if connectionId == 0 {
+ return nil, errors.BadInput.New("invalid connectionId")
+ }
+
+ var connection Conn
+ err := r.connHelper.First(&connection, input.Params)
+ if err != nil {
+ return nil, err
+ }
+
+ search, ok := input.Query["search"]
+ if !ok || len(search) == 0 {
+ search = []string{""}
+ }
+
+ var p int
+ var err1 error
+ page, ok := input.Query["page"]
+ if !ok || len(page) == 0 {
+ p = 1
+ } else {
+ p, err1 = strconv.Atoi(page[0])
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err1,
fmt.Sprintf("failed to Atoi page:%s", page[0]))
+ }
+ }
+ var ps int
+ pageSize, ok := input.Query["pageSize"]
+ if !ok || len(pageSize) == 0 {
+ ps = remoteScopesPerPage
+ } else {
+ ps, err1 = strconv.Atoi(pageSize[0])
+ if err1 != nil {
+ return nil, errors.BadInput.Wrap(err1,
fmt.Sprintf("failed to Atoi pageSize:%s", pageSize[0]))
+ }
+ }
+
+ queryData := &plugin.QueryData{
+ Page: p,
+ PerPage: ps,
+ Search: search,
+ }
+
+ var resBody []ApiScope
+ resBody, err = searchScope(r.basicRes, queryData, connection)
+ if err != nil {
+ return nil, err
+ }
+
+ outputBody := &SearchRemoteScopesOutput{}
+
+ // append project to output
+ for _, project := range resBody {
+ scope := project.ConvertApiScope()
+ child := RemoteScopesChild{
+ Type: TypeProject,
+ Id: scope.ScopeId(),
+ ParentId: nil,
+ Name: scope.ScopeName(),
+ Data: scope,
+ }
+
+ outputBody.Children = append(outputBody.Children, child)
+ }
+
+ outputBody.Page = p
+ outputBody.PageSize = ps
+
+ return &plugin.ApiResourceOutput{Body: outputBody, Status:
http.StatusOK}, nil
+}
+
+func getPageTokenFromPageData(pageData *plugin.QueryData) (string,
errors.Error) {
// Marshal json
pageTokenDecode, err := json.Marshal(pageData)
if err != nil {
@@ -212,9 +273,9 @@ func getPageTokenFromPageData(pageData *PageData) (string,
errors.Error) {
return base64.StdEncoding.EncodeToString(pageTokenDecode), nil
}
-func getPageDataFromPageToken(pageToken string) (*PageData, errors.Error) {
+func getPageDataFromPageToken(pageToken string) (*plugin.QueryData,
errors.Error) {
if pageToken == "" {
- return &PageData{
+ return &plugin.QueryData{
Page: 1,
PerPage: remoteScopesPerPage,
Tag: "group",
@@ -227,7 +288,7 @@ func getPageDataFromPageToken(pageToken string) (*PageData,
errors.Error) {
return nil, errors.Default.Wrap(err, fmt.Sprintf("decode
pageToken failed %s", pageToken))
}
// Unmarshal json
- pt := &PageData{}
+ pt := &plugin.QueryData{}
err = json.Unmarshal(pageTokenDecode, pt)
if err != nil {
return nil, errors.Default.Wrap(err, fmt.Sprintf("json
Unmarshal pageTokenDecode failed %s", pageTokenDecode))
@@ -235,21 +296,3 @@ func getPageDataFromPageToken(pageToken string)
(*PageData, errors.Error) {
return pt, nil
}
-
-func getQueryFromPageData(pageData *PageData) (url.Values, errors.Error) {
- query := url.Values{}
- query.Set("page", fmt.Sprintf("%v", pageData.Page))
- query.Set("per_page", fmt.Sprintf("%v", pageData.PerPage))
- return query, nil
-}
-
-func GetQueryForSearchProject(search string, page int, perPage int)
(url.Values, errors.Error) {
- query, err := getQueryFromPageData(&PageData{Page: page, PerPage:
perPage})
- if err != nil {
- return nil, err
- }
- query.Set("search", search)
- query.Set("scope", "projects")
-
- return query, nil
-}
diff --git a/backend/helpers/pluginhelper/api/scope_helper.go
b/backend/helpers/pluginhelper/api/scope_helper.go
index d34069ffd..749372831 100644
--- a/backend/helpers/pluginhelper/api/scope_helper.go
+++ b/backend/helpers/pluginhelper/api/scope_helper.go
@@ -86,7 +86,7 @@ func (c *ScopeApiHelper[Conn, Scope, Tr]) Put(input
*plugin.ApiResourceInput) (*
return nil, errors.BadInput.Wrap(err, "decoding Github repo
error")
}
// Extract the connection ID from the input.Params map
- connectionId, _ := ExtractFromReqParam(input.Params)
+ connectionId, _ := extractFromReqParam(input.Params)
if connectionId == 0 {
return nil, errors.BadInput.New("invalid connectionId or
scopeId")
}
@@ -127,7 +127,7 @@ func (c *ScopeApiHelper[Conn, Scope, Tr]) Put(input
*plugin.ApiResourceInput) (*
}
func (c *ScopeApiHelper[Conn, Scope, Tr]) Update(input
*plugin.ApiResourceInput, fieldName string) (*plugin.ApiResourceOutput,
errors.Error) {
- connectionId, scopeId := ExtractFromReqParam(input.Params)
+ connectionId, scopeId := extractFromReqParam(input.Params)
if connectionId == 0 || len(scopeId) == 0 || scopeId == "0" {
return &plugin.ApiResourceOutput{Body: nil, Status:
http.StatusInternalServerError}, errors.BadInput.New("invalid connectionId")
@@ -158,7 +158,7 @@ func (c *ScopeApiHelper[Conn, Scope, Tr]) Update(input
*plugin.ApiResourceInput,
}
func (c *ScopeApiHelper[Conn, Scope, Tr]) GetScopeList(input
*plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- connectionId, _ := ExtractFromReqParam(input.Params)
+ connectionId, _ := extractFromReqParam(input.Params)
if connectionId == 0 {
return nil, errors.BadInput.New("invalid path params")
}
@@ -204,7 +204,7 @@ func (c *ScopeApiHelper[Conn, Scope, Tr])
GetScopeList(input *plugin.ApiResource
}
func (c *ScopeApiHelper[Conn, Scope, Tr]) GetScope(input
*plugin.ApiResourceInput, fieldName string) (*plugin.ApiResourceOutput,
errors.Error) {
- connectionId, scopeId := ExtractFromReqParam(input.Params)
+ connectionId, scopeId := extractFromReqParam(input.Params)
if connectionId == 0 || len(scopeId) == 0 || scopeId == "0" {
return nil, errors.BadInput.New("invalid path params")
}
@@ -262,7 +262,7 @@ func (c *ScopeApiHelper[Conn, Scope, Tr]) save(scope
interface{}) errors.Error {
return nil
}
-func ExtractFromReqParam(params map[string]string) (uint64, string) {
+func extractFromReqParam(params map[string]string) (uint64, string) {
connectionId, err := strconv.ParseUint(params["connectionId"], 10, 64)
if err != nil {
return 0, ""
diff --git a/backend/plugins/bamboo/api/blueprint_v200.go
b/backend/plugins/bamboo/api/blueprint_v200.go
index d7fd92815..3ce938993 100644
--- a/backend/plugins/bamboo/api/blueprint_v200.go
+++ b/backend/plugins/bamboo/api/blueprint_v200.go
@@ -19,8 +19,6 @@ package api
import (
"fmt"
- "net/http"
-
"github.com/apache/incubator-devlake/plugins/bamboo/models"
"github.com/apache/incubator-devlake/core/errors"
@@ -31,7 +29,6 @@ import (
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
plugin "github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- aha
"github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
)
func MakePipelinePlanV200(
@@ -162,23 +159,3 @@ func GetTransformationRuleByproject(project
*models.BambooProject) (*models.Bamb
return transformationRules, nil
}
-
-func GetApiProject(
- projectKey string,
- apiClient aha.ApiClientAbstract,
-) (*models.ApiBambooProject, errors.Error) {
- projectRes := &models.ApiBambooProject{}
- res, err := apiClient.Get(fmt.Sprintf("project/%s.json", projectKey),
nil, nil)
- if err != nil {
- return nil, err
- }
- defer res.Body.Close()
- if res.StatusCode != http.StatusOK {
- return nil,
errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("unexpected status code when
requesting project detail from %s", res.Request.URL.String()))
- }
- err = helper.UnmarshalResponse(res, projectRes)
- if err != nil {
- return nil, err
- }
- return projectRes, nil
-}
diff --git a/backend/plugins/bamboo/api/remote.go
b/backend/plugins/bamboo/api/remote.go
index 395b560f9..a70b181bc 100644
--- a/backend/plugins/bamboo/api/remote.go
+++ b/backend/plugins/bamboo/api/remote.go
@@ -20,44 +20,16 @@ package api
import (
"context"
"fmt"
- "net/http"
- "net/url"
- "strconv"
-
+ context2 "github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ aha
"github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
"github.com/apache/incubator-devlake/plugins/bamboo/models"
+ "net/http"
+ "net/url"
)
-type RemoteScopesChild struct {
- Type string `json:"type"`
- ParentId *string `json:"parentId"`
- Id string `json:"id"`
- Name string `json:"name"`
- Data interface{} `json:"data"`
-}
-
-type RemoteScopesOutput struct {
- Children []RemoteScopesChild `json:"children"`
- NextPageToken string `json:"nextPageToken"`
-}
-
-type SearchRemoteScopesOutput struct {
- Children []RemoteScopesChild `json:"children"`
- Page int `json:"page"`
- PageSize int `json:"pageSize"`
-}
-
-type PageData struct {
- Page int `json:"page"`
- PageSize int `json:"per_page"`
-}
-
-const BambooRemoteScopesPerPage int = 100
-const TypeProject string = "scope"
-const TypeGroup string = "group"
-
// RemoteScopes list all available scope for users
// @Summary list all available scope for users
// @Description list all available scope for users
@@ -66,12 +38,33 @@ const TypeGroup string = "group"
// @Param connectionId path int false "connection ID"
// @Param groupId query string false "group ID"
// @Param pageToken query string false "page Token"
-// @Success 200 {object} RemoteScopesOutput
+// @Success 200 {object} api.RemoteScopesOutput
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/bamboo/connections/{connectionId}/remote-scopes [GET]
func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
- return remoteHelper.GetScopesFromRemote(input)
+ return remoteHelper.GetScopesFromRemote(input,
+ nil,
+ func(basicRes context2.BasicRes, gid string, queryData
*plugin.QueryData, connection models.BambooConnection)
([]models.ApiBambooProject, errors.Error) {
+ query := initialQuery(queryData)
+ // create api client
+ apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ if err != nil {
+ return nil, err
+ }
+ res, err := apiClient.Get("/project.json", query, nil)
+
+ if err != nil {
+ return nil, err
+ }
+
+ resBody := models.ApiBambooProjectResponse{}
+ err = api.UnmarshalResponse(res, &resBody)
+ if err != nil {
+ return nil, err
+ }
+ return resBody.Projects.Projects, err
+ })
}
// SearchRemoteScopes use the Search API and only return project
@@ -83,112 +76,68 @@ func RemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, er
// @Param search query string false "search"
// @Param page query int false "page number"
// @Param pageSize query int false "page size per page"
-// @Success 200 {object} SearchRemoteScopesOutput
+// @Success 200 {object} api.SearchRemoteScopesOutput
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/bamboo/connections/{connectionId}/search-remote-scopes
[GET]
func SearchRemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- connectionId, _ := extractParam(input.Params)
- if connectionId == 0 {
- return nil, errors.BadInput.New("invalid connectionId")
- }
-
- connection := &models.BambooConnection{}
- err := connectionHelper.First(connection, input.Params)
- if err != nil {
- return nil, err
- }
+ return remoteHelper.SearchRemoteScopes(input,
+ func(basicRes context2.BasicRes, queryData *plugin.QueryData,
connection models.BambooConnection) ([]models.ApiBambooProject, errors.Error) {
+ apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "failed
to get create apiClient")
+ }
+ query := initialQuery(queryData)
+ query.Set("searchTerm", queryData.Search[0])
+ // request search
+ res, err := apiClient.Get("search/projects.json",
query, nil)
+ if err != nil {
+ return nil, err
+ }
+ resBody := models.ApiBambooSearchProjectResponse{}
+ err = api.UnmarshalResponse(res, &resBody)
+ if err != nil {
+ return nil, err
+ }
+ var apiBambooProjects []models.ApiBambooProject
+ // append project to output
+ for _, apiResult := range resBody.SearchResults {
+ apiProject, err :=
GetApiProject(apiResult.SearchEntity.Key, apiClient)
+ if err != nil {
+ return nil, err
+ }
+
+ apiBambooProjects = append(apiBambooProjects,
*apiProject)
+ }
+ return apiBambooProjects, err
+ })
+}
- search, ok := input.Query["search"]
- if !ok || len(search) == 0 {
- search = []string{""}
- }
+func initialQuery(queryData *plugin.QueryData) url.Values {
+ query := url.Values{}
+ query.Set("showEmpty", fmt.Sprintf("%v", true))
+ query.Set("max-result", fmt.Sprintf("%v", queryData.PerPage))
+ query.Set("start-index", fmt.Sprintf("%v",
(queryData.Page-1)*queryData.PerPage))
+ return query
+}
- var p int
- var err1 error
- page, ok := input.Query["page"]
- if !ok || len(page) == 0 {
- p = 1
- } else {
- p, err1 = strconv.Atoi(page[0])
- if err != nil {
- return nil, errors.BadInput.Wrap(err1,
fmt.Sprintf("failed to Atoi page:%s", page[0]))
- }
- }
- var ps int
- pageSize, ok := input.Query["pageSize"]
- if !ok || len(pageSize) == 0 {
- ps = BambooRemoteScopesPerPage
- } else {
- ps, err1 = strconv.Atoi(pageSize[0])
- if err1 != nil {
- return nil, errors.BadInput.Wrap(err1,
fmt.Sprintf("failed to Atoi pageSize:%s", pageSize[0]))
- }
- }
- // create api client
- apiClient, err := api.NewApiClientFromConnection(context.TODO(),
basicRes, connection)
+// move from blueprint_v200 because of cycle import
+func GetApiProject(
+ projectKey string,
+ apiClient aha.ApiClientAbstract,
+) (*models.ApiBambooProject, errors.Error) {
+ projectRes := &models.ApiBambooProject{}
+ res, err := apiClient.Get(fmt.Sprintf("project/%s.json", projectKey),
nil, nil)
if err != nil {
return nil, err
}
-
- // set query
- query := GetQueryForSearchProject(search[0], p, ps)
-
- // request search
- res, err := apiClient.Get("search/projects.json", query, nil)
- if err != nil {
- return nil, err
+ defer res.Body.Close()
+ if res.StatusCode != http.StatusOK {
+ return nil,
errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("unexpected status code when
requesting project detail from %s", res.Request.URL.String()))
}
- resBody := models.ApiBambooSearchProjectResponse{}
- err = api.UnmarshalResponse(res, &resBody)
+ err = api.UnmarshalResponse(res, projectRes)
if err != nil {
return nil, err
}
-
- outputBody := &SearchRemoteScopesOutput{}
-
- // append project to output
- for _, apiResult := range resBody.SearchResults {
- var project models.BambooProject
- apiProject, err := GetApiProject(apiResult.SearchEntity.Key,
apiClient)
- if err != nil {
- return nil, err
- }
-
- project = apiProject.ConvertApiScope().(models.BambooProject)
- child := RemoteScopesChild{
- Type: TypeProject,
- Id: project.ProjectKey,
- ParentId: nil,
- Name: project.Name,
- Data: project,
- }
-
- outputBody.Children = append(outputBody.Children, child)
- }
-
- outputBody.Page = p
- outputBody.PageSize = ps
-
- return &plugin.ApiResourceOutput{Body: outputBody, Status:
http.StatusOK}, nil
-}
-
-func GetQueryFromPageData(pageData *PageData) url.Values {
- query := url.Values{}
- query.Set("showEmpty", fmt.Sprintf("%v", true))
- query.Set("max-result", fmt.Sprintf("%v", pageData.PageSize))
- query.Set("start-index", fmt.Sprintf("%v",
(pageData.Page-1)*pageData.PageSize))
- return query
-}
-
-func GetQueryForSearchProject(search string, page int, perPage int) url.Values
{
- query := GetQueryFromPageData(&PageData{Page: page, PageSize: perPage})
- query.Set("searchTerm", search)
-
- return query
-}
-func extractParam(params map[string]string) (uint64, string) {
- connectionId, _ := strconv.ParseUint(params["connectionId"], 10, 64)
- projectKey := params["projectKey"]
- return connectionId, projectKey
+ return projectRes, nil
}
diff --git a/backend/plugins/bamboo/impl/impl.go
b/backend/plugins/bamboo/impl/impl.go
index 115d7432e..be0a3f10b 100644
--- a/backend/plugins/bamboo/impl/impl.go
+++ b/backend/plugins/bamboo/impl/impl.go
@@ -19,7 +19,6 @@ package impl
import (
"fmt"
-
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
diff --git a/backend/plugins/bamboo/models/connection.go
b/backend/plugins/bamboo/models/connection.go
index 0e1e4fc5b..69694a522 100644
--- a/backend/plugins/bamboo/models/connection.go
+++ b/backend/plugins/bamboo/models/connection.go
@@ -18,11 +18,9 @@ limitations under the License.
package models
import (
- "context"
"fmt"
- context2 "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/plugin"
"net/http"
- "net/url"
"time"
"github.com/apache/incubator-devlake/core/errors"
@@ -30,6 +28,8 @@ import (
"github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
)
+var _ plugin.ApiConnection = (*BambooConnection)(nil)
+
type BambooConnection struct {
api.BaseConnection `mapstructure:",squash"`
BambooConn `mapstructure:",squash"`
@@ -92,27 +92,3 @@ type ApiRepository struct {
func (BambooConnection) TableName() string {
return "_tool_bamboo_connections"
}
-
-func (g BambooConnection) GetGroup(basicRes context2.BasicRes, gid string,
query url.Values) ([]GroupResponse, errors.Error) {
- return []GroupResponse{}, nil
-}
-
-func (g BambooConnection) GetScope(basicRes context2.BasicRes, gid string,
query url.Values) ([]ApiBambooProject, errors.Error) {
- // create api client
- apiClient, err := api.NewApiClientFromConnection(context.TODO(),
basicRes, &g)
- if err != nil {
- return nil, err
- }
- res, err := apiClient.Get("/project.json", query, nil)
-
- if err != nil {
- return nil, err
- }
-
- resBody := ApiBambooProjectResponse{}
- err = api.UnmarshalResponse(res, &resBody)
- if err != nil {
- return nil, err
- }
- return resBody.Projects.Projects, err
-}
diff --git a/backend/plugins/bamboo/models/project.go
b/backend/plugins/bamboo/models/project.go
index beeb2be56..2232604e6 100644
--- a/backend/plugins/bamboo/models/project.go
+++ b/backend/plugins/bamboo/models/project.go
@@ -19,9 +19,8 @@ package models
import (
"encoding/json"
- "github.com/apache/incubator-devlake/core/plugin"
-
"github.com/apache/incubator-devlake/core/models/common"
+ "github.com/apache/incubator-devlake/core/plugin"
)
var _ plugin.ToolLayerScope = (*BambooProject)(nil)
diff --git a/backend/plugins/bitbucket/api/remote.go
b/backend/plugins/bitbucket/api/remote.go
index 46c343b8a..68df6e4f1 100644
--- a/backend/plugins/bitbucket/api/remote.go
+++ b/backend/plugins/bitbucket/api/remote.go
@@ -20,44 +20,16 @@ package api
import (
"context"
"fmt"
+ context2 "github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/bitbucket/models"
"net/http"
"net/url"
- "strconv"
"strings"
)
-type RemoteScopesChild struct {
- Type string `json:"type"`
- ParentId *string `json:"parentId"`
- Id string `json:"id"`
- Name string `json:"name"`
- Data interface{} `json:"data"`
-}
-
-type RemoteScopesOutput struct {
- Children []RemoteScopesChild `json:"children"`
- NextPageToken string `json:"nextPageToken"`
-}
-
-type SearchRemoteScopesOutput struct {
- Children []RemoteScopesChild `json:"children"`
- Page int `json:"page"`
- PageSize int `json:"pageSize"`
-}
-
-type PageData struct {
- Page int `json:"page"`
- PerPage int `json:"per_page"`
-}
-
-const RemoteScopesPerPage int = 100
-const TypeScope string = "scope"
-const TypeGroup string = "group"
-
// RemoteScopes list all available scope for users
// @Summary list all available scope for users
// @Description list all available scope for users
@@ -66,12 +38,63 @@ const TypeGroup string = "group"
// @Param connectionId path int false "connection ID"
// @Param groupId query string false "group ID"
// @Param pageToken query string false "page Token"
-// @Success 200 {object} RemoteScopesOutput
+// @Success 200 {object} api.RemoteScopesOutput
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/bitbucket/connections/{connectionId}/remote-scopes [GET]
func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
- return remoteHelper.GetScopesFromRemote(input)
+ return remoteHelper.GetScopesFromRemote(input,
+ func(basicRes context2.BasicRes, gid string, queryData
*plugin.QueryData, connection models.BitbucketConnection)
([]models.GroupResponse, errors.Error) {
+ if gid != "" {
+ return nil, nil
+ }
+ query := initialQuery(queryData)
+
+ apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "failed
to get create apiClient")
+ }
+ var res *http.Response
+ query.Set("sort", "workspace.slug")
+ query.Set("fields",
"values.workspace.slug,values.workspace.name,pagelen,page,size")
+ res, err =
apiClient.Get("/user/permissions/workspaces", query, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ resBody := &models.WorkspaceResponse{}
+ err = api.UnmarshalResponse(res, resBody)
+ if err != nil {
+ return nil, err
+ }
+
+ return resBody.Values, err
+ },
+ func(basicRes context2.BasicRes, gid string, queryData
*plugin.QueryData, connection models.BitbucketConnection)
([]models.BitbucketApiRepo, errors.Error) {
+ if gid == "" {
+ return nil, nil
+ }
+ query := initialQuery(queryData)
+
+ apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "failed
to get create apiClient")
+ }
+ var res *http.Response
+ query.Set("fields",
"values.name,values.full_name,values.language,values.description,values.owner.username,values.created_on,values.updated_on,values.links.clone,values.links.self,pagelen,page,size")
+ // list projects part
+ res, err =
apiClient.Get(fmt.Sprintf("/repositories/%s", gid), query, nil)
+ if err != nil {
+ return nil, err
+ }
+ var resBody models.ReposResponse
+ err = api.UnmarshalResponse(res, &resBody)
+ if err != nil {
+ return nil, err
+ }
+ return resBody.Values, err
+ },
+ )
}
// SearchRemoteScopes use the Search API and only return project
@@ -83,106 +106,48 @@ func RemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, er
// @Param search query string false "search"
// @Param page query int false "page number"
// @Param pageSize query int false "page size per page"
-// @Success 200 {object} SearchRemoteScopesOutput
+// @Success 200 {object} api.SearchRemoteScopesOutput
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/bitbucket/connections/{connectionId}/search-remote-scopes
[GET]
func SearchRemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- connectionId, _ := extractParam(input.Params)
- if connectionId == 0 {
- return nil, errors.BadInput.New("invalid connectionId")
- }
-
- connection := &models.BitbucketConnection{}
- err := connectionHelper.First(connection, input.Params)
- if err != nil {
- return nil, err
- }
-
- search, ok := input.Query["search"]
- if !ok || len(search) == 0 {
- search = []string{""}
- }
- s := search[0]
-
- p := 1
- page, ok := input.Query["page"]
- if ok && len(page) != 0 {
- p, err = errors.Convert01(strconv.Atoi(page[0]))
- if err != nil {
- return nil, errors.BadInput.Wrap(err,
fmt.Sprintf("failed to Atoi page:%s", page[0]))
- }
- }
-
- ps := RemoteScopesPerPage
- pageSize, ok := input.Query["pageSize"]
- if ok && len(pageSize) != 0 {
- ps, err = errors.Convert01(strconv.Atoi(pageSize[0]))
- if err != nil {
- return nil, errors.BadInput.Wrap(err,
fmt.Sprintf("failed to Atoi pageSize:%s", pageSize[0]))
- }
- }
-
- // create api client
- apiClient, err := api.NewApiClientFromConnection(context.TODO(),
basicRes, connection)
- if err != nil {
- return nil, err
- }
-
- // set query
- query, err := GetQueryFromPageData(&PageData{p, ps})
- if err != nil {
- return nil, err
- }
-
- // request search
- query.Set("sort", "name")
- query.Set("fields",
"values.name,values.full_name,values.language,values.description,values.owner.username,values.created_on,values.updated_on,values.links.clone,values.links.self,pagelen,page,size")
-
- gid := ``
- if strings.Contains(s, `/`) {
- gid = strings.Split(s, `/`)[0]
- s = strings.Split(s, `/`)[0]
- }
- query.Set("q", fmt.Sprintf(`name~"%s"`, s))
- // list repos part
- res, err := apiClient.Get(fmt.Sprintf("/repositories/%s", gid), query,
nil)
- if err != nil {
- return nil, err
- }
- resBody := &models.ReposResponse{}
- err = api.UnmarshalResponse(res, &resBody)
- if err != nil {
- return nil, err
- }
-
- // set repos return
- outputBody := &SearchRemoteScopesOutput{Children: []RemoteScopesChild{}}
- for _, repo := range resBody.Values {
- child := RemoteScopesChild{
- Type: TypeScope,
- Id: repo.FullName,
- ParentId: nil,
- Name: repo.Name,
- Data: repo.ConvertApiScope(),
- }
- outputBody.Children = append(outputBody.Children, child)
- }
-
- outputBody.Page = p
- outputBody.PageSize = ps
-
- return &plugin.ApiResourceOutput{Body: outputBody, Status:
http.StatusOK}, nil
+ return remoteHelper.SearchRemoteScopes(input,
+ func(basicRes context2.BasicRes, queryData *plugin.QueryData,
connection models.BitbucketConnection) ([]models.BitbucketApiRepo,
errors.Error) {
+ // create api client
+ apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ if err != nil {
+ return nil, err
+ }
+ query := initialQuery(queryData)
+ s := queryData.Search[0]
+
+ // request search
+ query.Set("sort", "name")
+ query.Set("fields",
"values.name,values.full_name,values.language,values.description,values.owner.username,values.created_on,values.updated_on,values.links.clone,values.links.self,pagelen,page,size")
+ gid := ``
+ if strings.Contains(s, `/`) {
+ gid = strings.Split(s, `/`)[0]
+ s = strings.Split(s, `/`)[0]
+ }
+ query.Set("q", fmt.Sprintf(`name~"%s"`, s))
+ // list repos part
+ res, err :=
apiClient.Get(fmt.Sprintf("/repositories/%s", gid), query, nil)
+ if err != nil {
+ return nil, err
+ }
+ resBody := &models.ReposResponse{}
+ err = api.UnmarshalResponse(res, &resBody)
+ if err != nil {
+ return nil, err
+ }
+ return resBody.Values, err
+ },
+ )
}
-func GetQueryFromPageData(pageData *PageData) (url.Values, errors.Error) {
+func initialQuery(queryData *plugin.QueryData) url.Values {
query := url.Values{}
- query.Set("page", fmt.Sprintf("%v", pageData.Page))
- query.Set("pagelen", fmt.Sprintf("%v", pageData.PerPage))
- return query, nil
-}
-func extractParam(params map[string]string) (uint64, string) {
- connectionId, _ := strconv.ParseUint(params["connectionId"], 10, 64)
- fullName := strings.TrimLeft(params["repoId"], "/")
- return connectionId, fullName
+ query.Set("page", fmt.Sprintf("%v", queryData.Page))
+ query.Set("pagelen", fmt.Sprintf("%v", queryData.PerPage))
+ return query
}
diff --git a/backend/plugins/bitbucket/models/connection.go
b/backend/plugins/bitbucket/models/connection.go
index 6300a8deb..9db3befc5 100644
--- a/backend/plugins/bitbucket/models/connection.go
+++ b/backend/plugins/bitbucket/models/connection.go
@@ -18,18 +18,11 @@ limitations under the License.
package models
import (
- "context"
- "fmt"
- context2 "github.com/apache/incubator-devlake/core/context"
- "github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "net/http"
- "net/url"
)
-var _ plugin.ApiConnectionForRemote[GroupResponse, BitbucketApiRepo] =
(*BitbucketConnection)(nil)
-var _ plugin.ApiGroup = (*GroupResponse)(nil)
+var _ plugin.ApiConnection = (*BitbucketConnection)(nil)
// BitbucketConn holds the essential information to connect to the Bitbucket
API
type BitbucketConn struct {
@@ -46,51 +39,3 @@ type BitbucketConnection struct {
func (BitbucketConnection) TableName() string {
return "_tool_bitbucket_connections"
}
-
-func (g BitbucketConnection) GetGroup(basicRes context2.BasicRes, gid string,
query url.Values) ([]GroupResponse, errors.Error) {
- if gid != "" {
- return nil, nil
- }
- apiClient, err := api.NewApiClientFromConnection(context.TODO(),
basicRes, &g)
- if err != nil {
- return nil, errors.BadInput.Wrap(err, "failed to get create
apiClient")
- }
- var res *http.Response
- query.Set("sort", "workspace.slug")
- query.Set("fields",
"values.workspace.slug,values.workspace.name,pagelen,page,size")
- res, err = apiClient.Get("/user/permissions/workspaces", query, nil)
- if err != nil {
- return nil, err
- }
-
- resBody := &WorkspaceResponse{}
- err = api.UnmarshalResponse(res, resBody)
- if err != nil {
- return nil, err
- }
-
- return resBody.Values, err
-}
-
-func (g BitbucketConnection) GetScope(basicRes context2.BasicRes, gid string,
query url.Values) ([]BitbucketApiRepo, errors.Error) {
- if gid == "" {
- return nil, nil
- }
- apiClient, err := api.NewApiClientFromConnection(context.TODO(),
basicRes, &g)
- if err != nil {
- return nil, errors.BadInput.Wrap(err, "failed to get create
apiClient")
- }
- var res *http.Response
- query.Set("fields",
"values.name,values.full_name,values.language,values.description,values.owner.username,values.created_on,values.updated_on,values.links.clone,values.links.self,pagelen,page,size")
- // list projects part
- res, err = apiClient.Get(fmt.Sprintf("/repositories/%s", gid), query,
nil)
- if err != nil {
- return nil, err
- }
- var resBody ReposResponse
- err = api.UnmarshalResponse(res, &resBody)
- if err != nil {
- return nil, err
- }
- return resBody.Values, err
-}
diff --git a/backend/plugins/gitlab/api/remote.go
b/backend/plugins/gitlab/api/remote.go
index 69875646b..62c950834 100644
--- a/backend/plugins/gitlab/api/remote.go
+++ b/backend/plugins/gitlab/api/remote.go
@@ -20,46 +20,15 @@ package api
import (
"context"
"fmt"
- "net/http"
- "net/url"
- "strconv"
-
+ context2 "github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
- "github.com/apache/incubator-devlake/plugins/gitlab/tasks"
+ "net/http"
+ "net/url"
)
-type RemoteScopesChild struct {
- Type string `json:"type"`
- ParentId *string `json:"parentId"`
- Id string `json:"id"`
- Name string `json:"name"`
- Data interface{} `json:"data"`
-}
-
-type RemoteScopesOutput struct {
- Children []RemoteScopesChild `json:"children"`
- NextPageToken string `json:"nextPageToken"`
-}
-
-type SearchRemoteScopesOutput struct {
- Children []RemoteScopesChild `json:"children"`
- Page int `json:"page"`
- PageSize int `json:"pageSize"`
-}
-
-type PageData struct {
- Page int `json:"page"`
- PerPage int `json:"per_page"`
- Tag string `json:"tag"`
-}
-
-const GitlabRemoteScopesPerPage int = 100
-const TypeProject string = "scope"
-const TypeGroup string = "group"
-
// RemoteScopes list all available scope for users
// @Summary list all available scope for users
// @Description list all available scope for users
@@ -68,12 +37,64 @@ const TypeGroup string = "group"
// @Param connectionId path int false "connection ID"
// @Param groupId query string false "group ID"
// @Param pageToken query string false "page Token"
-// @Success 200 {object} RemoteScopesOutput
+// @Success 200 {object} api.RemoteScopesOutput
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/gitlab/connections/{connectionId}/remote-scopes [GET]
func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
- return remoteHelper.GetScopesFromRemote(input)
+ return remoteHelper.GetScopesFromRemote(input,
+ func(basicRes context2.BasicRes, gid string, queryData
*plugin.QueryData, connection models.GitlabConnection) ([]models.GroupResponse,
errors.Error) {
+ apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "failed
to get create apiClient")
+ }
+ query := initialQuery(queryData)
+ var res *http.Response
+ if gid == "" {
+ query.Set("top_level_only", "true")
+ res, err = apiClient.Get("groups", query, nil)
+ if err != nil {
+ return nil, err
+ }
+ } else {
+ res, err =
apiClient.Get(fmt.Sprintf("groups/%s/subgroups", gid), query, nil)
+ if err != nil {
+ return nil, err
+ }
+ }
+ var resBody []models.GroupResponse
+ err = api.UnmarshalResponse(res, &resBody)
+ if err != nil {
+ return nil, err
+ }
+ return resBody, err
+ },
+ func(basicRes context2.BasicRes, gid string, queryData
*plugin.QueryData, connection models.GitlabConnection)
([]models.GitlabApiProject, errors.Error) {
+ apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "failed
to get create apiClient")
+ }
+ query := initialQuery(queryData)
+ var res *http.Response
+ if gid == "" {
+ res, err =
apiClient.Get(fmt.Sprintf("users/%d/projects", apiClient.GetData("UserId")),
query, nil)
+ if err != nil {
+ return nil, err
+ }
+ } else {
+ query.Set("with_shared", "false")
+ res, err =
apiClient.Get(fmt.Sprintf("/groups/%s/projects", gid), query, nil)
+ if err != nil {
+ return nil, err
+ }
+ }
+ var resBody []models.GitlabApiProject
+ err = api.UnmarshalResponse(res, &resBody)
+ if err != nil {
+ return nil, err
+ }
+ return resBody, err
+ })
}
// SearchRemoteScopes use the Search API and only return project
@@ -85,112 +106,41 @@ func RemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, er
// @Param search query string false "search"
// @Param page query int false "page number"
// @Param pageSize query int false "page size per page"
-// @Success 200 {object} SearchRemoteScopesOutput
+// @Success 200 {object} api.SearchRemoteScopesOutput
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/gitlab/connections/{connectionId}/search-remote-scopes
[GET]
func SearchRemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- connectionId, _ := extractParam(input.Params)
- if connectionId == 0 {
- return nil, errors.BadInput.New("invalid connectionId")
- }
-
- connection := &models.GitlabConnection{}
- err := connectionHelper.First(connection, input.Params)
- if err != nil {
- return nil, err
- }
-
- search, ok := input.Query["search"]
- if !ok || len(search) == 0 {
- search = []string{""}
- }
-
- var p int
- var err1 error
- page, ok := input.Query["page"]
- if !ok || len(page) == 0 {
- p = 1
- } else {
- p, err1 = strconv.Atoi(page[0])
- if err != nil {
- return nil, errors.BadInput.Wrap(err1,
fmt.Sprintf("failed to Atoi page:%s", page[0]))
- }
- }
- var ps int
- pageSize, ok := input.Query["pageSize"]
- if !ok || len(pageSize) == 0 {
- ps = GitlabRemoteScopesPerPage
- } else {
- ps, err1 = strconv.Atoi(pageSize[0])
- if err1 != nil {
- return nil, errors.BadInput.Wrap(err1,
fmt.Sprintf("failed to Atoi pageSize:%s", pageSize[0]))
- }
- }
- // create api client
- apiClient, err := api.NewApiClientFromConnection(context.TODO(),
basicRes, connection)
- if err != nil {
- return nil, err
- }
-
- // set query
- query, err := GetQueryForSearchProject(search[0], p, ps)
- if err != nil {
- return nil, err
- }
-
- // request search
- res, err := apiClient.Get("search", query, nil)
- if err != nil {
- return nil, err
- }
- resBody := []tasks.GitlabApiProject{}
- err = api.UnmarshalResponse(res, &resBody)
- if err != nil {
- return nil, err
- }
-
- outputBody := &SearchRemoteScopesOutput{}
-
- // append project to output
- for _, project := range resBody {
- child := RemoteScopesChild{
- Type: TypeProject,
- Id: strconv.Itoa(project.GitlabId),
- ParentId: nil,
- Name: project.PathWithNamespace,
- Data: tasks.ConvertProject(&project),
- }
-
- outputBody.Children = append(outputBody.Children, child)
- }
-
- outputBody.Page = p
- outputBody.PageSize = ps
-
- return &plugin.ApiResourceOutput{Body: outputBody, Status:
http.StatusOK}, nil
+ return remoteHelper.SearchRemoteScopes(input,
+ func(basicRes context2.BasicRes, queryData *plugin.QueryData,
connection models.GitlabConnection) ([]models.GitlabApiProject, errors.Error) {
+ apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "failed
to get create apiClient")
+ }
+ query := initialQuery(queryData)
+ query.Set("search", queryData.Search[0])
+ query.Set("scope", "projects")
+ // request search
+ res, err := apiClient.Get("search", query, nil)
+ if err != nil {
+ return nil, err
+ }
+ var resBody []models.GitlabApiProject
+ err = api.UnmarshalResponse(res, &resBody)
+ if err != nil {
+ return nil, err
+ }
+ for i := 0; i < len(resBody); i++ {
+ // as we need to set PathWithNamespace to name
in SearchRemoteScopes, but interface.ScopeName will return name, so we switch it
+ resBody[i].Name, resBody[i].PathWithNamespace =
resBody[i].PathWithNamespace, resBody[i].Name
+ }
+ return resBody, err
+ })
}
-func GetQueryFromPageData(pageData *PageData) (url.Values, errors.Error) {
+func initialQuery(queryData *plugin.QueryData) url.Values {
query := url.Values{}
- query.Set("page", fmt.Sprintf("%v", pageData.Page))
- query.Set("per_page", fmt.Sprintf("%v", pageData.PerPage))
- return query, nil
-}
-
-func GetQueryForSearchProject(search string, page int, perPage int)
(url.Values, errors.Error) {
- query, err := GetQueryFromPageData(&PageData{Page: page, PerPage:
perPage})
- if err != nil {
- return nil, err
- }
- query.Set("search", search)
- query.Set("scope", "projects")
-
- return query, nil
-}
-
-func extractParam(params map[string]string) (uint64, uint64) {
- connectionId, _ := strconv.ParseUint(params["connectionId"], 10, 64)
- projectId, _ := strconv.ParseUint(params["projectId"], 10, 64)
- return connectionId, projectId
+ query.Set("page", fmt.Sprintf("%v", queryData.Page))
+ query.Set("per_page", fmt.Sprintf("%v", queryData.PerPage))
+ return query
}
diff --git a/backend/plugins/gitlab/models/connection.go
b/backend/plugins/gitlab/models/connection.go
index aa55a79d2..0b8f6fa91 100644
--- a/backend/plugins/gitlab/models/connection.go
+++ b/backend/plugins/gitlab/models/connection.go
@@ -18,16 +18,12 @@ limitations under the License.
package models
import (
- "context"
"fmt"
- context2 "github.com/apache/incubator-devlake/core/context"
- "github.com/apache/incubator-devlake/core/plugin"
- "net/http"
- "net/url"
-
"github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
+ "net/http"
)
// GitlabConn holds the essential information to connect to the Gitlab API
@@ -108,8 +104,7 @@ func (conn *GitlabConn) PrepareApiClient(apiClient
apihelperabstract.ApiClientAb
return nil
}
-var _ plugin.ApiConnectionForRemote[GroupResponse, GitlabApiProject] =
(*GitlabConnection)(nil)
-var _ plugin.ApiGroup = (*GroupResponse)(nil)
+var _ plugin.ApiConnection = (*GitlabConnection)(nil)
// GitlabConnection holds GitlabConn plus ID/Name for database storage
type GitlabConnection struct {
@@ -138,55 +133,3 @@ type ApiUserResponse struct {
func (GitlabConnection) TableName() string {
return "_tool_gitlab_connections"
}
-
-func (g GitlabConnection) GetGroup(basicRes context2.BasicRes, gid string,
query url.Values) ([]GroupResponse, errors.Error) {
- apiClient, err := api.NewApiClientFromConnection(context.TODO(),
basicRes, &g)
- if err != nil {
- return nil, errors.BadInput.Wrap(err, "failed to get create
apiClient")
- }
- var res *http.Response
- if gid == "" {
- query.Set("top_level_only", "true")
- res, err = apiClient.Get("groups", query, nil)
- if err != nil {
- return nil, err
- }
- } else {
- res, err = apiClient.Get(fmt.Sprintf("groups/%s/subgroups",
gid), query, nil)
- if err != nil {
- return nil, err
- }
- }
- var resBody []GroupResponse
- err = api.UnmarshalResponse(res, &resBody)
- if err != nil {
- return nil, err
- }
- return resBody, err
-}
-
-func (g GitlabConnection) GetScope(basicRes context2.BasicRes, gid string,
query url.Values) ([]GitlabApiProject, errors.Error) {
- apiClient, err := api.NewApiClientFromConnection(context.TODO(),
basicRes, &g)
- if err != nil {
- return nil, errors.BadInput.Wrap(err, "failed to get create
apiClient")
- }
- var res *http.Response
- if gid == "" {
- res, err = apiClient.Get(fmt.Sprintf("users/%d/projects",
apiClient.GetData("UserId")), query, nil)
- if err != nil {
- return nil, err
- }
- } else {
- query.Set("with_shared", "false")
- res, err = apiClient.Get(fmt.Sprintf("/groups/%s/projects",
gid), query, nil)
- if err != nil {
- return nil, err
- }
- }
- var resBody []GitlabApiProject
- err = api.UnmarshalResponse(res, &resBody)
- if err != nil {
- return nil, err
- }
- return resBody, err
-}
diff --git a/backend/plugins/gitlab/models/project.go
b/backend/plugins/gitlab/models/project.go
index af08f9128..9f9c1f1c0 100644
--- a/backend/plugins/gitlab/models/project.go
+++ b/backend/plugins/gitlab/models/project.go
@@ -82,6 +82,10 @@ func (gitlabApiProject GitlabApiProject) ConvertApiScope()
plugin.ToolLayerScope
p.ForkedFromProjectId =
gitlabApiProject.ForkedFromProject.GitlabId
p.ForkedFromProjectWebUrl =
gitlabApiProject.ForkedFromProject.WebUrl
}
+ // this might happen when GitlabConnection.SearchScopes
+ if len(p.Name) > len(p.PathWithNamespace) {
+ p.Name, p.PathWithNamespace = p.PathWithNamespace, p.Name
+ }
return p
}
