This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch release-v0.18
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/release-v0.18 by this push:
new c9e89d8d5 cherry-pick fix jenkins remote scopes API, add
GetRemoteScopesOutput for remoteHelper to v0.18(beta8) (#5959)
c9e89d8d5 is described below
commit c9e89d8d5cbb1748f698bf6b4becffd1bc8bc8d0
Author: Lynwee <linwei....@merico.dev>
AuthorDate: Mon Aug 28 11:32:26 2023 +0800
cherry-pick fix jenkins remote scopes API, add GetRemoteScopesOutput for
remoteHelper to v0.18(beta8) (#5959)
* fix(workflow): ignore reverted commit msg
* style(context): rename devlake's core context and golang's context
* fix(jenkins): fix remote scope list api
* fix(zentao): fix ci warn
---
.github/workflows/commit-msg.yml | 1 +
.../helpers/pluginhelper/api/remote_api_helper.go | 110 ++++++++++++++-------
backend/plugins/bamboo/api/remote.go | 12 +--
backend/plugins/bitbucket/api/remote.go | 16 +--
backend/plugins/github/api/remote.go | 16 +--
backend/plugins/gitlab/api/remote.go | 16 +--
backend/plugins/jenkins/api/jobs.go | 18 +---
backend/plugins/jenkins/api/remote.go | 97 ++++++++++--------
backend/plugins/jira/api/remote.go | 8 +-
backend/plugins/sonarqube/api/remote.go | 12 +--
backend/plugins/tapd/api/remote.go | 12 +--
backend/plugins/zentao/api/remote.go | 10 +-
12 files changed, 182 insertions(+), 146 deletions(-)
diff --git a/.github/workflows/commit-msg.yml b/.github/workflows/commit-msg.yml
index 2c8c664ac..18fefb2b3 100644
--- a/.github/workflows/commit-msg.yml
+++ b/.github/workflows/commit-msg.yml
@@ -36,4 +36,5 @@ jobs:
run: |
! git log --oneline ${{ github.event.pull_request.base.sha }}... \
| grep -vP '^\w{8,40} Merge ' \
+ | grep -vP '^\w{8,40} Revert ' \
| grep -vP '^\w{8,40}
(feat|fix|build|chore|docs|style|refactor|perf|test|ci)(\(\w+(-\w+)?\))?:(\s*).*'
diff --git a/backend/helpers/pluginhelper/api/remote_api_helper.go
b/backend/helpers/pluginhelper/api/remote_api_helper.go
index f0dab9bf4..76a0f540c 100644
--- a/backend/helpers/pluginhelper/api/remote_api_helper.go
+++ b/backend/helpers/pluginhelper/api/remote_api_helper.go
@@ -21,10 +21,11 @@ import (
"encoding/base64"
"encoding/json"
"fmt"
+ "github.com/apache/incubator-devlake/core/log"
"net/http"
"strconv"
- coreContext "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/go-playground/validator/v10"
@@ -64,14 +65,15 @@ type SearchRemoteScopesOutput struct {
// RemoteApiHelper is used to write the CURD of connection
type RemoteApiHelper[Conn plugin.ApiConnection, Scope plugin.ToolLayerScope,
ApiScope plugin.ApiScope, Group plugin.ApiGroup] struct {
- basicRes coreContext.BasicRes
+ basicRes context.BasicRes
validator *validator.Validate
connHelper *ConnectionApiHelper
+ logger log.Logger
}
// NewRemoteHelper creates a ScopeHelper for connection management
func NewRemoteHelper[Conn plugin.ApiConnection, Scope plugin.ToolLayerScope,
ApiScope plugin.ApiScope, Group plugin.ApiGroup](
- basicRes coreContext.BasicRes,
+ basicRes context.BasicRes,
vld *validator.Validate,
connHelper *ConnectionApiHelper,
) *RemoteApiHelper[Conn, Scope, ApiScope, Group] {
@@ -85,6 +87,7 @@ func NewRemoteHelper[Conn plugin.ApiConnection, Scope
plugin.ToolLayerScope, Api
basicRes: basicRes,
validator: vld,
connHelper: connHelper,
+ logger: basicRes.GetLogger(),
}
}
@@ -113,8 +116,11 @@ func (g BaseRemoteGroupResponse) GroupName() string {
}
const remoteScopesPerPage int = 100
-const TypeProject string = "scope"
-const TypeGroup string = "group"
+const (
+ TypeGroup string = "group" // group is just like a directory or a
folder, that holds some scopes.
+ TypeScope string = "scope" // scope, sometimes we call it project. But
scope is a more standard noun.
+ TypeMixed string = "mixed"
+)
// PrepareFirstPageToken prepares the first page token
func (r *RemoteApiHelper[Conn, Scope, ApiScope, Group])
PrepareFirstPageToken(customInfo string) (*plugin.ApiResourceOutput,
errors.Error) {
@@ -123,7 +129,7 @@ func (r *RemoteApiHelper[Conn, Scope, ApiScope, Group])
PrepareFirstPageToken(cu
Page: 1,
PerPage: remoteScopesPerPage,
CustomInfo: customInfo,
- Tag: "group",
+ Tag: TypeGroup,
})
if err != nil {
return nil, err
@@ -132,46 +138,77 @@ func (r *RemoteApiHelper[Conn, Scope, ApiScope, Group])
PrepareFirstPageToken(cu
return &plugin.ApiResourceOutput{Body: outputBody, Status:
http.StatusOK}, nil
}
-// GetScopesFromRemote gets the scopes from api
-func (r *RemoteApiHelper[Conn, Scope, ApiScope, Group])
GetScopesFromRemote(input *plugin.ApiResourceInput,
- getGroup func(basicRes coreContext.BasicRes, gid string, queryData
*RemoteQueryData, connection Conn) ([]Group, errors.Error),
- getScope func(basicRes coreContext.BasicRes, gid string, queryData
*RemoteQueryData, connection Conn) ([]ApiScope, errors.Error),
-) (*plugin.ApiResourceOutput, errors.Error) {
+func (r *RemoteApiHelper[Conn, Scope, ApiScope, ApiGroup])
GetRemoteScopesOutput(
+ input *plugin.ApiResourceInput,
+ getter func(basicRes context.BasicRes, groupId string, queryData
*RemoteQueryData, connection Conn) (*RemoteScopesOutput, errors.Error),
+) (*RemoteScopesOutput, errors.Error) {
connectionId, err :=
errors.Convert01(strconv.ParseUint(input.Params["connectionId"], 10, 64))
if err != nil || connectionId == 0 {
return nil, errors.BadInput.New("invalid connectionId")
}
-
var connection Conn
err = r.connHelper.First(&connection, input.Params)
if err != nil {
+ r.logger.Error(err, "find connection: %d", connectionId)
+ return nil, err
+ }
+ groupId := input.Query.Get("groupId")
+ pageToken := input.Query.Get("pageToken")
+ queryData, err := getPageDataFromPageTokenWithTag(pageToken, TypeMixed)
+ if err != nil {
+ r.logger.Error(err, "get page data from page token")
+ return nil, err
+ }
+ resp, err := getter(r.basicRes, groupId, queryData, connection)
+ if err != nil {
+ r.logger.Error(err, "call getter")
return nil, err
}
- groupId, ok := input.Query["groupId"]
- if !ok || len(groupId) == 0 {
- groupId = []string{""}
+ queryData.Page += 1
+ resp.NextPageToken, err = getPageTokenFromPageData(queryData)
+ if err != nil {
+ r.logger.Error(err, "get next page token")
+ return nil, err
}
+ if len(resp.Children) < queryData.PerPage {
+ // there are no more pages
+ resp.NextPageToken = ""
+ }
+ return resp, nil
+}
- pageToken, ok := input.Query["pageToken"]
- if !ok || len(pageToken) == 0 {
- pageToken = []string{""}
+// GetScopesFromRemote gets the scopes from api
+func (r *RemoteApiHelper[Conn, Scope, ApiScope, Group]) GetScopesFromRemote(
+ input *plugin.ApiResourceInput,
+ getGroup func(basicRes context.BasicRes, gid string, queryData
*RemoteQueryData, connection Conn) ([]Group, errors.Error),
+ getScope func(basicRes context.BasicRes, gid string, queryData
*RemoteQueryData, connection Conn) ([]ApiScope, errors.Error),
+) (*plugin.ApiResourceOutput, errors.Error) {
+
+ connectionId, err :=
errors.Convert01(strconv.ParseUint(input.Params["connectionId"], 10, 64))
+ if err != nil || connectionId == 0 {
+ return nil, errors.BadInput.New("invalid connectionId")
}
+ var connection Conn
+ err = r.connHelper.First(&connection, input.Params)
+ if err != nil {
+ return nil, err
+ }
+ // get groupId and pageData
+ groupId := input.Query.Get("groupId")
+ pageToken := input.Query.Get("pageToken")
- // get gid and pageData
- gid := groupId[0]
- queryData, err := getPageDataFromPageToken(pageToken[0])
+ queryData, err := getPageDataFromPageToken(pageToken)
if err != nil {
return nil, errors.BadInput.New("failed to get page token")
}
-
outputBody := &RemoteScopesOutput{}
// list groups part
if queryData.Tag == TypeGroup {
var resBody []Group
if getGroup != nil {
- resBody, err = getGroup(r.basicRes, gid, queryData,
connection)
+ resBody, err = getGroup(r.basicRes, groupId, queryData,
connection)
}
if err != nil {
return nil, err
@@ -186,7 +223,7 @@ func (r *RemoteApiHelper[Conn, Scope, ApiScope, Group])
GetScopesFromRemote(inpu
// don't need to save group into data
Data: nil,
}
- child.ParentId = &gid
+ child.ParentId = &groupId
if *child.ParentId == "" {
child.ParentId = nil
}
@@ -194,16 +231,16 @@ func (r *RemoteApiHelper[Conn, Scope, ApiScope, Group])
GetScopesFromRemote(inpu
}
// check groups count
if len(resBody) < queryData.PerPage {
- queryData.Tag = TypeProject
+ queryData.Tag = TypeScope
queryData.Page = 1
queryData.PerPage = queryData.PerPage - len(resBody)
}
}
// list projects part
- if queryData.Tag == TypeProject && getScope != nil {
+ if queryData.Tag == TypeScope && getScope != nil {
var resBody []ApiScope
- resBody, err = getScope(r.basicRes, gid, queryData, connection)
+ resBody, err = getScope(r.basicRes, groupId, queryData,
connection)
if err != nil {
return nil, err
}
@@ -212,21 +249,20 @@ func (r *RemoteApiHelper[Conn, Scope, ApiScope, Group])
GetScopesFromRemote(inpu
for _, project := range resBody {
scope := project.ConvertApiScope()
child := RemoteScopesChild{
- Type: TypeProject,
+ Type: TypeScope,
Id: scope.ScopeId(),
Name: scope.ScopeName(),
FullName: scope.ScopeFullName(),
Data: &scope,
}
- child.ParentId = &gid
+ child.ParentId = &groupId
if *child.ParentId == "" {
child.ParentId = nil
}
-
outputBody.Children = append(outputBody.Children, child)
}
- // check project count
+ // check scopes count
if len(resBody) < queryData.PerPage {
queryData = nil
}
@@ -236,18 +272,16 @@ func (r *RemoteApiHelper[Conn, Scope, ApiScope, Group])
GetScopesFromRemote(inpu
outputBody.NextPageToken = ""
if queryData != nil {
queryData.Page += 1
-
outputBody.NextPageToken, err =
getPageTokenFromPageData(queryData)
if err != nil {
return nil, err
}
}
-
return &plugin.ApiResourceOutput{Body: outputBody, Status:
http.StatusOK}, nil
}
func (r *RemoteApiHelper[Conn, Scope, ApiScope, Group])
SearchRemoteScopes(input *plugin.ApiResourceInput,
- searchScope func(basicRes coreContext.BasicRes, queryData
*RemoteQueryData, connection Conn) ([]ApiScope, errors.Error),
+ searchScope func(basicRes context.BasicRes, queryData *RemoteQueryData,
connection Conn) ([]ApiScope, errors.Error),
) (*plugin.ApiResourceOutput, errors.Error) {
connectionId, err :=
errors.Convert01(strconv.ParseUint(input.Params["connectionId"], 10, 64))
if err != nil || connectionId == 0 {
@@ -305,7 +339,7 @@ func (r *RemoteApiHelper[Conn, Scope, ApiScope, Group])
SearchRemoteScopes(input
for _, project := range resBody {
scope := project.ConvertApiScope()
child := RemoteScopesChild{
- Type: TypeProject,
+ Type: TypeScope,
Id: scope.ScopeId(),
ParentId: nil,
Name: scope.ScopeName(),
@@ -334,11 +368,15 @@ func getPageTokenFromPageData(pageData *RemoteQueryData)
(string, errors.Error)
}
func getPageDataFromPageToken(pageToken string) (*RemoteQueryData,
errors.Error) {
+ return getPageDataFromPageTokenWithTag(pageToken, TypeGroup)
+}
+
+func getPageDataFromPageTokenWithTag(pageToken string, queryTag string)
(*RemoteQueryData, errors.Error) {
if pageToken == "" {
return &RemoteQueryData{
Page: 1,
PerPage: remoteScopesPerPage,
- Tag: "group",
+ Tag: queryTag,
}, nil
}
diff --git a/backend/plugins/bamboo/api/remote.go
b/backend/plugins/bamboo/api/remote.go
index 7e1e8bbfe..e1fa0ef41 100644
--- a/backend/plugins/bamboo/api/remote.go
+++ b/backend/plugins/bamboo/api/remote.go
@@ -18,9 +18,9 @@ limitations under the License.
package api
import (
- "context"
+ gocontext "context"
"fmt"
- context2 "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -45,10 +45,10 @@ import (
func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
return remoteHelper.GetScopesFromRemote(input,
nil,
- func(basicRes context2.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.BambooConnection)
([]models.ApiBambooProject, errors.Error) {
+ func(basicRes context.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.BambooConnection)
([]models.ApiBambooProject, errors.Error) {
query := initialQuery(queryData)
// create api client
- apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ apiClient, err :=
api.NewApiClientFromConnection(gocontext.TODO(), basicRes, &connection)
if err != nil {
return nil, err
}
@@ -82,8 +82,8 @@ func RemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, er
// @Router /plugins/bamboo/connections/{connectionId}/search-remote-scopes
[GET]
func SearchRemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
return remoteHelper.SearchRemoteScopes(input,
- func(basicRes context2.BasicRes, queryData
*api.RemoteQueryData, connection models.BambooConnection)
([]models.ApiBambooProject, errors.Error) {
- apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ func(basicRes context.BasicRes, queryData *api.RemoteQueryData,
connection models.BambooConnection) ([]models.ApiBambooProject, errors.Error) {
+ apiClient, err :=
api.NewApiClientFromConnection(gocontext.TODO(), basicRes, &connection)
if err != nil {
return nil, errors.BadInput.Wrap(err, "failed
to get create apiClient")
}
diff --git a/backend/plugins/bitbucket/api/remote.go
b/backend/plugins/bitbucket/api/remote.go
index 7af42060d..60ddd72d0 100644
--- a/backend/plugins/bitbucket/api/remote.go
+++ b/backend/plugins/bitbucket/api/remote.go
@@ -18,13 +18,13 @@ limitations under the License.
package api
import (
- "context"
+ gocontext "context"
"fmt"
"net/http"
"net/url"
"strings"
- context2 "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -45,13 +45,13 @@ import (
// @Router /plugins/bitbucket/connections/{connectionId}/remote-scopes [GET]
func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
return remoteHelper.GetScopesFromRemote(input,
- func(basicRes context2.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.BitbucketConnection)
([]models.GroupResponse, errors.Error) {
+ func(basicRes context.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.BitbucketConnection)
([]models.GroupResponse, errors.Error) {
if gid != "" {
return nil, nil
}
query := initialQuery(queryData)
- apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ apiClient, err :=
api.NewApiClientFromConnection(gocontext.TODO(), basicRes, &connection)
if err != nil {
return nil, errors.BadInput.Wrap(err, "failed
to get create apiClient")
}
@@ -71,13 +71,13 @@ func RemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, er
return resBody.Values, err
},
- func(basicRes context2.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.BitbucketConnection)
([]models.BitbucketApiRepo, errors.Error) {
+ func(basicRes context.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.BitbucketConnection)
([]models.BitbucketApiRepo, errors.Error) {
if gid == "" {
return nil, nil
}
query := initialQuery(queryData)
- apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ apiClient, err :=
api.NewApiClientFromConnection(gocontext.TODO(), basicRes, &connection)
if err != nil {
return nil, errors.BadInput.Wrap(err, "failed
to get create apiClient")
}
@@ -113,9 +113,9 @@ func RemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, er
// @Router /plugins/bitbucket/connections/{connectionId}/search-remote-scopes
[GET]
func SearchRemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
return remoteHelper.SearchRemoteScopes(input,
- func(basicRes context2.BasicRes, queryData
*api.RemoteQueryData, connection models.BitbucketConnection)
([]models.BitbucketApiRepo, errors.Error) {
+ func(basicRes context.BasicRes, queryData *api.RemoteQueryData,
connection models.BitbucketConnection) ([]models.BitbucketApiRepo,
errors.Error) {
// create api client
- apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ apiClient, err :=
api.NewApiClientFromConnection(gocontext.TODO(), basicRes, &connection)
if err != nil {
return nil, err
}
diff --git a/backend/plugins/github/api/remote.go
b/backend/plugins/github/api/remote.go
index 961b19d39..2444b23d8 100644
--- a/backend/plugins/github/api/remote.go
+++ b/backend/plugins/github/api/remote.go
@@ -18,14 +18,14 @@ limitations under the License.
package api
import (
- "context"
+ gocontext "context"
"fmt"
"net/http"
"net/url"
"strconv"
"time"
- context2 "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -197,11 +197,11 @@ func (r repo) ConvertApiScope() plugin.ToolLayerScope {
// @Router /plugins/github/connections/{connectionId}/remote-scopes [GET]
func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
return remoteHelper.GetScopesFromRemote(input,
- func(basicRes context2.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.GithubConnection) ([]plugin.ApiGroup,
errors.Error) {
+ func(basicRes context.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.GithubConnection) ([]plugin.ApiGroup,
errors.Error) {
if gid != "" {
return nil, nil
}
- apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ apiClient, err :=
api.NewApiClientFromConnection(gocontext.TODO(), basicRes, &connection)
if err != nil {
return nil, errors.BadInput.Wrap(err, "failed
to get create apiClient")
}
@@ -231,11 +231,11 @@ func RemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, er
result = append(result, o)
return result, err
},
- func(basicRes context2.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.GithubConnection) ([]repo,
errors.Error) {
+ func(basicRes context.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.GithubConnection) ([]repo,
errors.Error) {
if gid == "" {
return nil, nil
}
- apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ apiClient, err :=
api.NewApiClientFromConnection(gocontext.TODO(), basicRes, &connection)
if err != nil {
return nil, errors.BadInput.Wrap(err, "failed
to get create apiClient")
}
@@ -276,8 +276,8 @@ func RemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, er
// @Router /plugins/github/connections/{connectionId}/search-remote-scopes
[GET]
func SearchRemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
return remoteHelper.SearchRemoteScopes(input,
- func(basicRes context2.BasicRes, queryData
*api.RemoteQueryData, connection models.GithubConnection) ([]repo,
errors.Error) {
- apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ func(basicRes context.BasicRes, queryData *api.RemoteQueryData,
connection models.GithubConnection) ([]repo, errors.Error) {
+ apiClient, err :=
api.NewApiClientFromConnection(gocontext.TODO(), basicRes, &connection)
if err != nil {
return nil, errors.BadInput.Wrap(err, "failed
to get create apiClient")
}
diff --git a/backend/plugins/gitlab/api/remote.go
b/backend/plugins/gitlab/api/remote.go
index 545e8d26f..12b2004c4 100644
--- a/backend/plugins/gitlab/api/remote.go
+++ b/backend/plugins/gitlab/api/remote.go
@@ -18,12 +18,12 @@ limitations under the License.
package api
import (
- "context"
+ gocontext "context"
"fmt"
"net/http"
"net/url"
- context2 "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -44,8 +44,8 @@ import (
// @Router /plugins/gitlab/connections/{connectionId}/remote-scopes [GET]
func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
return remoteHelper.GetScopesFromRemote(input,
- func(basicRes context2.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.GitlabConnection)
([]models.GroupResponse, errors.Error) {
- apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ func(basicRes context.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.GitlabConnection)
([]models.GroupResponse, errors.Error) {
+ apiClient, err :=
api.NewApiClientFromConnection(gocontext.TODO(), basicRes, &connection)
if err != nil {
return nil, errors.BadInput.Wrap(err, "failed
to get create apiClient")
}
@@ -74,8 +74,8 @@ func RemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, er
return resBody, err
},
- func(basicRes context2.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.GitlabConnection)
([]models.GitlabApiProject, errors.Error) {
- apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ func(basicRes context.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.GitlabConnection)
([]models.GitlabApiProject, errors.Error) {
+ apiClient, err :=
api.NewApiClientFromConnection(gocontext.TODO(), basicRes, &connection)
if err != nil {
return nil, errors.BadInput.Wrap(err, "failed
to get create apiClient")
}
@@ -120,8 +120,8 @@ func RemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, er
// @Router /plugins/gitlab/connections/{connectionId}/search-remote-scopes
[GET]
func SearchRemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
return remoteHelper.SearchRemoteScopes(input,
- func(basicRes context2.BasicRes, queryData
*api.RemoteQueryData, connection models.GitlabConnection)
([]models.GitlabApiProject, errors.Error) {
- apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ func(basicRes context.BasicRes, queryData *api.RemoteQueryData,
connection models.GitlabConnection) ([]models.GitlabApiProject, errors.Error) {
+ apiClient, err :=
api.NewApiClientFromConnection(gocontext.TODO(), basicRes, &connection)
if err != nil {
return nil, errors.BadInput.Wrap(err, "failed
to get create apiClient")
}
diff --git a/backend/plugins/jenkins/api/jobs.go
b/backend/plugins/jenkins/api/jobs.go
index f413a7e32..cccec06b2 100644
--- a/backend/plugins/jenkins/api/jobs.go
+++ b/backend/plugins/jenkins/api/jobs.go
@@ -30,18 +30,11 @@ import (
aha
"github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
)
-func GetJobsPage(
- apiClient aha.ApiClientAbstract,
- path string,
- page int,
- pageSize int,
- callback func(job *models.Job) errors.Error,
-) (int, errors.Error) {
+func GetJobsPage(apiClient aha.ApiClientAbstract, path string, page int,
pageSize int, callback func(job *models.Job) errors.Error) (int, errors.Error) {
i := page * pageSize
var data struct {
Jobs []json.RawMessage `json:"jobs"`
}
-
// set query
query := url.Values{}
treeValue :=
fmt.Sprintf("jobs[name,class,url,color,base,jobs,upstreamProjects[name]]{%d,%d}",
i, i+pageSize)
@@ -62,7 +55,6 @@ func GetJobsPage(
}
return 0, err
}
-
for _, rawJobs := range data.Jobs {
job := &models.Job{}
err1 := json.Unmarshal(rawJobs, job)
@@ -75,16 +67,10 @@ func GetJobsPage(
return len(data.Jobs), err
}
}
-
return len(data.Jobs), nil
}
-func GetJobs(
- apiClient aha.ApiClientAbstract,
- path string,
- pageSize int,
- callback func(job *models.Job) errors.Error,
-) errors.Error {
+func GetJobs(apiClient aha.ApiClientAbstract, path string, pageSize int,
callback func(job *models.Job) errors.Error) errors.Error {
for i := 0; ; i++ {
count, err := GetJobsPage(apiClient, path, i, pageSize,
callback)
if err != nil {
diff --git a/backend/plugins/jenkins/api/remote.go
b/backend/plugins/jenkins/api/remote.go
index 8df6646a9..cd932828a 100644
--- a/backend/plugins/jenkins/api/remote.go
+++ b/backend/plugins/jenkins/api/remote.go
@@ -18,10 +18,11 @@ limitations under the License.
package api
import (
- "context"
+ gocontext "context"
+ "net/http"
"strings"
- context2 "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -41,49 +42,59 @@ import (
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/jenkins/connections/{connectionId}/remote-scopes [GET]
func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
- return remoteHelper.GetScopesFromRemote(input,
- func(basicRes context2.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.JenkinsConnection) ([]models.Job,
errors.Error) {
- apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
- if err != nil {
- return nil, errors.BadInput.Wrap(err, "failed
to get create apiClient")
- }
-
- var resBody []models.Job
- _, err = GetJobsPage(apiClient, gid, queryData.Page-1,
queryData.PerPage, func(job *models.Job) errors.Error {
+ getter := func(basicRes context.BasicRes, groupId string, queryData
*api.RemoteQueryData, connection models.JenkinsConnection)
(*api.RemoteScopesOutput, errors.Error) {
+ var resBody []api.RemoteScopesChild
+ apiClient, err :=
api.NewApiClientFromConnection(gocontext.TODO(), basicRes, &connection)
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "failed to get
create apiClient")
+ }
+ getJobsPageCallBack := func(job *models.Job) errors.Error {
+ if job.Jobs != nil {
// this is a group
- if job.Jobs != nil {
- job.Path = gid
- resBody = append(resBody, *job)
+ job.Path = groupId
+ groupChild := api.RemoteScopesChild{
+ Type: api.TypeGroup,
+ Id: job.GroupId(),
+ Name: job.GroupName(),
+ // don't need to save group into data
+ Data: nil,
}
- return nil
- })
- if err != nil {
- return nil, errors.BadInput.Wrap(err, "failed
to GetJobsPage")
- }
-
- return resBody, err
- },
- func(basicRes context2.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.JenkinsConnection) ([]models.Job,
errors.Error) {
- apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
- if err != nil {
- return nil, errors.BadInput.Wrap(err, "failed
to get create apiClient")
- }
-
- var resBody []models.Job
- _, err = GetJobsPage(apiClient, gid, queryData.Page-1,
queryData.PerPage, func(job *models.Job) errors.Error {
- // this is only a job
- if job.Jobs == nil {
- job.Path = gid
- resBody = append(resBody, *job)
+ groupChild.ParentId = &groupId
+ if *groupChild.ParentId == "" {
+ groupChild.ParentId = nil
}
- return nil
- })
- if err != nil {
- return nil, errors.BadInput.Wrap(err, "failed
to GetJobsPage")
+ resBody = append(resBody, groupChild)
+ } else {
+ // this is a scope
+ scope := job.ConvertApiScope()
+ scopeChild := api.RemoteScopesChild{
+ Type: api.TypeScope,
+ Id: scope.ScopeId(),
+ Name: scope.ScopeName(),
+ FullName: scope.ScopeFullName(),
+ Data: &scope,
+ }
+ scopeChild.ParentId = &groupId
+ if *scopeChild.ParentId == "" {
+ scopeChild.ParentId = nil
+ }
+ resBody = append(resBody, scopeChild)
}
-
- return resBody, err
- })
+ return nil
+ }
+ _, err = GetJobsPage(apiClient, groupId, queryData.Page-1,
queryData.PerPage, getJobsPageCallBack)
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "failed to
GetJobsPage")
+ }
+ return &api.RemoteScopesOutput{
+ Children: resBody,
+ }, nil
+ }
+ remoteScopesOutput, err := remoteHelper.GetRemoteScopesOutput(input,
getter)
+ if err != nil {
+ return nil, err
+ }
+ return &plugin.ApiResourceOutput{Body: remoteScopesOutput, Status:
http.StatusOK}, nil
}
// SearchRemoteScopes use the Search API and only return project
@@ -101,8 +112,8 @@ func RemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, er
// @Router /plugins/jenkins/connections/{connectionId}/search-remote-scopes
[GET]
func SearchRemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
return remoteHelper.SearchRemoteScopes(input,
- func(basicRes context2.BasicRes, queryData
*api.RemoteQueryData, connection models.JenkinsConnection) ([]models.Job,
errors.Error) {
- apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ func(basicRes context.BasicRes, queryData *api.RemoteQueryData,
connection models.JenkinsConnection) ([]models.Job, errors.Error) {
+ apiClient, err :=
api.NewApiClientFromConnection(gocontext.TODO(), basicRes, &connection)
if err != nil {
return nil, errors.BadInput.Wrap(err, "failed
to get create apiClient")
}
diff --git a/backend/plugins/jira/api/remote.go
b/backend/plugins/jira/api/remote.go
index 90883b97b..441aa336d 100644
--- a/backend/plugins/jira/api/remote.go
+++ b/backend/plugins/jira/api/remote.go
@@ -18,11 +18,11 @@ limitations under the License.
package api
import (
- "context"
+ gocontext "context"
"fmt"
"net/url"
- context2 "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -45,10 +45,10 @@ import (
func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
return remoteHelper.GetScopesFromRemote(input,
nil,
- func(basicRes context2.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.JiraConnection) ([]apiv2models.Board,
errors.Error) {
+ func(basicRes context.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.JiraConnection) ([]apiv2models.Board,
errors.Error) {
query := initialQuery(queryData)
// create api client
- apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ apiClient, err :=
api.NewApiClientFromConnection(gocontext.TODO(), basicRes, &connection)
if err != nil {
return nil, err
}
diff --git a/backend/plugins/sonarqube/api/remote.go
b/backend/plugins/sonarqube/api/remote.go
index 7d885e9bc..f836ce093 100644
--- a/backend/plugins/sonarqube/api/remote.go
+++ b/backend/plugins/sonarqube/api/remote.go
@@ -18,9 +18,9 @@ limitations under the License.
package api
import (
- "context"
+ gocontext "context"
"fmt"
- context2 "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -42,10 +42,10 @@ import (
func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
return remoteHelper.GetScopesFromRemote(input,
nil,
- func(basicRes context2.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.SonarqubeConnection)
([]models.SonarqubeApiProject, errors.Error) {
+ func(basicRes context.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.SonarqubeConnection)
([]models.SonarqubeApiProject, errors.Error) {
query := initialQuery(queryData)
// create api client
- apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ apiClient, err :=
api.NewApiClientFromConnection(gocontext.TODO(), basicRes, &connection)
if err != nil {
return nil, err
}
@@ -82,11 +82,11 @@ func RemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, er
// @Router /plugins/sonarqube/connections/{connectionId}/search-remote-scopes
[GET]
func SearchRemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
return remoteHelper.SearchRemoteScopes(input,
- func(basicRes context2.BasicRes, queryData
*api.RemoteQueryData, connection models.SonarqubeConnection)
([]models.SonarqubeApiProject, errors.Error) {
+ func(basicRes context.BasicRes, queryData *api.RemoteQueryData,
connection models.SonarqubeConnection) ([]models.SonarqubeApiProject,
errors.Error) {
query := initialQuery(queryData)
query.Set("q", queryData.Search[0])
// create api client
- apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ apiClient, err :=
api.NewApiClientFromConnection(gocontext.TODO(), basicRes, &connection)
if err != nil {
return nil, err
}
diff --git a/backend/plugins/tapd/api/remote.go
b/backend/plugins/tapd/api/remote.go
index cf3d74ef5..cefa3f2ea 100644
--- a/backend/plugins/tapd/api/remote.go
+++ b/backend/plugins/tapd/api/remote.go
@@ -18,10 +18,10 @@ limitations under the License.
package api
import (
- "context"
+ gocontext "context"
"encoding/json"
"fmt"
- context2 "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -62,12 +62,12 @@ func PrepareFirstPageToken(input *plugin.ApiResourceInput)
(*plugin.ApiResourceO
// @Router /plugins/tapd/connections/{connectionId}/remote-scopes [GET]
func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
return remoteHelper.GetScopesFromRemote(input,
- func(basicRes context2.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.TapdConnection)
([]api.BaseRemoteGroupResponse, errors.Error) {
+ func(basicRes context.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.TapdConnection)
([]api.BaseRemoteGroupResponse, errors.Error) {
if gid == "" {
// if gid is empty, it means we need to query
company
gid = "1"
}
- apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ apiClient, err :=
api.NewApiClientFromConnection(gocontext.TODO(), basicRes, &connection)
if err != nil {
return nil, errors.BadInput.Wrap(err, "failed
to get create apiClient")
}
@@ -107,12 +107,12 @@ func RemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, er
return groups, err
},
- func(basicRes context2.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.TapdConnection)
([]models.TapdWorkspace, errors.Error) {
+ func(basicRes context.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.TapdConnection)
([]models.TapdWorkspace, errors.Error) {
if gid == "" {
return nil, nil
}
- apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ apiClient, err :=
api.NewApiClientFromConnection(gocontext.TODO(), basicRes, &connection)
if err != nil {
return nil, errors.BadInput.Wrap(err, "failed
to get create apiClient")
}
diff --git a/backend/plugins/zentao/api/remote.go
b/backend/plugins/zentao/api/remote.go
index e724e42d3..8e613004d 100644
--- a/backend/plugins/zentao/api/remote.go
+++ b/backend/plugins/zentao/api/remote.go
@@ -18,11 +18,11 @@ limitations under the License.
package api
import (
- "context"
+ gocontext "context"
"fmt"
"net/url"
- context2 "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -42,7 +42,7 @@ func (pr *ProjectResponse) ConvertFix() {
}
}
-func getGroup(basicRes context2.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.ZentaoConnection)
([]api.BaseRemoteGroupResponse, errors.Error) {
+func getGroup(basicRes context.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.ZentaoConnection)
([]api.BaseRemoteGroupResponse, errors.Error) {
return []api.BaseRemoteGroupResponse{
/*{
Id: `products`,
@@ -78,10 +78,10 @@ func RemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, er
} else if gid == `projects` {
return projectRemoteHelper.GetScopesFromRemote(input,
nil,
- func(basicRes context2.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.ZentaoConnection)
([]models.ZentaoProject, errors.Error) {
+ func(basicRes context.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.ZentaoConnection)
([]models.ZentaoProject, errors.Error) {
query := initialQuery(queryData)
// create api client
- apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ apiClient, err :=
api.NewApiClientFromConnection(gocontext.TODO(), basicRes, &connection)
if err != nil {
return nil, err
}
