This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new d7d6423a fix(bitbucket): modify blueprint
d7d6423a is described below
commit d7d6423a7510baf17ec4957b5907a55a8ee7140d
Author: Yingchu Chen <[email protected]>
AuthorDate: Thu Oct 13 20:35:18 2022 +0800
fix(bitbucket): modify blueprint
---
plugins/bitbucket/api/blueprint.go | 99 +++++++++++++++-------------
plugins/bitbucket/api/blueprint_test.go | 111 ++++++++++++++++++++++++++------
plugins/github/api/blueprint.go | 3 -
plugins/github/api/blueprint_test.go | 4 +-
plugins/gitlab/api/blueprint.go | 1 +
plugins/gitlab/api/blueprint_test.go | 4 +-
plugins/jenkins/api/blueprint_test.go | 5 +-
7 files changed, 154 insertions(+), 73 deletions(-)
diff --git a/plugins/bitbucket/api/blueprint.go
b/plugins/bitbucket/api/blueprint.go
index 825670e6..a79c584e 100644
--- a/plugins/bitbucket/api/blueprint.go
+++ b/plugins/bitbucket/api/blueprint.go
@@ -37,21 +37,15 @@ import (
"github.com/apache/incubator-devlake/utils"
)
-type repoGetter func(connectionId uint64, owner, repo string) (string, string,
errors.Error)
-
func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64,
scope []*core.BlueprintScopeV100) (core.PipelinePlan, errors.Error) {
- return makePipelinePlan(subtaskMetas, connectionId,
getBitbucketApiRepo, scope)
-}
-func getBitbucketApiRepo(connectionId uint64, owner, repo string) (string,
string, errors.Error) {
- // here is the tricky part, we have to obtain the repo id beforehand
connection := new(models.BitbucketConnection)
err := connectionHelper.FirstById(connection, connectionId)
if err != nil {
- return "", "", err
+ return nil, err
}
tokens := strings.Split(connection.GetEncodedToken(), ",")
if len(tokens) == 0 {
- return "", "", errors.Default.New("no token")
+ return nil, errors.Default.New("no token")
}
token := tokens[0]
apiClient, err := helper.NewApiClient(
@@ -65,40 +59,15 @@ func getBitbucketApiRepo(connectionId uint64, owner, repo
string) (string, strin
basicRes,
)
if err != nil {
- return "", "", err
- }
-
- res, err := apiClient.Get(path.Join("repositories", owner, repo), nil,
nil)
- if err != nil {
- return "", "", err
- }
- defer res.Body.Close()
- if res.StatusCode != http.StatusOK {
- return "", "", errors.Default.New(fmt.Sprintf(
- "unexpected status code when requesting repo detail %d
%s",
- res.StatusCode, res.Request.URL.String(),
- ))
- }
- body, err := errors.Convert01(io.ReadAll(res.Body))
- if err != nil {
- return "", "", err
- }
- apiRepo := new(tasks.BitbucketApiRepo)
- err = errors.Convert(json.Unmarshal(body, apiRepo))
- if err != nil {
- return "", "", err
+ return nil, err
}
- for _, u := range apiRepo.Links.Clone {
- if u.Name == "https" {
- return u.Href, connection.Password, nil
- }
- }
- return "", "", errors.Default.New("no clone url")
+ return makePipelinePlan(subtaskMetas, scope, apiClient, connection)
}
-func makePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64,
getter repoGetter, scope []*core.BlueprintScopeV100) (core.PipelinePlan,
errors.Error) {
+func makePipelinePlan(subtaskMetas []core.SubTaskMeta, scope
[]*core.BlueprintScopeV100, apiClient helper.ApiClientGetter, connection
*models.BitbucketConnection) (core.PipelinePlan, errors.Error) {
var err errors.Error
plan := make(core.PipelinePlan, len(scope))
+ var repo *tasks.BitbucketApiRepo
for i, scopeElem := range scope {
// handle taskOptions and transformationRules, by dumping them
to taskOptions
transformationRules := make(map[string]interface{})
@@ -130,13 +99,19 @@ func makePipelinePlan(subtaskMetas []core.SubTaskMeta,
connectionId uint64, gett
if err != nil {
return nil, err
}
- options["connectionId"] = connectionId
+ options["connectionId"] = connection.ID
options["transformationRules"] = transformationRules
// make sure task options is valid
op, err := tasks.DecodeAndValidateTaskOptions(options)
if err != nil {
return nil, err
}
+ memorizedGetApiRepo := func() (*tasks.BitbucketApiRepo,
errors.Error) {
+ if repo == nil {
+ repo, err = getApiRepo(op, apiClient)
+ }
+ return repo, err
+ }
// construct subtasks
subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas,
scopeElem.Entities)
if err != nil {
@@ -153,25 +128,61 @@ func makePipelinePlan(subtaskMetas []core.SubTaskMeta,
connectionId uint64, gett
})
// collect git data by gitextractor if CODE was requested
if utils.StringsContains(scopeElem.Entities,
core.DOMAIN_TYPE_CODE) {
- original, password, err1 := getter(connectionId,
op.Owner, op.Repo)
- if err1 != nil {
- return nil, err1
+ repo, err = memorizedGetApiRepo()
+ if err != nil {
+ return nil, err
+ }
+ originalUrl := ""
+ for _, u := range repo.Links.Clone {
+ if u.Name == "https" {
+ originalUrl = u.Href
+ }
}
- cloneUrl, err := errors.Convert01(url.Parse(original))
+ cloneUrl, err :=
errors.Convert01(url.Parse(originalUrl))
if err != nil {
return nil, err
}
- cloneUrl.User = url.UserPassword(op.Owner, password)
+ cloneUrl.User = url.UserPassword(op.Owner,
connection.Password)
stage = append(stage, &core.PipelineTask{
Plugin: "gitextractor",
Options: map[string]interface{}{
"url": cloneUrl.String(),
- "repoId":
didgen.NewDomainIdGenerator(&models.BitbucketRepo{}).Generate(connectionId,
fmt.Sprintf("%s/%s", op.Owner, op.Repo)),
+ "repoId":
didgen.NewDomainIdGenerator(&models.BitbucketRepo{}).Generate(connection.ID,
fmt.Sprintf("%s/%s", op.Owner, op.Repo)),
},
})
}
plan[i] = stage
+ repo = nil
}
return plan, nil
}
+
+func getApiRepo(op *tasks.BitbucketOptions, apiClient helper.ApiClientGetter)
(*tasks.BitbucketApiRepo, errors.Error) {
+ res, err := apiClient.Get(path.Join("repositories", op.Owner, op.Repo),
nil, nil)
+ if err != nil {
+ return nil, err
+ }
+ defer res.Body.Close()
+ if res.StatusCode != http.StatusOK {
+ return nil, errors.Default.New(fmt.Sprintf(
+ "unexpected status code when requesting repo detail %d
%s",
+ res.StatusCode, res.Request.URL.String(),
+ ))
+ }
+ body, err := errors.Convert01(io.ReadAll(res.Body))
+ if err != nil {
+ return nil, err
+ }
+ apiRepo := new(tasks.BitbucketApiRepo)
+ err = errors.Convert(json.Unmarshal(body, apiRepo))
+ if err != nil {
+ return nil, err
+ }
+ for _, u := range apiRepo.Links.Clone {
+ if u.Name == "https" {
+ return apiRepo, nil
+ }
+ }
+ return nil, errors.Default.New("no clone url")
+}
diff --git a/plugins/bitbucket/api/blueprint_test.go
b/plugins/bitbucket/api/blueprint_test.go
index 14e284a3..0e2e0932 100644
--- a/plugins/bitbucket/api/blueprint_test.go
+++ b/plugins/bitbucket/api/blueprint_test.go
@@ -18,19 +18,81 @@ limitations under the License.
package api
import (
+ "bytes"
+ "encoding/json"
"github.com/apache/incubator-devlake/mocks"
- "testing"
-
- "github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/models/common"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/models"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/tasks"
"github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/mock"
+ "io"
+ "net/http"
+ "path"
+ "testing"
)
func TestMakePipelinePlan(t *testing.T) {
- var mockGetter repoGetter
- mockGetter = func(connectionId uint64, owner, repo string) (string,
string, errors.Error) {
- return "https://[email protected]/thenicetgp/lake.git";,
"secret", nil
+ connection := &models.BitbucketConnection{
+ RestConnection: helper.RestConnection{
+ BaseConnection: helper.BaseConnection{
+ Name: "github-test",
+ Model: common.Model{
+ ID: 1,
+ },
+ },
+ Endpoint: "https://TestBitBucket/";,
+ Proxy: "",
+ RateLimitPerHour: 0,
+ },
+ BasicAuth: helper.BasicAuth{
+ Username: "Username",
+ Password: "Password",
+ },
}
+
+ mockApiCLient := mocks.NewApiClientGetter(t)
+ repo := &tasks.BitbucketApiRepo{
+ Links: struct {
+ Clone []struct {
+ Href string `json:"href"`
+ Name string `json:"name"`
+ } `json:"clone"`
+ Self struct {
+ Href string `json:"href"`
+ } `json:"self"`
+ Html struct {
+ Href string `json:"href"`
+ } `json:"html"`
+ }{
+ Clone: []struct {
+ Href string `json:"href"`
+ Name string `json:"name"`
+ }{
+ {
+ Href:
"https://bitbucket.org/thenicetgp/lake.git";,
+ Name: "https",
+ },
+ },
+ Self: struct {
+ Href string `json:"href"`
+ }{},
+ Html: struct {
+ Href string `json:"href"`
+ }{},
+ },
+ }
+ js, err := json.Marshal(repo)
+ assert.Nil(t, err)
+ res := &http.Response{}
+ res.Body = io.NopCloser(bytes.NewBuffer(js))
+ res.StatusCode = http.StatusOK
+ mockApiCLient.On("Get", path.Join("repositories", "thenicetgp",
"lake"), mock.Anything, mock.Anything).Return(res, nil)
+ mockMeta := mocks.NewPluginMeta(t)
+
mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/bitbucket")
+ err = core.RegisterPlugin("bitbucket", mockMeta)
scope := &core.BlueprintScopeV100{
Entities: []string{core.DOMAIN_TYPE_CODE,
core.DOMAIN_TYPE_TICKET, core.DOMAIN_TYPE_CODE_REVIEW, core.DOMAIN_TYPE_CROSS},
Options: []byte(`{
@@ -39,19 +101,32 @@ func TestMakePipelinePlan(t *testing.T) {
}`),
Transformation: nil,
}
- mockMeta := mocks.NewPluginMeta(t)
-
mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/bitbucket")
- err := core.RegisterPlugin("bitbucket", mockMeta)
+ scopes := make([]*core.BlueprintScopeV100, 0)
+ scopes = append(scopes, scope)
assert.Nil(t, err)
- plan, err := makePipelinePlan(nil, 1, mockGetter,
[]*core.BlueprintScopeV100{scope})
+ plan, err := makePipelinePlan(nil, scopes, mockApiCLient, connection)
assert.Nil(t, err)
- for _, stage := range plan {
- for _, task := range stage {
- if task.Plugin == "gitextractor" {
- assert.Equal(t, task.Options["url"],
"https://thenicetgp:[email protected]/thenicetgp/lake.git";)
- return
- }
- }
+
+ expectPlan := core.PipelinePlan{
+ core.PipelineStage{
+ {
+ Plugin: "bitbucket",
+ Subtasks: []string{},
+ Options: map[string]interface{}{
+ "connectionId": uint64(1),
+ "owner": "thenicetgp",
+ "repo": "lake",
+ "transformationRules":
map[string]interface{}{},
+ },
+ },
+ {
+ Plugin: "gitextractor",
+ Options: map[string]interface{}{
+ "repoId":
"bitbucket:BitbucketRepo:1:thenicetgp/lake",
+ "url":
"https://thenicetgp:[email protected]/thenicetgp/lake.git";,
+ },
+ },
+ },
}
- t.Fatal("no gitextractor plugin")
+ assert.Equal(t, expectPlan, plan)
}
diff --git a/plugins/github/api/blueprint.go b/plugins/github/api/blueprint.go
index 1cd7219b..aadabfbd 100644
--- a/plugins/github/api/blueprint.go
+++ b/plugins/github/api/blueprint.go
@@ -184,9 +184,6 @@ func makePipelinePlan(subtaskMetas []core.SubTaskMeta,
scope []*core.BlueprintSc
delete(transformationRules, "productionPattern")
}
plan[i] = stage
- if err != nil {
- return nil, err
- }
repo = nil
}
return plan, nil
diff --git a/plugins/github/api/blueprint_test.go
b/plugins/github/api/blueprint_test.go
index c663e3ba..9a35c14d 100644
--- a/plugins/github/api/blueprint_test.go
+++ b/plugins/github/api/blueprint_test.go
@@ -53,7 +53,7 @@ func TestProcessScope(t *testing.T) {
mockApiCLient := mocks.NewApiClientGetter(t)
repo := &tasks.GithubApiRepo{
GithubId: 12345,
- CloneUrl: "*this is cloneUrl*",
+ CloneUrl: "https://this_is_cloneUrl";,
}
js, err := json.Marshal(repo)
assert.Nil(t, err)
@@ -105,7 +105,7 @@ func TestProcessScope(t *testing.T) {
Options: map[string]interface{}{
"proxy": "",
"repoId": "github:GithubRepo:1:12345",
- "url":
"//git:123@%2Athis%20is%20cloneUrl%2A",
+ "url":
"https://git:123@this_is_cloneUrl";,
},
},
},
diff --git a/plugins/gitlab/api/blueprint.go b/plugins/gitlab/api/blueprint.go
index 5e22fb7c..d10ffbb3 100644
--- a/plugins/gitlab/api/blueprint.go
+++ b/plugins/gitlab/api/blueprint.go
@@ -183,6 +183,7 @@ func makePipelinePlan(subtaskMetas []core.SubTaskMeta,
scope []*core.BlueprintSc
delete(transformationRules, "productionPattern")
}
plan[i] = stage
+ repo = nil
}
return plan, nil
}
diff --git a/plugins/gitlab/api/blueprint_test.go
b/plugins/gitlab/api/blueprint_test.go
index 721a5e9a..4fbb255f 100644
--- a/plugins/gitlab/api/blueprint_test.go
+++ b/plugins/gitlab/api/blueprint_test.go
@@ -53,7 +53,7 @@ func TestProcessScope(t *testing.T) {
mockApiCLient := mocks.NewApiClientGetter(t)
repo := &tasks.GitlabApiProject{
GitlabId: 12345,
- HttpUrlToRepo: "*this is HttpUrlToRepo*",
+ HttpUrlToRepo: "https://this_is_HttpUrlToRepo";,
}
js, err := json.Marshal(repo)
assert.Nil(t, err)
@@ -104,7 +104,7 @@ func TestProcessScope(t *testing.T) {
Options: map[string]interface{}{
"proxy": "",
"repoId":
"gitlab:GitlabProject:1:12345",
- "url":
"//git:123@%2Athis%20is%20HttpUrlToRepo%2A",
+ "url":
"https://git:123@this_is_HttpUrlToRepo";,
},
},
},
diff --git a/plugins/jenkins/api/blueprint_test.go
b/plugins/jenkins/api/blueprint_test.go
index b509709a..2ebac544 100644
--- a/plugins/jenkins/api/blueprint_test.go
+++ b/plugins/jenkins/api/blueprint_test.go
@@ -45,10 +45,7 @@ func TestProcessScope(t *testing.T) {
Password: "Password",
},
}
- //mockMeta := mocks.NewPluginMeta(t)
-
//mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/jenkins")
- //err := core.RegisterPlugin("jenkins", mockMeta)
- //assert.Nil(t, err)
+
bs := &core.BlueprintScopeV100{
Entities: []string{"CICD"},
Options: json.RawMessage(`{
