This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 1f71ae6f fix(gitlab): modify makePipelinePlan
1f71ae6f is described below
commit 1f71ae6f000aa9b1dc41b124f1723f1d3db84afc
Author: Yingchu Chen <[email protected]>
AuthorDate: Thu Oct 13 19:03:18 2022 +0800
fix(gitlab): modify makePipelinePlan
relate to #3127
---
plugins/github/api/blueprint.go | 3 -
plugins/gitlab/api/blueprint.go | 258 +++++++++++++++++------------------
plugins/gitlab/api/blueprint_test.go | 80 +++++++++--
3 files changed, 192 insertions(+), 149 deletions(-)
diff --git a/plugins/github/api/blueprint.go b/plugins/github/api/blueprint.go
index e9dc5d1a..1cd7219b 100644
--- a/plugins/github/api/blueprint.go
+++ b/plugins/github/api/blueprint.go
@@ -163,9 +163,6 @@ func makePipelinePlan(subtaskMetas []core.SubTaskMeta,
scope []*core.BlueprintSc
if j == len(plan) {
plan = append(plan, nil)
}
- if err != nil {
- return nil, err
- }
repo, err = memorizedGetApiRepo()
if err != nil {
return nil, err
diff --git a/plugins/gitlab/api/blueprint.go b/plugins/gitlab/api/blueprint.go
index 8f424127..5e22fb7c 100644
--- a/plugins/gitlab/api/blueprint.go
+++ b/plugins/gitlab/api/blueprint.go
@@ -38,178 +38,172 @@ import (
func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64,
scope []*core.BlueprintScopeV100) (core.PipelinePlan, errors.Error) {
var err errors.Error
- plan := make(core.PipelinePlan, len(scope))
- for i, scopeElem := range scope {
- plan, err = processScope(subtaskMetas, connectionId, scopeElem,
i, plan, nil, nil)
- if err != nil {
- return nil, err
- }
- }
- return plan, nil
-}
-
-func processScope(subtaskMetas []core.SubTaskMeta, connectionId uint64,
scopeElem *core.BlueprintScopeV100, i int, plan core.PipelinePlan, apiRepo
*tasks.GitlabApiProject, connection *models.GitlabConnection)
(core.PipelinePlan, errors.Error) {
- var err errors.Error
- // handle taskOptions and transformationRules, by dumping them to
taskOptions
- transformationRules := make(map[string]interface{})
- if len(scopeElem.Transformation) > 0 {
- err = errors.Convert(json.Unmarshal(scopeElem.Transformation,
&transformationRules))
- if err != nil {
- return nil, err
- }
- }
- // refdiff
- if refdiffRules, ok := transformationRules["refdiff"]; ok &&
refdiffRules != nil {
- // add a new task to next stage
- j := i + 1
- if j == len(plan) {
- plan = append(plan, nil)
- }
- plan[j] = core.PipelineStage{
- {
- Plugin: "refdiff",
- Options: refdiffRules.(map[string]interface{}),
- },
- }
- // remove it from github transformationRules
- delete(transformationRules, "refdiff")
- }
- // construct task options for github
- options := make(map[string]interface{})
- err = errors.Convert(json.Unmarshal(scopeElem.Options, &options))
+ connection := new(models.GitlabConnection)
+ err = connectionHelper.FirstById(connection, connectionId)
if err != nil {
return nil, err
}
- options["connectionId"] = connectionId
- options["transformationRules"] = transformationRules
- // make sure task options is valid
- op, err := tasks.DecodeAndValidateTaskOptions(options)
+ token := strings.Split(connection.Token, ",")[0]
+
+ apiClient, err := helper.NewApiClient(
+ context.TODO(),
+ connection.Endpoint,
+ map[string]string{
+ "Authorization": fmt.Sprintf("Bearer %s", token),
+ },
+ 10*time.Second,
+ connection.Proxy,
+ BasicRes,
+ )
if err != nil {
return nil, err
}
- // construct subtasks
- subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas,
scopeElem.Entities)
+ plan, err := makePipelinePlan(subtaskMetas, scope, apiClient,
connection)
if err != nil {
return nil, err
}
- stage := plan[i]
- if stage == nil {
- stage = core.PipelineStage{}
- }
- stage = append(stage, &core.PipelineTask{
- Plugin: "gitlab",
- Subtasks: subtasks,
- Options: options,
- })
- // collect git data by gitextractor if CODE was requested
- if utils.StringsContains(scopeElem.Entities, core.DOMAIN_TYPE_CODE) {
- // here is the tricky part, we have to obtain the repo id
beforehand
- if connection == nil {
- connection = new(models.GitlabConnection)
- err = connectionHelper.FirstById(connection,
connectionId)
+ return plan, nil
+}
+
+func makePipelinePlan(subtaskMetas []core.SubTaskMeta, scope
[]*core.BlueprintScopeV100, apiClient helper.ApiClientGetter, connection
*models.GitlabConnection) (core.PipelinePlan, errors.Error) {
+ var err errors.Error
+ var repo *tasks.GitlabApiProject
+ plan := make(core.PipelinePlan, len(scope))
+ for i, scopeElem := range scope {
+ // handle taskOptions and transformationRules, by dumping them
to taskOptions
+ transformationRules := make(map[string]interface{})
+ if len(scopeElem.Transformation) > 0 {
+ err =
errors.Convert(json.Unmarshal(scopeElem.Transformation, &transformationRules))
if err != nil {
return nil, err
}
}
- token := strings.Split(connection.Token, ",")[0]
- if apiRepo == nil {
- apiRepo = new(tasks.GitlabApiProject)
- err = getApiRepo(connection, token, op, apiRepo)
- if err != nil {
- return nil, err
+ // refdiff
+ if refdiffRules, ok := transformationRules["refdiff"]; ok &&
refdiffRules != nil {
+ // add a new task to next stage
+ j := i + 1
+ if j == len(plan) {
+ plan = append(plan, nil)
+ }
+ plan[j] = core.PipelineStage{
+ {
+ Plugin: "refdiff",
+ Options:
refdiffRules.(map[string]interface{}),
+ },
}
+ // remove it from github transformationRules
+ delete(transformationRules, "refdiff")
}
- cloneUrl, err :=
errors.Convert01(url.Parse(apiRepo.HttpUrlToRepo))
+ // construct task options for github
+ options := make(map[string]interface{})
+ err = errors.Convert(json.Unmarshal(scopeElem.Options,
&options))
if err != nil {
return nil, err
}
- cloneUrl.User = url.UserPassword("git", token)
- stage = append(stage, &core.PipelineTask{
- Plugin: "gitextractor",
- Options: map[string]interface{}{
- "url": cloneUrl.String(),
- "repoId":
didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connectionId,
apiRepo.GitlabId),
- "proxy": connection.Proxy,
- },
- })
- }
- // dora
- if productionPattern, ok := transformationRules["productionPattern"];
ok && productionPattern != nil {
- j := i + 1
- // add a new task to next stage
- if j == len(plan) {
- plan = append(plan, nil)
+ options["connectionId"] = connection.ID
+ options["transformationRules"] = transformationRules
+ // make sure task options is valid
+ op, err := tasks.DecodeAndValidateTaskOptions(options)
+ if err != nil {
+ return nil, err
}
- if plan[j] != nil {
- j++
+
+ memorizedGetApiRepo := func() (*tasks.GitlabApiProject,
errors.Error) {
+ if repo == nil {
+ repo, err = getApiRepo(op, apiClient)
+ }
+ return repo, err
}
- if j == len(plan) {
- plan = append(plan, nil)
+
+ // construct subtasks
+ subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas,
scopeElem.Entities)
+ if err != nil {
+ return nil, err
}
- if apiRepo == nil {
- if connection == nil {
- connection = new(models.GitlabConnection)
- err = connectionHelper.FirstById(connection,
connectionId)
- if err != nil {
- return nil, err
- }
+ stage := plan[i]
+ if stage == nil {
+ stage = core.PipelineStage{}
+ }
+ stage = append(stage, &core.PipelineTask{
+ Plugin: "gitlab",
+ Subtasks: subtasks,
+ Options: options,
+ })
+ // collect git data by gitextractor if CODE was requested
+ if utils.StringsContains(scopeElem.Entities,
core.DOMAIN_TYPE_CODE) {
+ // here is the tricky part, we have to obtain the repo
id beforehand
+ repo, err = memorizedGetApiRepo()
+ if err != nil {
+ return nil, err
}
- token := strings.Split(connection.Token, ",")[0]
- apiRepo = new(tasks.GitlabApiProject)
- err = getApiRepo(connection, token, op, apiRepo)
+ cloneUrl, err :=
errors.Convert01(url.Parse(repo.HttpUrlToRepo))
if err != nil {
return nil, err
}
+ cloneUrl.User = url.UserPassword("git",
connection.Token)
+ stage = append(stage, &core.PipelineTask{
+ Plugin: "gitextractor",
+ Options: map[string]interface{}{
+ "url": cloneUrl.String(),
+ "repoId":
didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connection.ID,
repo.GitlabId),
+ "proxy": connection.Proxy,
+ },
+ })
}
- doraOption := make(map[string]interface{})
- doraOption["repoId"] =
didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connectionId,
apiRepo.GitlabId)
- doraOption["tasks"] = []string{"EnrichTaskEnv"}
- doraRules := make(map[string]interface{})
- doraRules["productionPattern"] = productionPattern
- doraOption["transformationRules"] = doraRules
- plan[j] = core.PipelineStage{
- {
- Plugin: "dora",
- Subtasks: []string{"EnrichTaskEnv"},
- Options: doraOption,
- },
+ // dora
+ if productionPattern, ok :=
transformationRules["productionPattern"]; ok && productionPattern != nil {
+ j := i + 1
+ // add a new task to next stage
+ if j == len(plan) {
+ plan = append(plan, nil)
+ }
+ if plan[j] != nil {
+ j++
+ }
+ if j == len(plan) {
+ plan = append(plan, nil)
+ }
+ repo, err = memorizedGetApiRepo()
+ if err != nil {
+ return nil, err
+ }
+ plan[j] = core.PipelineStage{
+ {
+ Plugin: "dora",
+ Subtasks: []string{"EnrichTaskEnv"},
+ Options: map[string]interface{}{
+ "repoId":
didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connection.ID,
repo.GitlabId),
+ "transformationRules":
map[string]interface{}{
+ "productionPattern":
productionPattern,
+ },
+ },
+ },
+ }
+ // remove it from github transformationRules
+ delete(transformationRules, "productionPattern")
}
- // remove it from github transformationRules
- delete(transformationRules, "productionPattern")
+ plan[i] = stage
}
- plan[i] = stage
return plan, nil
}
-func getApiRepo(connection *models.GitlabConnection, token string, op
*tasks.GitlabOptions, apiRepo *tasks.GitlabApiProject) errors.Error {
- apiClient, err := helper.NewApiClient(
- context.TODO(),
- connection.Endpoint,
- map[string]string{
- "Authorization": fmt.Sprintf("Bearer %s", token),
- },
- 10*time.Second,
- connection.Proxy,
- BasicRes,
- )
- if err != nil {
- return err
- }
+func getApiRepo(op *tasks.GitlabOptions, apiClient helper.ApiClientGetter)
(*tasks.GitlabApiProject, errors.Error) {
+ apiRepo := &tasks.GitlabApiProject{}
res, err := apiClient.Get(fmt.Sprintf("projects/%d", op.ProjectId),
nil, nil)
if err != nil {
- return err
+ return nil, err
}
- //defer res.Body.Close()
+ defer res.Body.Close()
if res.StatusCode != http.StatusOK {
- return
errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("unexpected status code when
requesting repo detail from %s", res.Request.URL.String()))
+ return nil,
errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("unexpected status code when
requesting repo detail from %s", res.Request.URL.String()))
}
body, err := errors.Convert01(io.ReadAll(res.Body))
if err != nil {
- return err
+ return nil, err
}
err = errors.Convert(json.Unmarshal(body, apiRepo))
if err != nil {
- return err
+ return nil, err
}
- return nil
+ return apiRepo, nil
}
diff --git a/plugins/gitlab/api/blueprint_test.go
b/plugins/gitlab/api/blueprint_test.go
index 2afe9848..721a5e9a 100644
--- a/plugins/gitlab/api/blueprint_test.go
+++ b/plugins/gitlab/api/blueprint_test.go
@@ -18,6 +18,7 @@ limitations under the License.
package api
import (
+ "bytes"
"encoding/json"
"github.com/apache/incubator-devlake/mocks"
"github.com/apache/incubator-devlake/models/common"
@@ -26,6 +27,9 @@ import (
"github.com/apache/incubator-devlake/plugins/gitlab/tasks"
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/mock"
+ "io"
+ "net/http"
"testing"
)
@@ -46,14 +50,26 @@ func TestProcessScope(t *testing.T) {
Token: "123",
},
}
+ mockApiCLient := mocks.NewApiClientGetter(t)
+ repo := &tasks.GitlabApiProject{
+ GitlabId: 12345,
+ HttpUrlToRepo: "*this is HttpUrlToRepo*",
+ }
+ js, err := json.Marshal(repo)
+ assert.Nil(t, err)
+ res := &http.Response{}
+ res.Body = io.NopCloser(bytes.NewBuffer(js))
+ res.StatusCode = http.StatusOK
+ mockApiCLient.On("Get", "projects/12345", mock.Anything,
mock.Anything).Return(res, nil)
+
mockMeta := mocks.NewPluginMeta(t)
mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/gitlab")
- err := core.RegisterPlugin("gitlab", mockMeta)
+ err = core.RegisterPlugin("gitlab", mockMeta)
assert.Nil(t, err)
bs := &core.BlueprintScopeV100{
Entities: []string{"CODE"},
Options: json.RawMessage(`{
- "projectId": 123
+ "projectId": 12345
}`),
Transformation: json.RawMessage(`{
"prType": "hey,man,wasup",
@@ -65,19 +81,55 @@ func TestProcessScope(t *testing.T) {
"productionPattern": "xxxx"
}`),
}
- apiRepo := &tasks.GitlabApiProject{
- GitlabId: 123,
- HttpUrlToRepo: "HttpUrlToRepo",
- }
scopes := make([]*core.BlueprintScopeV100, 0)
scopes = append(scopes, bs)
- plan := make(core.PipelinePlan, len(scopes))
- for i, scopeElem := range scopes {
- plan, err = processScope(nil, 1, scopeElem, i, plan, apiRepo,
connection)
- assert.Nil(t, err)
+ plan, err := makePipelinePlan(nil, scopes, mockApiCLient, connection)
+ assert.Nil(t, err)
+
+ expectPlan := core.PipelinePlan{
+ core.PipelineStage{
+ {
+ Plugin: "gitlab",
+ Subtasks: []string{},
+ Options: map[string]interface{}{
+ "connectionId": uint64(1),
+ "projectId": float64(12345),
+ "transformationRules":
map[string]interface{}{
+ "prType": "hey,man,wasup",
+ },
+ },
+ },
+ {
+ Plugin: "gitextractor",
+ Options: map[string]interface{}{
+ "proxy": "",
+ "repoId":
"gitlab:GitlabProject:1:12345",
+ "url":
"//git:123@%2Athis%20is%20HttpUrlToRepo%2A",
+ },
+ },
+ },
+ core.PipelineStage{
+ {
+ Plugin: "refdiff",
+ Options: map[string]interface{}{
+ "tagsLimit": float64(10),
+ "tagsOrder": "reverse semver",
+ "tagsPattern": "pattern",
+ },
+ },
+ },
+ core.PipelineStage{
+ {
+ Plugin: "dora",
+ Subtasks: []string{"EnrichTaskEnv"},
+ Options: map[string]interface{}{
+ "repoId":
"gitlab:GitlabProject:1:12345",
+ "transformationRules":
map[string]interface{}{
+ "productionPattern": "xxxx",
+ },
+ },
+ },
+ },
}
- planJson, err1 := json.Marshal(plan)
- assert.Nil(t, err1)
- expectPlan :=
`[[{"plugin":"gitlab","subtasks":[],"options":{"connectionId":1,"projectId":123,"transformationRules":{"prType":"hey,man,wasup"}}},{"plugin":"gitextractor","subtasks":null,"options":{"proxy":"","repoId":"gitlab:GitlabProject:1:123","url":"//git:123@HttpUrlToRepo"}}],[{"plugin":"refdiff","subtasks":null,"options":{"tagsLimit":10,"tagsOrder":"reverse
semver","tagsPattern":"pattern"}}],[{"plugin":"dora","subtasks":["EnrichTaskEnv"],"options":{"repoId":"gitlab:GitlabProject:1:
[...]
- assert.Equal(t, expectPlan, string(planJson))
+ assert.Equal(t, expectPlan, plan)
}
