This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new d302616e feat(github): modify unit test
d302616e is described below
commit d302616efa9e9123cb38d7fc8a70faa5a59a3654
Author: Yingchu Chen <[email protected]>
AuthorDate: Tue Sep 20 20:12:35 2022 +0800
feat(github): modify unit test
---
Makefile | 1 -
plugins/github/api/blueprint.go | 241 +++++++++++++++++------------------
plugins/github/api/blueprint_test.go | 72 ++++++-----
plugins/gitlab/api/blueprint.go | 1 +
4 files changed, 158 insertions(+), 157 deletions(-)
diff --git a/Makefile b/Makefile
index b9b70ebf..6bd35581 100644
--- a/Makefile
+++ b/Makefile
@@ -94,7 +94,6 @@ mock:
mockery --dir=./plugins/core --unroll-variadic=false --name='.*'
mockery --dir=./plugins/core/dal --unroll-variadic=false --name='.*'
mockery --dir=./plugins/helper --unroll-variadic=false --name='.*'
- mockery --dir=./plugins/github/api --unroll-variadic=false --name='.*'
test: unit-test e2e-test
diff --git a/plugins/github/api/blueprint.go b/plugins/github/api/blueprint.go
index 55f23592..e8eeffe2 100644
--- a/plugins/github/api/blueprint.go
+++ b/plugins/github/api/blueprint.go
@@ -37,146 +37,147 @@ import (
)
func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64,
scope []*core.BlueprintScopeV100) (core.PipelinePlan, errors.Error) {
- repoHelper := RepoHelper{}
- plan, err := DoMakePipeline(subtaskMetas, connectionId, scope,
repoHelper)
- if err != nil {
- return nil, err
- }
- return plan, nil
-}
-
-func DoMakePipeline(subtaskMetas []core.SubTaskMeta, connectionId uint64,
scope []*core.BlueprintScopeV100, repoHelper BpHelper) (core.PipelinePlan,
errors.Error) {
var err errors.Error
- apiRepo, _, _ := &tasks.GithubApiRepo{}, "", ""
plan := make(core.PipelinePlan, len(scope))
for i, scopeElem := range scope {
- // handle taskOptions and transformationRules, by dumping them
to taskOptions
- transformationRules := make(map[string]interface{})
- if len(scopeElem.Transformation) > 0 {
- err =
errors.Convert(json.Unmarshal(scopeElem.Transformation, &transformationRules))
- if err != nil {
- return nil, err
- }
- }
- // refdiff
- if refdiffRules, ok := transformationRules["refdiff"]; ok &&
refdiffRules != nil {
- // add a new task to next stage
- j := i + 1
- if j == len(plan) {
- plan = append(plan, nil)
- }
- plan[j] = core.PipelineStage{
- {
- Plugin: "refdiff",
- Options:
refdiffRules.(map[string]interface{}),
- },
- }
- // remove it from github transformationRules
- delete(transformationRules, "refdiff")
- }
- // construct task options for github
- options := make(map[string]interface{})
- err = errors.Convert(json.Unmarshal(scopeElem.Options,
&options))
+ plan, err = processScope(subtaskMetas, connectionId, scopeElem,
i, plan, nil, nil)
if err != nil {
return nil, err
}
- options["connectionId"] = connectionId
- options["transformationRules"] = transformationRules
- // make sure task options is valid
- op, err := tasks.DecodeAndValidateTaskOptions(options)
+ }
+ return plan, nil
+}
+func processScope(subtaskMetas []core.SubTaskMeta, connectionId uint64,
scopeElem *core.BlueprintScopeV100, i int, plan core.PipelinePlan, apiRepo
*tasks.GithubApiRepo, connection *models.GithubConnection) (core.PipelinePlan,
errors.Error) {
+ var err errors.Error
+ // handle taskOptions and transformationRules, by dumping them to
taskOptions
+ transformationRules := make(map[string]interface{})
+ if len(scopeElem.Transformation) > 0 {
+ err = errors.Convert(json.Unmarshal(scopeElem.Transformation,
&transformationRules))
if err != nil {
return nil, err
}
- // construct subtasks
- subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas,
scopeElem.Entities)
- if err != nil {
- return nil, err
+ }
+ // refdiff
+ if refdiffRules, ok := transformationRules["refdiff"]; ok &&
refdiffRules != nil {
+ // add a new task to next stage
+ j := i + 1
+ if j == len(plan) {
+ plan = append(plan, nil)
}
- stage := plan[i]
- if stage == nil {
- stage = core.PipelineStage{}
+ plan[j] = core.PipelineStage{
+ {
+ Plugin: "refdiff",
+ Options: refdiffRules.(map[string]interface{}),
+ },
}
- stage = append(stage, &core.PipelineTask{
- Plugin: "github",
- Subtasks: subtasks,
- Options: options,
- })
- // collect git data by gitextractor if CODE was requested
- if utils.StringsContains(scopeElem.Entities,
core.DOMAIN_TYPE_CODE) {
- // here is the tricky part, we have to obtain the repo
id beforehand
- res, token, proxy, err :=
repoHelper.GetApiRepo(connectionId, op)
- apiRepo = res.(*tasks.GithubApiRepo)
- if err != nil {
- return nil, err
- }
- cloneUrl, err :=
errors.Convert01(url.Parse(apiRepo.CloneUrl))
+ // remove it from github transformationRules
+ delete(transformationRules, "refdiff")
+ }
+ // construct task options for github
+ options := make(map[string]interface{})
+ err = errors.Convert(json.Unmarshal(scopeElem.Options, &options))
+ if err != nil {
+ return nil, err
+ }
+ options["connectionId"] = connectionId
+ options["transformationRules"] = transformationRules
+ // make sure task options is valid
+ op, err := tasks.DecodeAndValidateTaskOptions(options)
+ if err != nil {
+ return nil, err
+ }
+ // construct subtasks
+ subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas,
scopeElem.Entities)
+ if err != nil {
+ return nil, err
+ }
+ stage := plan[i]
+ if stage == nil {
+ stage = core.PipelineStage{}
+ }
+ stage = append(stage, &core.PipelineTask{
+ Plugin: "github",
+ Subtasks: subtasks,
+ Options: options,
+ })
+ // collect git data by gitextractor if CODE was requested
+ if utils.StringsContains(scopeElem.Entities, core.DOMAIN_TYPE_CODE) {
+ // here is the tricky part, we have to obtain the repo id
beforehand
+ if connection == nil {
+ connection = new(models.GithubConnection)
+ err = connectionHelper.FirstById(connection,
connectionId)
if err != nil {
return nil, err
}
- cloneUrl.User = url.UserPassword("git", token)
- stage = append(stage, &core.PipelineTask{
- Plugin: "gitextractor",
- Options: map[string]interface{}{
- "url": cloneUrl.String(),
- "repoId":
didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connectionId,
apiRepo.GithubId),
- "proxy": proxy,
- },
- })
}
- plan[i] = stage
- // dora
- if doraRules, ok := transformationRules["dora"]; ok &&
doraRules != nil {
- j := i + 1
- // add a new task to next stage
- if plan[j] != nil {
- j++
- }
- if j == len(plan) {
- plan = append(plan, nil)
- }
+ token := strings.Split(connection.Token, ",")[0]
+ if apiRepo == nil {
+ apiRepo = new(tasks.GithubApiRepo)
+ err = getApiRepo(connection, token, op, apiRepo)
if err != nil {
return nil, err
}
- if apiRepo.GithubId == 0 {
- res, _, _, err :=
repoHelper.GetApiRepo(connectionId, op)
+ }
+ cloneUrl, err := errors.Convert01(url.Parse(apiRepo.CloneUrl))
+ if err != nil {
+ return nil, err
+ }
+ cloneUrl.User = url.UserPassword("git", token)
+ stage = append(stage, &core.PipelineTask{
+ Plugin: "gitextractor",
+ Options: map[string]interface{}{
+ "url": cloneUrl.String(),
+ "repoId":
didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connectionId,
apiRepo.GithubId),
+ "proxy": connection.Proxy,
+ },
+ })
+ }
+ // dora
+ if doraRules, ok := transformationRules["dora"]; ok && doraRules != nil
{
+ j := i + 1
+ // add a new task to next stage
+ if plan[j] != nil {
+ j++
+ }
+ if j == len(plan) {
+ plan = append(plan, nil)
+ }
+ if err != nil {
+ return nil, err
+ }
+ if apiRepo == nil {
+ if connection == nil {
+ connection = new(models.GithubConnection)
+ err = connectionHelper.FirstById(connection,
connectionId)
if err != nil {
return nil, err
}
- apiRepo = res.(*tasks.GithubApiRepo)
-
}
- doraOption := make(map[string]interface{})
- doraOption["repoId"] =
didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connectionId,
apiRepo.GithubId)
- doraOption["tasks"] = []string{"EnrichTaskEnv"}
- doraOption["transformation"] = doraRules
- plan[j] = core.PipelineStage{
- {
- Plugin: "dora",
- Options: doraOption,
- },
+ token := strings.Split(connection.Token, ",")[0]
+ apiRepo = new(tasks.GithubApiRepo)
+ err = getApiRepo(connection, token, op, apiRepo)
+ if err != nil {
+ return nil, err
}
- // remove it from github transformationRules
- delete(transformationRules, "dora")
}
+ doraOption := make(map[string]interface{})
+ doraOption["repoId"] =
didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connectionId,
apiRepo.GithubId)
+ doraOption["tasks"] = []string{"EnrichTaskEnv"}
+ doraOption["transformation"] = doraRules
+ plan[j] = core.PipelineStage{
+ {
+ Plugin: "dora",
+ Options: doraOption,
+ },
+ }
+ // remove it from github transformationRules
+ delete(transformationRules, "dora")
}
+ plan[i] = stage
return plan, nil
}
-type BpHelper interface {
- GetApiRepo(connectionId uint64, op interface{}) (interface{}, string,
string, errors.Error)
-}
-
-type RepoHelper struct{}
-
-func (c RepoHelper) GetApiRepo(connectionId uint64, originOp interface{})
(interface{}, string, string, errors.Error) {
- op := originOp.(*tasks.GithubOptions)
- // here is the tricky part, we have to obtain the repo id beforehand
- connection := new(models.GithubConnection)
- err := connectionHelper.FirstById(connection, connectionId)
- if err != nil {
- return nil, "", "", err
- }
- token := strings.Split(connection.Token, ",")[0]
+func getApiRepo(connection *models.GithubConnection, token string, op
*tasks.GithubOptions, apiRepo *tasks.GithubApiRepo) errors.Error {
apiClient, err := helper.NewApiClient(
context.TODO(),
connection.Endpoint,
@@ -188,25 +189,23 @@ func (c RepoHelper) GetApiRepo(connectionId uint64,
originOp interface{}) (inter
basicRes,
)
if err != nil {
- return nil, "", "", err
+ return err
}
res, err := apiClient.Get(fmt.Sprintf("repos/%s/%s", op.Owner,
op.Repo), nil, nil)
if err != nil {
- return nil, "", "", err
+ return err
}
- defer res.Body.Close()
+ //defer res.Body.Close()
if res.StatusCode != http.StatusOK {
- return nil, "", "",
errors.HttpStatus(res.StatusCode).New(fmt.Sprintf(
- "unexpected status code when requesting repo detail
from %s", res.Request.URL.String()))
+ return
errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("unexpected status code when
requesting repo detail from %s", res.Request.URL.String()))
}
body, err := errors.Convert01(io.ReadAll(res.Body))
if err != nil {
- return nil, "", "", err
+ return err
}
- apiRepo := new(tasks.GithubApiRepo)
err = errors.Convert(json.Unmarshal(body, apiRepo))
if err != nil {
- return nil, "", "", errors.Convert(err)
+ return err
}
- return apiRepo, token, connection.Proxy, nil
+ return nil
}
diff --git a/plugins/github/api/blueprint_test.go
b/plugins/github/api/blueprint_test.go
index 24c93c0f..8d5f45aa 100644
--- a/plugins/github/api/blueprint_test.go
+++ b/plugins/github/api/blueprint_test.go
@@ -19,52 +19,49 @@ package api
import (
"encoding/json"
+ "github.com/apache/incubator-devlake/config"
+ "github.com/apache/incubator-devlake/logger"
"github.com/apache/incubator-devlake/mocks"
+ "github.com/apache/incubator-devlake/models/common"
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/github/models"
"github.com/apache/incubator-devlake/plugins/github/tasks"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/apache/incubator-devlake/runner"
"github.com/stretchr/testify/assert"
"testing"
)
-func TestMakePipelinePlan(t *testing.T) {
- option := &tasks.GithubOptions{
- ConnectionId: 1,
- Tasks: nil,
- Since: "",
- Owner: "merico-dev",
- Repo: "lake",
- TransformationRules: models.TransformationRules{
- PrType: "hey,man,wasup",
- PrComponent: "component/(.*)$",
- PrBodyClosePattern:
"(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and
)?(#|https:\\/\\/github.com\\/%s\\/%s\\/issues\\/)\\d+[ ]*)+)",
- IssueSeverity: "severity/(.*)$",
- IssuePriority: "^(highest|high|medium|low)$",
- IssueComponent: "component/(.*)$",
- IssueTypeBug: "^(bug|failure|error)$",
- IssueTypeIncident: "",
- IssueTypeRequirement:
"^(feat|feature|proposal|requirement)$",
- DeployTagPattern: "(?i)deploy",
+func TestProcessScope(t *testing.T) {
+ cfg := config.GetConfig()
+ log := logger.Global.Nested("github")
+ db, _ := runner.NewGormDb(cfg, log)
+ Init(cfg, log, db)
+ connection := &models.GithubConnection{
+ RestConnection: helper.RestConnection{
+ BaseConnection: helper.BaseConnection{
+ Name: "github-test",
+ Model: common.Model{
+ ID: 1,
+ },
+ },
+ Endpoint: "https://api.github.com/";,
+ Proxy: "",
+ RateLimitPerHour: 0,
+ },
+ AccessToken: helper.AccessToken{
+ Token: "123",
},
}
- mockConnHelper := mocks.NewBpHelper(t)
- mockConnHelper.On("GetApiRepo", uint64(1),
option).Return(&tasks.GithubApiRepo{
- Name: "test",
- GithubId: 1,
- CloneUrl: "CloneUrl",
- }, "", "", nil)
mockMeta := mocks.NewPluginMeta(t)
mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/github")
-
err := core.RegisterPlugin("github", mockMeta)
- if err != nil {
- panic(err)
- }
+ assert.Nil(t, err)
bs := &core.BlueprintScopeV100{
Entities: []string{"CODE"},
Options: json.RawMessage(`{
- "repo": "lake",
- "owner": "merico-dev"
+ "owner": "test",
+ "repo": "testRepo"
}`),
Transformation: json.RawMessage(`{
"prType": "hey,man,wasup",
@@ -79,14 +76,19 @@ func TestMakePipelinePlan(t *testing.T) {
}
}`),
}
+ apiRepo := &tasks.GithubApiRepo{
+ GithubId: 123,
+ CloneUrl: "HttpUrlToRepo",
+ }
scopes := make([]*core.BlueprintScopeV100, 0)
scopes = append(scopes, bs)
- plan, err := DoMakePipeline(nil, 1, scopes, mockConnHelper)
-
- assert.Nil(t, err)
+ plan := make(core.PipelinePlan, len(scopes))
+ for i, scopeElem := range scopes {
+ plan, err = processScope(nil, 1, scopeElem, i, plan, apiRepo,
connection)
+ assert.Nil(t, err)
+ }
planJson, err1 := json.Marshal(plan)
assert.Nil(t, err1)
- expectPlan :=
`[[{"plugin":"github","subtasks":[],"options":{"connectionId":1,"owner":"merico-dev","repo":"lake","transformationRules":{"prType":"hey,man,wasup"}}},{"plugin":"gitextractor","subtasks":null,"options":{"proxy":"","repoId":"github:GithubRepo:1:1","url":"//git:@CloneUrl"}}],[{"plugin":"refdiff","subtasks":null,"options":{"tagsLimit":10,"tagsOrder":"reverse
semver","tagsPattern":"pattern"}}],[{"plugin":"dora","subtasks":null,"options":{"repoId":"github:GithubRepo:1:1","tasks"
[...]
+ expectPlan :=
`[[{"plugin":"github","subtasks":[],"options":{"connectionId":1,"owner":"test","repo":"testRepo","transformationRules":{"prType":"hey,man,wasup"}}},{"plugin":"gitextractor","subtasks":null,"options":{"proxy":"","repoId":"github:GithubRepo:1:123","url":"//git:123@HttpUrlToRepo"}}],[{"plugin":"refdiff","subtasks":null,"options":{"tagsLimit":10,"tagsOrder":"reverse
semver","tagsPattern":"pattern"}}],[{"plugin":"dora","subtasks":null,"options":{"repoId":"github:GithubRepo:1:12
[...]
assert.Equal(t, expectPlan, string(planJson))
-
}
diff --git a/plugins/gitlab/api/blueprint.go b/plugins/gitlab/api/blueprint.go
index 3c1a1958..9112b403 100644
--- a/plugins/gitlab/api/blueprint.go
+++ b/plugins/gitlab/api/blueprint.go
@@ -113,6 +113,7 @@ func processScope(subtaskMetas []core.SubTaskMeta,
connectionId uint64, scopeEle
}
token := strings.Split(connection.Token, ",")[0]
if apiRepo == nil {
+ apiRepo = new(tasks.GitlabApiProject)
err = getApiRepo(connection, token, op, apiRepo)
if err != nil {
return nil, err
