This is an automated email from the ASF dual-hosted git repository. abeizn pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 0ce5b40aaa922a70cb8c443ee748b5abbb4437f4 Author: Yingchu Chen <[email protected]> AuthorDate: Wed Oct 12 22:41:35 2022 +0800 fix(github): modify bluprint test --- plugins/github/api/blueprint.go | 42 ++++++++++--------- plugins/github/api/blueprint_test.go | 80 ++++++++++++++++++++++++++++++------ plugins/helper/api_client.go | 9 ++++ 3 files changed, 98 insertions(+), 33 deletions(-) diff --git a/plugins/github/api/blueprint.go b/plugins/github/api/blueprint.go index 8acb4d7f..40e334fc 100644 --- a/plugins/github/api/blueprint.go +++ b/plugins/github/api/blueprint.go @@ -55,14 +55,25 @@ func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scop connection.Proxy, basicRes, ) - plan, err := makePipelinePlan(subtaskMetas, connectionId, scope, apiClient, connection) + if err != nil { + return nil, err + } + plan, err := makePipelinePlan(subtaskMetas, scope, apiClient, connection) if err != nil { return nil, err } return plan, nil } -func makePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scope []*core.BlueprintScopeV100, apiClient *helper.ApiClient, connection *models.GithubConnection) (core.PipelinePlan, errors.Error) { + +func makePipelinePlan(subtaskMetas []core.SubTaskMeta, scope []*core.BlueprintScopeV100, apiClient helper.ApiClientGetter, connection *models.GithubConnection) (core.PipelinePlan, errors.Error) { var err errors.Error + var repo *tasks.GithubApiRepo + getApiRepoIfNil := func(op *tasks.GithubOptions) (*tasks.GithubApiRepo, errors.Error) { + if repo == nil { + repo, err = getApiRepo(op, apiClient) + } + return repo, err + } plan := make(core.PipelinePlan, len(scope)) for i, scopeElem := range scope { // handle taskOptions and transformationRules, by dumping them to taskOptions @@ -95,7 +106,7 @@ func makePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scop if err != nil { return nil, err } - options["connectionId"] = connectionId + options["connectionId"] = connection.ID options["transformationRules"] = transformationRules // make sure task options is valid op, err := tasks.DecodeAndValidateTaskOptions(options) @@ -119,20 +130,12 @@ func makePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scop // collect git data by gitextractor if CODE was requested if utils.StringsContains(scopeElem.Entities, core.DOMAIN_TYPE_CODE) { // here is the tricky part, we have to obtain the repo id beforehand - if connection == nil { - connection = new(models.GithubConnection) - err = connectionHelper.FirstById(connection, connectionId) - if err != nil { - return nil, err - } - } token := strings.Split(connection.Token, ",")[0] - apiRepo, err := getApiRepo(op, apiClient) + repo, err = getApiRepoIfNil(op) if err != nil { return nil, err } - - cloneUrl, err := errors.Convert01(url.Parse(apiRepo.CloneUrl)) + cloneUrl, err := errors.Convert01(url.Parse(repo.CloneUrl)) if err != nil { return nil, err } @@ -141,7 +144,7 @@ func makePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scop Plugin: "gitextractor", Options: map[string]interface{}{ "url": cloneUrl.String(), - "repoId": didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connectionId, apiRepo.GithubId), + "repoId": didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID, repo.GithubId), "proxy": connection.Proxy, }, }) @@ -162,14 +165,13 @@ func makePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scop if err != nil { return nil, err } - - apiRepo, err := getApiRepo(op, apiClient) + repo, err = getApiRepoIfNil(op) if err != nil { return nil, err } doraOption := make(map[string]interface{}) - doraOption["repoId"] = didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connectionId, apiRepo.GithubId) + doraOption["repoId"] = didgen.NewDomainIdGenerator(&models.GithubRepo{}).Generate(connection.ID, repo.GithubId) doraRules := make(map[string]interface{}) doraRules["productionPattern"] = productionPattern doraOption["transformationRules"] = doraRules @@ -191,13 +193,13 @@ func makePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scop return plan, nil } -func getApiRepo(op *tasks.GithubOptions, apiClient *helper.ApiClient) (*tasks.GithubApiRepo, errors.Error) { - apiRepo := new(tasks.GithubApiRepo) +func getApiRepo(op *tasks.GithubOptions, apiClient helper.ApiClientGetter) (*tasks.GithubApiRepo, errors.Error) { + apiRepo := &tasks.GithubApiRepo{} res, err := apiClient.Get(fmt.Sprintf("repos/%s/%s", op.Owner, op.Repo), nil, nil) if err != nil { return nil, err } - //defer res.Body.Close() + defer res.Body.Close() if res.StatusCode != http.StatusOK { return nil, errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("unexpected status code when requesting repo detail from %s", res.Request.URL.String())) } diff --git a/plugins/github/api/blueprint_test.go b/plugins/github/api/blueprint_test.go index 9a1ad60a..08dd0390 100644 --- a/plugins/github/api/blueprint_test.go +++ b/plugins/github/api/blueprint_test.go @@ -18,6 +18,7 @@ limitations under the License. package api import ( + "bytes" "encoding/json" "github.com/apache/incubator-devlake/mocks" "github.com/apache/incubator-devlake/models/common" @@ -26,6 +27,9 @@ import ( "github.com/apache/incubator-devlake/plugins/github/tasks" "github.com/apache/incubator-devlake/plugins/helper" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "io" + "net/http" "testing" ) @@ -46,9 +50,20 @@ func TestProcessScope(t *testing.T) { Token: "123", }, } + mockApiCLient := mocks.NewApiClientGetter(t) + repo := &tasks.GithubApiRepo{ + GithubId: 12345, + CloneUrl: "*this is cloneUrl*", + } + js, err := json.Marshal(repo) + assert.Nil(t, err) + res := &http.Response{} + res.Body = io.NopCloser(bytes.NewBuffer(js)) + res.StatusCode = http.StatusOK + mockApiCLient.On("Get", "repos/test/testRepo", mock.Anything, mock.Anything).Return(res, nil) mockMeta := mocks.NewPluginMeta(t) mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/github") - err := core.RegisterPlugin("github", mockMeta) + err = core.RegisterPlugin("github", mockMeta) assert.Nil(t, err) bs := &core.BlueprintScopeV100{ Entities: []string{"CODE"}, @@ -66,19 +81,58 @@ func TestProcessScope(t *testing.T) { "productionPattern": "xxxx" }`), } - apiRepo := &tasks.GithubApiRepo{ - GithubId: 123, - CloneUrl: "HttpUrlToRepo", - } scopes := make([]*core.BlueprintScopeV100, 0) scopes = append(scopes, bs) - plan := make(core.PipelinePlan, len(scopes)) - for i, scopeElem := range scopes { - plan, err = processScope(nil, 1, scopeElem, i, plan, apiRepo, connection) - assert.Nil(t, err) + plan, err := makePipelinePlan(nil, scopes, mockApiCLient, connection) + assert.Nil(t, err) + + //planJson, err1 := json.Marshal(plan) + //assert.Nil(t, err1) + expectPlan := core.PipelinePlan{ + core.PipelineStage{ + { + Plugin: "github", + Subtasks: []string{}, + Options: map[string]interface{}{ + "connectionId": uint64(1), + "owner": "test", + "repo": "testRepo", + "transformationRules": map[string]interface{}{ + "prType": "hey,man,wasup", + }, + }, + }, + { + Plugin: "gitextractor", + Options: map[string]interface{}{ + "proxy": "", + "repoId": "github:GithubRepo:1:12345", + "url": "//git:123@%2Athis%20is%20cloneUrl%2A", + }, + }, + }, + core.PipelineStage{ + { + Plugin: "refdiff", + Options: map[string]interface{}{ + "tagsLimit": float64(10), + "tagsOrder": "reverse semver", + "tagsPattern": "pattern", + }, + }, + }, + core.PipelineStage{ + { + Plugin: "dora", + Subtasks: []string{"EnrichTaskEnv"}, + Options: map[string]interface{}{ + "repoId": "github:GithubRepo:1:12345", + "transformationRules": map[string]interface{}{ + "productionPattern": "xxxx", + }, + }, + }, + }, } - planJson, err1 := json.Marshal(plan) - assert.Nil(t, err1) - expectPlan := `[[{"plugin":"github","subtasks":[],"options":{"connectionId":1,"owner":"test","repo":"testRepo","transformationRules":{"prType":"hey,man,wasup"}}},{"plugin":"gitextractor","subtasks":null,"options":{"proxy":"","repoId":"github:GithubRepo:1:123","url":"//git:123@HttpUrlToRepo"}}],[{"plugin":"refdiff","subtasks":null,"options":{"tagsLimit":10,"tagsOrder":"reverse semver","tagsPattern":"pattern"}}],[{"plugin":"dora","subtasks":["EnrichTaskEnv"],"options":{"repoId":"github:Gi [...] - assert.Equal(t, expectPlan, string(planJson)) + assert.Equal(t, expectPlan, plan) } diff --git a/plugins/helper/api_client.go b/plugins/helper/api_client.go index 4708b8fc..09afbd55 100644 --- a/plugins/helper/api_client.go +++ b/plugins/helper/api_client.go @@ -40,6 +40,15 @@ import ( // ErrIgnoreAndContinue is a error which should be ignored var ErrIgnoreAndContinue = errors.Default.New("ignore and continue") +// ApiClientGetter will be used for uint test +type ApiClientGetter interface { + Get( + path string, + query url.Values, + headers http.Header, + ) (*http.Response, errors.Error) +} + // ApiClient is designed for simple api requests type ApiClient struct { client *http.Client
