klesh commented on code in PR #3847: URL: https://github.com/apache/incubator-devlake/pull/3847#discussion_r1044151933
########## plugins/gitlab/api/blueprint_V200_test.go: ########## @@ -0,0 +1,225 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "strconv" + "testing" + "time" + + "github.com/apache/incubator-devlake/errors" + "github.com/apache/incubator-devlake/helpers/unithelper" + "github.com/apache/incubator-devlake/mocks" + "github.com/apache/incubator-devlake/models/common" + "github.com/apache/incubator-devlake/models/domainlayer" + "github.com/apache/incubator-devlake/models/domainlayer/code" + "github.com/apache/incubator-devlake/models/domainlayer/devops" + "github.com/apache/incubator-devlake/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/gitlab/models" + "github.com/apache/incubator-devlake/plugins/gitlab/tasks" + "github.com/apache/incubator-devlake/plugins/helper" + "github.com/go-playground/validator/v10" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +func TestMakeDataSourcePipelinePlanV200(t *testing.T) { + const testConnectionID uint64 = 1 + const testTransformationRuleId uint64 = 2 + const testID int = 37 + const testGitlabEndPoint string = "https://gitlab.com/api/v4/"; + const testHttpUrlToRepo string = "https://this_is_cloneUrl"; + const testToken string = "nddtf" + const testName string = "gitlab-test" + const testTransformationRuleName string = "github transformation rule" + const testProxy string = "" + + bpScopes := []*core.BlueprintScopeV200{ + { + Entities: []string{core.DOMAIN_TYPE_CODE, core.DOMAIN_TYPE_TICKET, core.DOMAIN_TYPE_CICD}, + Id: strconv.Itoa(testID), + Name: testName, + }, + } + + var testGitlabProject *models.GitlabProject = &models.GitlabProject{ + ConnectionId: testConnectionID, + GitlabId: testID, + Name: testName, + + TransformationRuleId: testTransformationRuleId, + CreatedDate: time.Time{}, + HttpUrlToRepo: testHttpUrlToRepo, + } + + var testTransformationRule *models.GitlabTransformationRule = &models.GitlabTransformationRule{ + Model: common.Model{ + ID: testTransformationRuleId, + }, + Name: testTransformationRuleName, + PrType: "hey,man,wasup", + Refdiff: map[string]interface{}{ + "tagsPattern": "pattern", + "tagsLimit": 10, + "tagsOrder": "reverse semver", + }, + } + + var testGitlabConnection *models.GitlabConnection = &models.GitlabConnection{ + RestConnection: helper.RestConnection{ + BaseConnection: helper.BaseConnection{ + Name: testName, + Model: common.Model{ + ID: testConnectionID, + }, + }, + Endpoint: testGitlabEndPoint, + Proxy: testProxy, + RateLimitPerHour: 0, + }, + AccessToken: helper.AccessToken{ + Token: testToken, + }, + } + + var expectRepoId = "gitlab:GitlabProject:1:37" + + var testSubTaskMeta = []core.SubTaskMeta{ + tasks.CollectProjectMeta, + tasks.ExtractProjectMeta, + tasks.ConvertProjectMeta, + tasks.CollectApiIssuesMeta, + tasks.ExtractApiIssuesMeta, + tasks.ConvertIssuesMeta, + tasks.ConvertIssueLabelsMeta, + tasks.CollectApiJobsMeta, + tasks.ExtractApiJobsMeta, + tasks.CollectApiPipelinesMeta, + tasks.ExtractApiPipelinesMeta, + } + + var expectPlans core.PipelinePlan = core.PipelinePlan{ + { + { + Plugin: "refdiff", Review Comment: refdiff should be placed in next Stage which would be executed after `GitExtractor` ########## plugins/gitlab/api/blueprint_v200.go: ########## @@ -18,63 +18,206 @@ limitations under the License. package api import ( + "fmt" + "net/url" + "strconv" + "github.com/apache/incubator-devlake/errors" - "github.com/go-playground/validator/v10" + "github.com/apache/incubator-devlake/utils" + "github.com/apache/incubator-devlake/models/domainlayer" "github.com/apache/incubator-devlake/models/domainlayer/code" + "github.com/apache/incubator-devlake/models/domainlayer/devops" "github.com/apache/incubator-devlake/models/domainlayer/didgen" "github.com/apache/incubator-devlake/models/domainlayer/ticket" "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/core/dal" "github.com/apache/incubator-devlake/plugins/gitlab/models" "github.com/apache/incubator-devlake/plugins/helper" ) -func MakeDataSourcePipelinePlanV200(connectionId uint64, scopes []*core.BlueprintScopeV200) (pp core.PipelinePlan, sc []core.Scope, err errors.Error) { - pp = make(core.PipelinePlan, 0, 1) - sc = make([]core.Scope, 0, 3*len(scopes)) - err = nil - - connectionHelper := helper.NewConnectionHelper(BasicRes, validator.New()) +func MakePipelinePlanV200(subtaskMetas []core.SubTaskMeta, connectionId uint64, scope []*core.BlueprintScopeV200) (core.PipelinePlan, []core.Scope, errors.Error) { + var err errors.Error + connection := new(models.GitlabConnection) + err1 := connectionHelper.FirstById(connection, connectionId) + if err1 != nil { + return nil, nil, errors.Default.Wrap(err1, fmt.Sprintf("error on get connection by id[%d]", connectionId)) + } - // get the connection info for url - connection := &models.GitlabConnection{} - err = connectionHelper.FirstById(connection, connectionId) + sc, err := makeScopeV200(connectionId, scope) if err != nil { return nil, nil, err } - ps := make(core.PipelineStage, 0, len(scopes)) - for _, scope := range scopes { - var board ticket.Board - var repo code.Repo + pp, err := makePipelinePlanV200(subtaskMetas, scope, connection) + if err != nil { + return nil, nil, err + } - id := didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connectionId, scope.Id) + return pp, sc, nil +} - repo.Id = id - repo.Name = scope.Name +func makeScopeV200(connectionId uint64, scopes []*core.BlueprintScopeV200) ([]core.Scope, errors.Error) { + sc := make([]core.Scope, 0, 3*len(scopes)) - board.Id = id - board.Name = scope.Name + for _, scope := range scopes { + intScopeId, err1 := strconv.Atoi(scope.Id) + if err1 != nil { + return nil, errors.Default.Wrap(err1, fmt.Sprintf("Failed to strconv.Atoi for scope.Id [%s]", scope.Id)) + } + id := didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connectionId, intScopeId) + + // get repo from db + gitlabProject, err := GetRepoByConnectionIdAndscopeId(connectionId, scope.Id) + if err != nil { + return nil, err + } + + if utils.StringsContains(scope.Entities, core.DOMAIN_TYPE_CODE_REVIEW) || + utils.StringsContains(scope.Entities, core.DOMAIN_TYPE_CODE) || + utils.StringsContains(scope.Entities, core.DOMAIN_TYPE_CROSS) { + // if we don't need to collect gitex, we need to add repo to scopes here + scopeRepo := &code.Repo{ + DomainEntity: domainlayer.DomainEntity{ + Id: id, + }, + Name: gitlabProject.Name, + } + if gitlabProject.ForkedFromProjectWebUrl != "" { + scopeRepo.ForkedFrom = gitlabProject.ForkedFromProjectWebUrl + } + sc = append(sc, scopeRepo) + } + + // add cicd_scope to scopes + if utils.StringsContains(scope.Entities, core.DOMAIN_TYPE_CICD) { + scopeCICD := &devops.CicdScope{ + DomainEntity: domainlayer.DomainEntity{ + Id: id, + }, + Name: gitlabProject.Name, + } + sc = append(sc, scopeCICD) + } + + // add board to scopes + if utils.StringsContains(scope.Entities, core.DOMAIN_TYPE_TICKET) { + scopeTicket := &ticket.Board{ + DomainEntity: domainlayer.DomainEntity{ + Id: id, + }, + Name: gitlabProject.Name, + } + sc = append(sc, scopeTicket) + } + } - sc = append(sc, &repo) - sc = append(sc, &board) + return sc, nil +} - ps = append(ps, &core.PipelineTask{ - Plugin: "gitlab", - Options: map[string]interface{}{ - "name": scope.Name, - }, +func makePipelinePlanV200(subtaskMetas []core.SubTaskMeta, scopes []*core.BlueprintScopeV200, connection *models.GitlabConnection) (core.PipelinePlan, errors.Error) { + plans := make(core.PipelinePlan, 0, 3*len(scopes)) + for _, scope := range scopes { + var stage core.PipelineStage + var err errors.Error + // get repo + repo, err := GetRepoByConnectionIdAndscopeId(connection.ID, scope.Id) + if err != nil { + return nil, err + } + + // get transformationRuleId + transformationRules, err := GetTransformationRuleByRepo(repo) + if err != nil { + return nil, err + } + + // refdiff part + if transformationRules.Refdiff != nil { + task := &core.PipelineTask{ + Plugin: "refdiff", + Options: transformationRules.Refdiff, + } + stage = append(stage, task) + } + + // get int scopeId + intScopeId, err1 := strconv.Atoi(scope.Id) + if err != nil { + return nil, errors.Default.Wrap(err1, fmt.Sprintf("Failed to strconv.Atoi for scope.Id [%s]", scope.Id)) + } + + // gitlab main part + options := make(map[string]interface{}) + options["connectionId"] = connection.ID + options["scopeId"] = intScopeId Review Comment: Here says `scopeId`, but https://github.com/apache/incubator-devlake/pull/3847/files#diff-dbab24901795b0d9e4de519c6069101b6a6a739dd2dc15895e5b9ddf25d3bcfdR157 uses `ProjectId`, how does it work?  Did you test the code by requesting `/blueprints` with `settings.version=v2.0.0`? ########## plugins/gitlab/api/blueprint_v200.go: ########## @@ -18,63 +18,206 @@ limitations under the License. package api import ( + "fmt" + "net/url" + "strconv" + "github.com/apache/incubator-devlake/errors" - "github.com/go-playground/validator/v10" + "github.com/apache/incubator-devlake/utils" + "github.com/apache/incubator-devlake/models/domainlayer" "github.com/apache/incubator-devlake/models/domainlayer/code" + "github.com/apache/incubator-devlake/models/domainlayer/devops" "github.com/apache/incubator-devlake/models/domainlayer/didgen" "github.com/apache/incubator-devlake/models/domainlayer/ticket" "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/core/dal" "github.com/apache/incubator-devlake/plugins/gitlab/models" "github.com/apache/incubator-devlake/plugins/helper" ) -func MakeDataSourcePipelinePlanV200(connectionId uint64, scopes []*core.BlueprintScopeV200) (pp core.PipelinePlan, sc []core.Scope, err errors.Error) { - pp = make(core.PipelinePlan, 0, 1) - sc = make([]core.Scope, 0, 3*len(scopes)) - err = nil - - connectionHelper := helper.NewConnectionHelper(BasicRes, validator.New()) +func MakePipelinePlanV200(subtaskMetas []core.SubTaskMeta, connectionId uint64, scope []*core.BlueprintScopeV200) (core.PipelinePlan, []core.Scope, errors.Error) { + var err errors.Error + connection := new(models.GitlabConnection) + err1 := connectionHelper.FirstById(connection, connectionId) + if err1 != nil { + return nil, nil, errors.Default.Wrap(err1, fmt.Sprintf("error on get connection by id[%d]", connectionId)) + } - // get the connection info for url - connection := &models.GitlabConnection{} - err = connectionHelper.FirstById(connection, connectionId) + sc, err := makeScopeV200(connectionId, scope) if err != nil { return nil, nil, err } - ps := make(core.PipelineStage, 0, len(scopes)) - for _, scope := range scopes { - var board ticket.Board - var repo code.Repo + pp, err := makePipelinePlanV200(subtaskMetas, scope, connection) + if err != nil { + return nil, nil, err + } - id := didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connectionId, scope.Id) + return pp, sc, nil +} - repo.Id = id - repo.Name = scope.Name +func makeScopeV200(connectionId uint64, scopes []*core.BlueprintScopeV200) ([]core.Scope, errors.Error) { + sc := make([]core.Scope, 0, 3*len(scopes)) - board.Id = id - board.Name = scope.Name + for _, scope := range scopes { + intScopeId, err1 := strconv.Atoi(scope.Id) + if err1 != nil { + return nil, errors.Default.Wrap(err1, fmt.Sprintf("Failed to strconv.Atoi for scope.Id [%s]", scope.Id)) + } + id := didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connectionId, intScopeId) + + // get repo from db + gitlabProject, err := GetRepoByConnectionIdAndscopeId(connectionId, scope.Id) + if err != nil { + return nil, err + } + + if utils.StringsContains(scope.Entities, core.DOMAIN_TYPE_CODE_REVIEW) || + utils.StringsContains(scope.Entities, core.DOMAIN_TYPE_CODE) || + utils.StringsContains(scope.Entities, core.DOMAIN_TYPE_CROSS) { Review Comment: Why Repo is related to `DOMAIN_TYPE_CROSS`? it doesn't seem right. beside, I checked the DOMAIN TYPE declare in gitlab tasks, the follow three tasks should belong to CICD, not CROSS.  -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
