This is an automated email from the ASF dual-hosted git repository. markusb pushed a commit to branch feat/ado-disabled-repos in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 15a0545b469820c05a0d76bd0dc185ee172cbc3f Author: Markus Braunbeck <[email protected]> AuthorDate: Sun Jun 23 14:02:18 2024 +0200 feat(azuredevops): add support for disabled Azure DevOps repos --- .../plugins/azuredevops_go/api/blueprint_v200.go | 135 ++++++++++--------- .../azuredevops_go/api/blueprint_v200_test.go | 150 ++++++++++++++++++++- backend/plugins/azuredevops_go/api/init.go | 4 +- .../plugins/azuredevops_go/api/remote_helper.go | 13 +- 4 files changed, 229 insertions(+), 73 deletions(-) diff --git a/backend/plugins/azuredevops_go/api/blueprint_v200.go b/backend/plugins/azuredevops_go/api/blueprint_v200.go index a0fea7695..5f3ee38a8 100644 --- a/backend/plugins/azuredevops_go/api/blueprint_v200.go +++ b/backend/plugins/azuredevops_go/api/blueprint_v200.go @@ -20,8 +20,6 @@ package api import ( "net/url" - "golang.org/x/exp/slices" - "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/utils" @@ -71,51 +69,39 @@ func makeScopeV200( sc := make([]plugin.Scope, 0, 3*len(scopeDetails)) for _, scope := range scopeDetails { - azuredevopsRepo, scopeConfig := scope.Scope, scope.ScopeConfig - if azuredevopsRepo.Type != models.RepositoryTypeADO { + repo, scopeConfig := scope.Scope, scope.ScopeConfig + + if len(scopeConfig.Entities) == 0 { + logger.Printf("Precondition failed. Found empty ScopeConfig for Scope: %v. Skipping", repo.Name) continue } - id := didgen.NewDomainIdGenerator(&models.AzuredevopsRepo{}).Generate(connectionId, azuredevopsRepo.Id) - if !azuredevopsRepo.IsDisabled && (utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE_REVIEW) || - utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE)) { - scopeRepo := code.NewRepo(id, azuredevopsRepo.Name) + isDomainCode := utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE_REVIEW) || + utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE) + isDomainCICD := utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CICD) + isDomainTicket := utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_TICKET) + + id := didgen.NewDomainIdGenerator(&models.AzuredevopsRepo{}).Generate(connectionId, repo.Id) + + // DOMAIN_TYPE_CODE (i.e. gitextractor, rediff) only works if the repository is public and not disabled + if isDomainCode && !repo.IsDisabled && !repo.IsPrivate { + scopeRepo := code.NewRepo(id, repo.Name) sc = append(sc, scopeRepo) } // add cicd_scope to scopes - if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CICD) { - scopeCICD := devops.NewCicdScope(id, azuredevopsRepo.Name) + if isDomainCICD { + scopeCICD := devops.NewCicdScope(id, repo.Name) sc = append(sc, scopeCICD) } // add board to scopes - if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_TICKET) { - scopeTicket := ticket.NewBoard(id, azuredevopsRepo.Name) + if isDomainTicket { + scopeTicket := ticket.NewBoard(id, repo.Name) sc = append(sc, scopeTicket) } } - for _, scope := range scopeDetails { - azuredevopsRepo, scopeConfig := scope.Scope, scope.ScopeConfig - if azuredevopsRepo.Type == models.RepositoryTypeADO { - continue - } - id := didgen.NewDomainIdGenerator(&models.AzuredevopsRepo{}).Generate(connectionId, azuredevopsRepo.Id) - - // Azure DevOps Pipeline can be used with remote repositories such as GitHub and Bitbucket - if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CICD) { - scopeCICD := devops.NewCicdScope(id, azuredevopsRepo.Name) - sc = append(sc, scopeCICD) - } - - // DOMAIN_TYPE_CODE (i.e. gitextractor, rediff) only works if the repository is public - if !azuredevopsRepo.IsPrivate && utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE) { - scopeRepo := code.NewRepo(id, azuredevopsRepo.Name) - sc = append(sc, scopeRepo) - } - } - return sc, nil } @@ -125,62 +111,91 @@ func makePipelinePlanV200( scopeDetails []*srvhelper.ScopeDetail[models.AzuredevopsRepo, models.AzuredevopsScopeConfig], ) (coreModels.PipelinePlan, errors.Error) { plans := make(coreModels.PipelinePlan, 0, 3*len(scopeDetails)) + for _, scope := range scopeDetails { - azuredevopsRepo, scopeConfig := scope.Scope, scope.ScopeConfig - var stage coreModels.PipelineStage - var err errors.Error + repo, scopeConfig := scope.Scope, scope.ScopeConfig + + if len(scopeConfig.Entities) == 0 { + logger.Printf("Precondition failed. Found empty ScopeConfig for Scope: %v. Skipping", + repo.Name) + continue + } options := make(map[string]interface{}) - options["name"] = azuredevopsRepo.Name // this is solely for the FE to display the repo name of a task + options["name"] = repo.Name // this is solely for the FE to display the repo name of a task options["connectionId"] = connection.ID - options["organizationId"] = azuredevopsRepo.OrganizationId - options["projectId"] = azuredevopsRepo.ProjectId - options["externalId"] = azuredevopsRepo.ExternalId - options["repositoryId"] = azuredevopsRepo.Id - options["repositoryType"] = azuredevopsRepo.Type + options["organizationId"] = repo.OrganizationId + options["projectId"] = repo.ProjectId + options["externalId"] = repo.ExternalId + options["repositoryId"] = repo.Id + options["repositoryType"] = repo.Type // construct subtasks var entities []string - if scope.Scope.Type == models.RepositoryTypeADO { - entities = append(entities, scopeConfig.Entities...) - } else { - if i := slices.Index(scopeConfig.Entities, plugin.DOMAIN_TYPE_CICD); i >= 0 { - entities = append(entities, scopeConfig.Entities[i]) - } + var blockedEntities []string + + // We are unable to check out the code or gather pull requests for repositories that are disabled (DevOps) + // or private (GitHub) + if repo.IsDisabled || repo.IsPrivate { + blockedEntities = append(blockedEntities, []string{ + plugin.DOMAIN_TYPE_CODE, + plugin.DOMAIN_TYPE_CODE_REVIEW, + }...) + } - if i := slices.Index(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE); i >= 0 && !scope.Scope.IsPrivate { - entities = append(entities, scopeConfig.Entities[i]) + // We are unable to gather pull requests from repositories not hosted on DevOps. + // However, we can still check out the code if the repository is publicly available + if repo.Type != models.RepositoryTypeADO { + blockedEntities = append(blockedEntities, []string{ + plugin.DOMAIN_TYPE_CODE_REVIEW, + }...) + } + + for _, v := range scopeConfig.Entities { + if !utils.StringsContains(blockedEntities, v) { + entities = append(entities, v) } } - subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, entities) + var subtasks []string + var err errors.Error + if len(entities) > 0 { + // if entities is empty MakePipelinePlanSubtasks assumes that we want to + // enable all entity types + subtasks, err = helper.MakePipelinePlanSubtasks(subtaskMetas, entities) + } if err != nil { return nil, err } - stage = append(stage, &coreModels.PipelineTask{ - Plugin: "azuredevops_go", - Subtasks: subtasks, - Options: options, - }) + var stage []*coreModels.PipelineTask + if len(subtasks) > 0 { + stage = append(stage, &coreModels.PipelineTask{ + Plugin: "azuredevops_go", + Subtasks: subtasks, + Options: options, + }) + } else { + logger.Printf("Skipping azuredevops_go plugin due to empty subtasks. Please check your scope config") + } // collect git data by gitextractor if CODE was requested - if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE) && !scope.Scope.IsPrivate || len(scopeConfig.Entities) == 0 { - cloneUrl, err := errors.Convert01(url.Parse(azuredevopsRepo.RemoteUrl)) + if !repo.IsPrivate && !repo.IsDisabled && utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_CODE) { + cloneUrl, err := errors.Convert01(url.Parse(repo.RemoteUrl)) if err != nil { return nil, err } - if scope.Scope.Type == models.RepositoryTypeADO { + if repo.Type == models.RepositoryTypeADO { cloneUrl.User = url.UserPassword("git", connection.Token) } stage = append(stage, &coreModels.PipelineTask{ Plugin: "gitextractor", Options: map[string]interface{}{ "url": cloneUrl.String(), - "name": azuredevopsRepo.Name, - "repoId": didgen.NewDomainIdGenerator(&models.AzuredevopsRepo{}).Generate(connection.ID, azuredevopsRepo.Id), + "name": repo.Name, + "repoId": didgen.NewDomainIdGenerator(&models.AzuredevopsRepo{}).Generate(connection.ID, repo.Id), "proxy": connection.Proxy, "noShallowClone": true, }, diff --git a/backend/plugins/azuredevops_go/api/blueprint_v200_test.go b/backend/plugins/azuredevops_go/api/blueprint_v200_test.go index 2cb2a2513..d733689a9 100644 --- a/backend/plugins/azuredevops_go/api/blueprint_v200_test.go +++ b/backend/plugins/azuredevops_go/api/blueprint_v200_test.go @@ -19,6 +19,8 @@ package api import ( "fmt" + "github.com/apache/incubator-devlake/helpers/pluginhelper/subtaskmeta/sorter" + "github.com/apache/incubator-devlake/impls/logruslog" "reflect" "strings" "testing" @@ -38,6 +40,7 @@ import ( const ( connectionID uint64 = 1 azuredevopsRepoId = "ad05901f-c9b0-4938-bc8a-a22eb2467ceb" + azureDevOpsToken = "ado-pat" expectDomainScopeId = "azuredevops_go:AzuredevopsRepo:1:ad05901f-c9b0-4938-bc8a-a22eb2467ceb" ) @@ -47,6 +50,12 @@ func mockAzuredevopsPlugin(t *testing.T) { mockMeta.On("Name").Return("dummy").Maybe() err := plugin.RegisterPlugin("azuredevops_go", mockMeta) assert.Equal(t, err, nil) + + // The logger is assigned within the Init function, which is not executed during unit tests. + // To avoid a nil pointer, we need to manually set it here. + if logger == nil { + logger = logruslog.Global + } } func TestMakeScopes(t *testing.T) { @@ -191,12 +200,27 @@ func TestMakeRemoteRepoScopes(t *testing.T) { Type string Private bool Disabled bool + Entities []string // Data Entities configured in a scope config ExpectedScopes []string }{ - {Name: "Azure DevOps Repository", Type: models.RepositoryTypeADO, Private: false, ExpectedScopes: []string{"*code.Repo", "*ticket.Board", "*devops.CicdScope"}}, - {Name: "Azure DevOps disabled Repository", Type: models.RepositoryTypeADO, Disabled: true, ExpectedScopes: []string{"*ticket.Board", "*devops.CicdScope"}}, - {Name: "Public GitHub Repository", Type: models.RepositoryTypeGithub, Private: false, ExpectedScopes: []string{"*code.Repo", "*devops.CicdScope"}}, - {Name: "Private GitHub Repository", Type: models.RepositoryTypeGithub, Private: true, ExpectedScopes: []string{"*devops.CicdScope"}}, + {Name: "Azure DevOps Repository w/o Scope Config", Type: models.RepositoryTypeADO, Private: false, + Entities: plugin.DOMAIN_TYPES, ExpectedScopes: []string{"*code.Repo", "*ticket.Board", "*devops.CicdScope"}}, + {Name: "Azure DevOps Repository w/ Scope Config", Type: models.RepositoryTypeADO, Private: false, + Entities: []string{plugin.DOMAIN_TYPE_CODE}, ExpectedScopes: []string{"*code.Repo"}}, + {Name: "Azure DevOps disabled Repository w/o Scope Config", Type: models.RepositoryTypeADO, Disabled: true, + Entities: plugin.DOMAIN_TYPES, ExpectedScopes: []string{"*ticket.Board", "*devops.CicdScope"}}, + {Name: "Azure DevOps disabled Repository w/ Scope Config", Type: models.RepositoryTypeADO, Disabled: true, + Entities: []string{plugin.DOMAIN_TYPE_CODE}, ExpectedScopes: []string{}}, + {Name: "Azure DevOps disabled Repository w/ Scope Config", Type: models.RepositoryTypeADO, Disabled: true, + Entities: []string{plugin.DOMAIN_TYPE_CROSS}, ExpectedScopes: []string{}}, + {Name: "Azure DevOps disabled Repository w/ Scope Config", Type: models.RepositoryTypeADO, Disabled: true, + Entities: []string{plugin.DOMAIN_TYPE_CICD}, ExpectedScopes: []string{"*devops.CicdScope"}}, + {Name: "Public GitHub Repository", Type: models.RepositoryTypeGithub, Private: false, + Entities: plugin.DOMAIN_TYPES, ExpectedScopes: []string{"*code.Repo", "*devops.CicdScope", "*ticket.Board"}}, + {Name: "Private GitHub Repository w/ Scope Config", Type: models.RepositoryTypeGithub, Private: true, + Entities: plugin.DOMAIN_TYPES, ExpectedScopes: []string{"*devops.CicdScope", "*ticket.Board"}}, + {Name: "Private GitHub Repository w/o Scope Config", Type: models.RepositoryTypeGithub, Private: true, + ExpectedScopes: []string{}}, } for _, d := range data { @@ -220,8 +244,7 @@ func TestMakeRemoteRepoScopes(t *testing.T) { }, ScopeConfig: &models.AzuredevopsScopeConfig{ ScopeConfig: common.ScopeConfig{ - Entities: []string{plugin.DOMAIN_TYPE_CODE, plugin.DOMAIN_TYPE_TICKET, - plugin.DOMAIN_TYPE_CICD, plugin.DOMAIN_TYPE_CODE_REVIEW}, + Entities: d.Entities, }, }, }, @@ -240,3 +263,118 @@ func TestMakeRemoteRepoScopes(t *testing.T) { } } + +func TestSubtasks(t *testing.T) { + mockAzuredevopsPlugin(t) + + allSubtasks, err := sorter.NewTableSorter(tasks.SubTaskMetaList).Sort() + if err != nil { + t.Errorf("failed to sort subtasks: %v", err) + } + + data := []struct { + Name string + Type string + Private bool + Disabled bool + Entities []string // Data Entities configured in a scope config + ValidEntities []string + }{ + {Name: "Active Azure DevOps Repository", Type: models.RepositoryTypeADO, + Entities: plugin.DOMAIN_TYPES, ValidEntities: plugin.DOMAIN_TYPES}, + {Name: "Disabled Azure DevOps Repository", Type: models.RepositoryTypeADO, Disabled: true, Entities: plugin.DOMAIN_TYPES, + ValidEntities: []string{plugin.DOMAIN_TYPE_TICKET, plugin.DOMAIN_TYPE_CICD, plugin.DOMAIN_TYPE_CROSS}}, + {Name: "Public GitHub Repository", Type: models.RepositoryTypeGithub, Entities: plugin.DOMAIN_TYPES, + ValidEntities: []string{plugin.DOMAIN_TYPE_CICD, plugin.DOMAIN_TYPE_CROSS, plugin.DOMAIN_TYPE_CODE}}, + {Name: "Private GitHub Repository", Type: models.RepositoryTypeGithub, Entities: plugin.DOMAIN_TYPES, Private: true, + ValidEntities: []string{plugin.DOMAIN_TYPE_CICD, plugin.DOMAIN_TYPE_CROSS}}, + } + + for _, d := range data { + t.Run(d.Name, func(t *testing.T) { + id := strings.ToLower(d.Name) + id = strings.ReplaceAll(id, " ", "-") + actualPlans, err := makePipelinePlanV200( + allSubtasks, + adoConnection(connectionID, azureDevOpsToken), + []*srvhelper.ScopeDetail[models.AzuredevopsRepo, models.AzuredevopsScopeConfig]{ + { + Scope: adoRepo(d.Type, d.Private, d.Disabled), + ScopeConfig: adoScopeConfig(d.Entities), + }, + }, + ) + assert.Nil(t, err) + + validSubtasks, err := api.MakePipelinePlanSubtasks(allSubtasks, d.ValidEntities) + assert.Nil(t, err) + + var count int + for _, stage := range actualPlans { + for _, task := range stage { + if task.Plugin == "azuredevops_go" { + for _, subtask := range task.Subtasks { + assert.Contains(t, validSubtasks, subtask) + count++ + } + assert.Equal(t, count, len(validSubtasks)) + + } + } + } + }) + } +} + +func adoConnection(connectionID uint64, pat string) *models.AzuredevopsConnection { + return &models.AzuredevopsConnection{ + BaseConnection: api.BaseConnection{ + Model: common.Model{ + ID: connectionID, + }, + }, + AzuredevopsConn: models.AzuredevopsConn{ + AzuredevopsAccessToken: models.AzuredevopsAccessToken{ + Token: pat, + }, + }, + } +} + +func adoRepo(repoType string, isPrivate, isDisabled bool) models.AzuredevopsRepo { + const ( + httpUrlToRepo = "https://this_is_cloneUrl"; + azureDevOpsProjectName = "azuredevops-test-project" + azureDevOpsOrgName = "azuredevops-test-org" + ) + + return models.AzuredevopsRepo{ + Id: fmt.Sprint(azuredevopsRepoId), + AzureDevOpsPK: models.AzureDevOpsPK{ + ProjectId: azureDevOpsProjectName, + OrganizationId: azureDevOpsOrgName, + }, + Name: azureDevOpsProjectName, + Url: httpUrlToRepo, + RemoteUrl: httpUrlToRepo, + Type: repoType, + IsPrivate: isPrivate, + IsDisabled: isDisabled, + } +} + +func adoScopeConfig(entities []string) *models.AzuredevopsScopeConfig { + return &models.AzuredevopsScopeConfig{ + ScopeConfig: common.ScopeConfig{ + Entities: entities, + }, + DeploymentPattern: "(?i)deploy", + ProductionPattern: "(?i)prod", + Refdiff: map[string]interface{}{ + "tagsPattern": "pattern", + "tagsLimit": 10, + "tagsOrder": "reverse semver", + }, + } + +} diff --git a/backend/plugins/azuredevops_go/api/init.go b/backend/plugins/azuredevops_go/api/init.go index 0401e2b36..163398928 100644 --- a/backend/plugins/azuredevops_go/api/init.go +++ b/backend/plugins/azuredevops_go/api/init.go @@ -19,6 +19,7 @@ package api import ( "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/log" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" @@ -32,6 +33,7 @@ var dsHelper *api.DsHelper[models.AzuredevopsConnection, models.AzuredevopsRepo, var raProxy *api.DsRemoteApiProxyHelper[models.AzuredevopsConnection] var raScopeList *api.DsRemoteApiScopeListHelper[models.AzuredevopsConnection, models.AzuredevopsRepo, AzuredevopsRemotePagination] var raScopeSearch *api.DsRemoteApiScopeSearchHelper[models.AzuredevopsConnection, models.AzuredevopsRepo] +var logger log.Logger func Init(br context.BasicRes, p plugin.PluginMeta) { vld = validator.New() @@ -53,5 +55,5 @@ func Init(br context.BasicRes, p plugin.PluginMeta) { raProxy = api.NewDsRemoteApiProxyHelper[models.AzuredevopsConnection](dsHelper.ConnApi.ModelApiHelper) raScopeList = api.NewDsRemoteApiScopeListHelper[models.AzuredevopsConnection, models.AzuredevopsRepo, AzuredevopsRemotePagination](raProxy, listAzuredevopsRemoteScopes) raScopeSearch = api.NewDsRemoteApiScopeSearchHelper[models.AzuredevopsConnection, models.AzuredevopsRepo](raProxy, nil) - + logger = br.GetLogger() } diff --git a/backend/plugins/azuredevops_go/api/remote_helper.go b/backend/plugins/azuredevops_go/api/remote_helper.go index 4cf26cba6..ee88c149e 100644 --- a/backend/plugins/azuredevops_go/api/remote_helper.go +++ b/backend/plugins/azuredevops_go/api/remote_helper.go @@ -151,12 +151,13 @@ func listAzuredevopsRepos( for _, v := range repos { pID := orgId + idSeparator + projectId repo := models.AzuredevopsRepo{ - Id: v.Id, - Type: models.RepositoryTypeADO, - Name: v.Name, - Url: v.Url, - RemoteUrl: v.RemoteUrl, - IsFork: false, + Id: v.Id, + Type: models.RepositoryTypeADO, + Name: v.Name, + Url: v.Url, + RemoteUrl: v.RemoteUrl, + IsFork: false, + IsDisabled: v.IsDisabled, } repo.ProjectId = projectId repo.OrganizationId = orgId
