This is an automated email from the ASF dual-hosted git repository. warren pushed a commit to branch fix-8670 in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit a0d0388986269222fbb1966cc3a1500b58cd57f1 Author: warren <[email protected]> AuthorDate: Wed Dec 31 19:56:35 2025 +0800 feat(azuredevops): add support for Release Pipelines (Classic Pipelines) (#8670) Add support for Azure DevOps Release Pipelines (Classic/XAML pipelines) to collect deployment data from the Release Management API. Changes: - Add Release and ReleaseDeployment data models - Add collectors for releases and deployments using vsrm.dev.azure.com API - Add extractors to parse raw API responses - Add converter to transform deployments into CICDPipeline domain model - Add database migration for new tables The deployments are converted to DevLake's standard CICDPipeline format with type set to DEPLOYMENT, enabling DORA metrics calculation for classic release pipelines. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <[email protected]> --- backend/plugins/azuredevops_go/impl/impl.go | 2 + ...{register.go => 20241231_add_release_tables.go} | 29 +++- .../models/migrationscripts/archived/release.go | 69 +++++++++ .../models/migrationscripts/register.go | 1 + backend/plugins/azuredevops_go/models/release.go | 151 ++++++++++++++++++++ .../azuredevops_go/tasks/release_collector.go | 93 ++++++++++++ .../tasks/release_deployment_collector.go | 93 ++++++++++++ .../tasks/release_deployment_converter.go | 157 +++++++++++++++++++++ .../tasks/release_deployment_extractor.go | 91 ++++++++++++ .../azuredevops_go/tasks/release_extractor.go | 83 +++++++++++ 10 files changed, 762 insertions(+), 7 deletions(-) diff --git a/backend/plugins/azuredevops_go/impl/impl.go b/backend/plugins/azuredevops_go/impl/impl.go index 44cc01e52..b5dc254fb 100644 --- a/backend/plugins/azuredevops_go/impl/impl.go +++ b/backend/plugins/azuredevops_go/impl/impl.go @@ -88,6 +88,8 @@ func (p Azuredevops) GetTablesInfo() []dal.Tabler { &models.AzuredevopsPrLabel{}, &models.AzuredevopsProject{}, &models.AzuredevopsPullRequest{}, + &models.AzuredevopsRelease{}, + &models.AzuredevopsReleaseDeployment{}, &models.AzuredevopsRepo{}, &models.AzuredevopsRepoCommit{}, &models.AzuredevopsScopeConfig{}, diff --git a/backend/plugins/azuredevops_go/models/migrationscripts/register.go b/backend/plugins/azuredevops_go/models/migrationscripts/20241231_add_release_tables.go similarity index 54% copy from backend/plugins/azuredevops_go/models/migrationscripts/register.go copy to backend/plugins/azuredevops_go/models/migrationscripts/20241231_add_release_tables.go index 59fa7832f..d4934434a 100644 --- a/backend/plugins/azuredevops_go/models/migrationscripts/register.go +++ b/backend/plugins/azuredevops_go/models/migrationscripts/20241231_add_release_tables.go @@ -18,13 +18,28 @@ limitations under the License. package migrationscripts import ( - "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/models/migrationscripts/archived" ) -// All return all the migration scripts -func All() []plugin.MigrationScript { - return []plugin.MigrationScript{ - new(addInitTables), - new(extendRepoTable), - } +type addReleaseTables struct { +} + +func (u *addReleaseTables) Up(basicRes context.BasicRes) errors.Error { + err := migrationhelper.AutoMigrateTables( + basicRes, + &archived.AzuredevopsRelease{}, + &archived.AzuredevopsReleaseDeployment{}, + ) + return err +} + +func (*addReleaseTables) Version() uint64 { + return 20241231000001 +} + +func (*addReleaseTables) Name() string { + return "Add Azure DevOps Release Pipeline tables" } diff --git a/backend/plugins/azuredevops_go/models/migrationscripts/archived/release.go b/backend/plugins/azuredevops_go/models/migrationscripts/archived/release.go new file mode 100644 index 000000000..d771a1426 --- /dev/null +++ b/backend/plugins/azuredevops_go/models/migrationscripts/archived/release.go @@ -0,0 +1,69 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package archived + +import ( + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + "time" +) + +type AzuredevopsRelease struct { + archived.NoPKModel + + ConnectionId uint64 `gorm:"primaryKey"` + AzuredevopsId int `gorm:"primaryKey"` + ProjectId string `gorm:"type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + Status string `gorm:"type:varchar(100)"` + ReleaseDefinitionId int + ReleaseDefinitionName string `gorm:"type:varchar(255)"` + Description string `gorm:"type:text"` + CreatedOn *time.Time + ModifiedOn *time.Time +} + +func (AzuredevopsRelease) TableName() string { + return "_tool_azuredevops_go_releases" +} + +type AzuredevopsReleaseDeployment struct { + archived.NoPKModel + + ConnectionId uint64 `gorm:"primaryKey"` + AzuredevopsId int `gorm:"primaryKey"` + ReleaseId int `gorm:"index"` + ProjectId string `gorm:"type:varchar(255)"` + Name string `gorm:"type:varchar(255)"` + Status string `gorm:"type:varchar(100)"` + OperationStatus string `gorm:"type:varchar(100)"` + DeploymentStatus string `gorm:"type:varchar(100)"` + DefinitionName string `gorm:"type:varchar(255)"` + DefinitionId int + EnvironmentId int + EnvironmentName string `gorm:"type:varchar(255)"` + AttemptNumber int + Reason string `gorm:"type:varchar(100)"` + QueuedOn *time.Time + StartedOn *time.Time + CompletedOn *time.Time + LastModifiedOn *time.Time +} + +func (AzuredevopsReleaseDeployment) TableName() string { + return "_tool_azuredevops_go_release_deployments" +} diff --git a/backend/plugins/azuredevops_go/models/migrationscripts/register.go b/backend/plugins/azuredevops_go/models/migrationscripts/register.go index 59fa7832f..c9a409c4c 100644 --- a/backend/plugins/azuredevops_go/models/migrationscripts/register.go +++ b/backend/plugins/azuredevops_go/models/migrationscripts/register.go @@ -26,5 +26,6 @@ func All() []plugin.MigrationScript { return []plugin.MigrationScript{ new(addInitTables), new(extendRepoTable), + new(addReleaseTables), } } diff --git a/backend/plugins/azuredevops_go/models/release.go b/backend/plugins/azuredevops_go/models/release.go new file mode 100644 index 000000000..cace21e94 --- /dev/null +++ b/backend/plugins/azuredevops_go/models/release.go @@ -0,0 +1,151 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" + "time" +) + +// AzuredevopsRelease represents a release from Azure DevOps Release Pipelines (Classic) +type AzuredevopsRelease struct { + common.NoPKModel + + ConnectionId uint64 `gorm:"primaryKey"` + AzuredevopsId int `json:"id" gorm:"primaryKey"` + ProjectId string + Name string + Status string + ReleaseDefinitionId int + ReleaseDefinitionName string + Description string + CreatedOn *time.Time + ModifiedOn *time.Time +} + +func (AzuredevopsRelease) TableName() string { + return "_tool_azuredevops_go_releases" +} + +// AzuredevopsReleaseDeployment represents a deployment (environment) within a release +type AzuredevopsReleaseDeployment struct { + common.NoPKModel + + ConnectionId uint64 `gorm:"primaryKey"` + AzuredevopsId int `json:"id" gorm:"primaryKey"` + ReleaseId int `gorm:"index"` + ProjectId string + Name string + Status string + OperationStatus string + DeploymentStatus string + DefinitionName string + DefinitionId int + EnvironmentId int + EnvironmentName string + AttemptNumber int + Reason string + QueuedOn *time.Time + StartedOn *time.Time + CompletedOn *time.Time + LastModifiedOn *time.Time +} + +func (AzuredevopsReleaseDeployment) TableName() string { + return "_tool_azuredevops_go_release_deployments" +} + +// AzuredevopsApiRelease is the API response structure from Azure DevOps Release API +type AzuredevopsApiRelease struct { + Id int `json:"id"` + Name string `json:"name"` + Status string `json:"status"` + CreatedOn *time.Time `json:"createdOn"` + ModifiedOn *time.Time `json:"modifiedOn"` + Description string `json:"description"` + ReleaseDefinition struct { + Id int `json:"id"` + Name string `json:"name"` + Url string `json:"url"` + Path string `json:"path"` + } `json:"releaseDefinition"` + Environments []AzuredevopsApiReleaseEnvironment `json:"environments"` + ProjectReference struct { + Id string `json:"id"` + Name string `json:"name"` + } `json:"projectReference"` +} + +// AzuredevopsApiReleaseEnvironment represents an environment in the release +type AzuredevopsApiReleaseEnvironment struct { + Id int `json:"id"` + ReleaseId int `json:"releaseId"` + Name string `json:"name"` + Status string `json:"status"` + DeploySteps []AzuredevopsApiDeployStep `json:"deploySteps"` + PreDeployApprovals []struct { + Status string `json:"status"` + } `json:"preDeployApprovals"` + PostDeployApprovals []struct { + Status string `json:"status"` + } `json:"postDeployApprovals"` +} + +// AzuredevopsApiDeployStep represents a deployment step +type AzuredevopsApiDeployStep struct { + Id int `json:"id"` + DeploymentId int `json:"deploymentId"` + Attempt int `json:"attempt"` + Reason string `json:"reason"` + Status string `json:"status"` + OperationStatus string `json:"operationStatus"` + QueuedOn *time.Time `json:"queuedOn"` + LastModifiedOn *time.Time `json:"lastModifiedOn"` + HasStarted bool `json:"hasStarted"` +} + +// AzuredevopsApiDeployment is the API response structure for deployments +type AzuredevopsApiDeployment struct { + Id int `json:"id"` + Release struct { + Id int `json:"id"` + Name string `json:"name"` + } `json:"release"` + ReleaseDefinition struct { + Id int `json:"id"` + Name string `json:"name"` + Path string `json:"path"` + } `json:"releaseDefinition"` + ReleaseEnvironment struct { + Id int `json:"id"` + Name string `json:"name"` + } `json:"releaseEnvironment"` + ProjectReference struct { + Id string `json:"id"` + Name string `json:"name"` + } `json:"projectReference"` + DefinitionEnvironmentId int `json:"definitionEnvironmentId"` + Attempt int `json:"attempt"` + Reason string `json:"reason"` + DeploymentStatus string `json:"deploymentStatus"` + OperationStatus string `json:"operationStatus"` + QueuedOn *time.Time `json:"queuedOn"` + StartedOn *time.Time `json:"startedOn"` + CompletedOn *time.Time `json:"completedOn"` + LastModifiedOn *time.Time `json:"lastModifiedOn"` +} \ No newline at end of file diff --git a/backend/plugins/azuredevops_go/tasks/release_collector.go b/backend/plugins/azuredevops_go/tasks/release_collector.go new file mode 100644 index 000000000..7744cb85b --- /dev/null +++ b/backend/plugins/azuredevops_go/tasks/release_collector.go @@ -0,0 +1,93 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/url" + "strconv" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +func init() { + RegisterSubtaskMeta(&CollectReleasesMeta) +} + +const RawReleaseTable = "azuredevops_go_api_releases" + +var CollectReleasesMeta = plugin.SubTaskMeta{ + Name: "collectApiReleases", + EntryPoint: CollectReleases, + EnabledByDefault: true, + Description: "Collect Release Pipeline data from Azure DevOps Release API (Classic Pipelines)", + DomainTypes: []string{plugin.DOMAIN_TYPE_CICD}, + ProductTables: []string{RawReleaseTable}, +} + +func CollectReleases(taskCtx plugin.SubTaskContext) errors.Error { + rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RawReleaseTable) + + collector, err := api.NewStatefulApiCollectorForFinalizableEntity(api.FinalizableApiCollectorArgs{ + RawDataSubTaskArgs: *rawDataSubTaskArgs, + ApiClient: data.ApiClient, + CollectNewRecordsByList: api.FinalizableApiCollectorListArgs{ + GetNextPageCustomData: ExtractContToken, + PageSize: 100, + FinalizableApiCollectorCommonArgs: api.FinalizableApiCollectorCommonArgs{ + // Azure DevOps Release API uses a different base URL: vsrm.dev.azure.com + UrlTemplate: "https://vsrm.dev.azure.com/{{ .Params.OrganizationId }}/{{ .Params.ProjectId }}/_apis/release/releases?api-version=7.1", + Query: func(reqData *api.RequestData, createdAfter *time.Time) (url.Values, errors.Error) { + query := url.Values{} + query.Set("$top", strconv.Itoa(reqData.Pager.Size)) + query.Set("$expand", "environments") + if reqData.CustomData != nil { + pag := reqData.CustomData.(CustomPageDate) + query.Set("continuationToken", pag.ContinuationToken) + } + + if createdAfter != nil { + query.Set("minCreatedTime", createdAfter.Format(time.RFC3339)) + } + return query, nil + }, + ResponseParser: ParseRawMessageFromValue, + AfterResponse: change203To401, + }, + GetCreated: func(item json.RawMessage) (time.Time, errors.Error) { + var release struct { + CreatedOn time.Time `json:"createdOn"` + } + err := json.Unmarshal(item, &release) + if err != nil { + return time.Time{}, errors.BadInput.Wrap(err, "failed to unmarshal Azure DevOps Release") + } + return release.CreatedOn, nil + }, + }, + }) + + if err != nil { + return err + } + + return collector.Execute() +} diff --git a/backend/plugins/azuredevops_go/tasks/release_deployment_collector.go b/backend/plugins/azuredevops_go/tasks/release_deployment_collector.go new file mode 100644 index 000000000..b69d0db99 --- /dev/null +++ b/backend/plugins/azuredevops_go/tasks/release_deployment_collector.go @@ -0,0 +1,93 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/url" + "strconv" + "time" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" +) + +func init() { + RegisterSubtaskMeta(&CollectReleaseDeploymentsMeta) +} + +const RawReleaseDeploymentTable = "azuredevops_go_api_release_deployments" + +var CollectReleaseDeploymentsMeta = plugin.SubTaskMeta{ + Name: "collectApiReleaseDeployments", + EntryPoint: CollectReleaseDeployments, + EnabledByDefault: true, + Description: "Collect Release Deployment data from Azure DevOps Release API (Classic Pipelines)", + DomainTypes: []string{plugin.DOMAIN_TYPE_CICD}, + ProductTables: []string{RawReleaseDeploymentTable}, +} + +func CollectReleaseDeployments(taskCtx plugin.SubTaskContext) errors.Error { + rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RawReleaseDeploymentTable) + + collector, err := api.NewStatefulApiCollectorForFinalizableEntity(api.FinalizableApiCollectorArgs{ + RawDataSubTaskArgs: *rawDataSubTaskArgs, + ApiClient: data.ApiClient, + CollectNewRecordsByList: api.FinalizableApiCollectorListArgs{ + GetNextPageCustomData: ExtractContToken, + PageSize: 100, + FinalizableApiCollectorCommonArgs: api.FinalizableApiCollectorCommonArgs{ + // Azure DevOps Release API uses a different base URL: vsrm.dev.azure.com + UrlTemplate: "https://vsrm.dev.azure.com/{{ .Params.OrganizationId }}/{{ .Params.ProjectId }}/_apis/release/deployments?api-version=7.1", + Query: func(reqData *api.RequestData, createdAfter *time.Time) (url.Values, errors.Error) { + query := url.Values{} + query.Set("$top", strconv.Itoa(reqData.Pager.Size)) + query.Set("queryOrder", "descending") + if reqData.CustomData != nil { + pag := reqData.CustomData.(CustomPageDate) + query.Set("continuationToken", pag.ContinuationToken) + } + + if createdAfter != nil { + query.Set("minStartedTime", createdAfter.Format(time.RFC3339)) + } + return query, nil + }, + ResponseParser: ParseRawMessageFromValue, + AfterResponse: change203To401, + }, + GetCreated: func(item json.RawMessage) (time.Time, errors.Error) { + var deployment struct { + QueuedOn time.Time `json:"queuedOn"` + } + err := json.Unmarshal(item, &deployment) + if err != nil { + return time.Time{}, errors.BadInput.Wrap(err, "failed to unmarshal Azure DevOps Release Deployment") + } + return deployment.QueuedOn, nil + }, + }, + }) + + if err != nil { + return err + } + + return collector.Execute() +} diff --git a/backend/plugins/azuredevops_go/tasks/release_deployment_converter.go b/backend/plugins/azuredevops_go/tasks/release_deployment_converter.go new file mode 100644 index 000000000..805e06489 --- /dev/null +++ b/backend/plugins/azuredevops_go/tasks/release_deployment_converter.go @@ -0,0 +1,157 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/devops" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" +) + +func init() { + RegisterSubtaskMeta(&ConvertReleaseDeploymentsMeta) +} + +var ConvertReleaseDeploymentsMeta = plugin.SubTaskMeta{ + Name: "convertApiReleaseDeployments", + EntryPoint: ConvertReleaseDeployments, + EnabledByDefault: true, + Description: "Convert tool layer table azuredevops_release_deployments into domain layer table cicd_pipelines", + DomainTypes: []string{plugin.DOMAIN_TYPE_CICD}, + DependencyTables: []string{ + models.AzuredevopsReleaseDeployment{}.TableName(), + }, +} + +// Release deployment status and operation status mappings +// Reference: https://learn.microsoft.com/en-us/rest/api/azure/devops/release/deployments/list +const ( + releaseDeploymentStatusSucceeded = "succeeded" + releaseDeploymentStatusFailed = "failed" + releaseDeploymentStatusNotDeployed = "notDeployed" + releaseDeploymentStatusPartiallySucceeded = "partiallySucceeded" + releaseOperationStatusApproved = "Approved" + releaseOperationStatusCanceled = "Canceled" + releaseOperationStatusCancelling = "Cancelling" + releaseOperationStatusDeferred = "Deferred" + releaseOperationStatusEvaluatingGates = "EvaluatingGates" + releaseOperationStatusGateFailed = "GateFailed" + releaseOperationStatusManualInterventionPending = "ManualInterventionPending" + releaseOperationStatusPending = "Pending" + releaseOperationStatusPhaseCanceled = "PhaseCanceled" + releaseOperationStatusPhaseFailed = "PhaseFailed" + releaseOperationStatusPhaseInProgress = "PhaseInProgress" + releaseOperationStatusPhasePartiallySucceeded = "PhasePartiallySucceeded" + releaseOperationStatusPhaseSucceeded = "PhaseSucceeded" + releaseOperationStatusQueued = "Queued" + releaseOperationStatusRejected = "Rejected" + releaseOperationStatusScheduled = "Scheduled" + releaseOperationStatusUndefined = "Undefined" +) + +var releaseDeploymentResultRule = devops.ResultRule{ + Success: []string{releaseDeploymentStatusSucceeded, releaseDeploymentStatusPartiallySucceeded}, + Failure: []string{releaseDeploymentStatusFailed, releaseDeploymentStatusNotDeployed}, + Default: devops.RESULT_DEFAULT, +} + +var releaseDeploymentStatusRule = devops.StatusRule{ + Done: []string{releaseOperationStatusApproved, releaseOperationStatusCanceled, releaseOperationStatusRejected, releaseOperationStatusPhaseCanceled, releaseOperationStatusPhaseFailed, releaseOperationStatusPhasePartiallySucceeded, releaseOperationStatusPhaseSucceeded, releaseOperationStatusGateFailed}, + InProgress: []string{releaseOperationStatusPending, releaseOperationStatusQueued, releaseOperationStatusScheduled, releaseOperationStatusDeferred, releaseOperationStatusCancelling, releaseOperationStatusEvaluatingGates, releaseOperationStatusManualInterventionPending, releaseOperationStatusPhaseInProgress, releaseOperationStatusUndefined}, + Default: devops.STATUS_OTHER, +} + +func ConvertReleaseDeployments(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RawReleaseDeploymentTable) + clauses := []dal.Clause{ + dal.Select("*"), + dal.From(&models.AzuredevopsReleaseDeployment{}), + dal.Where("project_id = ? and connection_id = ?", + data.Options.ProjectId, data.Options.ConnectionId), + } + + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + defer cursor.Close() + + deploymentIdGen := didgen.NewDomainIdGenerator(&models.AzuredevopsReleaseDeployment{}) + projectIdGen := didgen.NewDomainIdGenerator(&models.AzuredevopsProject{}) + + converter, err := api.NewDataConverter(api.DataConverterArgs{ + RawDataSubTaskArgs: *rawDataSubTaskArgs, + InputRowType: reflect.TypeOf(models.AzuredevopsReleaseDeployment{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + deployment := inputRow.(*models.AzuredevopsReleaseDeployment) + duration := 0.0 + + if deployment.CompletedOn != nil && deployment.StartedOn != nil { + duration = float64(deployment.CompletedOn.Sub(*deployment.StartedOn).Milliseconds() / 1e3) + } + + // Create a unique pipeline name combining release definition and environment + pipelineName := deployment.DefinitionName + if deployment.EnvironmentName != "" { + pipelineName = deployment.DefinitionName + " - " + deployment.EnvironmentName + } + + domainPipeline := &devops.CICDPipeline{ + DomainEntity: domainlayer.DomainEntity{ + Id: deploymentIdGen.Generate(data.Options.ConnectionId, deployment.AzuredevopsId), + }, + Name: pipelineName, + Result: devops.GetResult(&releaseDeploymentResultRule, deployment.DeploymentStatus), + Status: devops.GetStatus(&releaseDeploymentStatusRule, deployment.OperationStatus), + OriginalStatus: deployment.OperationStatus, + OriginalResult: deployment.DeploymentStatus, + CicdScopeId: projectIdGen.Generate(data.Options.ConnectionId, data.Options.ProjectId), + Environment: data.RegexEnricher.ReturnNameIfMatched(devops.PRODUCTION, pipelineName+";"+deployment.EnvironmentName), + Type: devops.DEPLOYMENT, + DurationSec: duration, + } + + if deployment.QueuedOn != nil { + domainPipeline.TaskDatesInfo = devops.TaskDatesInfo{ + CreatedDate: *deployment.QueuedOn, + QueuedDate: deployment.QueuedOn, + StartedDate: deployment.StartedOn, + FinishedDate: deployment.CompletedOn, + } + } + + return []interface{}{ + domainPipeline, + }, nil + }, + }) + if err != nil { + return err + } + + return converter.Execute() +} diff --git a/backend/plugins/azuredevops_go/tasks/release_deployment_extractor.go b/backend/plugins/azuredevops_go/tasks/release_deployment_extractor.go new file mode 100644 index 000000000..af75bdc1f --- /dev/null +++ b/backend/plugins/azuredevops_go/tasks/release_deployment_extractor.go @@ -0,0 +1,91 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" +) + +func init() { + RegisterSubtaskMeta(&ExtractApiReleaseDeploymentsMeta) +} + +var ExtractApiReleaseDeploymentsMeta = plugin.SubTaskMeta{ + Name: "extractApiReleaseDeployments", + EntryPoint: ExtractApiReleaseDeployments, + EnabledByDefault: true, + Description: "Extract raw release deployment data into tool layer table", + DomainTypes: []string{plugin.DOMAIN_TYPE_CICD}, + DependencyTables: []string{RawReleaseDeploymentTable}, + ProductTables: []string{ + models.AzuredevopsReleaseDeployment{}.TableName(), + }, +} + +func ExtractApiReleaseDeployments(taskCtx plugin.SubTaskContext) errors.Error { + rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RawReleaseDeploymentTable) + + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: *rawDataSubTaskArgs, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + results := make([]interface{}, 0, 1) + + deploymentApi := &models.AzuredevopsApiDeployment{} + err := errors.Convert(json.Unmarshal(row.Data, deploymentApi)) + if err != nil { + return nil, err + } + + deployment := &models.AzuredevopsReleaseDeployment{ + ConnectionId: data.Options.ConnectionId, + AzuredevopsId: deploymentApi.Id, + ReleaseId: deploymentApi.Release.Id, + ProjectId: data.Options.ProjectId, + Name: deploymentApi.Release.Name, + Status: deploymentApi.OperationStatus, + OperationStatus: deploymentApi.OperationStatus, + DeploymentStatus: deploymentApi.DeploymentStatus, + DefinitionName: deploymentApi.ReleaseDefinition.Name, + DefinitionId: deploymentApi.ReleaseDefinition.Id, + EnvironmentId: deploymentApi.ReleaseEnvironment.Id, + EnvironmentName: deploymentApi.ReleaseEnvironment.Name, + AttemptNumber: deploymentApi.Attempt, + Reason: deploymentApi.Reason, + QueuedOn: deploymentApi.QueuedOn, + StartedOn: deploymentApi.StartedOn, + CompletedOn: deploymentApi.CompletedOn, + LastModifiedOn: deploymentApi.LastModifiedOn, + } + + results = append(results, deployment) + + return results, nil + }, + }) + + if err != nil { + return err + } + + return extractor.Execute() +} diff --git a/backend/plugins/azuredevops_go/tasks/release_extractor.go b/backend/plugins/azuredevops_go/tasks/release_extractor.go new file mode 100644 index 000000000..ee01a06ce --- /dev/null +++ b/backend/plugins/azuredevops_go/tasks/release_extractor.go @@ -0,0 +1,83 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/azuredevops_go/models" +) + +func init() { + RegisterSubtaskMeta(&ExtractApiReleasesMeta) +} + +var ExtractApiReleasesMeta = plugin.SubTaskMeta{ + Name: "extractApiReleases", + EntryPoint: ExtractApiReleases, + EnabledByDefault: true, + Description: "Extract raw release data into tool layer table", + DomainTypes: []string{plugin.DOMAIN_TYPE_CICD}, + DependencyTables: []string{RawReleaseTable}, + ProductTables: []string{ + models.AzuredevopsRelease{}.TableName(), + }, +} + +func ExtractApiReleases(taskCtx plugin.SubTaskContext) errors.Error { + rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx, RawReleaseTable) + + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: *rawDataSubTaskArgs, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + results := make([]interface{}, 0, 1) + + releaseApi := &models.AzuredevopsApiRelease{} + err := errors.Convert(json.Unmarshal(row.Data, releaseApi)) + if err != nil { + return nil, err + } + + release := &models.AzuredevopsRelease{ + ConnectionId: data.Options.ConnectionId, + AzuredevopsId: releaseApi.Id, + ProjectId: data.Options.ProjectId, + Name: releaseApi.Name, + Status: releaseApi.Status, + ReleaseDefinitionId: releaseApi.ReleaseDefinition.Id, + ReleaseDefinitionName: releaseApi.ReleaseDefinition.Name, + Description: releaseApi.Description, + CreatedOn: releaseApi.CreatedOn, + ModifiedOn: releaseApi.ModifiedOn, + } + + results = append(results, release) + + return results, nil + }, + }) + + if err != nil { + return err + } + + return extractor.Execute() +}
