This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new a7217dbf2 feat(gitlab): add deployment entity (#6165)
a7217dbf2 is described below
commit a7217dbf29a1ec847b5e0275183be4c75ea4944d
Author: Lynwee <[email protected]>
AuthorDate: Sun Oct 8 11:09:20 2023 +0800
feat(gitlab): add deployment entity (#6165)
* feat(gitlab): add deployment entity
* feat(gitlab): support transformation for deployment's environment field
* fix(gitlab): move GitlabDeployment to migration's archived package and
add gitlab_id field
* fix(gitlab): make DeployableDuration *float64 instead of float64
* fix(gitlab): remove unused if statements
* fix(gitlab): update DeployableDuration, make it *float64 in migration
script
* fix(gitlab): make deployment to support incremental update
* fix(gitlab): remove unused codes
---
backend/plugins/github/utils/utils.go | 2 +-
backend/plugins/gitlab/impl/impl.go | 1 +
backend/plugins/gitlab/models/deployment.go | 98 +++++++++
.../{register.go => 20230926_add_deployment.go} | 37 ++--
.../models/migrationscripts/archived/deployment.go | 98 +++++++++
.../gitlab/models/migrationscripts/register.go | 1 +
.../plugins/gitlab/tasks/deployment_collector.go | 75 +++++++
.../plugins/gitlab/tasks/deployment_convertor.go | 138 ++++++++++++
.../plugins/gitlab/tasks/deployment_extractor.go | 236 +++++++++++++++++++++
9 files changed, 664 insertions(+), 22 deletions(-)
diff --git a/backend/plugins/github/utils/utils.go
b/backend/plugins/github/utils/utils.go
index f2c1e7158..85125e602 100644
--- a/backend/plugins/github/utils/utils.go
+++ b/backend/plugins/github/utils/utils.go
@@ -75,7 +75,7 @@ func GetRateLimitPerSecond(info RateLimitInfo) int {
unixResetTime := info.ResetTime.Unix()
unixNow := info.Date.Unix()
timeBetweenNowAndReset := unixResetTime - unixNow
- // Adjust the remaining to be less then actual to avoid hitting the
limit exactly.
+ // Adjust the remaining to be less than actual to avoid hitting the
limit exactly.
multiplier := 0.98
adjustedRemaining := float64(info.Remaining) * multiplier
return int(adjustedRemaining / float64(timeBetweenNowAndReset)) //*
multiplier
diff --git a/backend/plugins/gitlab/impl/impl.go
b/backend/plugins/gitlab/impl/impl.go
index 5b1f79dc5..d4799ab3e 100644
--- a/backend/plugins/gitlab/impl/impl.go
+++ b/backend/plugins/gitlab/impl/impl.go
@@ -101,6 +101,7 @@ func (p Gitlab) GetTablesInfo() []dal.Tabler {
&models.GitlabTag{},
&models.GitlabIssueAssignee{},
&models.GitlabScopeConfig{},
+ &models.GitlabDeployment{},
}
}
diff --git a/backend/plugins/gitlab/models/deployment.go
b/backend/plugins/gitlab/models/deployment.go
new file mode 100644
index 000000000..2eb353b4c
--- /dev/null
+++ b/backend/plugins/gitlab/models/deployment.go
@@ -0,0 +1,98 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "github.com/apache/incubator-devlake/core/models/common"
+ "time"
+)
+
+type GitlabDeployment struct {
+ common.NoPKModel `swaggerignore:"true" json:"-" mapstructure:"-"`
+
+ ConnectionId uint64 `json:"connection_id" gorm:"primaryKey"`
+ GitlabId int `json:"gitlab_id" gorm:"primaryKey"`
+
+ CreatedDate time.Time `json:"created_date"`
+ UpdatedDate time.Time `json:"updated_date"`
+ Status string `json:"status"` //created, running, success,
failed, canceled, or blocked
+
+ DeploymentId int `json:"id" gorm:"primaryKey"`
+ Iid int `json:"iid"`
+ Ref string `json:"ref"`
+ Sha string `json:"sha"`
+ Environment string `json:"environment" gorm:"type:varchar(255)"`
+ Name string `json:"name" gorm:"type:varchar(255)"`
+
+ DeployableCommitAuthorEmail string
`json:"deployable_commit_author_email" gorm:"type:varchar(255)"`
+ DeployableCommitAuthorName string
`json:"deployable_commit_author_name" gorm:"type:varchar(255)"`
+ DeployableCommitCreatedAt time.Time
`json:"deployable_commit_created_at"`
+ DeployableCommitID string `json:"deployable_commit_id"
gorm:"type:varchar(255)"`
+ DeployableCommitMessage string `json:"deployable_commit_message"
gorm:"type:varchar(255)"`
+ DeployableCommitShortID string
`json:"deployable_commit_short_id" gorm:"type:varchar(255)"`
+ DeployableCommitTitle string `json:"deployable_commit_title"
gorm:"type:varchar(255)"`
+
+ //DeployableCoverage any `json:"deployable_coverage"`
+ DeployableCreatedAt *time.Time `json:"deployable_created_at"`
+ DeployableFinishedAt *time.Time `json:"deployable_finished_at"`
+ DeployableID int `json:"deployable_id"`
+ DeployableName string `json:"deployable_name"
gorm:"type:varchar(255)"`
+ DeployableRef string `json:"deployable_ref"
gorm:"type:varchar(255)"`
+ //DeployableRunner any `json:"deployable_runner"`
+ DeployableStage string `json:"deployable_stage"
gorm:"type:varchar(255)"`
+ DeployableStartedAt *time.Time `json:"deployable_started_at"`
+ DeployableStatus string `json:"deployable_status"
gorm:"type:varchar(255)"`
+ DeployableTag bool `json:"deployable_tag"`
+ DeployableDuration *float64 `json:"deployable_duration"`
+ QueuedDuration float64 `json:"queued_duration"`
+
+ DeployableUserID int `json:"deployable_user_id"`
+ DeployableUserName string `json:"deployable_user_name"
gorm:"type:varchar(255)"`
+ DeployableUserUsername string `json:"deployable_user_username"
gorm:"type:varchar(255)"`
+ DeployableUserState string `json:"deployable_user_state"
gorm:"type:varchar(255)"`
+ DeployableUserAvatarURL string `json:"deployable_user_avatar_url"
gorm:"type:varchar(255)"`
+ DeployableUserWebURL string `json:"deployable_user_web_url"
gorm:"type:varchar(255)"`
+ DeployableUserCreatedAt time.Time `json:"deployable_user_created_at"`
+ //DeployableUserBio any `json:"deployable_user_bio"`
+ //DeployableUserLocation any `json:"deployable_user_location"`
+ DeployableUserPublicEmail string `json:"deployable_user_public_email"
gorm:"type:varchar(255)"`
+ DeployableUserSkype string `json:"deployable_user_skype"
gorm:"type:varchar(255)"`
+ DeployableUserLinkedin string `json:"deployable_user_linkedin"
gorm:"type:varchar(255)"`
+ DeployableUserTwitter string `json:"deployable_user_twitter"
gorm:"type:varchar(255)"`
+ DeployableUserWebsiteURL string `json:"deployable_user_website_url"
gorm:"type:varchar(255)"`
+ DeployableUserOrganization string `json:"deployable_user_organization"
gorm:"type:varchar(255)"`
+
+ DeployablePipelineCreatedAt time.Time
`json:"deployable_pipeline_created_at"`
+ DeployablePipelineID int `json:"deployable_pipeline_id"`
+ DeployablePipelineRef string `json:"deployable_pipeline_ref"
gorm:"type:varchar(255)"`
+ DeployablePipelineSha string `json:"deployable_pipeline_sha"
gorm:"type:varchar(255)"`
+ DeployablePipelineStatus string
`json:"deployable_pipeline_status" gorm:"type:varchar(255)"`
+ DeployablePipelineUpdatedAt time.Time
`json:"deployable_pipeline_updated_at"`
+ DeployablePipelineWebURL string
`json:"deployable_pipeline_web_url" gorm:"type:varchar(255)"`
+
+ UserAvatarURL string `json:"user_avatar_url" gorm:"type:varchar(255)"`
+ UserID int `json:"user_id"`
+ UserName string `json:"user_name" gorm:"type:varchar(255)"`
+ UserState string `json:"user_state" gorm:"type:varchar(255)"`
+ UserUsername string `json:"user_username" gorm:"type:varchar(255)"`
+ UserWebURL string `json:"user_web_url" gorm:"type:varchar(255)"`
+}
+
+func (GitlabDeployment) TableName() string {
+ return "_tool_gitlab_deployments"
+}
diff --git a/backend/plugins/gitlab/models/migrationscripts/register.go
b/backend/plugins/gitlab/models/migrationscripts/20230926_add_deployment.go
similarity index 55%
copy from backend/plugins/gitlab/models/migrationscripts/register.go
copy to
backend/plugins/gitlab/models/migrationscripts/20230926_add_deployment.go
index 9287ac8ea..b8674c9b3 100644
--- a/backend/plugins/gitlab/models/migrationscripts/register.go
+++ b/backend/plugins/gitlab/models/migrationscripts/20230926_add_deployment.go
@@ -18,27 +18,22 @@ limitations under the License.
package migrationscripts
import (
- "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+
"github.com/apache/incubator-devlake/plugins/gitlab/models/migrationscripts/archived"
)
-// All return all the migration scripts
-func All() []plugin.MigrationScript {
- return []plugin.MigrationScript{
- new(addInitTables),
- new(addGitlabCI),
- new(addPipelineID),
- new(addPipelineProjects),
- new(fixDurationToFloat8),
- new(addTransformationRule20221125),
- new(addStdTypeToIssue221230),
- new(addIsDetailRequired20230210),
- new(addConnectionIdToTransformationRule),
- new(addGitlabCommitAuthorInfo),
- new(addTypeEnvToPipeline),
- new(renameTr2ScopeConfig),
- new(addGitlabIssueAssignee),
- new(addMrCommitSha),
- new(addRawParamTableForScope),
- new(addProjectArchived),
- }
+type addDeployment struct {
+}
+
+func (addDeployment) Up(basicRes context.BasicRes) errors.Error {
+ return basicRes.GetDal().AutoMigrate(&archived.GitlabDeployment{})
+}
+
+func (addDeployment) Version() uint64 {
+ return 20230926140000
+}
+
+func (addDeployment) Name() string {
+ return "add deployment table in tool layer"
}
diff --git
a/backend/plugins/gitlab/models/migrationscripts/archived/deployment.go
b/backend/plugins/gitlab/models/migrationscripts/archived/deployment.go
new file mode 100644
index 000000000..60f9c285b
--- /dev/null
+++ b/backend/plugins/gitlab/models/migrationscripts/archived/deployment.go
@@ -0,0 +1,98 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+import (
+
"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+ "time"
+)
+
+type GitlabDeployment struct {
+ archived.NoPKModel `swaggerignore:"true" json:"-" mapstructure:"-"`
+
+ ConnectionId uint64 `json:"connection_id" gorm:"primaryKey"`
+ GitlabId int `json:"gitlab_id" gorm:"primaryKey"`
+
+ CreatedDate time.Time `json:"created_date"`
+ UpdatedDate time.Time `json:"updated_date"`
+ Status string `json:"status"`
+
+ DeploymentId int `json:"id" gorm:"primaryKey"`
+ Iid int `json:"iid"`
+ Ref string `json:"ref"`
+ Sha string `json:"sha"`
+ Environment string `json:"environment" gorm:"type:varchar(255)"`
+ Name string `json:"name" gorm:"type:varchar(255)"`
+
+ DeployableCommitAuthorEmail string
`json:"deployable_commit_author_email" gorm:"type:varchar(255)"`
+ DeployableCommitAuthorName string
`json:"deployable_commit_author_name" gorm:"type:varchar(255)"`
+ DeployableCommitCreatedAt time.Time
`json:"deployable_commit_created_at"`
+ DeployableCommitID string `json:"deployable_commit_id"
gorm:"type:varchar(255)"`
+ DeployableCommitMessage string `json:"deployable_commit_message"
gorm:"type:varchar(255)"`
+ DeployableCommitShortID string
`json:"deployable_commit_short_id" gorm:"type:varchar(255)"`
+ DeployableCommitTitle string `json:"deployable_commit_title"
gorm:"type:varchar(255)"`
+
+ //DeployableCoverage any `json:"deployable_coverage"`
+ DeployableCreatedAt *time.Time `json:"deployable_created_at"`
+ DeployableFinishedAt *time.Time `json:"deployable_finished_at"`
+ DeployableID int `json:"deployable_id"`
+ DeployableName string `json:"deployable_name"
gorm:"type:varchar(255)"`
+ DeployableRef string `json:"deployable_ref"
gorm:"type:varchar(255)"`
+ //DeployableRunner any `json:"deployable_runner"`
+ DeployableStage string `json:"deployable_stage"
gorm:"type:varchar(255)"`
+ DeployableStartedAt *time.Time `json:"deployable_started_at"`
+ DeployableStatus string `json:"deployable_status"
gorm:"type:varchar(255)"`
+ DeployableTag bool `json:"deployable_tag"`
+ DeployableDuration *float64 `json:"deployable_duration"`
+ QueuedDuration float64 `json:"queued_duration"`
+
+ DeployableUserID int `json:"deployable_user_id"`
+ DeployableUserName string `json:"deployable_user_name"
gorm:"type:varchar(255)"`
+ DeployableUserUsername string `json:"deployable_user_username"
gorm:"type:varchar(255)"`
+ DeployableUserState string `json:"deployable_user_state"
gorm:"type:varchar(255)"`
+ DeployableUserAvatarURL string `json:"deployable_user_avatar_url"
gorm:"type:varchar(255)"`
+ DeployableUserWebURL string `json:"deployable_user_web_url"
gorm:"type:varchar(255)"`
+ DeployableUserCreatedAt time.Time `json:"deployable_user_created_at"`
+ //DeployableUserBio any `json:"deployable_user_bio"`
+ //DeployableUserLocation any `json:"deployable_user_location"`
+ DeployableUserPublicEmail string `json:"deployable_user_public_email"
gorm:"type:varchar(255)"`
+ DeployableUserSkype string `json:"deployable_user_skype"
gorm:"type:varchar(255)"`
+ DeployableUserLinkedin string `json:"deployable_user_linkedin"
gorm:"type:varchar(255)"`
+ DeployableUserTwitter string `json:"deployable_user_twitter"
gorm:"type:varchar(255)"`
+ DeployableUserWebsiteURL string `json:"deployable_user_website_url"
gorm:"type:varchar(255)"`
+ DeployableUserOrganization string `json:"deployable_user_organization"
gorm:"type:varchar(255)"`
+
+ DeployablePipelineCreatedAt time.Time
`json:"deployable_pipeline_created_at"`
+ DeployablePipelineID int `json:"deployable_pipeline_id"`
+ DeployablePipelineRef string `json:"deployable_pipeline_ref"
gorm:"type:varchar(255)"`
+ DeployablePipelineSha string `json:"deployable_pipeline_sha"
gorm:"type:varchar(255)"`
+ DeployablePipelineStatus string
`json:"deployable_pipeline_status" gorm:"type:varchar(255)"`
+ DeployablePipelineUpdatedAt time.Time
`json:"deployable_pipeline_updated_at"`
+ DeployablePipelineWebURL string
`json:"deployable_pipeline_web_url" gorm:"type:varchar(255)"`
+
+ UserAvatarURL string `json:"user_avatar_url" gorm:"type:varchar(255)"`
+ UserID int `json:"user_id"`
+ UserName string `json:"user_name" gorm:"type:varchar(255)"`
+ UserState string `json:"user_state" gorm:"type:varchar(255)"`
+ UserUsername string `json:"user_username" gorm:"type:varchar(255)"`
+ UserWebURL string `json:"user_web_url" gorm:"type:varchar(255)"`
+}
+
+func (GitlabDeployment) TableName() string {
+ return "_tool_gitlab_deployments"
+}
diff --git a/backend/plugins/gitlab/models/migrationscripts/register.go
b/backend/plugins/gitlab/models/migrationscripts/register.go
index 9287ac8ea..c6c547d2e 100644
--- a/backend/plugins/gitlab/models/migrationscripts/register.go
+++ b/backend/plugins/gitlab/models/migrationscripts/register.go
@@ -40,5 +40,6 @@ func All() []plugin.MigrationScript {
new(addMrCommitSha),
new(addRawParamTableForScope),
new(addProjectArchived),
+ new(addDeployment),
}
}
diff --git a/backend/plugins/gitlab/tasks/deployment_collector.go
b/backend/plugins/gitlab/tasks/deployment_collector.go
new file mode 100644
index 000000000..721ab558e
--- /dev/null
+++ b/backend/plugins/gitlab/tasks/deployment_collector.go
@@ -0,0 +1,75 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "net/url"
+ "time"
+)
+
+var _ plugin.SubTaskEntryPoint = CollectDeployment
+
+const (
+ RAW_DEPLOYMENT = "gitlab_deployments"
+)
+
+func init() {
+ RegisterSubtaskMeta(CollectDeploymentMeta)
+}
+
+var CollectDeploymentMeta = &plugin.SubTaskMeta{
+ Name: "CollectDeployment",
+ EntryPoint: CollectDeployment,
+ EnabledByDefault: true,
+ Description: "Collect gitlab deployment from api into raw layer
table",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+ Dependencies: []*plugin.SubTaskMeta{},
+}
+
+func CollectDeployment(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_DEPLOYMENT)
+ collectorWithState, err :=
helper.NewStatefulApiCollector(*rawDataSubTaskArgs)
+ if err != nil {
+ return err
+ }
+ err = collectorWithState.InitCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ UrlTemplate: "projects/{{ .Params.ProjectId
}}/deployments",
+ Query: func(reqData *helper.RequestData) (url.Values,
errors.Error) {
+ query, err := GetQuery(reqData)
+ if err != nil {
+ return query, err
+ }
+ if collectorWithState.Since != nil {
+ query.Set("updated_after",
collectorWithState.Since.Format(time.RFC3339))
+ }
+ return query, nil
+ },
+ GetTotalPages: GetTotalPagesFromResponse,
+ ResponseParser: GetRawMessageFromResponse,
+ })
+ if err != nil {
+ return err
+ }
+ return collectorWithState.Execute()
+}
diff --git a/backend/plugins/gitlab/tasks/deployment_convertor.go
b/backend/plugins/gitlab/tasks/deployment_convertor.go
new file mode 100644
index 000000000..a5faf8986
--- /dev/null
+++ b/backend/plugins/gitlab/tasks/deployment_convertor.go
@@ -0,0 +1,138 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "fmt"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/domainlayer"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/gitlab/models"
+ "github.com/spf13/cast"
+ "reflect"
+)
+
+var _ plugin.SubTaskEntryPoint = ConvertDeployment
+
+func init() {
+ RegisterSubtaskMeta(ConvertDeploymentMeta)
+}
+
+var ConvertDeploymentMeta = &plugin.SubTaskMeta{
+ Name: "ConvertDeployment",
+ EntryPoint: ConvertDeployment,
+ EnabledByDefault: true,
+ Description: "Convert gitlab deployment from tool layer to domain
layer",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+ Dependencies: []*plugin.SubTaskMeta{ExtractDeploymentMeta},
+}
+
+func ConvertDeployment(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_DEPLOYMENT)
+ db := taskCtx.GetDal()
+
+ repo := &models.GitlabProject{}
+ err := db.First(repo, dal.Where("gitlab_id = ? and connection_id = ?",
data.Options.ProjectId, data.Options.ConnectionId))
+ if err != nil {
+ return err
+ }
+
+ projectIdGen := didgen.NewDomainIdGenerator(&models.GitlabProject{})
+
+ cursor, err := db.Cursor(
+ dal.From(&models.GitlabDeployment{}),
+ dal.Where("connection_id = ? AND gitlab_id = ?",
data.Options.ConnectionId, data.Options.ProjectId),
+ )
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ idGen := didgen.NewDomainIdGenerator(&models.GitlabDeployment{})
+ //pipelineIdGen :=
didgen.NewDomainIdGenerator(&models.BitbucketPipeline{})
+
+ converter, err := api.NewDataConverter(api.DataConverterArgs{
+ InputRowType: reflect.TypeOf(models.GitlabDeployment{}),
+ Input: cursor,
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
+ gitlabDeployment := inputRow.(*models.GitlabDeployment)
+
+ var duration *uint64
+ if gitlabDeployment.DeployableDuration != nil {
+ deployableDuration :=
cast.ToUint64(*gitlabDeployment.DeployableDuration)
+ duration = &deployableDuration
+ }
+ if duration == nil || *duration == 0 {
+ if gitlabDeployment.DeployableFinishedAt != nil
&& gitlabDeployment.DeployableCreatedAt != nil {
+ deployableDuration :=
uint64(gitlabDeployment.DeployableFinishedAt.Sub(*gitlabDeployment.DeployableCreatedAt).Seconds())
+ duration = &deployableDuration
+ }
+ }
+ domainDeployCommit := &devops.CicdDeploymentCommit{
+ DomainEntity:
domainlayer.NewDomainEntity(idGen.Generate(data.Options.ConnectionId,
gitlabDeployment.DeploymentId)),
+ CicdScopeId:
projectIdGen.Generate(data.Options.ConnectionId, data.Options.ProjectId),
+ CicdDeploymentId:
idGen.Generate(data.Options.ConnectionId, gitlabDeployment.DeploymentId),
+ Name: fmt.Sprintf("%s:%d",
gitlabDeployment.Name, gitlabDeployment.DeploymentId),
+ Result: devops.GetResult(&devops.ResultRule{
+ Failed: []string{"UNDEPLOYED",
"failed"},
+ Success: []string{"COMPLETED",
"success"},
+ Abort: []string{"created",
"canceled"},
+ Manual: []string{"running", "blocked"},
+ Default: gitlabDeployment.Status,
+ }, gitlabDeployment.Status),
+ Status:
devops.GetStatus(&devops.StatusRule[string]{
+ Done: []string{"COMPLETED",
"UNDEPLOYED", "failed", "success", "canceled"},
+ InProgress: []string{"running"},
+ NotStarted: []string{"created"},
+ Manual: []string{"blocked"},
+ Default: gitlabDeployment.Status,
+ }, gitlabDeployment.Status),
+ Environment: gitlabDeployment.Environment,
+ CreatedDate: gitlabDeployment.CreatedDate,
+ StartedDate:
gitlabDeployment.DeployableStartedAt,
+ FinishedDate:
gitlabDeployment.DeployableFinishedAt,
+ CommitSha: gitlabDeployment.Sha,
+ RefName: gitlabDeployment.Ref,
+ RepoId:
didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(data.Options.ConnectionId,
data.Options.ProjectId),
+ RepoUrl: repo.WebUrl,
+ }
+ if duration != nil {
+ domainDeployCommit.DurationSec = duration
+ }
+ if data.RegexEnricher != nil {
+ domainDeployCommit.Environment =
data.RegexEnricher.ReturnNameIfOmittedOrMatched(devops.PRODUCTION,
gitlabDeployment.Environment)
+ }
+
+ return []interface{}{
+ domainDeployCommit,
+ domainDeployCommit.ToDeployment(),
+ }, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/backend/plugins/gitlab/tasks/deployment_extractor.go
b/backend/plugins/gitlab/tasks/deployment_extractor.go
new file mode 100644
index 000000000..2e8124301
--- /dev/null
+++ b/backend/plugins/gitlab/tasks/deployment_extractor.go
@@ -0,0 +1,236 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/common"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/gitlab/models"
+ "time"
+)
+
+var _ plugin.SubTaskEntryPoint = ExtractDeployment
+
+func init() {
+ RegisterSubtaskMeta(ExtractDeploymentMeta)
+}
+
+var ExtractDeploymentMeta = &plugin.SubTaskMeta{
+ Name: "ExtractDeployment",
+ EntryPoint: ExtractDeployment,
+ EnabledByDefault: true,
+ Description: "Extract gitlab deployment from raw layer to tool
layer",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_CICD},
+ Dependencies: []*plugin.SubTaskMeta{CollectDeploymentMeta},
+}
+
+func ExtractDeployment(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_DEPLOYMENT)
+
+ extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
+ deploymentResp := &GitlabDeploymentResp{}
+ err := errors.Convert(json.Unmarshal(row.Data,
deploymentResp))
+ if err != nil {
+ return nil, err
+ }
+ gitlabDeployment :=
deploymentResp.toGitlabDeployment(data.Options.ConnectionId,
data.Options.ProjectId)
+ return []interface{}{
+ gitlabDeployment,
+ }, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
+
+type GitlabDeploymentResp struct {
+ CreatedAt time.Time `json:"created_at"`
+ UpdatedAt time.Time `json:"updated_at"`
+ Status string `json:"status"`
+ Deployable GitlabDeploymentDeployable `json:"deployable"`
+ Environment GitlabDeploymentEnvironment `json:"environment"`
+ ID int `json:"id"`
+ Iid int `json:"iid"`
+ Ref string `json:"ref"`
+ Sha string `json:"sha"`
+ User GitlabDeploymentSimpleUser `json:"user"`
+}
+
+func (r GitlabDeploymentResp) toGitlabDeployment(connectionId uint64, gitlabId
int) *models.GitlabDeployment {
+ return &models.GitlabDeployment{
+ NoPKModel: common.NewNoPKModel(),
+ ConnectionId: connectionId,
+ GitlabId: gitlabId,
+ CreatedDate: r.CreatedAt,
+ UpdatedDate: r.UpdatedAt,
+ Status: r.Status,
+ DeploymentId: r.ID,
+ Iid: r.Iid,
+ Ref: r.Ref,
+ Sha: r.Sha,
+ Environment: r.Environment.Name,
+ Name: r.Deployable.Name,
+ DeployableCommitAuthorEmail: r.Deployable.Commit.AuthorEmail,
+ DeployableCommitAuthorName: r.Deployable.Commit.AuthorName,
+ DeployableCommitCreatedAt: r.Deployable.Commit.CreatedAt,
+ DeployableCommitID: r.Deployable.Commit.ID,
+ DeployableCommitMessage: r.Deployable.Commit.Message,
+ DeployableCommitShortID: r.Deployable.Commit.ShortID,
+ DeployableCommitTitle: r.Deployable.Commit.Title,
+ //DeployableCoverage: r.Deployable.Coverage,
+ DeployableID: r.Deployable.ID,
+ DeployableName: r.Deployable.Name,
+ DeployableRef: r.Deployable.Ref,
+ //DeployableRunner: r.Deployable.Runner,
+ DeployableStage: r.Deployable.Stage,
+ DeployableStatus: r.Deployable.Status,
+ DeployableTag: r.Deployable.Tag,
+ DeployableUserID: r.Deployable.User.ID,
+ DeployableUserName: r.Deployable.User.Name,
+ DeployableUserUsername: r.Deployable.User.Username,
+ DeployableUserState: r.Deployable.User.State,
+ DeployableUserAvatarURL: r.Deployable.User.AvatarURL,
+ DeployableUserWebURL: r.Deployable.User.WebURL,
+ DeployableUserCreatedAt: r.Deployable.User.CreatedAt,
+ //DeployableUserBio: r.Deployable.User.Bio,
+ //DeployableUserLocation: r.Deployable.User.Location,
+ DeployableUserPublicEmail: r.Deployable.User.PublicEmail,
+ DeployableUserSkype: r.Deployable.User.Skype,
+ DeployableUserLinkedin: r.Deployable.User.Linkedin,
+ DeployableUserTwitter: r.Deployable.User.Twitter,
+ DeployableUserWebsiteURL: r.Deployable.User.WebsiteURL,
+ DeployableUserOrganization: r.Deployable.User.Organization,
+ DeployablePipelineCreatedAt: r.Deployable.Pipeline.CreatedAt,
+ DeployablePipelineID: r.Deployable.Pipeline.ID,
+ DeployablePipelineRef: r.Deployable.Pipeline.Ref,
+ DeployablePipelineSha: r.Deployable.Pipeline.Sha,
+ DeployablePipelineStatus: r.Deployable.Pipeline.Status,
+ DeployablePipelineUpdatedAt: r.Deployable.Pipeline.UpdatedAt,
+ DeployablePipelineWebURL: r.Deployable.Pipeline.WebURL,
+ UserAvatarURL: r.User.AvatarURL,
+ UserID: r.User.ID,
+ UserName: r.User.Name,
+ UserState: r.User.State,
+ UserUsername: r.User.Username,
+ UserWebURL: r.User.WebURL,
+ DeployableStartedAt: r.Deployable.StartedAt,
+ DeployableCreatedAt: r.Deployable.CreatedAt,
+ DeployableFinishedAt: r.Deployable.FinishedAt,
+ DeployableDuration: r.Deployable.Duration,
+ }
+}
+
+type GitlabDeploymentCommit struct {
+ AuthorEmail string `json:"author_email"`
+ AuthorName string `json:"author_name"`
+ CreatedAt time.Time `json:"created_at"`
+ ID string `json:"id"`
+ Message string `json:"message"`
+ ShortID string `json:"short_id"`
+ Title string `json:"title"`
+}
+
+type GitlabDeploymentProject struct {
+ CiJobTokenScopeEnabled bool `json:"ci_job_token_scope_enabled"`
+}
+
+type GitlabDeploymentFullUser struct {
+ ID int `json:"id"`
+ Name string `json:"name"`
+ Username string `json:"username"`
+ State string `json:"state"`
+ AvatarURL string `json:"avatar_url"`
+ WebURL string `json:"web_url"`
+ CreatedAt time.Time `json:"created_at"`
+ Bio any `json:"bio"`
+ Location any `json:"location"`
+ PublicEmail string `json:"public_email"`
+ Skype string `json:"skype"`
+ Linkedin string `json:"linkedin"`
+ Twitter string `json:"twitter"`
+ WebsiteURL string `json:"website_url"`
+ Organization string `json:"organization"`
+}
+
+type GitlabDeploymentPipeline struct {
+ CreatedAt time.Time `json:"created_at"`
+ ID int `json:"id"`
+ Ref string `json:"ref"`
+ Sha string `json:"sha"`
+ Status string `json:"status"`
+ UpdatedAt time.Time `json:"updated_at"`
+ WebURL string `json:"web_url"`
+}
+
+type GitlabDeploymentDeployable struct {
+ Commit GitlabDeploymentCommit `json:"commit"`
+ Coverage any `json:"coverage"`
+ CreatedAt *time.Time `json:"created_at"`
+ FinishedAt *time.Time `json:"finished_at"`
+ ID int `json:"id"`
+ Name string `json:"name"`
+ Ref string `json:"ref"`
+ Runner any `json:"runner"`
+ Stage string `json:"stage"`
+ StartedAt *time.Time `json:"started_at"`
+ Status string `json:"status"`
+ Tag bool `json:"tag"`
+ Project GitlabDeploymentProject `json:"project"`
+ User GitlabDeploymentFullUser `json:"user"`
+ Pipeline GitlabDeploymentPipeline `json:"pipeline"`
+
+ AllowFailure bool `json:"allow_failure"`
+ ErasedAt *time.Time `json:"erased_at"`
+
+ Duration *float64 `json:"duration"`
+ QueuedDuration *float64 `json:"queued_duration"`
+ ArtifactsExpireAt *time.Time
`json:"artifacts_expire_at "`
+ TagList []string `json:"tag_list"`
+ Artifacts []GitlabDeploymentArtifacts `json:"artifacts"`
+}
+
+type GitlabDeploymentEnvironment struct {
+ ExternalURL string `json:"external_url"`
+ ID int `json:"id"`
+ Name string `json:"name"`
+}
+
+type GitlabDeploymentArtifacts struct {
+ FileType string `json:"file_type"`
+ Size int `json:"size"`
+ Filename string `json:"filename"`
+ FileFormat any `json:"file_format"`
+}
+
+type GitlabDeploymentSimpleUser struct {
+ AvatarURL string `json:"avatar_url"`
+ ID int `json:"id"`
+ Name string `json:"name"`
+ State string `json:"state"`
+ Username string `json:"username"`
+ WebURL string `json:"web_url"`
+}
