This is an automated email from the ASF dual-hosted git repository. abeizn pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 7db91aeaa31955618cb441bd0f35a56f1ca546b1 Author: Yingchu Chen <[email protected]> AuthorDate: Fri Sep 16 17:32:22 2022 +0800 fix(plugins): use didgen closes #3087 --- plugins/bitbucket/tasks/commit_extractor.go | 2 +- plugins/jenkins/api/connection.go | 2 +- plugins/jenkins/api/init.go | 6 +- plugins/jenkins/e2e/builds_test.go | 2 +- plugins/jenkins/models/build.go | 6 +- .../20220916_modify_jenkins_build.go | 146 +++++++++++++++++++++ plugins/jenkins/tasks/build_cicd_convertor.go | 17 +-- plugins/jenkins/tasks/build_extractor.go | 4 +- plugins/jenkins/tasks/build_repo_convertor.go | 8 +- plugins/jenkins/tasks/stage_collector.go | 2 +- plugins/jenkins/tasks/stage_convertor.go | 2 +- 11 files changed, 172 insertions(+), 25 deletions(-) diff --git a/plugins/bitbucket/tasks/commit_extractor.go b/plugins/bitbucket/tasks/commit_extractor.go index c6c44c43..75aee98e 100644 --- a/plugins/bitbucket/tasks/commit_extractor.go +++ b/plugins/bitbucket/tasks/commit_extractor.go @@ -99,7 +99,7 @@ func ExtractApiCommits(taskCtx core.SubTaskContext) errors.Error { } //if commit.Author != nil { - // bitbucketCommit.AuthorName = commit.Author.User.DisplayName + // bitbucketCommit.AuthorName = commit.Author.User.FullDisplayName // bitbucketCommit.AuthorId = commit.Author.User.AccountId // results = append(results, commit.Author) //} diff --git a/plugins/jenkins/api/connection.go b/plugins/jenkins/api/connection.go index 6cd53092..bd52567f 100644 --- a/plugins/jenkins/api/connection.go +++ b/plugins/jenkins/api/connection.go @@ -58,7 +58,7 @@ func TestConnection(input *core.ApiResourceInput) (*core.ApiResourceOutput, erro }, 3*time.Second, connection.Proxy, - basicRes, + BasicRes, ) if err != nil { return nil, err diff --git a/plugins/jenkins/api/init.go b/plugins/jenkins/api/init.go index 6774e148..f1fb68f9 100644 --- a/plugins/jenkins/api/init.go +++ b/plugins/jenkins/api/init.go @@ -27,13 +27,13 @@ import ( var vld *validator.Validate var connectionHelper *helper.ConnectionApiHelper -var basicRes core.BasicRes +var BasicRes core.BasicRes func Init(config *viper.Viper, logger core.Logger, database *gorm.DB) { - basicRes = helper.NewDefaultBasicRes(config, logger, database) + BasicRes = helper.NewDefaultBasicRes(config, logger, database) vld = validator.New() connectionHelper = helper.NewConnectionHelper( - basicRes, + BasicRes, vld, ) } diff --git a/plugins/jenkins/e2e/builds_test.go b/plugins/jenkins/e2e/builds_test.go index 81978338..40112595 100644 --- a/plugins/jenkins/e2e/builds_test.go +++ b/plugins/jenkins/e2e/builds_test.go @@ -55,7 +55,7 @@ func TestJenkinsBuildsDataFlow(t *testing.T) { "connection_id", "job_name", "duration", - "display_name", + "full_display_name", "estimated_duration", "number", "result", diff --git a/plugins/jenkins/models/build.go b/plugins/jenkins/models/build.go index fc347bdb..8e3a4fad 100644 --- a/plugins/jenkins/models/build.go +++ b/plugins/jenkins/models/build.go @@ -28,11 +28,11 @@ type JenkinsBuild struct { common.NoPKModel // collected fields ConnectionId uint64 `gorm:"primaryKey"` - JobName string `gorm:"primaryKey;type:varchar(255)"` + JobName string `gorm:"index;type:varchar(255)"` Duration float64 // build time - DisplayName string `gorm:"type:varchar(255)"` // "#7" + FullDisplayName string `gorm:"primaryKey;type:varchar(255)"` // "job name #7" EstimatedDuration float64 // EstimatedDuration - Number int64 `gorm:"primaryKey"` + Number int64 `gorm:"index"` Result string // Result Timestamp int64 // start time StartTime time.Time // convered by timestamp diff --git a/plugins/jenkins/models/migrationscripts/20220916_modify_jenkins_build.go b/plugins/jenkins/models/migrationscripts/20220916_modify_jenkins_build.go new file mode 100644 index 00000000..9d0e6700 --- /dev/null +++ b/plugins/jenkins/models/migrationscripts/20220916_modify_jenkins_build.go @@ -0,0 +1,146 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "context" + "fmt" + "github.com/apache/incubator-devlake/errors" + "github.com/apache/incubator-devlake/models/migrationscripts/archived" + "github.com/apache/incubator-devlake/plugins/helper" + "github.com/apache/incubator-devlake/plugins/jenkins/api" + "gorm.io/gorm" + "reflect" + "strings" + "time" +) + +type modifyJenkinsBuild struct{} + +type JenkinsBuildOld struct { + archived.NoPKModel + // collected fields + ConnectionId uint64 `gorm:"primaryKey"` + JobName string `gorm:"primaryKey;type:varchar(255)"` + Duration float64 // build time + DisplayName string `gorm:"type:varchar(255)"` // "#7" + EstimatedDuration float64 // EstimatedDuration + Number int64 `gorm:"primaryKey"` + Result string // Result + Timestamp int64 // start time + StartTime time.Time // convered by timestamp + CommitSha string `gorm:"type:varchar(255)"` + Type string `gorm:"index;type:varchar(255)"` + Class string `gorm:"index;type:varchar(255)" ` + TriggeredBy string `gorm:"type:varchar(255)"` + Building bool + HasStages bool +} + +func (JenkinsBuildOld) TableName() string { + return "_tool_jenkins_builds" +} + +type JenkinsBuil0916 struct { + archived.NoPKModel + // collected fields + ConnectionId uint64 `gorm:"primaryKey"` + JobName string `gorm:"index;type:varchar(255)"` + Duration float64 // build time + FullDisplayName string `gorm:"primaryKey;type:varchar(255)"` // "#7" + EstimatedDuration float64 // EstimatedDuration + Number int64 `gorm:"index"` + Result string // Result + Timestamp int64 // start time + StartTime time.Time // convered by timestamp + Type string `gorm:"index;type:varchar(255)"` + Class string `gorm:"index;type:varchar(255)" ` + TriggeredBy string `gorm:"type:varchar(255)"` + Building bool + HasStages bool +} + +func (JenkinsBuil0916) TableName() string { + return "_tool_jenkins_builds" +} + +func (*modifyJenkinsBuild) Up(ctx context.Context, db *gorm.DB) errors.Error { + cursor, err := db.Model(&JenkinsBuildOld{}).Rows() + if err != nil { + return errors.Convert(err) + } + err = db.Migrator().RenameTable(&JenkinsBuildOld{}, "_tool_jenkins_builds_old") + if err != nil { + return errors.Default.Wrap(err, "fail to rename _tool_jenkins_builds") + } + err = db.Migrator().AutoMigrate(&JenkinsBuil0916{}) + if err != nil { + return errors.Default.Wrap(err, "fail to create _tool_jenkins_builds") + } + batch, err := helper.NewBatchSave(api.BasicRes, reflect.TypeOf(&JenkinsBuil0916{}), 300) + if err != nil { + return errors.Default.Wrap(err, "error getting batch from table") + } + defer batch.Close() + for cursor.Next() { + build := JenkinsBuildOld{} + err = db.ScanRows(cursor, &build) + if err != nil { + return errors.Convert(err) + } + newBuild := &JenkinsBuil0916{ + NoPKModel: build.NoPKModel, + ConnectionId: build.ConnectionId, + JobName: build.JobName, + Duration: build.Duration, + FullDisplayName: build.DisplayName, + EstimatedDuration: build.EstimatedDuration, + Number: build.Number, + Result: build.Result, + Timestamp: build.Timestamp, + StartTime: build.StartTime, + Type: build.Type, + Class: build.Class, + TriggeredBy: build.TriggeredBy, + Building: build.Building, + HasStages: build.HasStages, + } + if strings.Contains(build.DisplayName, build.JobName) { + newBuild.FullDisplayName = build.DisplayName + } else { + newBuild.FullDisplayName = fmt.Sprintf("%s %s", build.JobName, build.DisplayName) + } + err = batch.Add(&newBuild) + if err != nil { + return errors.Convert(err) + } + } + if err != nil { + return errors.Convert(err) + } + + return nil +} + +func (*modifyJenkinsBuild) Version() uint64 { + return 20220916231237 +} + +func (*modifyJenkinsBuild) Name() string { + return "Jenkins modify build primary key" +} diff --git a/plugins/jenkins/tasks/build_cicd_convertor.go b/plugins/jenkins/tasks/build_cicd_convertor.go index 97486132..ea524e6c 100644 --- a/plugins/jenkins/tasks/build_cicd_convertor.go +++ b/plugins/jenkins/tasks/build_cicd_convertor.go @@ -20,6 +20,7 @@ package tasks import ( "fmt" "github.com/apache/incubator-devlake/errors" + "github.com/apache/incubator-devlake/models/domainlayer/didgen" "github.com/apache/incubator-devlake/plugins/jenkins/models" "reflect" "time" @@ -85,10 +86,11 @@ func ConvertBuildsToCICD(taskCtx core.SubTaskContext) errors.Error { finishTime := jenkinsBuild.StartTime.Add(time.Duration(durationSec * int64(time.Second))) jenkinsPipelineFinishedDate = &finishTime } + buildIdGen := didgen.NewDomainIdGenerator(&models.JenkinsBuild{}) jenkinsPipeline := &devops.CICDPipeline{ DomainEntity: domainlayer.DomainEntity{ - Id: fmt.Sprintf("%s:%s:%d:%s", "jenkins", "JenkinsPipeline", jenkinsBuild.ConnectionId, - jenkinsBuild.DisplayName), + Id: buildIdGen.Generate(jenkinsBuild.ConnectionId, + jenkinsBuild.FullDisplayName), }, Name: jenkinsBuild.JobName, Result: jenkinsPipelineResult, @@ -100,10 +102,10 @@ func ConvertBuildsToCICD(taskCtx core.SubTaskContext) errors.Error { if jenkinsBuild.TriggeredBy != "" { domainPipelineRelation := &devops.CICDPipelineRelationship{ - ParentPipelineId: fmt.Sprintf("%s:%s:%d:%s", "jenkins", "JenkinsPipeline", jenkinsBuild.ConnectionId, + ParentPipelineId: buildIdGen.Generate(jenkinsBuild.ConnectionId, jenkinsBuild.TriggeredBy), - ChildPipelineId: fmt.Sprintf("%s:%s:%d:%s", "jenkins", "JenkinsPipeline", jenkinsBuild.ConnectionId, - jenkinsBuild.DisplayName), + ChildPipelineId: buildIdGen.Generate(jenkinsBuild.ConnectionId, + jenkinsBuild.FullDisplayName), } results = append(results, domainPipelineRelation) } @@ -114,7 +116,7 @@ func ConvertBuildsToCICD(taskCtx core.SubTaskContext) errors.Error { jenkinsTask := &devops.CICDTask{ DomainEntity: domainlayer.DomainEntity{ Id: fmt.Sprintf("%s:%s:%d:%s", "jenkins", "JenkinsTask", jenkinsBuild.ConnectionId, - jenkinsBuild.DisplayName), + jenkinsBuild.FullDisplayName), }, Name: jenkinsBuild.JobName, Result: jenkinsPipelineResult, @@ -124,8 +126,7 @@ func ConvertBuildsToCICD(taskCtx core.SubTaskContext) errors.Error { FinishedDate: jenkinsPipelineFinishedDate, } - jenkinsTask.PipelineId = fmt.Sprintf("%s:%s:%d:%s", "jenkins", "JenkinsPipeline", - jenkinsBuild.ConnectionId, jenkinsBuild.DisplayName) + jenkinsTask.PipelineId = buildIdGen.Generate(jenkinsBuild.ConnectionId, jenkinsBuild.FullDisplayName) jenkinsTask.RawDataOrigin = jenkinsBuild.RawDataOrigin results = append(results, jenkinsTask) diff --git a/plugins/jenkins/tasks/build_extractor.go b/plugins/jenkins/tasks/build_extractor.go index ba51475e..2d1d233f 100644 --- a/plugins/jenkins/tasks/build_extractor.go +++ b/plugins/jenkins/tasks/build_extractor.go @@ -76,7 +76,7 @@ func ExtractApiBuilds(taskCtx core.SubTaskContext) errors.Error { ConnectionId: data.Options.ConnectionId, JobName: input.Name, Duration: body.Duration, - DisplayName: body.DisplayName, + FullDisplayName: body.DisplayName, EstimatedDuration: body.EstimatedDuration, Number: body.Number, Result: body.Result, @@ -103,7 +103,7 @@ func ExtractApiBuilds(taskCtx core.SubTaskContext) errors.Error { if url != "" { buildCommitRemoteUrl := models.JenkinsBuildRepo{ ConnectionId: data.Options.ConnectionId, - BuildName: build.DisplayName, + BuildName: build.FullDisplayName, CommitSha: sha, RepoUrl: url, Branch: branch, diff --git a/plugins/jenkins/tasks/build_repo_convertor.go b/plugins/jenkins/tasks/build_repo_convertor.go index ac0b68eb..0dd10749 100644 --- a/plugins/jenkins/tasks/build_repo_convertor.go +++ b/plugins/jenkins/tasks/build_repo_convertor.go @@ -18,7 +18,6 @@ limitations under the License. package tasks import ( - "fmt" "github.com/apache/incubator-devlake/errors" "reflect" @@ -64,9 +63,10 @@ func ConvertBuildRepos(taskCtx core.SubTaskContext) errors.Error { }, Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { jenkinsBuildRepo := inputRow.(*models.JenkinsBuildRepo) - build := &devops.CiCDPipelineCommit{ - PipelineId: fmt.Sprintf("%s:%s:%d:%s", "jenkins", "JenkinsTask", jenkinsBuildRepo.ConnectionId, - jenkinsBuildRepo.BuildName), + buildIdGen := didgen.NewDomainIdGenerator(&models.JenkinsBuild{}) + build := &devops.CiCDPipelineRepo{ + PipelineId: buildIdGen.Generate(jenkinsBuild.ConnectionId, + jenkinsBuildRepo.BuildName), CommitSha: jenkinsBuildRepo.CommitSha, Branch: jenkinsBuildRepo.Branch, RepoUrl: jenkinsBuildRepo.RepoUrl, diff --git a/plugins/jenkins/tasks/stage_collector.go b/plugins/jenkins/tasks/stage_collector.go index 43c810ef..3ba652ea 100644 --- a/plugins/jenkins/tasks/stage_collector.go +++ b/plugins/jenkins/tasks/stage_collector.go @@ -50,7 +50,7 @@ func CollectApiStages(taskCtx core.SubTaskContext) errors.Error { db := taskCtx.GetDal() data := taskCtx.GetData().(*JenkinsTaskData) clauses := []dal.Clause{ - dal.Select("tjj.path,tjb.job_name,tjb.number,tjb.display_name"), + dal.Select("tjj.path,tjb.job_name,tjb.number,tjb.full_display_name"), dal.From("_tool_jenkins_builds as tjb,_tool_jenkins_jobs as tjj"), dal.Where(`tjb.connection_id = ? and tjb.class = ? and tjb.job_name = tjj.name`, data.Options.ConnectionId, "WorkflowRun"), diff --git a/plugins/jenkins/tasks/stage_convertor.go b/plugins/jenkins/tasks/stage_convertor.go index b68e6168..f300b181 100644 --- a/plugins/jenkins/tasks/stage_convertor.go +++ b/plugins/jenkins/tasks/stage_convertor.go @@ -68,7 +68,7 @@ func ConvertStages(taskCtx core.SubTaskContext) errors.Error { tjs.pause_duration_millis, tjs.type, tjb.triggered_by, tjb.building`), dal.From("_tool_jenkins_builds tjb"), - dal.Join("left join _tool_jenkins_stages tjs on tjs.build_name = tjb.display_name"), + dal.Join("left join _tool_jenkins_stages tjs on tjs.build_name = tjb.full_display_name"), dal.Where("tjb.connection_id = ? ", data.Options.ConnectionId), }
