This is an automated email from the ASF dual-hosted git repository. abeizn pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit ba5c53ba6351b255a08c96b033b70ab6e221f536 Author: Yingchu Chen <[email protected]> AuthorDate: Fri Aug 5 10:00:29 2022 +0800 feat(jenkins): stage convertor --- docker-compose.yml | 6 +- plugins/jenkins/impl/impl.go | 1 + .../migrationscripts/20220714_add_init_tables.go | 53 ++++---- ...d_convertor_cicd.go => build_cicd_convertor.go} | 30 +++-- plugins/jenkins/tasks/build_enricher.go | 21 +++- plugins/jenkins/tasks/stage_collector.go | 2 +- plugins/jenkins/tasks/stage_convertor.go | 139 +++++++++++++++++++++ 7 files changed, 206 insertions(+), 46 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index f15d5f67..aee2117d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -42,7 +42,7 @@ services: # POSTGRES_PASSWORD: merico grafana: - image: mericodev/devlake-dashboard:v0.12.1-beta + image: mericodev/devlake-dashboard:latest build: context: grafana/ ports: @@ -60,7 +60,7 @@ services: - mysql devlake: - image: mericodev/devlake:v0.12.1-beta + image: mericodev/devlake:latest build: context: "." args: @@ -75,7 +75,7 @@ services: - mysql config-ui: - image: mericodev/devlake-config-ui:v0.12.1-beta + image: mericodev/devlake-config-ui:latest build: context: "config-ui" ports: diff --git a/plugins/jenkins/impl/impl.go b/plugins/jenkins/impl/impl.go index 30036228..299a43a1 100644 --- a/plugins/jenkins/impl/impl.go +++ b/plugins/jenkins/impl/impl.go @@ -59,6 +59,7 @@ func (plugin Jenkins) SubTaskMetas() []core.SubTaskMeta { tasks.ExtractApiStagesMeta, tasks.EnrichApiBuildsMeta, tasks.ConvertBuildsToCICDMeta, + tasks.ConvertStagesMeta, tasks.ConvertJobsMeta, tasks.ConvertBuildsMeta, } diff --git a/plugins/jenkins/models/migrationscripts/20220714_add_init_tables.go b/plugins/jenkins/models/migrationscripts/20220714_add_init_tables.go index 79055179..5830eec2 100644 --- a/plugins/jenkins/models/migrationscripts/20220714_add_init_tables.go +++ b/plugins/jenkins/models/migrationscripts/20220714_add_init_tables.go @@ -19,6 +19,9 @@ package migrationscripts import ( "context" + "github.com/apache/incubator-devlake/config" + "github.com/apache/incubator-devlake/plugins/core" + "gorm.io/gorm/clause" "github.com/apache/incubator-devlake/plugins/jenkins/models/migrationscripts/archived" "gorm.io/gorm" @@ -40,37 +43,37 @@ func (*addInitTables) Up(ctx context.Context, db *gorm.DB) error { err = db.Migrator().AutoMigrate( &archived.JenkinsJob{}, &archived.JenkinsBuild{}, - //&archived.JenkinsConnection{}, + &archived.JenkinsConnection{}, ) if err != nil { return err } - //v := config.GetConfig() + v := config.GetConfig() - //encKey := v.GetString("ENCODE_KEY") - //endPoint := v.GetString("JENKINS_ENDPOINT") - //useName := v.GetString("JENKINS_USERNAME") - //passWord := v.GetString("JENKINS_PASSWORD") - //if encKey == "" || endPoint == "" || useName == "" || passWord == "" { - // return nil - //} - //conn := &archived.JenkinsConnection{} - //conn.Name = "init jenkins connection" - //conn.ID = 1 - //conn.Endpoint = endPoint - //conn.Proxy = v.GetString("JENKINS_PROXY") - //conn.RateLimitPerHour = v.GetInt("JENKINS_API_REQUESTS_PER_HOUR") - //conn.Username = useName - //conn.Password, err = core.Encrypt(encKey, passWord) - //if err != nil { - // return err - //} - //err = db.Clauses(clause.OnConflict{DoNothing: true}).Create(conn).Error - // - //if err != nil { - // return err - //} + encKey := v.GetString("ENCODE_KEY") + endPoint := v.GetString("JENKINS_ENDPOINT") + useName := v.GetString("JENKINS_USERNAME") + passWord := v.GetString("JENKINS_PASSWORD") + if encKey == "" || endPoint == "" || useName == "" || passWord == "" { + return nil + } + conn := &archived.JenkinsConnection{} + conn.Name = "init jenkins connection" + conn.ID = 1 + conn.Endpoint = endPoint + conn.Proxy = v.GetString("JENKINS_PROXY") + conn.RateLimitPerHour = v.GetInt("JENKINS_API_REQUESTS_PER_HOUR") + conn.Username = useName + conn.Password, err = core.Encrypt(encKey, passWord) + if err != nil { + return err + } + err = db.Clauses(clause.OnConflict{DoNothing: true}).Create(conn).Error + + if err != nil { + return err + } return nil } diff --git a/plugins/jenkins/tasks/build_convertor_cicd.go b/plugins/jenkins/tasks/build_cicd_convertor.go similarity index 89% rename from plugins/jenkins/tasks/build_convertor_cicd.go rename to plugins/jenkins/tasks/build_cicd_convertor.go index a227fdc9..3fe3d645 100644 --- a/plugins/jenkins/tasks/build_convertor_cicd.go +++ b/plugins/jenkins/tasks/build_cicd_convertor.go @@ -46,7 +46,7 @@ type JenkinsBuildWithRepo struct { Building bool Branch string `gorm:"type:varchar(255)"` RepoUrl string `gorm:"type:varchar(255)"` - StageCount int + HasStages bool common.NoPKModel } @@ -64,8 +64,9 @@ func ConvertBuildsToCICD(taskCtx core.SubTaskContext) error { clauses := []dal.Clause{ dal.Select(`tjb.connection_id, tjb.duration, tjb.display_name, tjb.estimated_duration, tjb.number, + tjb._raw_data_remark, tjb._raw_data_id, tjb._raw_data_table, tjb._raw_data_params, tjb.result, tjb.timestamp, tjb.start_time, tjbr.commit_sha, tjb.type, tjb.class, - tjb.triggered_by, tjb.building, tjbr.branch, tjbr.repo_url`), + tjb.triggered_by, tjb.building, tjbr.branch, tjbr.repo_url, tjb.has_stages`), dal.From("_tool_jenkins_builds tjb"), dal.Join("left join _tool_jenkins_build_repos tjbr on tjbr.build_name = tjb.display_name"), dal.Where("tjb.connection_id = ?", data.Options.ConnectionId), @@ -76,9 +77,6 @@ func ConvertBuildsToCICD(taskCtx core.SubTaskContext) error { } defer cursor.Close() - //jobIdGen := didgen.NewDomainIdGenerator(&models.JenkinsJob{}) - //buildIdGen := didgen.NewDomainIdGenerator(&models.JenkinsBuild{}) - converter, err := helper.NewDataConverter(helper.DataConverterArgs{ InputRowType: reflect.TypeOf(JenkinsBuildWithRepo{}), Input: cursor, @@ -124,25 +122,26 @@ func ConvertBuildsToCICD(taskCtx core.SubTaskContext) error { Result: jenkinsPipelineResult, Status: jenkinsPipelineStatus, FinishedDate: jenkinsPipelineFinishedDate, - //Type: "", - DurationSec: uint64(durationSec), - CreatedDate: jenkinsBuildWithRepo.StartTime, + Type: "CI/CD", + DurationSec: uint64(durationSec), + CreatedDate: jenkinsBuildWithRepo.StartTime, } if jenkinsPipelineFinishedDate != nil { } + jenkinsPipeline.RawDataOrigin = jenkinsBuildWithRepo.RawDataOrigin results = append(results, jenkinsPipeline) } - if jenkinsBuildWithRepo.StageCount == 0 { + if !jenkinsBuildWithRepo.HasStages { jenkinsTask := &devops.CICDTask{ DomainEntity: domainlayer.DomainEntity{ Id: fmt.Sprintf("%s:%s:%d:%s", "jenkins", "JenkinsTask", jenkinsBuildWithRepo.ConnectionId, jenkinsBuildWithRepo.DisplayName), }, - Name: jenkinsBuildWithRepo.DisplayName, - Result: jenkinsPipelineResult, - Status: jenkinsPipelineStatus, - //Type: "", + Name: jenkinsBuildWithRepo.DisplayName, + Result: jenkinsPipelineResult, + Status: jenkinsPipelineStatus, + Type: "CI/CD", DurationSec: uint64(durationSec), StatedDate: jenkinsBuildWithRepo.StartTime, FinishedDate: jenkinsPipelineFinishedDate, @@ -150,9 +149,7 @@ func ConvertBuildsToCICD(taskCtx core.SubTaskContext) error { if jenkinsBuildWithRepo.TriggeredBy != "" { tmp := make([]*JenkinsBuildWithRepo, 0) clauses := []dal.Clause{ - dal.Select(`tjb.connection_id, tjb.duration, tjb.display_name, tjb.estimated_duration, tjb.number, - tjb.result, tjb.timestamp, tjbr.commit_sha, tjb.type, tjb.class, - tjb.triggered_by, tjb.building, tjbr.branch, tjbr.repo_url`), + dal.Select(`tjb.display_name, tjb.result, tjb.timestamp, tjbr.commit_sha`), dal.From("_tool_jenkins_builds tjb"), dal.Join("left join _tool_jenkins_build_repos tjbr on tjbr.build_name = tjb.display_name"), dal.Where("tjb.connection_id = ? and tjb.display_name = ?", data.Options.ConnectionId, jenkinsBuildWithRepo.TriggeredBy), @@ -169,6 +166,7 @@ func ConvertBuildsToCICD(taskCtx core.SubTaskContext) error { jenkinsTask.PipelineId = fmt.Sprintf("%s:%s:%d:%s:%s", "jenkins", "JenkinsPipeline", jenkinsBuildWithRepo.ConnectionId, jenkinsBuildWithRepo.CommitSha, jenkinsBuildWithRepo.DisplayName) } + jenkinsTask.RawDataOrigin = jenkinsBuildWithRepo.RawDataOrigin results = append(results, jenkinsTask) } diff --git a/plugins/jenkins/tasks/build_enricher.go b/plugins/jenkins/tasks/build_enricher.go index ceed2726..76efb5fc 100644 --- a/plugins/jenkins/tasks/build_enricher.go +++ b/plugins/jenkins/tasks/build_enricher.go @@ -21,6 +21,8 @@ import ( "github.com/apache/incubator-devlake/plugins/core" "github.com/apache/incubator-devlake/plugins/core/dal" "github.com/apache/incubator-devlake/plugins/jenkins/models" + "strconv" + "strings" ) // this struct should be moved to `gitub_api_common.go` @@ -43,6 +45,9 @@ func EnrichApiBuilds(taskCtx core.SubTaskContext) error { dal.Groupby("build_name"), } cursor, err := db.Cursor(clauses...) + defer cursor.Close() + taskCtx.SetProgress(0, -1) + for cursor.Next() { var buildName string err = cursor.Scan(&buildName) @@ -53,11 +58,25 @@ func EnrichApiBuilds(taskCtx core.SubTaskContext) error { continue } build := &models.JenkinsBuild{} + build.ConnectionId = data.Options.ConnectionId + str := strings.Split(buildName, "#") + build.JobName = strings.TrimSpace(str[0]) + number, err := strconv.Atoi(strings.TrimSpace(str[1])) + if err != nil { + return err + } + build.Number = int64(number) + err = db.First(build) + if err != nil { + return err + } build.HasStages = true - err = db.Update(&models.JenkinsBuild{}, dal.Where("connection_id = ? and build_name = ?", data.Options.ConnectionId, buildName)) + + err = db.Update(build) if err != nil { return err } + taskCtx.IncProgress(1) } return nil } diff --git a/plugins/jenkins/tasks/stage_collector.go b/plugins/jenkins/tasks/stage_collector.go index 19f6a83f..a3d05928 100644 --- a/plugins/jenkins/tasks/stage_collector.go +++ b/plugins/jenkins/tasks/stage_collector.go @@ -50,7 +50,7 @@ func CollectApiStages(taskCtx core.SubTaskContext) error { clauses := []dal.Clause{ dal.Select("tjb.job_name,tjb.number, tjb.display_name"), dal.From("_tool_jenkins_builds tjb"), - dal.Where(`tjb.connection_id = ? and tjb.type = ?`, + dal.Where(`tjb.connection_id = ? and tjb.class = ?`, data.Options.ConnectionId, "WorkflowRun"), } diff --git a/plugins/jenkins/tasks/stage_convertor.go b/plugins/jenkins/tasks/stage_convertor.go new file mode 100644 index 00000000..e3e1bca8 --- /dev/null +++ b/plugins/jenkins/tasks/stage_convertor.go @@ -0,0 +1,139 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "fmt" + "github.com/apache/incubator-devlake/models/common" + "github.com/apache/incubator-devlake/models/domainlayer" + "github.com/apache/incubator-devlake/models/domainlayer/devops" + "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/core/dal" + "github.com/apache/incubator-devlake/plugins/helper" + "reflect" + "time" +) + +type JenkinsBuildWithRepoStage struct { + // collected fields + ConnectionId uint64 `gorm:"primaryKey"` + ID string `json:"id" gorm:"primaryKey;type:varchar(255)"` + Name string `json:"name" gorm:"type:varchar(255)"` + ExecNode string `json:"execNode" gorm:"type:varchar(255)"` + CommitSha string `gorm:"type:varchar(255)"` + Result string // Result + Status string `json:"status" gorm:"type:varchar(255)"` + StartTimeMillis int64 `json:"startTimeMillis"` + DurationMillis int `json:"durationMillis"` + PauseDurationMillis int `json:"pauseDurationMillis"` + Type string `gorm:"index;type:varchar(255)"` + BuildName string `gorm:"primaryKey;type:varchar(255)"` + Branch string `gorm:"type:varchar(255)"` + RepoUrl string `gorm:"type:varchar(255)"` + common.NoPKModel +} + +var ConvertStagesMeta = core.SubTaskMeta{ + Name: "convertStages", + EntryPoint: ConvertStages, + EnabledByDefault: true, + Description: "convert jenkins_stages", + DomainTypes: []string{core.DOMAIN_TYPE_CICD}, +} + +func ConvertStages(taskCtx core.SubTaskContext) error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*JenkinsTaskData) + clauses := []dal.Clause{ + dal.Select(`tjb.connection_id, tjs.build_name, tjs.name, tjs._raw_data_remark, + tjs._raw_data_id, tjs._raw_data_table, tjs._raw_data_params, + tjs.status, tjs.start_time_millis, tjs.duration_millis, + tjs.pause_duration_millis, tjs.type, + tjbr.commit_sha, tjb.class, + tjb.triggered_by, tjb.building, tjbr.branch, tjbr.repo_url`), + dal.From("_tool_jenkins_builds tjb"), + dal.Join("left join _tool_jenkins_build_repos tjbr on tjbr.build_name = tjb.display_name"), + dal.Join("left join _tool_jenkins_stages tjs on tjs.build_name = tjb.display_name"), + dal.Where("tjb.connection_id = ? ", data.Options.ConnectionId), + } + + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + defer cursor.Close() + + convertor, err := helper.NewDataConverter(helper.DataConverterArgs{ + InputRowType: reflect.TypeOf(JenkinsBuildWithRepoStage{}), + Input: cursor, + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Params: JenkinsApiParams{ + ConnectionId: data.Options.ConnectionId, + }, + Ctx: taskCtx, + Table: RAW_STAGE_TABLE, + }, + Convert: func(inputRow interface{}) ([]interface{}, error) { + body := inputRow.(*JenkinsBuildWithRepoStage) + if body.Name == "" { + return nil, err + } + durationSec := int64(body.DurationMillis / 1000) + jenkinsTaskResult := "" + jenkinsTaskStatus := "DONE" + var jenkinsTaskFinishedDate *time.Time + results := make([]interface{}, 0) + if body.Result == "SUCCESS" { + jenkinsTaskResult = devops.SUCCESS + } else if body.Result == "FAILED" { + jenkinsTaskResult = devops.FAILURE + } else { + jenkinsTaskResult = devops.ABORT + } + startedDate := time.Unix(body.StartTimeMillis/1000, 0) + finishedDate := startedDate.Add(time.Duration(durationSec * int64(time.Second))) + jenkinsTaskFinishedDate = &finishedDate + + jenkinsTask := &devops.CICDTask{ + DomainEntity: domainlayer.DomainEntity{ + Id: fmt.Sprintf("%s:%s:%d:%s:%s", "jenkins", "JenkinsTask", body.ConnectionId, + body.BuildName, body.Name), + }, + Name: body.Name, + PipelineId: fmt.Sprintf("%s:%s:%d:%s:%s", "jenkins", "JenkinsPipeline", body.ConnectionId, + body.CommitSha, body.BuildName), + Result: jenkinsTaskResult, + Status: jenkinsTaskStatus, + Type: "CI/CD", + DurationSec: uint64(body.DurationMillis / 1000), + StatedDate: time.Unix(durationSec, 0), + FinishedDate: jenkinsTaskFinishedDate, + } + jenkinsTask.RawDataOrigin = body.RawDataOrigin + + results = append(results, jenkinsTask) + return results, nil + }, + }) + + if err != nil { + return err + } + + return convertor.Execute() +}
