This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 0d143121 feat(jenkins): add jenkins pipeline convertor
0d143121 is described below
commit 0d143121bbddda5e028dd747e2361d1f5a359bbe
Author: Yingchu Chen <[email protected]>
AuthorDate: Fri Aug 5 02:23:18 2022 +0800
feat(jenkins): add jenkins pipeline convertor
---
models/domainlayer/devops/cicd_pipeline.go | 10 +-
models/domainlayer/devops/cicd_task.go | 2 +-
plugins/jenkins/impl/impl.go | 2 +
plugins/jenkins/models/build.go | 2 +-
plugins/jenkins/models/build_repo.go | 2 +-
.../20220729_modify_all_entities.go | 7 +-
plugins/jenkins/models/pipeline.go | 27 +++
.../cicd_task.go => plugins/jenkins/models/task.go | 8 +-
plugins/jenkins/tasks/build_convertor_cicd.go | 184 +++++++++++++++++++++
plugins/jenkins/tasks/build_enricher.go | 63 +++++++
10 files changed, 298 insertions(+), 9 deletions(-)
diff --git a/models/domainlayer/devops/cicd_pipeline.go
b/models/domainlayer/devops/cicd_pipeline.go
index 5550f29c..0b0d266a 100644
--- a/models/domainlayer/devops/cicd_pipeline.go
+++ b/models/domainlayer/devops/cicd_pipeline.go
@@ -16,9 +16,17 @@ type CICDPipeline struct {
Type string `gorm:"type:varchar(100);comment: to indicate this
is CI or CD"`
DurationSec uint64
CreatedDate time.Time
- FinishedDate time.Time
+ FinishedDate *time.Time
}
func (CICDPipeline) TableName() string {
return "cicd_pipelines"
}
+
+const (
+ SUCCESS = "SUCCESS"
+ FAILURE = "FAILURE"
+ ABORT = "ABORT"
+ IN_PROGRESS = "IN_PROGRESS"
+ DONE = "DONE"
+)
diff --git a/models/domainlayer/devops/cicd_task.go
b/models/domainlayer/devops/cicd_task.go
index 03af484d..bb025b62 100644
--- a/models/domainlayer/devops/cicd_task.go
+++ b/models/domainlayer/devops/cicd_task.go
@@ -14,7 +14,7 @@ type CICDTask struct {
Type string `gorm:"type:varchar(100);comment: to indicate this
is CI or CD"`
DurationSec uint64
StatedDate time.Time
- FinishedDate time.Time
+ FinishedDate *time.Time
}
func (CICDTask) TableName() string {
diff --git a/plugins/jenkins/impl/impl.go b/plugins/jenkins/impl/impl.go
index 167a7355..30036228 100644
--- a/plugins/jenkins/impl/impl.go
+++ b/plugins/jenkins/impl/impl.go
@@ -57,6 +57,8 @@ func (plugin Jenkins) SubTaskMetas() []core.SubTaskMeta {
tasks.ExtractApiBuildsMeta,
tasks.CollectApiStagesMeta,
tasks.ExtractApiStagesMeta,
+ tasks.EnrichApiBuildsMeta,
+ tasks.ConvertBuildsToCICDMeta,
tasks.ConvertJobsMeta,
tasks.ConvertBuildsMeta,
}
diff --git a/plugins/jenkins/models/build.go b/plugins/jenkins/models/build.go
index f3af1a10..fc347bdb 100644
--- a/plugins/jenkins/models/build.go
+++ b/plugins/jenkins/models/build.go
@@ -26,7 +26,6 @@ import (
// JenkinsBuild db entity for jenkins build
type JenkinsBuild struct {
common.NoPKModel
-
// collected fields
ConnectionId uint64 `gorm:"primaryKey"`
JobName string `gorm:"primaryKey;type:varchar(255)"`
@@ -42,6 +41,7 @@ type JenkinsBuild struct {
Class string `gorm:"index;type:varchar(255)" `
TriggeredBy string `gorm:"type:varchar(255)"`
Building bool
+ HasStages bool
}
func (JenkinsBuild) TableName() string {
diff --git a/plugins/jenkins/models/build_repo.go
b/plugins/jenkins/models/build_repo.go
index 7f26c54a..104fa3f7 100644
--- a/plugins/jenkins/models/build_repo.go
+++ b/plugins/jenkins/models/build_repo.go
@@ -7,7 +7,7 @@ type JenkinsBuildRepo struct {
BuildName string `gorm:"primaryKey;type:varchar(255)"`
CommitSha string `gorm:"primaryKey;type:varchar(255)"`
Branch string `gorm:"type:varchar(255)"`
- RepoUrl string `gorm:"primaryKey;type:varchar(255)"`
+ RepoUrl string `gorm:"type:varchar(255)"`
common.NoPKModel
}
diff --git
a/plugins/jenkins/models/migrationscripts/20220729_modify_all_entities.go
b/plugins/jenkins/models/migrationscripts/20220729_modify_all_entities.go
index 33a00e19..7ca40e3f 100644
--- a/plugins/jenkins/models/migrationscripts/20220729_modify_all_entities.go
+++ b/plugins/jenkins/models/migrationscripts/20220729_modify_all_entities.go
@@ -53,7 +53,8 @@ type JenkinsBuildRepo0729 struct {
BuildName string `gorm:"primaryKey;type:varchar(255)"`
CommitSha string `gorm:"primaryKey;type:varchar(255)"`
Branch string `gorm:"type:varchar(255)"`
- RepoUrl string `gorm:"primaryKey;type:varchar(255)"`
+ RepoUrl string `gorm:"type:varchar(255)"`
+ HasStages bool
archived.NoPKModel
}
@@ -84,6 +85,10 @@ func (*modifyAllEntities) Up(ctx context.Context, db
*gorm.DB) error {
if err != nil {
return err
}
+ err = db.Migrator().AddColumn(JenkinsBuild0729{}, "class")
+ if err != nil {
+ return err
+ }
err = db.Migrator().AddColumn(JenkinsBuild0729{}, "triggered_by")
if err != nil {
return err
diff --git a/plugins/jenkins/models/pipeline.go
b/plugins/jenkins/models/pipeline.go
new file mode 100644
index 00000000..4b5b21ad
--- /dev/null
+++ b/plugins/jenkins/models/pipeline.go
@@ -0,0 +1,27 @@
+package models
+
+import (
+ "github.com/apache/incubator-devlake/models/common"
+ "time"
+)
+
+type JenkinsPipeline struct {
+ common.NoPKModel
+ // collected fields
+ ConnectionId uint64 `gorm:"primaryKey"`
+ DurationSec uint64
+ Name string `gorm:"type:varchar(255);primaryKey"`
+ Result string // Result
+ Status string // Result
+ Timestamp int64 // start time
+ CreatedDate time.Time // convered by timestamp
+ CommitSha string `gorm:"primaryKey;type:varchar(255)"`
+ Type string `gorm:"index;type:varchar(255)"`
+ Building bool
+ Repo string `gorm:"type:varchar(255);index"`
+ FinishedDate *time.Time
+}
+
+func (JenkinsPipeline) TableName() string {
+ return "_tool_jenkins_pipelines"
+}
diff --git a/models/domainlayer/devops/cicd_task.go
b/plugins/jenkins/models/task.go
similarity index 80%
copy from models/domainlayer/devops/cicd_task.go
copy to plugins/jenkins/models/task.go
index 03af484d..4d35b501 100644
--- a/models/domainlayer/devops/cicd_task.go
+++ b/plugins/jenkins/models/task.go
@@ -1,11 +1,11 @@
-package devops
+package models
import (
"github.com/apache/incubator-devlake/models/domainlayer"
"time"
)
-type CICDTask struct {
+type JenkinsTask struct {
domainlayer.DomainEntity
Name string `gorm:"type:varchar(255)"`
PipelineId string `gorm:"index;type:varchar(255)"`
@@ -17,6 +17,6 @@ type CICDTask struct {
FinishedDate time.Time
}
-func (CICDTask) TableName() string {
- return "cicd_tasks"
+func (JenkinsTask) TableName() string {
+ return "_tool_jenkins_tasks"
}
diff --git a/plugins/jenkins/tasks/build_convertor_cicd.go
b/plugins/jenkins/tasks/build_convertor_cicd.go
new file mode 100644
index 00000000..a227fdc9
--- /dev/null
+++ b/plugins/jenkins/tasks/build_convertor_cicd.go
@@ -0,0 +1,184 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "fmt"
+ "github.com/apache/incubator-devlake/models/common"
+ "github.com/apache/incubator-devlake/models/domainlayer"
+ "github.com/apache/incubator-devlake/models/domainlayer/devops"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "reflect"
+ "time"
+)
+
+type JenkinsBuildWithRepo struct {
+ // collected fields
+ ConnectionId uint64 `gorm:"primaryKey"`
+ Duration float64 // build time
+ DisplayName string `gorm:"type:varchar(255)"` // "#7"
+ EstimatedDuration float64 // EstimatedDuration
+ Number int64 `gorm:"primaryKey"`
+ Result string // Result
+ Timestamp int64 // start time
+ StartTime time.Time // convered by timestamp
+ CommitSha string `gorm:"type:varchar(255)"`
+ Type string `gorm:"index;type:varchar(255)"`
+ Class string `gorm:"index;type:varchar(255)" `
+ TriggeredBy string `gorm:"type:varchar(255)"`
+ Building bool
+ Branch string `gorm:"type:varchar(255)"`
+ RepoUrl string `gorm:"type:varchar(255)"`
+ StageCount int
+ common.NoPKModel
+}
+
+var ConvertBuildsToCICDMeta = core.SubTaskMeta{
+ Name: "convertBuildsToCICD",
+ EntryPoint: ConvertBuildsToCICD,
+ EnabledByDefault: true,
+ Description: "convert builds to cicd",
+ DomainTypes: []string{core.DOMAIN_TYPE_CICD},
+}
+
+func ConvertBuildsToCICD(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDal()
+ data := taskCtx.GetData().(*JenkinsTaskData)
+
+ clauses := []dal.Clause{
+ dal.Select(`tjb.connection_id, tjb.duration, tjb.display_name,
tjb.estimated_duration, tjb.number,
+ tjb.result, tjb.timestamp, tjb.start_time,
tjbr.commit_sha, tjb.type, tjb.class,
+ tjb.triggered_by, tjb.building, tjbr.branch,
tjbr.repo_url`),
+ dal.From("_tool_jenkins_builds tjb"),
+ dal.Join("left join _tool_jenkins_build_repos tjbr on
tjbr.build_name = tjb.display_name"),
+ dal.Where("tjb.connection_id = ?", data.Options.ConnectionId),
+ }
+ cursor, err := db.Cursor(clauses...)
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ //jobIdGen := didgen.NewDomainIdGenerator(&models.JenkinsJob{})
+ //buildIdGen := didgen.NewDomainIdGenerator(&models.JenkinsBuild{})
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ InputRowType: reflect.TypeOf(JenkinsBuildWithRepo{}),
+ Input: cursor,
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Params: JenkinsApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ },
+ Ctx: taskCtx,
+ Table: RAW_BUILD_TABLE,
+ },
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ jenkinsBuildWithRepo := inputRow.(*JenkinsBuildWithRepo)
+ durationSec := int64(jenkinsBuildWithRepo.Duration /
1000)
+ jenkinsPipelineResult := ""
+ jenkinsPipelineStatus := ""
+ var jenkinsPipelineFinishedDate *time.Time
+ results := make([]interface{}, 0)
+ if jenkinsBuildWithRepo.Result == "SUCCESS" {
+ jenkinsPipelineResult = devops.SUCCESS
+ } else if jenkinsBuildWithRepo.Result == "FAILURE" {
+ jenkinsPipelineResult = devops.FAILURE
+ } else {
+ jenkinsPipelineResult = devops.ABORT
+ }
+
+ if jenkinsBuildWithRepo.Building {
+ jenkinsPipelineStatus = devops.IN_PROGRESS
+ } else {
+ jenkinsPipelineStatus = devops.DONE
+ finishTime :=
jenkinsBuildWithRepo.StartTime.Add(time.Duration(durationSec *
int64(time.Second)))
+ jenkinsPipelineFinishedDate = &finishTime
+ }
+ if jenkinsBuildWithRepo.TriggeredBy == "" {
+ jenkinsPipeline := &devops.CICDPipeline{
+ DomainEntity: domainlayer.DomainEntity{
+ Id:
fmt.Sprintf("%s:%s:%d:%s:%s", "jenkins", "JenkinsPipeline",
jenkinsBuildWithRepo.ConnectionId,
+
jenkinsBuildWithRepo.CommitSha, jenkinsBuildWithRepo.DisplayName),
+ },
+ Name:
jenkinsBuildWithRepo.DisplayName,
+ CommitSha:
jenkinsBuildWithRepo.CommitSha,
+ Branch:
jenkinsBuildWithRepo.Branch,
+ Repo:
jenkinsBuildWithRepo.RepoUrl,
+ Result: jenkinsPipelineResult,
+ Status: jenkinsPipelineStatus,
+ FinishedDate:
jenkinsPipelineFinishedDate,
+ //Type: "",
+ DurationSec: uint64(durationSec),
+ CreatedDate:
jenkinsBuildWithRepo.StartTime,
+ }
+ if jenkinsPipelineFinishedDate != nil {
+ }
+ results = append(results, jenkinsPipeline)
+ }
+
+ if jenkinsBuildWithRepo.StageCount == 0 {
+ jenkinsTask := &devops.CICDTask{
+ DomainEntity: domainlayer.DomainEntity{
+ Id: fmt.Sprintf("%s:%s:%d:%s",
"jenkins", "JenkinsTask", jenkinsBuildWithRepo.ConnectionId,
+
jenkinsBuildWithRepo.DisplayName),
+ },
+ Name:
jenkinsBuildWithRepo.DisplayName,
+ Result: jenkinsPipelineResult,
+ Status: jenkinsPipelineStatus,
+ //Type: "",
+ DurationSec: uint64(durationSec),
+ StatedDate:
jenkinsBuildWithRepo.StartTime,
+ FinishedDate:
jenkinsPipelineFinishedDate,
+ }
+ if jenkinsBuildWithRepo.TriggeredBy != "" {
+ tmp := make([]*JenkinsBuildWithRepo, 0)
+ clauses := []dal.Clause{
+ dal.Select(`tjb.connection_id,
tjb.duration, tjb.display_name, tjb.estimated_duration, tjb.number,
+ tjb.result,
tjb.timestamp, tjbr.commit_sha, tjb.type, tjb.class,
+ tjb.triggered_by,
tjb.building, tjbr.branch, tjbr.repo_url`),
+ dal.From("_tool_jenkins_builds
tjb"),
+ dal.Join("left join
_tool_jenkins_build_repos tjbr on tjbr.build_name = tjb.display_name"),
+ dal.Where("tjb.connection_id =
? and tjb.display_name = ?", data.Options.ConnectionId,
jenkinsBuildWithRepo.TriggeredBy),
+ }
+ err = db.All(&tmp, clauses...)
+ if err != nil {
+ return nil, err
+ }
+ if len(tmp) > 0 {
+ jenkinsTask.PipelineId =
fmt.Sprintf("%s:%s:%d:%s:%s", "jenkins", "JenkinsPipeline",
jenkinsBuildWithRepo.ConnectionId,
+ tmp[0].CommitSha,
tmp[0].DisplayName)
+ }
+ } else {
+ jenkinsTask.PipelineId =
fmt.Sprintf("%s:%s:%d:%s:%s", "jenkins", "JenkinsPipeline",
jenkinsBuildWithRepo.ConnectionId,
+ jenkinsBuildWithRepo.CommitSha,
jenkinsBuildWithRepo.DisplayName)
+ }
+ results = append(results, jenkinsTask)
+
+ }
+
+ return results, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/plugins/jenkins/tasks/build_enricher.go
b/plugins/jenkins/tasks/build_enricher.go
new file mode 100644
index 00000000..ceed2726
--- /dev/null
+++ b/plugins/jenkins/tasks/build_enricher.go
@@ -0,0 +1,63 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+ "github.com/apache/incubator-devlake/plugins/jenkins/models"
+)
+
+// this struct should be moved to `gitub_api_common.go`
+
+var EnrichApiBuildsMeta = core.SubTaskMeta{
+ Name: "enrichApiBuilds",
+ EntryPoint: EnrichApiBuilds,
+ EnabledByDefault: true,
+ Description: "Enrich jenkins_builds",
+ DomainTypes: []string{core.DOMAIN_TYPE_CICD},
+}
+
+func EnrichApiBuilds(taskCtx core.SubTaskContext) error {
+ data := taskCtx.GetData().(*JenkinsTaskData)
+ db := taskCtx.GetDal()
+ clauses := []dal.Clause{
+ dal.Select("distinct build_name"),
+ dal.From(&models.JenkinsStage{}),
+ dal.Where("connection_id = ?", data.Options.ConnectionId),
+ dal.Groupby("build_name"),
+ }
+ cursor, err := db.Cursor(clauses...)
+ for cursor.Next() {
+ var buildName string
+ err = cursor.Scan(&buildName)
+ if err != nil {
+ return err
+ }
+ if buildName == "" {
+ continue
+ }
+ build := &models.JenkinsBuild{}
+ build.HasStages = true
+ err = db.Update(&models.JenkinsBuild{},
dal.Where("connection_id = ? and build_name = ?", data.Options.ConnectionId,
buildName))
+ if err != nil {
+ return err
+ }
+ }
+ return nil
+}
