This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new f50da973 feat(jenkins): convert build_repo mapping to domain
f50da973 is described below
commit f50da973fb659fe7a2dc872f5686515cc70eb1c9
Author: Yingchu Chen <[email protected]>
AuthorDate: Thu Aug 18 17:32:12 2022 +0800
feat(jenkins): convert build_repo mapping to domain
---
models/domainlayer/devops/cicd_pipeline_repo.go | 14 ++++
.../{20220803_add_cicd.go => 20220818_add_cicd.go} | 14 +++-
plugins/azure/models/build.go | 51 ------------
plugins/jenkins/impl/impl.go | 4 +-
plugins/jenkins/tasks/build_cicd_convertor.go | 96 ++++++----------------
...richer.go => build_origin_triggger_enricher.go} | 61 +++++++-------
plugins/jenkins/tasks/build_repo_convertor.go | 86 +++++++++++++++++++
...{build_enricher.go => build_stages_enricher.go} | 10 +--
8 files changed, 177 insertions(+), 159 deletions(-)
diff --git a/models/domainlayer/devops/cicd_pipeline_repo.go
b/models/domainlayer/devops/cicd_pipeline_repo.go
new file mode 100644
index 00000000..55b8954b
--- /dev/null
+++ b/models/domainlayer/devops/cicd_pipeline_repo.go
@@ -0,0 +1,14 @@
+package devops
+
+import "github.com/apache/incubator-devlake/models/domainlayer"
+
+type CiCDPipelineRepo struct {
+ domainlayer.DomainEntity
+ CommitSha string `gorm:"primaryKey;type:varchar(255)"`
+ Branch string `gorm:"type:varchar(255)"`
+ RepoUrl string `gorm:"type:varchar(255)"`
+}
+
+func (CiCDPipelineRepo) TableName() string {
+ return "cicd_pipeline_repos"
+}
diff --git a/models/migrationscripts/20220803_add_cicd.go
b/models/migrationscripts/20220818_add_cicd.go
similarity index 87%
rename from models/migrationscripts/20220803_add_cicd.go
rename to models/migrationscripts/20220818_add_cicd.go
index 475caedd..ce0d55aa 100644
--- a/models/migrationscripts/20220803_add_cicd.go
+++ b/models/migrationscripts/20220818_add_cicd.go
@@ -29,6 +29,7 @@ type addCICD struct{}
func (*addCICD) Up(ctx context.Context, db *gorm.DB) error {
err := db.Migrator().AutoMigrate(
+ &CICDPipelineRepo{},
&CICDPipeline{},
&CICDTask{},
)
@@ -40,7 +41,7 @@ func (*addCICD) Up(ctx context.Context, db *gorm.DB) error {
}
func (*addCICD) Version() uint64 {
- return 20220803232735
+ return 20220818232735
}
func (*addCICD) Name() string {
@@ -80,3 +81,14 @@ type CICDTask struct {
func (CICDTask) TableName() string {
return "cicd_tasks"
}
+
+type CICDPipelineRepo struct {
+ archived.DomainEntity
+ CommitSha string `gorm:"primaryKey;type:varchar(255)"`
+ Branch string `gorm:"type:varchar(255)"`
+ RepoUrl string `gorm:"type:varchar(255)"`
+}
+
+func (CICDPipelineRepo) TableName() string {
+ return "cicd_pipeline_repos"
+}
diff --git a/plugins/azure/models/build.go b/plugins/azure/models/build.go
index aeeace28..06109b15 100644
--- a/plugins/azure/models/build.go
+++ b/plugins/azure/models/build.go
@@ -23,54 +23,3 @@ type AzureBuild struct {
func (AzureBuild) TableName() string {
return "_tool_azure_builds"
}
-
-type AutoGenerated struct {
- Quality string `json:"quality"`
- AuthoredBy struct {
- DisplayName string `json:"displayName"`
- URL string `json:"url"`
- Links struct {
- Avatar struct {
- Href string `json:"href"`
- } `json:"avatar"`
- } `json:"_links"`
- ID string `json:"id"`
- UniqueName string `json:"uniqueName"`
- ImageURL string `json:"imageUrl"`
- Descriptor string `json:"descriptor"`
- } `json:"authoredBy"`
- Drafts []interface{} `json:"drafts"`
- Queue struct {
- Links struct {
- Self struct {
- Href string `json:"href"`
- } `json:"self"`
- } `json:"_links"`
- ID int `json:"id"`
- Name string `json:"name"`
- URL string `json:"url"`
- Pool struct {
- ID int `json:"id"`
- Name string `json:"name"`
- IsHosted bool `json:"isHosted"`
- } `json:"pool"`
- } `json:"queue"`
- ID int `json:"id"`
- Name string `json:"name"`
- URL string `json:"url"`
- URI string `json:"uri"`
- Path string `json:"path"`
- Type string `json:"type"`
- QueueStatus string `json:"queueStatus"`
- Revision int `json:"revision"`
- CreatedDate time.Time `json:"createdDate"`
- Project struct {
- ID string `json:"id"`
- Name string `json:"name"`
- URL string `json:"url"`
- State string `json:"state"`
- Revision int `json:"revision"`
- Visibility string `json:"visibility"`
- LastUpdateTime time.Time `json:"lastUpdateTime"`
- } `json:"project"`
-}
diff --git a/plugins/jenkins/impl/impl.go b/plugins/jenkins/impl/impl.go
index 677a8d07..04503a08 100644
--- a/plugins/jenkins/impl/impl.go
+++ b/plugins/jenkins/impl/impl.go
@@ -71,9 +71,11 @@ func (plugin Jenkins) SubTaskMetas() []core.SubTaskMeta {
tasks.ExtractApiBuildsMeta,
tasks.CollectApiStagesMeta,
tasks.ExtractApiStagesMeta,
- tasks.EnrichApiBuildsMeta,
+ tasks.EnrichApiBuildOriginTriggerMeta,
+ tasks.EnrichApiBuildWithStagesMeta,
tasks.ConvertBuildsToCICDMeta,
tasks.ConvertStagesMeta,
+ tasks.ConvertBuildReposMeta,
tasks.ConvertJobsMeta,
tasks.ConvertBuildsMeta,
}
diff --git a/plugins/jenkins/tasks/build_cicd_convertor.go
b/plugins/jenkins/tasks/build_cicd_convertor.go
index cc6ee371..e8c72237 100644
--- a/plugins/jenkins/tasks/build_cicd_convertor.go
+++ b/plugins/jenkins/tasks/build_cicd_convertor.go
@@ -19,10 +19,10 @@ package tasks
import (
"fmt"
+ "github.com/apache/incubator-devlake/plugins/jenkins/models"
"reflect"
"time"
- "github.com/apache/incubator-devlake/models/common"
"github.com/apache/incubator-devlake/models/domainlayer"
"github.com/apache/incubator-devlake/models/domainlayer/devops"
"github.com/apache/incubator-devlake/plugins/core"
@@ -30,27 +30,6 @@ import (
"github.com/apache/incubator-devlake/plugins/helper"
)
-type JenkinsBuildWithRepo struct {
- // collected fields
- ConnectionId uint64 `gorm:"primaryKey"`
- Duration float64 // build time
- DisplayName string `gorm:"type:varchar(255)"` // "#7"
- EstimatedDuration float64 // EstimatedDuration
- Number int64 `gorm:"primaryKey"`
- Result string // Result
- Timestamp int64 // start time
- StartTime time.Time // convered by timestamp
- CommitSha string `gorm:"type:varchar(255)"`
- Type string `gorm:"index;type:varchar(255)"`
- Class string `gorm:"index;type:varchar(255)" `
- TriggeredBy string `gorm:"type:varchar(255)"`
- Building bool
- Branch string `gorm:"type:varchar(255)"`
- RepoUrl string `gorm:"type:varchar(255)"`
- HasStages bool
- common.NoPKModel
-}
-
var ConvertBuildsToCICDMeta = core.SubTaskMeta{
Name: "convertBuildsToCICD",
EntryPoint: ConvertBuildsToCICD,
@@ -64,13 +43,8 @@ func ConvertBuildsToCICD(taskCtx core.SubTaskContext) error {
data := taskCtx.GetData().(*JenkinsTaskData)
clauses := []dal.Clause{
- dal.Select(`tjb.connection_id, tjb.duration, tjb.display_name,
tjb.estimated_duration, tjb.number,
- tjb._raw_data_remark, tjb._raw_data_id,
tjb._raw_data_table, tjb._raw_data_params,
- tjb.result, tjb.timestamp, tjb.start_time,
tjbr.commit_sha, tjb.type, tjb.class,
- tjb.triggered_by, tjb.building, tjbr.branch,
tjbr.repo_url, tjb.has_stages`),
- dal.From("_tool_jenkins_builds tjb"),
- dal.Join("left join _tool_jenkins_build_repos tjbr on
tjbr.build_name = tjb.display_name"),
- dal.Where("tjb.connection_id = ?", data.Options.ConnectionId),
+ dal.From("_tool_jenkins_builds"),
+ dal.Where("_tool_jenkins_builds.connection_id = ?",
data.Options.ConnectionId),
}
cursor, err := db.Cursor(clauses...)
if err != nil {
@@ -79,7 +53,7 @@ func ConvertBuildsToCICD(taskCtx core.SubTaskContext) error {
defer cursor.Close()
converter, err := helper.NewDataConverter(helper.DataConverterArgs{
- InputRowType: reflect.TypeOf(JenkinsBuildWithRepo{}),
+ InputRowType: reflect.TypeOf(models.JenkinsBuild{}),
Input: cursor,
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Params: JenkinsApiParams{
@@ -89,85 +63,69 @@ func ConvertBuildsToCICD(taskCtx core.SubTaskContext) error
{
Table: RAW_BUILD_TABLE,
},
Convert: func(inputRow interface{}) ([]interface{}, error) {
- jenkinsBuildWithRepo := inputRow.(*JenkinsBuildWithRepo)
- durationSec := int64(jenkinsBuildWithRepo.Duration /
1000)
+ jenkinsBuild := inputRow.(*models.JenkinsBuild)
+ durationSec := int64(jenkinsBuild.Duration / 1000)
jenkinsPipelineResult := ""
jenkinsPipelineStatus := ""
var jenkinsPipelineFinishedDate *time.Time
results := make([]interface{}, 0)
- if jenkinsBuildWithRepo.Result == "SUCCESS" {
+ if jenkinsBuild.Result == "SUCCESS" {
jenkinsPipelineResult = devops.SUCCESS
- } else if jenkinsBuildWithRepo.Result == "FAILURE" {
+ } else if jenkinsBuild.Result == "FAILURE" {
jenkinsPipelineResult = devops.FAILURE
} else {
jenkinsPipelineResult = devops.ABORT
}
- if jenkinsBuildWithRepo.Building {
+ if jenkinsBuild.Building {
jenkinsPipelineStatus = devops.IN_PROGRESS
} else {
jenkinsPipelineStatus = devops.DONE
- finishTime :=
jenkinsBuildWithRepo.StartTime.Add(time.Duration(durationSec *
int64(time.Second)))
+ finishTime :=
jenkinsBuild.StartTime.Add(time.Duration(durationSec * int64(time.Second)))
jenkinsPipelineFinishedDate = &finishTime
}
- if jenkinsBuildWithRepo.TriggeredBy == "" {
+ if jenkinsBuild.TriggeredBy == "" {
jenkinsPipeline := &devops.CICDPipeline{
DomainEntity: domainlayer.DomainEntity{
- Id:
fmt.Sprintf("%s:%s:%d:%s:%s", "jenkins", "JenkinsPipeline",
jenkinsBuildWithRepo.ConnectionId,
-
jenkinsBuildWithRepo.CommitSha, jenkinsBuildWithRepo.DisplayName),
+ Id: fmt.Sprintf("%s:%s:%d:%s",
"jenkins", "JenkinsPipeline", jenkinsBuild.ConnectionId,
+
jenkinsBuild.DisplayName),
},
- Name:
jenkinsBuildWithRepo.DisplayName,
- CommitSha:
jenkinsBuildWithRepo.CommitSha,
- Branch:
jenkinsBuildWithRepo.Branch,
- Repo:
jenkinsBuildWithRepo.RepoUrl,
+ Name: jenkinsBuild.DisplayName,
Result: jenkinsPipelineResult,
Status: jenkinsPipelineStatus,
FinishedDate:
jenkinsPipelineFinishedDate,
Type: "CI/CD",
DurationSec: uint64(durationSec),
- CreatedDate:
jenkinsBuildWithRepo.StartTime,
+ CreatedDate: jenkinsBuild.StartTime,
}
if jenkinsPipelineFinishedDate != nil {
}
- jenkinsPipeline.RawDataOrigin =
jenkinsBuildWithRepo.RawDataOrigin
+ jenkinsPipeline.RawDataOrigin =
jenkinsBuild.RawDataOrigin
results = append(results, jenkinsPipeline)
}
- if !jenkinsBuildWithRepo.HasStages {
+ if !jenkinsBuild.HasStages {
jenkinsTask := &devops.CICDTask{
DomainEntity: domainlayer.DomainEntity{
- Id: fmt.Sprintf("%s:%s:%d:%s",
"jenkins", "JenkinsTask", jenkinsBuildWithRepo.ConnectionId,
-
jenkinsBuildWithRepo.DisplayName),
+ Id: fmt.Sprintf("%s:%s:%d:%s",
"jenkins", "JenkinsTask", jenkinsBuild.ConnectionId,
+
jenkinsBuild.DisplayName),
},
- Name:
jenkinsBuildWithRepo.DisplayName,
+ Name: jenkinsBuild.DisplayName,
Result: jenkinsPipelineResult,
Status: jenkinsPipelineStatus,
Type: "CI/CD",
DurationSec: uint64(durationSec),
- StartedDate:
jenkinsBuildWithRepo.StartTime,
+ StartedDate: jenkinsBuild.StartTime,
FinishedDate:
jenkinsPipelineFinishedDate,
}
- if jenkinsBuildWithRepo.TriggeredBy != "" {
- tmp := make([]*JenkinsBuildWithRepo, 0)
- clauses := []dal.Clause{
- dal.Select(`tjb.display_name,
tjb.result, tjb.timestamp, tjbr.commit_sha`),
- dal.From("_tool_jenkins_builds
tjb"),
- dal.Join("left join
_tool_jenkins_build_repos tjbr on tjbr.build_name = tjb.display_name"),
- dal.Where("tjb.connection_id =
? and tjb.display_name = ?", data.Options.ConnectionId,
jenkinsBuildWithRepo.TriggeredBy),
- }
- err = db.All(&tmp, clauses...)
- if err != nil {
- return nil, err
- }
- if len(tmp) > 0 {
- jenkinsTask.PipelineId =
fmt.Sprintf("%s:%s:%d:%s:%s", "jenkins", "JenkinsPipeline",
jenkinsBuildWithRepo.ConnectionId,
- tmp[0].CommitSha,
tmp[0].DisplayName)
- }
+ if jenkinsBuild.TriggeredBy != "" {
+ jenkinsTask.PipelineId =
fmt.Sprintf("%s:%s:%d:%s", "jenkins", "JenkinsPipeline",
+ jenkinsBuild.ConnectionId,
jenkinsBuild.TriggeredBy)
} else {
- jenkinsTask.PipelineId =
fmt.Sprintf("%s:%s:%d:%s:%s", "jenkins", "JenkinsPipeline",
jenkinsBuildWithRepo.ConnectionId,
- jenkinsBuildWithRepo.CommitSha,
jenkinsBuildWithRepo.DisplayName)
+ jenkinsTask.PipelineId =
fmt.Sprintf("%s:%s:%d:%s", "jenkins", "JenkinsPipeline",
+ jenkinsBuild.ConnectionId,
jenkinsBuild.DisplayName)
}
- jenkinsTask.RawDataOrigin =
jenkinsBuildWithRepo.RawDataOrigin
+ jenkinsTask.RawDataOrigin =
jenkinsBuild.RawDataOrigin
results = append(results, jenkinsTask)
}
diff --git a/plugins/jenkins/tasks/build_enricher.go
b/plugins/jenkins/tasks/build_origin_triggger_enricher.go
similarity index 60%
copy from plugins/jenkins/tasks/build_enricher.go
copy to plugins/jenkins/tasks/build_origin_triggger_enricher.go
index 76efb5fc..c95fd6d4 100644
--- a/plugins/jenkins/tasks/build_enricher.go
+++ b/plugins/jenkins/tasks/build_origin_triggger_enricher.go
@@ -20,58 +20,55 @@ package tasks
import (
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/core/dal"
+ "github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/jenkins/models"
- "strconv"
- "strings"
+ "reflect"
)
// this struct should be moved to `gitub_api_common.go`
-var EnrichApiBuildsMeta = core.SubTaskMeta{
- Name: "enrichApiBuilds",
- EntryPoint: EnrichApiBuilds,
+var EnrichApiBuildOriginTriggerMeta = core.SubTaskMeta{
+ Name: "enrichApiBuildOriginTrigger",
+ EntryPoint: EnrichApiBuildOriginTrigger,
EnabledByDefault: true,
- Description: "Enrich jenkins_builds",
+ Description: "Enrich jenkins build with origin trigger",
DomainTypes: []string{core.DOMAIN_TYPE_CICD},
}
-func EnrichApiBuilds(taskCtx core.SubTaskContext) error {
+func EnrichApiBuildOriginTrigger(taskCtx core.SubTaskContext) error {
data := taskCtx.GetData().(*JenkinsTaskData)
db := taskCtx.GetDal()
clauses := []dal.Clause{
- dal.Select("distinct build_name"),
- dal.From(&models.JenkinsStage{}),
+ dal.From(&models.JenkinsBuild{}),
dal.Where("connection_id = ?", data.Options.ConnectionId),
- dal.Groupby("build_name"),
}
cursor, err := db.Cursor(clauses...)
- defer cursor.Close()
taskCtx.SetProgress(0, -1)
+ iterator, err := helper.NewDalCursorIterator(db, cursor,
reflect.TypeOf(models.JenkinsBuild{}))
+ if err != nil {
+ return err
+ }
+ defer iterator.Close()
+ for iterator.HasNext() {
+ i, err := iterator.Fetch()
+ build := i.(*models.JenkinsBuild)
- for cursor.Next() {
- var buildName string
- err = cursor.Scan(&buildName)
- if err != nil {
- return err
- }
- if buildName == "" {
+ if build.TriggeredBy == "" {
continue
}
- build := &models.JenkinsBuild{}
- build.ConnectionId = data.Options.ConnectionId
- str := strings.Split(buildName, "#")
- build.JobName = strings.TrimSpace(str[0])
- number, err := strconv.Atoi(strings.TrimSpace(str[1]))
- if err != nil {
- return err
- }
- build.Number = int64(number)
- err = db.First(build)
- if err != nil {
- return err
+ buildTmp := &models.JenkinsBuild{}
+ buildTmp.DisplayName = build.TriggeredBy
+ for {
+ err = db.First(buildTmp)
+ if err != nil {
+ return err
+ }
+ if buildTmp.TriggeredBy == "" {
+ break
+ }
+ buildTmp.DisplayName = buildTmp.TriggeredBy
}
- build.HasStages = true
-
+ build.TriggeredBy = buildTmp.DisplayName
err = db.Update(build)
if err != nil {
return err
diff --git a/plugins/jenkins/tasks/build_repo_convertor.go
b/plugins/jenkins/tasks/build_repo_convertor.go
new file mode 100644
index 00000000..1d3e96e4
--- /dev/null
+++ b/plugins/jenkins/tasks/build_repo_convertor.go
@@ -0,0 +1,86 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "fmt"
+ "reflect"
+
+ "github.com/apache/incubator-devlake/models/domainlayer"
+ "github.com/apache/incubator-devlake/models/domainlayer/devops"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/apache/incubator-devlake/plugins/jenkins/models"
+)
+
+var ConvertBuildReposMeta = core.SubTaskMeta{
+ Name: "convertBuildRepos",
+ EntryPoint: ConvertBuildRepos,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table jenkins_builds into domain
layer table builds",
+ DomainTypes: []string{core.DOMAIN_TYPE_CICD},
+}
+
+func ConvertBuildRepos(taskCtx core.SubTaskContext) error {
+ db := taskCtx.GetDal()
+ data := taskCtx.GetData().(*JenkinsTaskData)
+
+ clauses := []dal.Clause{
+ dal.Select("*"),
+ dal.From(&models.JenkinsBuildRepo{}),
+ dal.Where("connection_id = ?", data.Options.ConnectionId),
+ }
+ cursor, err := db.Cursor(clauses...)
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ InputRowType: reflect.TypeOf(models.JenkinsBuildRepo{}),
+ Input: cursor,
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Params: JenkinsApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ },
+ Ctx: taskCtx,
+ Table: RAW_BUILD_TABLE,
+ },
+ Convert: func(inputRow interface{}) ([]interface{}, error) {
+ jenkinsBuildRepo := inputRow.(*models.JenkinsBuildRepo)
+ build := &devops.CiCDPipelineRepo{
+ DomainEntity: domainlayer.DomainEntity{
+ Id: fmt.Sprintf("%s:%s:%d:%s",
"jenkins", "JenkinsTask", jenkinsBuildRepo.ConnectionId,
+ jenkinsBuildRepo.BuildName),
+ },
+ CommitSha: jenkinsBuildRepo.CommitSha,
+ Branch: jenkinsBuildRepo.Branch,
+ RepoUrl: jenkinsBuildRepo.RepoUrl,
+ }
+ return []interface{}{
+ build,
+ }, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/plugins/jenkins/tasks/build_enricher.go
b/plugins/jenkins/tasks/build_stages_enricher.go
similarity index 88%
rename from plugins/jenkins/tasks/build_enricher.go
rename to plugins/jenkins/tasks/build_stages_enricher.go
index 76efb5fc..e0af3128 100644
--- a/plugins/jenkins/tasks/build_enricher.go
+++ b/plugins/jenkins/tasks/build_stages_enricher.go
@@ -27,15 +27,15 @@ import (
// this struct should be moved to `gitub_api_common.go`
-var EnrichApiBuildsMeta = core.SubTaskMeta{
- Name: "enrichApiBuilds",
- EntryPoint: EnrichApiBuilds,
+var EnrichApiBuildWithStagesMeta = core.SubTaskMeta{
+ Name: "enrichApiBuildWithStages",
+ EntryPoint: EnrichApiBuildWithStages,
EnabledByDefault: true,
- Description: "Enrich jenkins_builds",
+ Description: "Enrich jenkins build with stages",
DomainTypes: []string{core.DOMAIN_TYPE_CICD},
}
-func EnrichApiBuilds(taskCtx core.SubTaskContext) error {
+func EnrichApiBuildWithStages(taskCtx core.SubTaskContext) error {
data := taskCtx.GetData().(*JenkinsTaskData)
db := taskCtx.GetDal()
clauses := []dal.Clause{
