This is an automated email from the ASF dual-hosted git repository. abeizn pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit beede0343119cdd7ecf614959fb78be2a3888291 Author: Yingchu Chen <[email protected]> AuthorDate: Mon Aug 1 21:35:44 2022 +0800 feat(jenkins): modify builds --- plugins/jenkins/models/build.go | 3 + plugins/jenkins/models/build_build.go | 14 ----- .../{build_commit_repo_url.go => build_repo.go} | 9 +-- plugins/jenkins/models/job.go | 13 ++--- .../jenkins/models/{up_down_job.go => job_dag.go} | 6 +- .../20220729_modify_all_entities.go | 66 +++++++++------------- plugins/jenkins/models/response.go | 25 ++++++-- plugins/jenkins/models/stage.go | 1 + plugins/jenkins/tasks/build_collector.go | 2 +- plugins/jenkins/tasks/build_extractor.go | 27 ++++----- plugins/jenkins/tasks/job_extractor.go | 13 ++--- 11 files changed, 85 insertions(+), 94 deletions(-) diff --git a/plugins/jenkins/models/build.go b/plugins/jenkins/models/build.go index 6506d9ef..f3af1a10 100644 --- a/plugins/jenkins/models/build.go +++ b/plugins/jenkins/models/build.go @@ -39,6 +39,9 @@ type JenkinsBuild struct { StartTime time.Time // convered by timestamp CommitSha string `gorm:"type:varchar(255)"` Type string `gorm:"index;type:varchar(255)"` + Class string `gorm:"index;type:varchar(255)" ` + TriggeredBy string `gorm:"type:varchar(255)"` + Building bool } func (JenkinsBuild) TableName() string { diff --git a/plugins/jenkins/models/build_build.go b/plugins/jenkins/models/build_build.go deleted file mode 100644 index be6e2500..00000000 --- a/plugins/jenkins/models/build_build.go +++ /dev/null @@ -1,14 +0,0 @@ -package models - -import "github.com/apache/incubator-devlake/models/common" - -type JenkinsBuildTriggeredBuilds struct { - ConnectionId uint64 `gorm:"primaryKey"` - BuildName string `gorm:"primaryKey;type:varchar(255)"` - TriggeredBuildName string `gorm:"primaryKey;type:varchar(255)"` - common.NoPKModel -} - -func (JenkinsBuildTriggeredBuilds) TableName() string { - return "_tool_jenkins_build_triggered_builds" -} diff --git a/plugins/jenkins/models/build_commit_repo_url.go b/plugins/jenkins/models/build_repo.go similarity index 54% rename from plugins/jenkins/models/build_commit_repo_url.go rename to plugins/jenkins/models/build_repo.go index de81ba38..7f26c54a 100644 --- a/plugins/jenkins/models/build_commit_repo_url.go +++ b/plugins/jenkins/models/build_repo.go @@ -2,14 +2,15 @@ package models import "github.com/apache/incubator-devlake/models/common" -type JenkinsBuildCommitRepoUrl struct { +type JenkinsBuildRepo struct { ConnectionId uint64 `gorm:"primaryKey"` BuildName string `gorm:"primaryKey;type:varchar(255)"` CommitSha string `gorm:"primaryKey;type:varchar(255)"` - RemoteUrl string `gorm:"primaryKey;type:varchar(255)"` + Branch string `gorm:"type:varchar(255)"` + RepoUrl string `gorm:"primaryKey;type:varchar(255)"` common.NoPKModel } -func (JenkinsBuildCommitRepoUrl) TableName() string { - return "_tool_jenkins_build_commit_repo_urls" +func (JenkinsBuildRepo) TableName() string { + return "_tool_jenkins_build_repos" } diff --git a/plugins/jenkins/models/job.go b/plugins/jenkins/models/job.go index f525343b..43687ce2 100644 --- a/plugins/jenkins/models/job.go +++ b/plugins/jenkins/models/job.go @@ -25,13 +25,12 @@ import ( // JenkinsJobProps current used jenkins job props type JenkinsJobProps struct { // collected fields - ConnectionId uint64 `gorm:"primaryKey"` - Name string `gorm:"primaryKey;type:varchar(255)"` - Path string `gorm:"primaryKey;type:varchar(511)"` - Class string `gorm:"type:varchar(255)"` - Color string `gorm:"type:varchar(255)"` - Base string `gorm:"type:varchar(255)"` - HasUpstreamProjects bool + ConnectionId uint64 `gorm:"primaryKey"` + Name string `gorm:"primaryKey;type:varchar(255)"` + Path string `gorm:"primaryKey;type:varchar(511)"` + Class string `gorm:"type:varchar(255)"` + Color string `gorm:"type:varchar(255)"` + Base string `gorm:"type:varchar(255)"` } // JenkinsJob db entity for jenkins job diff --git a/plugins/jenkins/models/up_down_job.go b/plugins/jenkins/models/job_dag.go similarity index 69% rename from plugins/jenkins/models/up_down_job.go rename to plugins/jenkins/models/job_dag.go index fcca32f2..dcbe7253 100644 --- a/plugins/jenkins/models/up_down_job.go +++ b/plugins/jenkins/models/job_dag.go @@ -2,13 +2,13 @@ package models import "github.com/apache/incubator-devlake/models/common" -type JenkinsUpDownJob struct { +type JenkinsJobDag struct { ConnetionId uint64 `gorm:"primaryKey"` UpstreamJob string `gorm:"primaryKey;type:varchar(255)"` DownstreamJob string `gorm:"primaryKey;type:varchar(255)"` common.NoPKModel } -func (JenkinsUpDownJob) TableName() string { - return "_tool_jenkins_up_down_jobs" +func (JenkinsJobDag) TableName() string { + return "_tool_jenkins_job_dags" } diff --git a/plugins/jenkins/models/migrationscripts/20220729_modify_all_entities.go b/plugins/jenkins/models/migrationscripts/20220729_modify_all_entities.go index 43d8b4e7..33a00e19 100644 --- a/plugins/jenkins/models/migrationscripts/20220729_modify_all_entities.go +++ b/plugins/jenkins/models/migrationscripts/20220729_modify_all_entities.go @@ -19,62 +19,46 @@ package migrationscripts import ( "context" - "github.com/apache/incubator-devlake/models/common" "github.com/apache/incubator-devlake/models/migrationscripts/archived" "gorm.io/gorm" ) type modifyAllEntities struct{} -// JenkinsBuild db entity for jenkins build -type JenkinsBuild0729 struct { - Type string `gorm:"index;type:varchar(255)" ` -} - -func (JenkinsBuild0729) TableName() string { - return "_tool_jenkins_builds" -} - -type JenkinsJob0729 struct { - HasUpstreamProjects bool -} - -func (JenkinsJob0729) TableName() string { - return "_tool_jenkins_jobs" -} - -type JenkinsUpDownJob0729 struct { +type JenkinsJobDag0729 struct { ConnetionId uint64 `gorm:"primaryKey"` UpstreamJob string `gorm:"primaryKey;type:varchar(255)"` DownstreamJob string `gorm:"primaryKey;type:varchar(255)"` archived.NoPKModel } -func (JenkinsUpDownJob0729) TableName() string { - return "_tool_jenkins_up_down_jobs" +func (JenkinsJobDag0729) TableName() string { + return "_tool_jenkins_job_dags" } -type JenkinsBuildCommitRepoUrl0729 struct { - ConnectionId uint64 `gorm:"primaryKey"` - BuildName string `gorm:"primaryKey;type:varchar(255)"` - CommitSha string `gorm:"primaryKey;type:varchar(255)"` - RemoteUrl string `gorm:"primaryKey;type:varchar(255)"` - common.NoPKModel +// JenkinsBuild db entity for jenkins build +type JenkinsBuild0729 struct { + TriggeredBy string `gorm:"type:varchar(255)"` + Type string `gorm:"index;type:varchar(255)" ` + Class string `gorm:"index;type:varchar(255)" ` + Building bool } -func (JenkinsBuildCommitRepoUrl0729) TableName() string { - return "_tool_jenkins_build_commit_repo_urls" +func (JenkinsBuild0729) TableName() string { + return "_tool_jenkins_builds" } -type JenkinsBuildTriggeredBuilds0729 struct { - ConnectionId uint64 `gorm:"primaryKey"` - BuildName string `gorm:"primaryKey;type:varchar(255)"` - TriggeredBuildName string `gorm:"primaryKey;type:varchar(255)"` +type JenkinsBuildRepo0729 struct { + ConnectionId uint64 `gorm:"primaryKey"` + BuildName string `gorm:"primaryKey;type:varchar(255)"` + CommitSha string `gorm:"primaryKey;type:varchar(255)"` + Branch string `gorm:"type:varchar(255)"` + RepoUrl string `gorm:"primaryKey;type:varchar(255)"` archived.NoPKModel } -func (JenkinsBuildTriggeredBuilds0729) TableName() string { - return "_tool_jenkins_build_triggered_builds" +func (JenkinsBuildRepo0729) TableName() string { + return "_tool_jenkins_build_repos" } type JenkinsStage0729 struct { @@ -88,6 +72,7 @@ type JenkinsStage0729 struct { DurationMillis int `json:"durationMillis"` PauseDurationMillis int `json:"pauseDurationMillis"` BuildName string `gorm:"primaryKey;type:varchar(255)"` + Type string `gorm:"index;type:varchar(255)"` } func (JenkinsStage0729) TableName() string { @@ -99,14 +84,17 @@ func (*modifyAllEntities) Up(ctx context.Context, db *gorm.DB) error { if err != nil { return err } - err = db.Migrator().AddColumn(JenkinsJob0729{}, "has_upstream_projects") + err = db.Migrator().AddColumn(JenkinsBuild0729{}, "triggered_by") + if err != nil { + return err + } + err = db.Migrator().AddColumn(JenkinsBuild0729{}, "building") if err != nil { return err } err = db.Migrator().AutoMigrate( - JenkinsUpDownJob0729{}, - JenkinsBuildCommitRepoUrl0729{}, - JenkinsBuildTriggeredBuilds0729{}, + JenkinsJobDag0729{}, + JenkinsBuildRepo0729{}, JenkinsStage0729{}, ) if err != nil { diff --git a/plugins/jenkins/models/response.go b/plugins/jenkins/models/response.go index 3171bf27..1727c197 100644 --- a/plugins/jenkins/models/response.go +++ b/plugins/jenkins/models/response.go @@ -81,15 +81,16 @@ type ApiBuildResponse struct { ChangeSet ChangeSet `json:"changeSet"` } type LastBuiltRevision struct { - SHA1 string `json:"SHA1"` + SHA1 string `json:"SHA1"` + Branches []Branch `json:"branch"` } type Action struct { - Class string `json:"_class,omitempty"` - LastBuiltRevision LastBuiltRevision `json:"lastBuiltRevision,omitempty"` - MercurialRevisionNumber string `json:"mercurialRevisionNumber"` - RemoteUrls []string `json:"remoteUrls"` - TriggeredBuilds []ApiBuildResponse `json:"triggeredBuilds"` + Class string `json:"_class,omitempty"` + LastBuiltRevision LastBuiltRevision `json:"lastBuiltRevision,omitempty"` + MercurialRevisionNumber string `json:"mercurialRevisionNumber"` + RemoteUrls []string `json:"remoteUrls"` + Causes []Cause `json:"causes"` } type ChangeSet struct { Class string `json:"_class"` @@ -97,6 +98,10 @@ type ChangeSet struct { Revisions []Revision `json:"revision"` } +type Branch struct { + Name string `json:"name"` +} + type Revision struct { Module string Revision int @@ -116,3 +121,11 @@ type Stage struct { DurationMillis int `json:"durationMillis"` PauseDurationMillis int `json:"pauseDurationMillis"` } + +type Cause struct { + Class string `json:"_class"` + ShortDescription string `json:"shortDescription"` + UpstreamBuild int `json:"upstreamBuild"` + UpstreamProject string `json:"upstreamProject"` + UpstreamURL string `json:"upstreamUrl"` +} diff --git a/plugins/jenkins/models/stage.go b/plugins/jenkins/models/stage.go index a8654d90..108b1952 100644 --- a/plugins/jenkins/models/stage.go +++ b/plugins/jenkins/models/stage.go @@ -31,6 +31,7 @@ type JenkinsStage struct { StartTimeMillis int64 `json:"startTimeMillis"` DurationMillis int `json:"durationMillis"` PauseDurationMillis int `json:"pauseDurationMillis"` + Type string `gorm:"index;type:varchar(255)"` BuildName string `gorm:"primaryKey;type:varchar(255)"` } diff --git a/plugins/jenkins/tasks/build_collector.go b/plugins/jenkins/tasks/build_collector.go index 86877c24..7e7d3667 100644 --- a/plugins/jenkins/tasks/build_collector.go +++ b/plugins/jenkins/tasks/build_collector.go @@ -82,7 +82,7 @@ func CollectApiBuilds(taskCtx core.SubTaskContext) error { Query: func(reqData *helper.RequestData) (url.Values, error) { query := url.Values{} treeValue := fmt.Sprintf( - "allBuilds[number,timestamp,duration,estimatedDuration,fullDisplayName,result,actions[lastBuiltRevision[SHA1],remoteUrls,mercurialRevisionNumber,triggeredBuilds[fullDisplayName,number,url,result,status,duration]],changeSet[kind,revisions[revision]]]{%d,%d}", + "allBuilds[number,timestamp,duration,estimatedDuration,fullDisplayName,result,actions[lastBuiltRevision[SHA1,branch[name]],remoteUrls,mercurialRevisionNumber,causes[*]],changeSet[kind,revisions[revision]]]{%d,%d}", reqData.Pager.Skip, reqData.Pager.Skip+reqData.Pager.Size) query.Set("tree", treeValue) return query, nil diff --git a/plugins/jenkins/tasks/build_extractor.go b/plugins/jenkins/tasks/build_extractor.go index 66a818d4..cbe48f71 100644 --- a/plugins/jenkins/tasks/build_extractor.go +++ b/plugins/jenkins/tasks/build_extractor.go @@ -19,6 +19,7 @@ package tasks import ( "encoding/json" + "fmt" "strconv" "strings" "time" @@ -72,7 +73,6 @@ func ExtractApiBuilds(taskCtx core.SubTaskContext) error { class := strList[len(strList)-1] build := &models.JenkinsBuild{ ConnectionId: data.Options.ConnectionId, - Type: class, JobName: input.Name, Duration: body.Duration, DisplayName: body.DisplayName, @@ -80,12 +80,14 @@ func ExtractApiBuilds(taskCtx core.SubTaskContext) error { Number: body.Number, Result: body.Result, Timestamp: body.Timestamp, + Class: class, StartTime: time.Unix(body.Timestamp/1000, 0), } vcs := body.ChangeSet.Kind if vcs == "git" || vcs == "hg" { for _, a := range body.Actions { sha := "" + branch := "" if a.LastBuiltRevision.SHA1 != "" { sha = a.LastBuiltRevision.SHA1 } @@ -93,28 +95,27 @@ func ExtractApiBuilds(taskCtx core.SubTaskContext) error { sha = a.MercurialRevisionNumber } build.CommitSha = sha + if len(a.LastBuiltRevision.Branches) > 0 { + branch = a.LastBuiltRevision.Branches[0].Name + } for _, url := range a.RemoteUrls { if url != "" { - buildCommitRemoteUrl := models.JenkinsBuildCommitRepoUrl{ + buildCommitRemoteUrl := models.JenkinsBuildRepo{ ConnectionId: data.Options.ConnectionId, BuildName: build.DisplayName, CommitSha: sha, - RemoteUrl: url, + RepoUrl: url, + Branch: branch, } results = append(results, &buildCommitRemoteUrl) } } - if a.TriggeredBuilds != nil && len(a.TriggeredBuilds) > 0 { - for _, b := range a.TriggeredBuilds { - if b.DisplayName == "" { - continue - } - buildTrigger := models.JenkinsBuildTriggeredBuilds{ - ConnectionId: data.Options.ConnectionId, - BuildName: build.DisplayName, - TriggeredBuildName: b.DisplayName, + if len(a.Causes) > 0 { + for _, cause := range a.Causes { + if cause.UpstreamProject != "" { + triggeredByBuild := fmt.Sprintf("%s #%d", cause.UpstreamProject, cause.UpstreamBuild) + build.TriggeredBy = triggeredByBuild } - results = append(results, &buildTrigger) } } } diff --git a/plugins/jenkins/tasks/job_extractor.go b/plugins/jenkins/tasks/job_extractor.go index f78742be..3b715960 100644 --- a/plugins/jenkins/tasks/job_extractor.go +++ b/plugins/jenkins/tasks/job_extractor.go @@ -68,16 +68,15 @@ func ExtractApiJobs(taskCtx core.SubTaskContext) error { job := &models.JenkinsJob{ JenkinsJobProps: models.JenkinsJobProps{ - ConnectionId: data.Options.ConnectionId, - Name: body.Name, - Path: input.Path, - Class: body.Class, - Color: body.Color, - HasUpstreamProjects: len(body.UpstreamProjects) > 0, + ConnectionId: data.Options.ConnectionId, + Name: body.Name, + Path: input.Path, + Class: body.Class, + Color: body.Color, }, } for _, upstreamProject := range body.UpstreamProjects { - upDownJob := models.JenkinsUpDownJob{ + upDownJob := models.JenkinsJobDag{ ConnetionId: data.Options.ConnectionId, UpstreamJob: upstreamProject.Name, DownstreamJob: job.Name,
