This is an automated email from the ASF dual-hosted git repository. abeizn pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit e3941b6b4eb89cc1492c058c5e7c55d6c61001df Author: Yingchu Chen <[email protected]> AuthorDate: Fri Jul 29 21:52:47 2022 +0800 feat(jenkins): collect more fields add builds_triggered builds and up_down_jobs closes #2637 --- .../e2e/snapshot_tables/_tool_jenkins_builds.csv | 16 +-- plugins/jenkins/e2e/snapshot_tables/builds.csv | 16 +-- plugins/jenkins/impl/impl.go | 2 + plugins/jenkins/models/build.go | 1 + plugins/jenkins/models/build_build.go | 14 +++ plugins/jenkins/models/build_commit_repo_url.go | 15 +++ plugins/jenkins/models/job.go | 13 ++- .../20220729_modify_all_entities.go | 125 +++++++++++++++++++++ .../jenkins/models/migrationscripts/register.go | 1 + plugins/jenkins/models/response.go | 44 ++++++-- plugins/jenkins/models/{build.go => stage.go} | 30 ++--- plugins/jenkins/models/up_down_job.go | 14 +++ plugins/jenkins/tasks/build_collector.go | 2 +- plugins/jenkins/tasks/build_extractor.go | 37 +++++- plugins/jenkins/tasks/job_collector.go | 2 +- plugins/jenkins/tasks/job_extractor.go | 23 ++-- .../{build_collector.go => stage_collector.go} | 41 ++++--- .../tasks/{job_extractor.go => stage_extractor.go} | 40 +++---- 18 files changed, 331 insertions(+), 105 deletions(-) diff --git a/plugins/jenkins/e2e/snapshot_tables/_tool_jenkins_builds.csv b/plugins/jenkins/e2e/snapshot_tables/_tool_jenkins_builds.csv index b61c1434..e6133952 100644 --- a/plugins/jenkins/e2e/snapshot_tables/_tool_jenkins_builds.csv +++ b/plugins/jenkins/e2e/snapshot_tables/_tool_jenkins_builds.csv @@ -1,9 +1,9 @@ connection_id,job_name,number,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,duration,display_name,estimated_duration,result,timestamp,start_time,commit_sha -1,这是一个改了名字的测试任务,1,"{""ConnectionId"":1}",_raw_jenkins_api_builds,82,,57,#1,10,SUCCESS,1650017153775,2022-04-15T10:05:53.000+00:00, -1,这是一个改了名字的测试任务,2,"{""ConnectionId"":1}",_raw_jenkins_api_builds,81,,4,#2,10,SUCCESS,1650017177939,2022-04-15T10:06:17.000+00:00, -1,这是一个改了名字的测试任务,3,"{""ConnectionId"":1}",_raw_jenkins_api_builds,80,,3,#3,10,SUCCESS,1650017186253,2022-04-15T10:06:26.000+00:00, -1,这是一个改了名字的测试任务,4,"{""ConnectionId"":1}",_raw_jenkins_api_builds,79,,6,#4,10,SUCCESS,1650022556910,2022-04-15T11:35:56.000+00:00, -1,这是一个改了名字的测试任务,5,"{""ConnectionId"":1}",_raw_jenkins_api_builds,78,,6,#5,10,SUCCESS,1650022558491,2022-04-15T11:35:58.000+00:00, -1,这是一个改了名字的测试任务,6,"{""ConnectionId"":1}",_raw_jenkins_api_builds,77,,10,#6,10,SUCCESS,1650022560954,2022-04-15T11:36:00.000+00:00, -1,这是一个改了名字的测试任务,7,"{""ConnectionId"":1}",_raw_jenkins_api_builds,76,,8,#7,10,SUCCESS,1650023883294,2022-04-15T11:58:03.000+00:00, -1,这是一个改了名字的测试任务,8,"{""ConnectionId"":1}",_raw_jenkins_api_builds,75,,11,#8,10,SUCCESS,1650023894336,2022-04-15T11:58:14.000+00:00, +1,这是一个改了名字的测试任务,1,"{""ConnectionId"":1}",_raw_jenkins_api_builds,82,,57,,10,SUCCESS,1650017153775,2022-04-15T10:05:53.000+00:00, +1,这是一个改了名字的测试任务,2,"{""ConnectionId"":1}",_raw_jenkins_api_builds,81,,4,,10,SUCCESS,1650017177939,2022-04-15T10:06:17.000+00:00, +1,这是一个改了名字的测试任务,3,"{""ConnectionId"":1}",_raw_jenkins_api_builds,80,,3,,10,SUCCESS,1650017186253,2022-04-15T10:06:26.000+00:00, +1,这是一个改了名字的测试任务,4,"{""ConnectionId"":1}",_raw_jenkins_api_builds,79,,6,,10,SUCCESS,1650022556910,2022-04-15T11:35:56.000+00:00, +1,这是一个改了名字的测试任务,5,"{""ConnectionId"":1}",_raw_jenkins_api_builds,78,,6,,10,SUCCESS,1650022558491,2022-04-15T11:35:58.000+00:00, +1,这是一个改了名字的测试任务,6,"{""ConnectionId"":1}",_raw_jenkins_api_builds,77,,10,,10,SUCCESS,1650022560954,2022-04-15T11:36:00.000+00:00, +1,这是一个改了名字的测试任务,7,"{""ConnectionId"":1}",_raw_jenkins_api_builds,76,,8,,10,SUCCESS,1650023883294,2022-04-15T11:58:03.000+00:00, +1,这是一个改了名字的测试任务,8,"{""ConnectionId"":1}",_raw_jenkins_api_builds,75,,11,,10,SUCCESS,1650023894336,2022-04-15T11:58:14.000+00:00, diff --git a/plugins/jenkins/e2e/snapshot_tables/builds.csv b/plugins/jenkins/e2e/snapshot_tables/builds.csv index 38476b9b..a2b50a07 100644 --- a/plugins/jenkins/e2e/snapshot_tables/builds.csv +++ b/plugins/jenkins/e2e/snapshot_tables/builds.csv @@ -1,9 +1,9 @@ id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,job_id,name,commit_sha,duration_sec,status,started_date -jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:1,"{""ConnectionId"":1}",_raw_jenkins_api_builds,82,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,#1,,0,SUCCESS,2022-04-15T10:05:53.000+00:00 -jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:2,"{""ConnectionId"":1}",_raw_jenkins_api_builds,81,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,#2,,0,SUCCESS,2022-04-15T10:06:17.000+00:00 -jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:3,"{""ConnectionId"":1}",_raw_jenkins_api_builds,80,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,#3,,0,SUCCESS,2022-04-15T10:06:26.000+00:00 -jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:4,"{""ConnectionId"":1}",_raw_jenkins_api_builds,79,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,#4,,0,SUCCESS,2022-04-15T11:35:56.000+00:00 -jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:5,"{""ConnectionId"":1}",_raw_jenkins_api_builds,78,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,#5,,0,SUCCESS,2022-04-15T11:35:58.000+00:00 -jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:6,"{""ConnectionId"":1}",_raw_jenkins_api_builds,77,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,#6,,0,SUCCESS,2022-04-15T11:36:00.000+00:00 -jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:7,"{""ConnectionId"":1}",_raw_jenkins_api_builds,76,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,#7,,0,SUCCESS,2022-04-15T11:58:03.000+00:00 -jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:8,"{""ConnectionId"":1}",_raw_jenkins_api_builds,75,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,#8,,0,SUCCESS,2022-04-15T11:58:14.000+00:00 +jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:1,"{""ConnectionId"":1}",_raw_jenkins_api_builds,82,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,,,0,SUCCESS,2022-04-15T10:05:53.000+00:00 +jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:2,"{""ConnectionId"":1}",_raw_jenkins_api_builds,81,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,,,0,SUCCESS,2022-04-15T10:06:17.000+00:00 +jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:3,"{""ConnectionId"":1}",_raw_jenkins_api_builds,80,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,,,0,SUCCESS,2022-04-15T10:06:26.000+00:00 +jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:4,"{""ConnectionId"":1}",_raw_jenkins_api_builds,79,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,,,0,SUCCESS,2022-04-15T11:35:56.000+00:00 +jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:5,"{""ConnectionId"":1}",_raw_jenkins_api_builds,78,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,,,0,SUCCESS,2022-04-15T11:35:58.000+00:00 +jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:6,"{""ConnectionId"":1}",_raw_jenkins_api_builds,77,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,,,0,SUCCESS,2022-04-15T11:36:00.000+00:00 +jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:7,"{""ConnectionId"":1}",_raw_jenkins_api_builds,76,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,,,0,SUCCESS,2022-04-15T11:58:03.000+00:00 +jenkins:JenkinsBuild:1:这是一个改了名字的测试任务:8,"{""ConnectionId"":1}",_raw_jenkins_api_builds,75,,jenkins:JenkinsJob:1:这是一个改了名字的测试任务,,,0,SUCCESS,2022-04-15T11:58:14.000+00:00 diff --git a/plugins/jenkins/impl/impl.go b/plugins/jenkins/impl/impl.go index d9d6f92d..167a7355 100644 --- a/plugins/jenkins/impl/impl.go +++ b/plugins/jenkins/impl/impl.go @@ -55,6 +55,8 @@ func (plugin Jenkins) SubTaskMetas() []core.SubTaskMeta { tasks.ExtractApiJobsMeta, tasks.CollectApiBuildsMeta, tasks.ExtractApiBuildsMeta, + tasks.CollectApiStagesMeta, + tasks.ExtractApiStagesMeta, tasks.ConvertJobsMeta, tasks.ConvertBuildsMeta, } diff --git a/plugins/jenkins/models/build.go b/plugins/jenkins/models/build.go index 98d4b974..6506d9ef 100644 --- a/plugins/jenkins/models/build.go +++ b/plugins/jenkins/models/build.go @@ -38,6 +38,7 @@ type JenkinsBuild struct { Timestamp int64 // start time StartTime time.Time // convered by timestamp CommitSha string `gorm:"type:varchar(255)"` + Type string `gorm:"index;type:varchar(255)"` } func (JenkinsBuild) TableName() string { diff --git a/plugins/jenkins/models/build_build.go b/plugins/jenkins/models/build_build.go new file mode 100644 index 00000000..be6e2500 --- /dev/null +++ b/plugins/jenkins/models/build_build.go @@ -0,0 +1,14 @@ +package models + +import "github.com/apache/incubator-devlake/models/common" + +type JenkinsBuildTriggeredBuilds struct { + ConnectionId uint64 `gorm:"primaryKey"` + BuildName string `gorm:"primaryKey;type:varchar(255)"` + TriggeredBuildName string `gorm:"primaryKey;type:varchar(255)"` + common.NoPKModel +} + +func (JenkinsBuildTriggeredBuilds) TableName() string { + return "_tool_jenkins_build_triggered_builds" +} diff --git a/plugins/jenkins/models/build_commit_repo_url.go b/plugins/jenkins/models/build_commit_repo_url.go new file mode 100644 index 00000000..de81ba38 --- /dev/null +++ b/plugins/jenkins/models/build_commit_repo_url.go @@ -0,0 +1,15 @@ +package models + +import "github.com/apache/incubator-devlake/models/common" + +type JenkinsBuildCommitRepoUrl struct { + ConnectionId uint64 `gorm:"primaryKey"` + BuildName string `gorm:"primaryKey;type:varchar(255)"` + CommitSha string `gorm:"primaryKey;type:varchar(255)"` + RemoteUrl string `gorm:"primaryKey;type:varchar(255)"` + common.NoPKModel +} + +func (JenkinsBuildCommitRepoUrl) TableName() string { + return "_tool_jenkins_build_commit_repo_urls" +} diff --git a/plugins/jenkins/models/job.go b/plugins/jenkins/models/job.go index 43687ce2..f525343b 100644 --- a/plugins/jenkins/models/job.go +++ b/plugins/jenkins/models/job.go @@ -25,12 +25,13 @@ import ( // JenkinsJobProps current used jenkins job props type JenkinsJobProps struct { // collected fields - ConnectionId uint64 `gorm:"primaryKey"` - Name string `gorm:"primaryKey;type:varchar(255)"` - Path string `gorm:"primaryKey;type:varchar(511)"` - Class string `gorm:"type:varchar(255)"` - Color string `gorm:"type:varchar(255)"` - Base string `gorm:"type:varchar(255)"` + ConnectionId uint64 `gorm:"primaryKey"` + Name string `gorm:"primaryKey;type:varchar(255)"` + Path string `gorm:"primaryKey;type:varchar(511)"` + Class string `gorm:"type:varchar(255)"` + Color string `gorm:"type:varchar(255)"` + Base string `gorm:"type:varchar(255)"` + HasUpstreamProjects bool } // JenkinsJob db entity for jenkins job diff --git a/plugins/jenkins/models/migrationscripts/20220729_modify_all_entities.go b/plugins/jenkins/models/migrationscripts/20220729_modify_all_entities.go new file mode 100644 index 00000000..43d8b4e7 --- /dev/null +++ b/plugins/jenkins/models/migrationscripts/20220729_modify_all_entities.go @@ -0,0 +1,125 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "context" + "github.com/apache/incubator-devlake/models/common" + "github.com/apache/incubator-devlake/models/migrationscripts/archived" + "gorm.io/gorm" +) + +type modifyAllEntities struct{} + +// JenkinsBuild db entity for jenkins build +type JenkinsBuild0729 struct { + Type string `gorm:"index;type:varchar(255)" ` +} + +func (JenkinsBuild0729) TableName() string { + return "_tool_jenkins_builds" +} + +type JenkinsJob0729 struct { + HasUpstreamProjects bool +} + +func (JenkinsJob0729) TableName() string { + return "_tool_jenkins_jobs" +} + +type JenkinsUpDownJob0729 struct { + ConnetionId uint64 `gorm:"primaryKey"` + UpstreamJob string `gorm:"primaryKey;type:varchar(255)"` + DownstreamJob string `gorm:"primaryKey;type:varchar(255)"` + archived.NoPKModel +} + +func (JenkinsUpDownJob0729) TableName() string { + return "_tool_jenkins_up_down_jobs" +} + +type JenkinsBuildCommitRepoUrl0729 struct { + ConnectionId uint64 `gorm:"primaryKey"` + BuildName string `gorm:"primaryKey;type:varchar(255)"` + CommitSha string `gorm:"primaryKey;type:varchar(255)"` + RemoteUrl string `gorm:"primaryKey;type:varchar(255)"` + common.NoPKModel +} + +func (JenkinsBuildCommitRepoUrl0729) TableName() string { + return "_tool_jenkins_build_commit_repo_urls" +} + +type JenkinsBuildTriggeredBuilds0729 struct { + ConnectionId uint64 `gorm:"primaryKey"` + BuildName string `gorm:"primaryKey;type:varchar(255)"` + TriggeredBuildName string `gorm:"primaryKey;type:varchar(255)"` + archived.NoPKModel +} + +func (JenkinsBuildTriggeredBuilds0729) TableName() string { + return "_tool_jenkins_build_triggered_builds" +} + +type JenkinsStage0729 struct { + archived.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + ID string `json:"id" gorm:"primaryKey;type:varchar(255)"` + Name string `json:"name" gorm:"type:varchar(255)"` + ExecNode string `json:"execNode" gorm:"type:varchar(255)"` + Status string `json:"status" gorm:"type:varchar(255)"` + StartTimeMillis int64 `json:"startTimeMillis"` + DurationMillis int `json:"durationMillis"` + PauseDurationMillis int `json:"pauseDurationMillis"` + BuildName string `gorm:"primaryKey;type:varchar(255)"` +} + +func (JenkinsStage0729) TableName() string { + return "_tool_jenkins_stages" +} + +func (*modifyAllEntities) Up(ctx context.Context, db *gorm.DB) error { + err := db.Migrator().AddColumn(JenkinsBuild0729{}, "type") + if err != nil { + return err + } + err = db.Migrator().AddColumn(JenkinsJob0729{}, "has_upstream_projects") + if err != nil { + return err + } + err = db.Migrator().AutoMigrate( + JenkinsUpDownJob0729{}, + JenkinsBuildCommitRepoUrl0729{}, + JenkinsBuildTriggeredBuilds0729{}, + JenkinsStage0729{}, + ) + if err != nil { + return err + } + + return nil +} + +func (*modifyAllEntities) Version() uint64 { + return 20220729231237 +} + +func (*modifyAllEntities) Name() string { + return "Jenkins modify build and job" +} diff --git a/plugins/jenkins/models/migrationscripts/register.go b/plugins/jenkins/models/migrationscripts/register.go index c1365f7d..63eb2733 100644 --- a/plugins/jenkins/models/migrationscripts/register.go +++ b/plugins/jenkins/models/migrationscripts/register.go @@ -25,5 +25,6 @@ import ( func All() []migration.Script { return []migration.Script{ new(addInitTables), + new(modifyAllEntities), } } diff --git a/plugins/jenkins/models/response.go b/plugins/jenkins/models/response.go index ace21939..3171bf27 100644 --- a/plugins/jenkins/models/response.go +++ b/plugins/jenkins/models/response.go @@ -37,12 +37,19 @@ type ApiResponse struct { NodeDescription string `json:"nodeDescription"` } type Job struct { - URL string `json:"url"` - Name string `json:"name"` - Color string `json:"color"` + URL string `json:"url"` + Name string `json:"name"` + Color string `json:"color"` + Class string `json:"_class"` + Jobs *[]Job `json:"jobs"` + UpstreamProjects []Project `json:"upstreamProjects"` +} + +type Project struct { Class string `json:"_class"` - Jobs *[]Job `json:"jobs"` + Name string `json:"name"` } + type Views struct { URL string `json:"url"` Name string `json:"name"` @@ -66,10 +73,10 @@ type ApiBuildResponse struct { Class string `json:"_class"` Number int64 `json:"number"` Result string `json:"result"` - Actions []Actions `json:"actions"` + Actions []Action `json:"actions"` Duration float64 `json:"duration"` Timestamp int64 `json:"timestamp"` - DisplayName string `json:"displayName"` + DisplayName string `json:"fullDisplayName"` EstimatedDuration float64 `json:"estimatedDuration"` ChangeSet ChangeSet `json:"changeSet"` } @@ -77,10 +84,12 @@ type LastBuiltRevision struct { SHA1 string `json:"SHA1"` } -type Actions struct { - Class string `json:"_class,omitempty"` - LastBuiltRevision LastBuiltRevision `json:"lastBuiltRevision,omitempty"` - MercurialRevisionNumber string `json:"mercurialRevisionNumber"` +type Action struct { + Class string `json:"_class,omitempty"` + LastBuiltRevision LastBuiltRevision `json:"lastBuiltRevision,omitempty"` + MercurialRevisionNumber string `json:"mercurialRevisionNumber"` + RemoteUrls []string `json:"remoteUrls"` + TriggeredBuilds []ApiBuildResponse `json:"triggeredBuilds"` } type ChangeSet struct { Class string `json:"_class"` @@ -92,3 +101,18 @@ type Revision struct { Module string Revision int } + +type Stage struct { + Links struct { + Self struct { + Href string `json:"href"` + } `json:"self"` + } `json:"_links"` + ID string `json:"id"` + Name string `json:"name"` + ExecNode string `json:"execNode"` + Status string `json:"status"` + StartTimeMillis int64 `json:"startTimeMillis"` + DurationMillis int `json:"durationMillis"` + PauseDurationMillis int `json:"pauseDurationMillis"` +} diff --git a/plugins/jenkins/models/build.go b/plugins/jenkins/models/stage.go similarity index 55% copy from plugins/jenkins/models/build.go copy to plugins/jenkins/models/stage.go index 98d4b974..a8654d90 100644 --- a/plugins/jenkins/models/build.go +++ b/plugins/jenkins/models/stage.go @@ -18,28 +18,22 @@ limitations under the License. package models import ( - "time" - "github.com/apache/incubator-devlake/models/common" ) -// JenkinsBuild db entity for jenkins build -type JenkinsBuild struct { +type JenkinsStage struct { common.NoPKModel - - // collected fields - ConnectionId uint64 `gorm:"primaryKey"` - JobName string `gorm:"primaryKey;type:varchar(255)"` - Duration float64 // build time - DisplayName string `gorm:"type:varchar(255)"` // "#7" - EstimatedDuration float64 // EstimatedDuration - Number int64 `gorm:"primaryKey"` - Result string // Result - Timestamp int64 // start time - StartTime time.Time // convered by timestamp - CommitSha string `gorm:"type:varchar(255)"` + ConnectionId uint64 `gorm:"primaryKey"` + ID string `json:"id" gorm:"primaryKey;type:varchar(255)"` + Name string `json:"name" gorm:"type:varchar(255)"` + ExecNode string `json:"execNode" gorm:"type:varchar(255)"` + Status string `json:"status" gorm:"type:varchar(255)"` + StartTimeMillis int64 `json:"startTimeMillis"` + DurationMillis int `json:"durationMillis"` + PauseDurationMillis int `json:"pauseDurationMillis"` + BuildName string `gorm:"primaryKey;type:varchar(255)"` } -func (JenkinsBuild) TableName() string { - return "_tool_jenkins_builds" +func (JenkinsStage) TableName() string { + return "_tool_jenkins_stages" } diff --git a/plugins/jenkins/models/up_down_job.go b/plugins/jenkins/models/up_down_job.go new file mode 100644 index 00000000..fcca32f2 --- /dev/null +++ b/plugins/jenkins/models/up_down_job.go @@ -0,0 +1,14 @@ +package models + +import "github.com/apache/incubator-devlake/models/common" + +type JenkinsUpDownJob struct { + ConnetionId uint64 `gorm:"primaryKey"` + UpstreamJob string `gorm:"primaryKey;type:varchar(255)"` + DownstreamJob string `gorm:"primaryKey;type:varchar(255)"` + common.NoPKModel +} + +func (JenkinsUpDownJob) TableName() string { + return "_tool_jenkins_up_down_jobs" +} diff --git a/plugins/jenkins/tasks/build_collector.go b/plugins/jenkins/tasks/build_collector.go index dc601421..86877c24 100644 --- a/plugins/jenkins/tasks/build_collector.go +++ b/plugins/jenkins/tasks/build_collector.go @@ -82,7 +82,7 @@ func CollectApiBuilds(taskCtx core.SubTaskContext) error { Query: func(reqData *helper.RequestData) (url.Values, error) { query := url.Values{} treeValue := fmt.Sprintf( - "allBuilds[number,timestamp,duration,estimatedDuration,displayName,result,actions[lastBuiltRevision[SHA1],mercurialRevisionNumber],changeSet[kind,revisions[revision]]]{%d,%d}", + "allBuilds[number,timestamp,duration,estimatedDuration,fullDisplayName,result,actions[lastBuiltRevision[SHA1],remoteUrls,mercurialRevisionNumber,triggeredBuilds[fullDisplayName,number,url,result,status,duration]],changeSet[kind,revisions[revision]]]{%d,%d}", reqData.Pager.Skip, reqData.Pager.Skip+reqData.Pager.Size) query.Set("tree", treeValue) return query, nil diff --git a/plugins/jenkins/tasks/build_extractor.go b/plugins/jenkins/tasks/build_extractor.go index bfc8e2ed..66a818d4 100644 --- a/plugins/jenkins/tasks/build_extractor.go +++ b/plugins/jenkins/tasks/build_extractor.go @@ -20,6 +20,7 @@ package tasks import ( "encoding/json" "strconv" + "strings" "time" "github.com/apache/incubator-devlake/plugins/core" @@ -66,10 +67,12 @@ func ExtractApiBuilds(taskCtx core.SubTaskContext) error { return nil, err } - results := make([]interface{}, 0, 1) - + results := make([]interface{}, 0) + strList := strings.Split(body.Class, ".") + class := strList[len(strList)-1] build := &models.JenkinsBuild{ ConnectionId: data.Options.ConnectionId, + Type: class, JobName: input.Name, Duration: body.Duration, DisplayName: body.DisplayName, @@ -82,11 +85,37 @@ func ExtractApiBuilds(taskCtx core.SubTaskContext) error { vcs := body.ChangeSet.Kind if vcs == "git" || vcs == "hg" { for _, a := range body.Actions { + sha := "" if a.LastBuiltRevision.SHA1 != "" { - build.CommitSha = a.LastBuiltRevision.SHA1 + sha = a.LastBuiltRevision.SHA1 } if a.MercurialRevisionNumber != "" { - build.CommitSha = a.MercurialRevisionNumber + sha = a.MercurialRevisionNumber + } + build.CommitSha = sha + for _, url := range a.RemoteUrls { + if url != "" { + buildCommitRemoteUrl := models.JenkinsBuildCommitRepoUrl{ + ConnectionId: data.Options.ConnectionId, + BuildName: build.DisplayName, + CommitSha: sha, + RemoteUrl: url, + } + results = append(results, &buildCommitRemoteUrl) + } + } + if a.TriggeredBuilds != nil && len(a.TriggeredBuilds) > 0 { + for _, b := range a.TriggeredBuilds { + if b.DisplayName == "" { + continue + } + buildTrigger := models.JenkinsBuildTriggeredBuilds{ + ConnectionId: data.Options.ConnectionId, + BuildName: build.DisplayName, + TriggeredBuildName: b.DisplayName, + } + results = append(results, &buildTrigger) + } } } } else if vcs == "svn" { diff --git a/plugins/jenkins/tasks/job_collector.go b/plugins/jenkins/tasks/job_collector.go index 59abde31..dd5b7c2e 100644 --- a/plugins/jenkins/tasks/job_collector.go +++ b/plugins/jenkins/tasks/job_collector.go @@ -69,7 +69,7 @@ func CollectApiJobs(taskCtx core.SubTaskContext) error { Query: func(reqData *helper.RequestData) (url.Values, error) { query := url.Values{} treeValue := fmt.Sprintf( - "jobs[name,class,url,color,base,jobs]{%d,%d}", + "jobs[name,class,url,color,base,jobs,upstreamProjects[name]]{%d,%d}", reqData.Pager.Skip, reqData.Pager.Skip+reqData.Pager.Size) query.Set("tree", treeValue) return query, nil diff --git a/plugins/jenkins/tasks/job_extractor.go b/plugins/jenkins/tasks/job_extractor.go index 1955c8fb..f78742be 100644 --- a/plugins/jenkins/tasks/job_extractor.go +++ b/plugins/jenkins/tasks/job_extractor.go @@ -19,7 +19,6 @@ package tasks import ( "encoding/json" - "github.com/apache/incubator-devlake/plugins/core" "github.com/apache/incubator-devlake/plugins/helper" "github.com/apache/incubator-devlake/plugins/jenkins/models" @@ -65,17 +64,27 @@ func ExtractApiJobs(taskCtx core.SubTaskContext) error { return nil, err } - results := make([]interface{}, 0, 1) + results := make([]interface{}, 0, 1+len(body.UpstreamProjects)) job := &models.JenkinsJob{ JenkinsJobProps: models.JenkinsJobProps{ - ConnectionId: data.Options.ConnectionId, - Name: body.Name, - Path: input.Path, - Class: body.Class, - Color: body.Color, + ConnectionId: data.Options.ConnectionId, + Name: body.Name, + Path: input.Path, + Class: body.Class, + Color: body.Color, + HasUpstreamProjects: len(body.UpstreamProjects) > 0, }, } + for _, upstreamProject := range body.UpstreamProjects { + upDownJob := models.JenkinsUpDownJob{ + ConnetionId: data.Options.ConnectionId, + UpstreamJob: upstreamProject.Name, + DownstreamJob: job.Name, + } + results = append(results, &upDownJob) + } + results = append(results, job) return results, nil diff --git a/plugins/jenkins/tasks/build_collector.go b/plugins/jenkins/tasks/stage_collector.go similarity index 68% copy from plugins/jenkins/tasks/build_collector.go copy to plugins/jenkins/tasks/stage_collector.go index dc601421..19f6a83f 100644 --- a/plugins/jenkins/tasks/build_collector.go +++ b/plugins/jenkins/tasks/stage_collector.go @@ -19,7 +19,6 @@ package tasks import ( "encoding/json" - "fmt" "net/http" "net/url" "reflect" @@ -29,28 +28,30 @@ import ( "github.com/apache/incubator-devlake/plugins/helper" ) -const RAW_BUILD_TABLE = "jenkins_api_builds" +const RAW_STAGE_TABLE = "jenkins_api_stages" -var CollectApiBuildsMeta = core.SubTaskMeta{ - Name: "collectApiBuilds", - EntryPoint: CollectApiBuilds, +var CollectApiStagesMeta = core.SubTaskMeta{ + Name: "collectApiStages", + EntryPoint: CollectApiStages, EnabledByDefault: true, - Description: "Collect builds data from jenkins api", + Description: "Collect stages data from jenkins api", DomainTypes: []string{core.DOMAIN_TYPE_CICD}, } -type SimpleJob struct { - Name string - Path string +type SimpleBuild struct { + JobName string + Number string + DisplayName string } -func CollectApiBuilds(taskCtx core.SubTaskContext) error { +func CollectApiStages(taskCtx core.SubTaskContext) error { db := taskCtx.GetDal() data := taskCtx.GetData().(*JenkinsTaskData) clauses := []dal.Clause{ - dal.Select("tjj.name,tjj.path"), - dal.From("_tool_jenkins_jobs tjj"), - dal.Where(`tjj.connection_id = ?`, data.Options.ConnectionId), + dal.Select("tjb.job_name,tjb.number, tjb.display_name"), + dal.From("_tool_jenkins_builds tjb"), + dal.Where(`tjb.connection_id = ? and tjb.type = ?`, + data.Options.ConnectionId, "WorkflowRun"), } cursor, err := db.Cursor(clauses...) @@ -59,7 +60,7 @@ func CollectApiBuilds(taskCtx core.SubTaskContext) error { } defer cursor.Close() - iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleJob{})) + iterator, err := helper.NewDalCursorIterator(db, cursor, reflect.TypeOf(SimpleBuild{})) if err != nil { return err } @@ -70,32 +71,28 @@ func CollectApiBuilds(taskCtx core.SubTaskContext) error { ConnectionId: data.Options.ConnectionId, }, Ctx: taskCtx, - Table: RAW_BUILD_TABLE, + Table: RAW_STAGE_TABLE, }, ApiClient: data.ApiClient, PageSize: 100, Input: iterator, - UrlTemplate: "{{ .Input.Path }}job/{{ .Input.Name }}/api/json", + UrlTemplate: "job/{{ .Input.JobName }}/{{ .Input.Number }}/wfapi/describe", /* (Optional) Return query string for request, or you can plug them into UrlTemplate directly */ Query: func(reqData *helper.RequestData) (url.Values, error) { query := url.Values{} - treeValue := fmt.Sprintf( - "allBuilds[number,timestamp,duration,estimatedDuration,displayName,result,actions[lastBuiltRevision[SHA1],mercurialRevisionNumber],changeSet[kind,revisions[revision]]]{%d,%d}", - reqData.Pager.Skip, reqData.Pager.Skip+reqData.Pager.Size) - query.Set("tree", treeValue) return query, nil }, ResponseParser: func(res *http.Response) ([]json.RawMessage, error) { var data struct { - Builds []json.RawMessage `json:"allBuilds"` + Stages []json.RawMessage `json:"stages"` } err := helper.UnmarshalResponse(res, &data) if err != nil { return nil, err } - return data.Builds, nil + return data.Stages, nil }, }) diff --git a/plugins/jenkins/tasks/job_extractor.go b/plugins/jenkins/tasks/stage_extractor.go similarity index 69% copy from plugins/jenkins/tasks/job_extractor.go copy to plugins/jenkins/tasks/stage_extractor.go index 1955c8fb..cbe0634f 100644 --- a/plugins/jenkins/tasks/job_extractor.go +++ b/plugins/jenkins/tasks/stage_extractor.go @@ -19,7 +19,6 @@ package tasks import ( "encoding/json" - "github.com/apache/incubator-devlake/plugins/core" "github.com/apache/incubator-devlake/plugins/helper" "github.com/apache/incubator-devlake/plugins/jenkins/models" @@ -27,15 +26,15 @@ import ( // this struct should be moved to `gitub_api_common.go` -var ExtractApiJobsMeta = core.SubTaskMeta{ - Name: "extractApiJobs", - EntryPoint: ExtractApiJobs, +var ExtractApiStagesMeta = core.SubTaskMeta{ + Name: "extractApiStages", + EntryPoint: ExtractApiStages, EnabledByDefault: true, - Description: "Extract raw jobs data into tool layer table jenkins_jobs", + Description: "Extract raw stages data into tool layer table jenkins_stages", DomainTypes: []string{core.DOMAIN_TYPE_CICD}, } -func ExtractApiJobs(taskCtx core.SubTaskContext) error { +func ExtractApiStages(taskCtx core.SubTaskContext) error { data := taskCtx.GetData().(*JenkinsTaskData) extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ @@ -50,34 +49,35 @@ func ExtractApiJobs(taskCtx core.SubTaskContext) error { /* Table store raw data */ - Table: RAW_JOB_TABLE, + Table: RAW_STAGE_TABLE, }, Extract: func(row *helper.RawData) ([]interface{}, error) { - body := &models.Job{} + body := &models.Stage{} err := json.Unmarshal(row.Data, body) if err != nil { return nil, err } - - input := &models.FolderInput{} + input := &SimpleBuild{} err = json.Unmarshal(row.Input, input) if err != nil { return nil, err } - results := make([]interface{}, 0, 1) + results := make([]interface{}, 0) - job := &models.JenkinsJob{ - JenkinsJobProps: models.JenkinsJobProps{ - ConnectionId: data.Options.ConnectionId, - Name: body.Name, - Path: input.Path, - Class: body.Class, - Color: body.Color, - }, + stage := &models.JenkinsStage{ + ConnectionId: data.Options.ConnectionId, + ID: body.ID, + Name: body.Name, + ExecNode: body.ExecNode, + Status: body.Status, + StartTimeMillis: body.StartTimeMillis, + DurationMillis: body.DurationMillis, + PauseDurationMillis: body.PauseDurationMillis, + BuildName: input.DisplayName, } - results = append(results, job) + results = append(results, stage) return results, nil }, })
