This is an automated email from the ASF dual-hosted git repository. mintsweet pushed a commit to branch feat-dora-config in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit d69d2262979397819ac0f6b32347d8bee2d7a4bd Author: abeizn <[email protected]> AuthorDate: Wed Sep 11 16:50:40 2024 +0800 feat: jenkins dora config (#8033) --- backend/plugins/github/api/connection_api.go | 4 +- backend/plugins/jenkins/api/connection_api.go | 94 ++++++++++++++++++++++ backend/plugins/jenkins/impl/impl.go | 3 + backend/plugins/jenkins/models/build.go | 20 ++--- ...{register.go => 20240911_add_url_for_builds.go} | 40 +++++---- .../jenkins/models/migrationscripts/register.go | 1 + backend/plugins/jenkins/models/response.go | 1 + backend/plugins/jenkins/tasks/build_collector.go | 2 +- backend/plugins/jenkins/tasks/build_extractor.go | 1 + 9 files changed, 137 insertions(+), 29 deletions(-) diff --git a/backend/plugins/github/api/connection_api.go b/backend/plugins/github/api/connection_api.go index 02c0265de..de0991ef6 100644 --- a/backend/plugins/github/api/connection_api.go +++ b/backend/plugins/github/api/connection_api.go @@ -410,7 +410,7 @@ func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResource // @Tags plugins/github // @Param id path int true "id" // @Param connectionId path int true "connectionId" -// @Success 200 {object} models.GithubScopeConfigDeployment +// @Success 200 {array} string "List of Environment Names" // @Failure 400 {object} shared.ApiBody "Bad Request" // @Failure 500 {object} shared.ApiBody "Internal Error" // @Router /plugins/github/connections/{connectionId}/deployments [GET] @@ -437,7 +437,7 @@ func GetConnectionDeployments(input *plugin.ApiResourceInput) (*plugin.ApiResour // @Tags plugins/github // @Param id path int true "id" // @Param connectionId path int true "connectionId" -// @Success 200 {object} models.GithubScopeConfigDeployment +// @Success 200 {object} map[string]interface{} // @Failure 400 {object} shared.ApiBody "Bad Request" // @Failure 500 {object} shared.ApiBody "Internal Error" // @Router /plugins/github/connections/{connectionId}/transform-to-deployments [POST] diff --git a/backend/plugins/jenkins/api/connection_api.go b/backend/plugins/jenkins/api/connection_api.go index b213bb068..4b5dd5ceb 100644 --- a/backend/plugins/jenkins/api/connection_api.go +++ b/backend/plugins/jenkins/api/connection_api.go @@ -19,6 +19,7 @@ package api import ( "context" + "fmt" "net/http" "strings" @@ -176,3 +177,96 @@ func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { return dsHelper.ConnApi.GetDetail(input) } + +// GetConnectionTransformToDeployments return one connection deployments +// @Summary return one connection deployments +// @Description return one connection deployments +// @Tags plugins/jenkins +// @Param id path int true "id" +// @Param connectionId path int true "connectionId" +// @Success 200 {object} map[string]interface{} +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/jenkins/connections/{connectionId}/transform-to-deployments [POST] +func GetConnectionTransformToDeployments(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + db := basicRes.GetDal() + connectionId := input.Params["connectionId"] + deploymentPattern := input.Body["deploymentPattern"] + productionPattern := input.Body["productionPattern"] + page, err := api.ParsePageParam(input.Body, "page", 1) + if err != nil { + return nil, errors.Default.New("invalid page value") + } + pageSize, err := api.ParsePageParam(input.Body, "pageSize", 10) + if err != nil { + return nil, errors.Default.New("invalid pageSize value") + } + + cursor, err := db.RawCursor(` + SELECT DISTINCT number, job_name, full_name, url, start_time + FROM( + SELECT number, job_name, full_name, url, start_time + FROM _tool_jenkins_builds + WHERE connection_id = ? AND full_name REGEXP ? + AND full_name REGEXP ? + UNION + SELECT number, job_name, full_name, url, start_time + FROM _tool_jenkins_stages s + LEFT JOIN _tool_jenkins_builds b ON b.full_name = s.build_name + WHERE s.connection_id = ? AND s.name REGEXP ? + AND s.name REGEXP ? + ) AS t + ORDER BY start_time DESC + `, connectionId, deploymentPattern, productionPattern, connectionId, deploymentPattern, productionPattern) + if err != nil { + return nil, errors.Default.Wrap(err, "error on get") + } + defer cursor.Close() + + type selectFileds struct { + Number int + JobName string + FullName string + URL string + } + type transformedFields struct { + Name string + URL string + } + var allRuns []transformedFields + for cursor.Next() { + sf := &selectFileds{} + err = db.Fetch(cursor, sf) + if err != nil { + return nil, errors.Default.Wrap(err, "error on fetch") + } + // Directly transform and append to allRuns + transformed := transformedFields{ + Name: fmt.Sprintf("#%d - %s", sf.Number, sf.JobName), + URL: sf.URL, + } + allRuns = append(allRuns, transformed) + } + // Calculate total count + totalCount := len(allRuns) + + // Paginate in memory + start := (page - 1) * pageSize + end := start + pageSize + if start > totalCount { + start = totalCount + } + if end > totalCount { + end = totalCount + } + pagedRuns := allRuns[start:end] + + // Return result containing paged runs and total count + result := map[string]interface{}{ + "total": totalCount, + "data": pagedRuns, + } + return &plugin.ApiResourceOutput{ + Body: result, + }, nil +} diff --git a/backend/plugins/jenkins/impl/impl.go b/backend/plugins/jenkins/impl/impl.go index 7411f7543..7be75a8e1 100644 --- a/backend/plugins/jenkins/impl/impl.go +++ b/backend/plugins/jenkins/impl/impl.go @@ -216,6 +216,9 @@ func (p Jenkins) ApiResources() map[string]map[string]plugin.ApiResourceHandler "POST": api.CreateScopeConfig, "GET": api.GetScopeConfigList, }, + "connections/:connectionId/transform-to-deployments": { + "POST": api.GetConnectionTransformToDeployments, + }, "connections/:connectionId/scope-configs/:scopeConfigId": { "PATCH": api.UpdateScopeConfig, "GET": api.GetScopeConfig, diff --git a/backend/plugins/jenkins/models/build.go b/backend/plugins/jenkins/models/build.go index 203d81d00..804c5521c 100644 --- a/backend/plugins/jenkins/models/build.go +++ b/backend/plugins/jenkins/models/build.go @@ -18,22 +18,24 @@ limitations under the License. package models import ( - "github.com/apache/incubator-devlake/core/models/common" "time" + + "github.com/apache/incubator-devlake/core/models/common" ) // JenkinsBuild db entity for jenkins build type JenkinsBuild struct { common.NoPKModel // collected fields - ConnectionId uint64 `gorm:"primaryKey"` - JobName string `gorm:"index;type:varchar(255)"` - JobPath string `gorm:"index;type:varchar(255)"` - Duration float64 // build time - FullName string `gorm:"primaryKey;type:varchar(255)"` // "path/job name#7" - EstimatedDuration float64 // EstimatedDuration - Number int64 `gorm:"index"` - Result string // Result + ConnectionId uint64 `gorm:"primaryKey"` + JobName string `gorm:"index;type:varchar(255)"` + JobPath string `gorm:"index;type:varchar(255)"` + Duration float64 // build time + FullName string `gorm:"primaryKey;type:varchar(255)"` // "path/job name#7" + EstimatedDuration float64 // EstimatedDuration + Number int64 `gorm:"index"` + Result string // Result + Url string Timestamp int64 // start time StartTime time.Time // convered by timestamp Type string `gorm:"index;type:varchar(255)"` diff --git a/backend/plugins/jenkins/models/migrationscripts/register.go b/backend/plugins/jenkins/models/migrationscripts/20240911_add_url_for_builds.go similarity index 59% copy from backend/plugins/jenkins/models/migrationscripts/register.go copy to backend/plugins/jenkins/models/migrationscripts/20240911_add_url_for_builds.go index 1dc1eb84b..d3959b55a 100644 --- a/backend/plugins/jenkins/models/migrationscripts/register.go +++ b/backend/plugins/jenkins/models/migrationscripts/20240911_add_url_for_builds.go @@ -18,23 +18,29 @@ limitations under the License. package migrationscripts import ( - "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" ) -// All return all the migration scripts -func All() []plugin.MigrationScript { - return []plugin.MigrationScript{ - new(addInitTables), - new(modifyAllEntities), - new(modifyJenkinsBuild), - new(addJobFields), - new(addJobPathForBuilds), - new(changeIndexOfJobPath), - new(addTransformationRule20221128), - new(addFullNameForBuilds), - new(addConnectionIdToTransformationRule), - new(renameTr2ScopeConfig), - new(addRawParamTableForScope), - new(addNumberToJenkinsBuildCommit), - } +type addUrlForBuilds struct{} + +type jenkinsBuild20240911 struct { + Url string +} + +func (jenkinsBuild20240911) TableName() string { + return "_tool_jenkins_builds" +} + +func (script *addUrlForBuilds) Up(basicRes context.BasicRes) errors.Error { + db := basicRes.GetDal() + return db.AutoMigrate(&jenkinsBuild20240911{}) +} + +func (*addUrlForBuilds) Version() uint64 { + return 20240911231237 +} + +func (*addUrlForBuilds) Name() string { + return "add url for builds" } diff --git a/backend/plugins/jenkins/models/migrationscripts/register.go b/backend/plugins/jenkins/models/migrationscripts/register.go index 1dc1eb84b..ff2c9ac7a 100644 --- a/backend/plugins/jenkins/models/migrationscripts/register.go +++ b/backend/plugins/jenkins/models/migrationscripts/register.go @@ -36,5 +36,6 @@ func All() []plugin.MigrationScript { new(renameTr2ScopeConfig), new(addRawParamTableForScope), new(addNumberToJenkinsBuildCommit), + new(addUrlForBuilds), } } diff --git a/backend/plugins/jenkins/models/response.go b/backend/plugins/jenkins/models/response.go index 643bec691..962a875af 100644 --- a/backend/plugins/jenkins/models/response.go +++ b/backend/plugins/jenkins/models/response.go @@ -102,6 +102,7 @@ type ApiBuildResponse struct { Class string `json:"_class"` Number int64 `json:"number"` Result string `json:"result"` + Url string `json:"url"` Building bool `json:"building"` Actions []Action `json:"actions"` Duration float64 `json:"duration"` diff --git a/backend/plugins/jenkins/tasks/build_collector.go b/backend/plugins/jenkins/tasks/build_collector.go index 0d792c614..c0b47f95f 100644 --- a/backend/plugins/jenkins/tasks/build_collector.go +++ b/backend/plugins/jenkins/tasks/build_collector.go @@ -90,7 +90,7 @@ func collectSingleJobApiBuilds(taskCtx plugin.SubTaskContext) errors.Error { Query: func(reqData *helper.RequestData, createdAfter *time.Time) (url.Values, errors.Error) { query := url.Values{} treeValue := fmt.Sprintf( - "allBuilds[timestamp,number,duration,building,estimatedDuration,fullDisplayName,result,actions[lastBuiltRevision[SHA1,branch[name]],remoteUrls,mercurialRevisionNumber,causes[*]],changeSet[kind,revisions[revision]]]{%d,%d}", + "allBuilds[timestamp,number,duration,building,estimatedDuration,fullDisplayName,result,url,actions[lastBuiltRevision[SHA1,branch[name]],remoteUrls,mercurialRevisionNumber,causes[*]],changeSet[kind,revisions[revision]]]{%d,%d}", reqData.Pager.Skip, reqData.Pager.Skip+reqData.Pager.Size) query.Set("tree", treeValue) return query, nil diff --git a/backend/plugins/jenkins/tasks/build_extractor.go b/backend/plugins/jenkins/tasks/build_extractor.go index d039034ab..842c46d2c 100644 --- a/backend/plugins/jenkins/tasks/build_extractor.go +++ b/backend/plugins/jenkins/tasks/build_extractor.go @@ -81,6 +81,7 @@ func ExtractApiBuilds(taskCtx plugin.SubTaskContext) errors.Error { EstimatedDuration: body.EstimatedDuration, Number: body.Number, Result: body.Result, + Url: body.Url, Timestamp: body.Timestamp, Class: class, Building: body.Building,
