This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 75cbde6e4 feat: add remote api for jenkins (#5358)
75cbde6e4 is described below
commit 75cbde6e4464f37f77c76ccfe60ad0c346efd293
Author: mappjzc <[email protected]>
AuthorDate: Tue Jun 6 15:26:22 2023 +0800
feat: add remote api for jenkins (#5358)
* feat: add remote api for jenkins
Add RemoteScopes API for jenkins.
Add SearchRemoteScopes API for jenkins.
Nddtfjiang <[email protected]>
* fix: fix remote error
Fix remote error.
Nddtfjiang <[email protected]>
---
backend/plugins/jenkins/api/init.go | 7 ++
backend/plugins/jenkins/api/jobs.go | 84 ++++++++++--------
backend/plugins/jenkins/api/remote.go | 134 +++++++++++++++++++++++++++++
backend/plugins/jenkins/impl/impl.go | 28 +++---
backend/plugins/jenkins/models/job.go | 8 ++
backend/plugins/jenkins/models/response.go | 26 ++++++
6 files changed, 236 insertions(+), 51 deletions(-)
diff --git a/backend/plugins/jenkins/api/init.go
b/backend/plugins/jenkins/api/init.go
index ae2a1e10d..c6914494a 100644
--- a/backend/plugins/jenkins/api/init.go
+++ b/backend/plugins/jenkins/api/init.go
@@ -27,6 +27,8 @@ import (
var vld *validator.Validate
var connectionHelper *api.ConnectionApiHelper
var scopeHelper *api.ScopeApiHelper[models.JenkinsConnection,
models.JenkinsJob, models.JenkinsScopeConfig]
+var remoteHelper *api.RemoteApiHelper[models.JenkinsConnection,
models.JenkinsJob, models.Job, models.Job]
+
var basicRes context.BasicRes
var scHelper *api.ScopeConfigHelper[models.JenkinsScopeConfig]
@@ -50,6 +52,11 @@ func Init(br context.BasicRes) {
params,
nil,
)
+ remoteHelper = api.NewRemoteHelper[models.JenkinsConnection,
models.JenkinsJob, models.Job, models.Job](
+ basicRes,
+ vld,
+ connectionHelper,
+ )
scHelper = api.NewScopeConfigHelper[models.JenkinsScopeConfig](
basicRes,
vld,
diff --git a/backend/plugins/jenkins/api/jobs.go
b/backend/plugins/jenkins/api/jobs.go
index 992c11449..f413a7e32 100644
--- a/backend/plugins/jenkins/api/jobs.go
+++ b/backend/plugins/jenkins/api/jobs.go
@@ -30,55 +30,71 @@ import (
aha
"github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
)
-func GetJobs(
+func GetJobsPage(
apiClient aha.ApiClientAbstract,
path string,
+ page int,
pageSize int,
callback func(job *models.Job) errors.Error,
-) errors.Error {
- for i := 0; ; i += pageSize {
- var data struct {
- Jobs []json.RawMessage `json:"jobs"`
- }
+) (int, errors.Error) {
+ i := page * pageSize
+ var data struct {
+ Jobs []json.RawMessage `json:"jobs"`
+ }
- // set query
- query := url.Values{}
- treeValue :=
fmt.Sprintf("jobs[name,class,url,color,base,jobs,upstreamProjects[name]]{%d,%d}",
i, i+pageSize)
- query.Set("tree", treeValue)
+ // set query
+ query := url.Values{}
+ treeValue :=
fmt.Sprintf("jobs[name,class,url,color,base,jobs,upstreamProjects[name]]{%d,%d}",
i, i+pageSize)
+ query.Set("tree", treeValue)
- res, err := apiClient.Get(path+"/api/json", query, nil)
- if err != nil {
- return err
+ res, err := apiClient.Get(path+"/api/json", query, nil)
+ if err != nil {
+ return 0, err
+ }
+
+ err = helper.UnmarshalResponse(res, &data)
+ if err != nil {
+ // In some directories, after testing
+ // it is found that if the second page is empty, a 500 error
will be returned.
+ // So we don't think it's an error to return 500 in this case
+ if i > 0 && res.StatusCode == http.StatusInternalServerError {
+ return 0, nil
+ }
+ return 0, err
+ }
+
+ for _, rawJobs := range data.Jobs {
+ job := &models.Job{}
+ err1 := json.Unmarshal(rawJobs, job)
+ if err1 != nil {
+ return len(data.Jobs), errors.Convert(err1)
}
- err = helper.UnmarshalResponse(res, &data)
+ err = callback(job)
if err != nil {
- // In some directories, after testing
- // it is found that if the second page is empty, a 500
error will be returned.
- // So we don't think it's an error to return 500 in
this case
- if i > 0 && res.StatusCode ==
http.StatusInternalServerError {
- return nil
- }
- return err
+ return len(data.Jobs), err
}
+ }
- for _, rawJobs := range data.Jobs {
- job := &models.Job{}
- err1 := json.Unmarshal(rawJobs, job)
- if err1 != nil {
- return errors.Convert(err1)
- }
+ return len(data.Jobs), nil
+}
- err = callback(job)
- if err != nil {
- return err
- }
+func GetJobs(
+ apiClient aha.ApiClientAbstract,
+ path string,
+ pageSize int,
+ callback func(job *models.Job) errors.Error,
+) errors.Error {
+ for i := 0; ; i++ {
+ count, err := GetJobsPage(apiClient, path, i, pageSize,
callback)
+ if err != nil {
+ return err
}
-
- if len(data.Jobs) < pageSize {
- return nil
+ if count < pageSize {
+ break
}
}
+ return nil
}
func GetJob(
diff --git a/backend/plugins/jenkins/api/remote.go
b/backend/plugins/jenkins/api/remote.go
new file mode 100644
index 000000000..8df6646a9
--- /dev/null
+++ b/backend/plugins/jenkins/api/remote.go
@@ -0,0 +1,134 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "context"
+ "strings"
+
+ context2 "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/jenkins/models"
+)
+
+// RemoteScopes list all available scope for users
+// @Summary list all available scope for users
+// @Description list all available scope for users
+// @Tags plugins/jenkins
+// @Accept application/json
+// @Param connectionId path int false "connection ID"
+// @Param groupId query string false "group ID"
+// @Param pageToken query string false "page Token"
+// @Success 200 {object} api.RemoteScopesOutput
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/jenkins/connections/{connectionId}/remote-scopes [GET]
+func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
+ return remoteHelper.GetScopesFromRemote(input,
+ func(basicRes context2.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.JenkinsConnection) ([]models.Job,
errors.Error) {
+ apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "failed
to get create apiClient")
+ }
+
+ var resBody []models.Job
+ _, err = GetJobsPage(apiClient, gid, queryData.Page-1,
queryData.PerPage, func(job *models.Job) errors.Error {
+ // this is a group
+ if job.Jobs != nil {
+ job.Path = gid
+ resBody = append(resBody, *job)
+ }
+ return nil
+ })
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "failed
to GetJobsPage")
+ }
+
+ return resBody, err
+ },
+ func(basicRes context2.BasicRes, gid string, queryData
*api.RemoteQueryData, connection models.JenkinsConnection) ([]models.Job,
errors.Error) {
+ apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "failed
to get create apiClient")
+ }
+
+ var resBody []models.Job
+ _, err = GetJobsPage(apiClient, gid, queryData.Page-1,
queryData.PerPage, func(job *models.Job) errors.Error {
+ // this is only a job
+ if job.Jobs == nil {
+ job.Path = gid
+ resBody = append(resBody, *job)
+ }
+ return nil
+ })
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "failed
to GetJobsPage")
+ }
+
+ return resBody, err
+ })
+}
+
+// SearchRemoteScopes use the Search API and only return project
+// @Summary use the Search API and only return project
+// @Description use the Search API and only return project
+// @Tags plugins/jenkins
+// @Accept application/json
+// @Param connectionId path int false "connection ID"
+// @Param search query string false "search"
+// @Param page query int false "page number"
+// @Param pageSize query int false "page size per page"
+// @Success 200 {object} api.SearchRemoteScopesOutput
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/jenkins/connections/{connectionId}/search-remote-scopes
[GET]
+func SearchRemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ return remoteHelper.SearchRemoteScopes(input,
+ func(basicRes context2.BasicRes, queryData
*api.RemoteQueryData, connection models.JenkinsConnection) ([]models.Job,
errors.Error) {
+ apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "failed
to get create apiClient")
+ }
+
+ var resBody []models.Job
+ breakError := errors.Default.New("we need break from
get all jobs for page full")
+ count := 0
+ pageoOffset := (queryData.Page - 1) * queryData.PerPage
+ err = GetAllJobs(apiClient, "", "", queryData.PerPage,
func(job *models.Job, isPath bool) errors.Error {
+ if job.Jobs == nil {
+ if strings.Contains(job.FullName,
queryData.Search[0]) {
+ if count >= pageoOffset {
+ resBody =
append(resBody, *job)
+ }
+ count++
+ }
+ if len(resBody) > queryData.PerPage {
+ return breakError
+ }
+ }
+ return nil
+ })
+ if err != nil && err != breakError {
+ return nil, errors.BadInput.Wrap(err, "failed
to get create apiClient")
+ }
+
+ return resBody, nil
+ })
+}
diff --git a/backend/plugins/jenkins/impl/impl.go
b/backend/plugins/jenkins/impl/impl.go
index 62dbe55ea..e3b03cf4e 100644
--- a/backend/plugins/jenkins/impl/impl.go
+++ b/backend/plugins/jenkins/impl/impl.go
@@ -180,6 +180,12 @@ func (p Jenkins) ApiResources()
map[string]map[string]plugin.ApiResourceHandler
"DELETE": api.DeleteConnection,
"GET": api.GetConnection,
},
+ "connections/:connectionId/remote-scopes": {
+ "GET": api.RemoteScopes,
+ },
+ "connections/:connectionId/search-remote-scopes": {
+ "GET": api.SearchRemoteScopes,
+ },
"connections/:connectionId/scopes/*scopeId": {
"GET": api.GetScope,
"PATCH": api.UpdateScope,
@@ -240,7 +246,11 @@ func EnrichOptions(taskCtx plugin.TaskContext,
err = api.GetJob(apiClient, op.JobPath, op.JobName, op.JobFullName,
100, func(job *models.Job, isPath bool) errors.Error {
log.Debug(fmt.Sprintf("Current job: %s", job.FullName))
op.JobPath = job.Path
- jenkinsJob := ConvertJobToJenkinsJob(job, op)
+ jenkinsJob := job.ConvertApiScope().(models.JenkinsJob)
+
+ jenkinsJob.ConnectionId = op.ConnectionId
+ jenkinsJob.ScopeConfigId = op.ScopeConfigId
+
err = taskCtx.GetDal().CreateIfNotExist(jenkinsJob)
return err
})
@@ -267,19 +277,3 @@ func EnrichOptions(taskCtx plugin.TaskContext,
return nil
}
-
-func ConvertJobToJenkinsJob(job *models.Job, op *tasks.JenkinsOptions)
*models.JenkinsJob {
- return &models.JenkinsJob{
- ConnectionId: op.ConnectionId,
- FullName: job.FullName,
- ScopeConfigId: op.ScopeConfigId,
- Name: job.Name,
- Path: job.Path,
- Class: job.Class,
- Color: job.Color,
- Base: job.Base,
- Url: job.URL,
- Description: job.Description,
- PrimaryView: job.URL + job.Path + job.Class,
- }
-}
diff --git a/backend/plugins/jenkins/models/job.go
b/backend/plugins/jenkins/models/job.go
index 197211577..313ec0caa 100644
--- a/backend/plugins/jenkins/models/job.go
+++ b/backend/plugins/jenkins/models/job.go
@@ -40,3 +40,11 @@ type JenkinsJob struct {
func (JenkinsJob) TableName() string {
return "_tool_jenkins_jobs"
}
+
+func (j JenkinsJob) ScopeId() string {
+ return j.FullName
+}
+
+func (j JenkinsJob) ScopeName() string {
+ return j.Name
+}
diff --git a/backend/plugins/jenkins/models/response.go
b/backend/plugins/jenkins/models/response.go
index 31db53db0..18390249a 100644
--- a/backend/plugins/jenkins/models/response.go
+++ b/backend/plugins/jenkins/models/response.go
@@ -17,6 +17,10 @@ limitations under the License.
package models
+import (
+ "github.com/apache/incubator-devlake/core/plugin"
+)
+
type Job struct {
FullName string `gorm:"primaryKey;type:varchar(255)"`
Path string `gorm:"primaryKey;type:varchar(511)"`
@@ -31,6 +35,28 @@ type Job struct {
*PrimaryView `json:"primaryView"`
}
+func (j Job) GroupId() string {
+ return j.Path + "job/" + j.Name + "/"
+}
+
+func (j Job) GroupName() string {
+ return j.Name
+}
+
+func (j Job) ConvertApiScope() plugin.ToolLayerScope {
+ return &JenkinsJob{
+ FullName: j.FullName,
+ Name: j.Name,
+ Path: j.Path,
+ Class: j.Class,
+ Color: j.Color,
+ Base: j.Base,
+ Url: j.URL,
+ Description: j.Description,
+ PrimaryView: j.URL + j.Path + j.Class,
+ }
+}
+
type Project struct {
Class string `json:"_class"`
Name string `json:"name"`
