This is an automated email from the ASF dual-hosted git repository.
likyh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new b424c9dea feat: add enrich options for jenkins prepare (#3959)
b424c9dea is described below
commit b424c9dea320fc5b6a9f52305edfe53fd82ba316
Author: mappjzc <[email protected]>
AuthorDate: Fri Dec 16 20:17:52 2022 +0800
feat: add enrich options for jenkins prepare (#3959)
add enrich options for jenkins prepare
Nddtfjiang <[email protected]>
---
plugins/jenkins/api/jobs.go | 66 +++++++++++++++------
plugins/jenkins/api/jobs_test.go | 78 ++++++++++++++++++++++++
plugins/jenkins/impl/impl.go | 124 +++++++++++++++++++++++++++++----------
plugins/jenkins/jenkins.go | 2 +-
4 files changed, 218 insertions(+), 52 deletions(-)
diff --git a/plugins/jenkins/api/jobs.go b/plugins/jenkins/api/jobs.go
index c4e0f1b1b..fd33d5268 100644
--- a/plugins/jenkins/api/jobs.go
+++ b/plugins/jenkins/api/jobs.go
@@ -29,8 +29,7 @@ import (
"github.com/apache/incubator-devlake/plugins/helper"
)
-// request all jobs
-func GetAllJobs(apiClient helper.ApiClientGetter, path string, beforename
string, pageSize int, callback func(job *models.Job, isPath bool) errors.Error)
errors.Error {
+func GetJobs(apiClient helper.ApiClientGetter, path string, pageSize int,
callback func(job *models.Job) errors.Error) errors.Error {
for i := 0; ; i += pageSize {
var data struct {
Jobs []json.RawMessage `json:"jobs"`
@@ -52,7 +51,7 @@ func GetAllJobs(apiClient helper.ApiClientGetter, path
string, beforename string
// it is found that if the second page is empty, a 500
error will be returned.
// So we don't think it's an error to return 500 in
this case
if i > 0 && res.StatusCode ==
http.StatusInternalServerError {
- break
+ return nil
}
return err
}
@@ -64,29 +63,58 @@ func GetAllJobs(apiClient helper.ApiClientGetter, path
string, beforename string
return errors.Convert(err1)
}
- job.Path = path
- job.FullName = beforename + job.Name
-
- if job.Jobs != nil {
- err = callback(job, true)
- if err != nil {
- return err
- }
- err = GetAllJobs(apiClient,
path+"job/"+job.Name+"/", beforename+job.Name+"/", pageSize, callback)
- } else {
- err = callback(job, false)
- }
-
+ err = callback(job)
if err != nil {
return err
}
}
- // break with empty data
if len(data.Jobs) < pageSize {
- break
+ return nil
}
}
+}
+
+func GetJob(apiClient helper.ApiClientGetter, path string, name string,
fullName string, pageSize int, callback func(job *models.Job, isPath bool)
errors.Error) errors.Error {
+ var err errors.Error
+
+ return GetJobs(apiClient, path, pageSize, func(job *models.Job)
errors.Error {
+ if job.Name != name {
+ return nil
+ }
+ job.Path = path
+ job.FullName = fullName
+
+ if job.Jobs != nil {
+ err = callback(job, true)
+ if err != nil {
+ return err
+ }
+ } else {
+ err = callback(job, false)
+ }
+
+ return err
+ })
+}
+
+// request all jobs
+func GetAllJobs(apiClient helper.ApiClientGetter, path string, beforename
string, pageSize int, callback func(job *models.Job, isPath bool) errors.Error)
errors.Error {
+ var err errors.Error
+ return GetJobs(apiClient, path, pageSize, func(job *models.Job)
errors.Error {
+ job.Path = path
+ job.FullName = beforename + job.Name
+
+ if job.Jobs != nil {
+ err = callback(job, true)
+ if err != nil {
+ return err
+ }
+ err = GetAllJobs(apiClient, path+"job/"+job.Name+"/",
beforename+job.Name+"/", pageSize, callback)
+ } else {
+ err = callback(job, false)
+ }
- return nil
+ return err
+ })
}
diff --git a/plugins/jenkins/api/jobs_test.go b/plugins/jenkins/api/jobs_test.go
index c1119eb8a..1fe1e3d27 100644
--- a/plugins/jenkins/api/jobs_test.go
+++ b/plugins/jenkins/api/jobs_test.go
@@ -32,6 +32,84 @@ import (
"github.com/stretchr/testify/mock"
)
+func TestGetJob(t *testing.T) {
+ const testPageSize int = 100
+ var remoteData []*models.Job = []*models.Job{
+ {
+ Name: "free",
+ Color: "blue",
+ Class: "hudson.model.FreeStyleProject",
+ Base: "",
+ URL:
"https://test.nddtf.com/job/dir-test/job/dir-test-2/job/free/";,
+ Description: "",
+ },
+ {
+ Name: "free1",
+ Color: "blue",
+ Class: "hudson.model.FreeStyleProject",
+ Base: "",
+ URL:
"https://test.nddtf.com/job/dir-test/job/dir-test-2/job/free1/";,
+ Description: "",
+ },
+ }
+
+ var expectJobs []*models.Job = []*models.Job{
+ {
+ FullName: "dir-test/dir-test-2/free1",
+ Path: "job/dir-test/job/dir-test-2/",
+ Name: "free1",
+ Color: "blue",
+ Class: "hudson.model.FreeStyleProject",
+ Base: "",
+ URL:
"https://test.nddtf.com/job/dir-test/job/dir-test-2/job/free1/";,
+ Description: "",
+ },
+ }
+
+ mockApiClient := mocks.NewApiClientGetter(t)
+
+ var data struct {
+ Jobs []json.RawMessage `json:"jobs"`
+ }
+
+ // jobs
+ data.Jobs = []json.RawMessage{}
+ js, err1 := json.Marshal(remoteData[0])
+ assert.Nil(t, err1)
+ data.Jobs = append(data.Jobs, js)
+
+ js, err1 = json.Marshal(remoteData[1])
+ assert.Nil(t, err1)
+ data.Jobs = append(data.Jobs, js)
+
+ js, err1 = json.Marshal(data)
+ assert.Nil(t, err1)
+
+ res := &http.Response{}
+ res.Body = io.NopCloser(bytes.NewBuffer(js))
+ res.StatusCode = http.StatusOK
+
+ mockApiClient.On("Get", mock.Anything, mock.Anything,
mock.Anything).Return(res, nil).Once()
+
+ basicRes = unithelper.DummyBasicRes(func(mockDal *mocks.Dal) {})
+
+ var jobs []*models.Job
+
+ var paths []*models.Job
+
+ err := GetJob(mockApiClient, "job/dir-test/job/dir-test-2/", "free1",
"dir-test/dir-test-2/free1", testPageSize, func(job *models.Job, isPath bool)
errors.Error {
+ if isPath {
+ paths = append(paths, job)
+ } else {
+ jobs = append(jobs, job)
+ }
+ return nil
+ })
+
+ assert.Equal(t, err, nil)
+ assert.Equal(t, expectJobs, jobs)
+}
+
func TestGetAllJobs(t *testing.T) {
const testPageSize int = 100
diff --git a/plugins/jenkins/impl/impl.go b/plugins/jenkins/impl/impl.go
index 278906673..973b5e89d 100644
--- a/plugins/jenkins/impl/impl.go
+++ b/plugins/jenkins/impl/impl.go
@@ -98,39 +98,7 @@ func (plugin Jenkins) PrepareTaskData(taskCtx
core.TaskContext, options map[stri
if err != nil {
return nil, err
}
- // If this is from BpV200, we should set JobFullName to scopeId
- if op.JobFullName == "" {
- op.JobFullName = op.ScopeId
- }
- // Validate op and convert JobFullName to JobPath and JobName
- op, err = tasks.ValidateTaskOptions(op)
- if err != nil {
- return nil, err
- }
- // get jenkinsJob from db
- jenkinsJob := &models.JenkinsJob{}
- err = taskCtx.GetDal().First(jenkinsJob,
- dal.Where(`connection_id = ? and full_name = ?`,
- op.ConnectionId, op.ScopeId))
- if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find
jenkinsJob%s", op.ScopeId))
- }
- if op.TransformationRuleId == 0 {
- op.TransformationRuleId = jenkinsJob.TransformationRuleId
- }
- if !strings.HasSuffix(op.JobPath, "/") {
- op.JobPath = fmt.Sprintf("%s/", op.JobPath)
- }
- // We only set op.JenkinsTransformationRule when it's nil and we have
op.TransformationRuleId != 0
- if op.JenkinsTransformationRule == nil && op.TransformationRuleId != 0 {
- var transformationRule models.JenkinsTransformationRule
- err = taskCtx.GetDal().First(&transformationRule, dal.Where("id
= ?", op.TransformationRuleId))
- if err != nil {
- return nil, errors.BadInput.Wrap(err, "fail to get
transformationRule")
- }
- op.JenkinsTransformationRule = &transformationRule
- }
logger := taskCtx.GetLogger()
logger.Debug("%v", options)
connection := &models.JenkinsConnection{}
@@ -151,6 +119,11 @@ func (plugin Jenkins) PrepareTaskData(taskCtx
core.TaskContext, options map[stri
return nil, err
}
+ err = EnrichOptions(taskCtx, op, apiClient)
+ if err != nil {
+ return nil, err
+ }
+
var createdDateAfter time.Time
if op.CreatedDateAfter != "" {
createdDateAfter, err =
errors.Convert01(time.Parse(time.RFC3339, op.CreatedDateAfter))
@@ -230,3 +203,90 @@ func (plugin Jenkins) Close(taskCtx core.TaskContext)
errors.Error {
data.ApiClient.Release()
return nil
}
+
+func EnrichOptions(taskCtx core.TaskContext,
+ op *tasks.JenkinsOptions,
+ apiClient *helper.ApiAsyncClient) errors.Error {
+ jenkinsJob := &models.JenkinsJob{}
+ // If this is from BpV200, we should set JobFullName to scopeId
+ if op.JobFullName == "" {
+ op.JobFullName = op.ScopeId
+ }
+ // validate the op and set name=owner/repo if this is from advanced
mode or bpV100
+ op, err := tasks.ValidateTaskOptions(op)
+ if err != nil {
+ return err
+ }
+ log := taskCtx.GetLogger()
+
+ // We only set op.JenkinsTransformationRule when it's nil and we have
op.TransformationRuleId != 0
+ if op.JenkinsTransformationRule == nil && op.TransformationRuleId != 0 {
+ var transformationRule models.JenkinsTransformationRule
+ err = taskCtx.GetDal().First(&transformationRule, dal.Where("id
= ?", op.TransformationRuleId))
+ if err != nil {
+ return errors.BadInput.Wrap(err, "fail to get
transformationRule")
+ }
+ op.JenkinsTransformationRule = &transformationRule
+ }
+
+ if op.JenkinsTransformationRule == nil && op.TransformationRuleId == 0 {
+ op.JenkinsTransformationRule =
new(models.JenkinsTransformationRule)
+ }
+
+ // for advanced mode or others which we only have name, for bp v200, we
have githubId
+ err = taskCtx.GetDal().First(jenkinsJob,
+ dal.Where(`connection_id = ? and full_name = ?`,
+ op.ConnectionId, op.JobFullName))
+ if err == nil {
+ op.Name = jenkinsJob.Name
+ op.JobPath = jenkinsJob.Path
+ if op.TransformationRuleId == 0 {
+ op.TransformationRuleId =
jenkinsJob.TransformationRuleId
+ }
+ } else {
+ if taskCtx.GetDal().IsErrorNotFound(err) && op.JobFullName !=
"" {
+
+ pathSplit := strings.Split(op.JobFullName, "/")
+ lastOne := len(pathSplit)
+
+ path := "job/" + strings.Join(pathSplit[0:lastOne-1],
"job/")
+ name := pathSplit[lastOne-1]
+
+ err = api.GetJob(apiClient, path, name, op.JobFullName,
100, func(job *models.Job, isPath bool) errors.Error {
+ log.Debug(fmt.Sprintf("Current job: %s",
job.FullName))
+ op.Name = job.Name
+ op.JobPath = job.Path
+ jenkinsJob := ConvertJobToJenkinsJob(job, op)
+ err =
taskCtx.GetDal().CreateIfNotExist(jenkinsJob)
+ return err
+ })
+ if err != nil {
+ return err
+ }
+ } else {
+ return errors.Default.Wrap(err, fmt.Sprintf("fail to
find repo %s", op.Name))
+ }
+ }
+
+ if !strings.HasSuffix(op.JobPath, "/") {
+ op.JobPath = fmt.Sprintf("%s/", op.JobPath)
+ }
+
+ return nil
+}
+
+func ConvertJobToJenkinsJob(job *models.Job, op *tasks.JenkinsOptions)
*models.JenkinsJob {
+ return &models.JenkinsJob{
+ ConnectionId: op.ConnectionId,
+ FullName: job.FullName,
+ TransformationRuleId: op.TransformationRuleId,
+ Name: job.Name,
+ Path: job.Path,
+ Class: job.Class,
+ Color: job.Color,
+ Base: job.Base,
+ Url: job.URL,
+ Description: job.Description,
+ PrimaryView: job.URL + job.Path + job.Class,
+ }
+}
diff --git a/plugins/jenkins/jenkins.go b/plugins/jenkins/jenkins.go
index cce331af0..4af374e0c 100644
--- a/plugins/jenkins/jenkins.go
+++ b/plugins/jenkins/jenkins.go
@@ -35,7 +35,7 @@ func main() {
jenkinsCmd.Run = func(cmd *cobra.Command, args []string) {
runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
"connectionId": *connectionId,
- "jobFullName": jobFullName,
+ "jobFullName": *jobFullName,
"deployTagPattern": *deployTagPattern,
"createdDateAfter": *CreatedDateAfter,
})
