This is an automated email from the ASF dual-hosted git repository.
likyh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 0c78a4b1c Jenkins add GetAllJobs (#3931)
0c78a4b1c is described below
commit 0c78a4b1cb6307a7daaa9a4129a339fb7abd0eb5
Author: mappjzc <[email protected]>
AuthorDate: Thu Dec 15 19:07:55 2022 +0800
Jenkins add GetAllJobs (#3931)
* feat: add get all jobs
Add get all jobs
Nddtfjiang <[email protected]>
* feat: add unit test for get all jobs
add unit test for get all jobs
Nddtfjiang <[email protected]>
* feat: add jobs to jenkins v100
add jobs to jenkins v100
Nddtfjiang <[email protected]>
---
plugins/jenkins/api/blueprint_v100.go | 128 +++++++++++------
plugins/jenkins/api/blueprint_v100_test.go | 48 ++++++-
plugins/jenkins/api/jobs.go | 92 ++++++++++++
plugins/jenkins/api/jobs_test.go | 218 +++++++++++++++++++++++++++++
plugins/jenkins/models/response.go | 1 +
5 files changed, 436 insertions(+), 51 deletions(-)
diff --git a/plugins/jenkins/api/blueprint_v100.go
b/plugins/jenkins/api/blueprint_v100.go
index cb3c2e4e1..e5aa9a57d 100644
--- a/plugins/jenkins/api/blueprint_v100.go
+++ b/plugins/jenkins/api/blueprint_v100.go
@@ -18,7 +18,11 @@ limitations under the License.
package api
import (
+ "context"
"encoding/json"
+ "fmt"
+ "time"
+
"github.com/apache/incubator-devlake/plugins/jenkins/models"
"github.com/apache/incubator-devlake/errors"
@@ -34,17 +38,37 @@ func MakePipelinePlanV100(subtaskMetas []core.SubTaskMeta,
connectionId uint64,
if err != nil {
return nil, err
}
- plan, err := makePipelinePlanV100(subtaskMetas, scope, connection)
+
+ apiClient, err := helper.NewApiClient(
+ context.Background(),
+ connection.Endpoint,
+ map[string]string{
+ "Authorization": fmt.Sprintf("Basic %s",
connection.GetEncodedToken()),
+ },
+ 10*time.Second,
+ connection.Proxy,
+ basicRes,
+ )
+ if err != nil {
+ return nil, err
+ }
+
+ plan, err := makePipelinePlanV100(subtaskMetas, scope, connection,
apiClient)
if err != nil {
return nil, err
}
return plan, nil
}
-func makePipelinePlanV100(subtaskMetas []core.SubTaskMeta, scope
[]*core.BlueprintScopeV100, connection *models.JenkinsConnection)
(core.PipelinePlan, errors.Error) {
+func makePipelinePlanV100(subtaskMetas []core.SubTaskMeta, scope
[]*core.BlueprintScopeV100, connection *models.JenkinsConnection, apiClient
helper.ApiClientGetter) (core.PipelinePlan, errors.Error) {
var err errors.Error
- plan := make(core.PipelinePlan, len(scope))
- for i, scopeElem := range scope {
+ plans := make(core.PipelinePlan, 0, len(scope))
+
+ if err != nil {
+ return nil, err
+ }
+
+ for _, scopeElem := range scope {
// handle taskOptions and transformationRules, by dumping them
to taskOptions
transformationRules := make(map[string]interface{})
if len(scopeElem.Transformation) > 0 {
@@ -53,64 +77,78 @@ func makePipelinePlanV100(subtaskMetas []core.SubTaskMeta,
scope []*core.Bluepri
return nil, err
}
}
- taskOptions := make(map[string]interface{})
- err = errors.Convert(json.Unmarshal(scopeElem.Options,
&taskOptions))
- if err != nil {
- return nil, err
- }
- taskOptions["connectionId"] = connection.ID
- taskOptions["transformationRules"] = transformationRules
- op, err := tasks.DecodeTaskOptions(taskOptions)
- if err != nil {
- return nil, err
- }
- _, err = tasks.ValidateTaskOptions(op)
- if err != nil {
- return nil, err
- }
- // subtasks
- subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas,
scopeElem.Entities)
- if err != nil {
- return nil, err
- }
- stage := core.PipelineStage{
- {
- Plugin: "jenkins",
- Subtasks: subtasks,
- Options: taskOptions,
- },
+
+ // check productionPattern and separate it from
transformationRules
+ productionPattern, ok :=
transformationRules["productionPattern"]
+ if ok && productionPattern != nil {
+ delete(transformationRules, "productionPattern")
+ } else {
+ productionPattern = nil
}
- if productionPattern, ok :=
transformationRules["productionPattern"]; ok && productionPattern != nil {
- j := i + 1
- if j == len(plan) {
- plan = append(plan, nil)
+
+ err = GetAllJobs(apiClient, "", "", 100, func(job *models.Job,
isPath bool) errors.Error {
+ // do not mind path
+ if isPath {
+ return nil
+ }
+
+ taskOptions := make(map[string]interface{})
+ err = errors.Convert(json.Unmarshal(scopeElem.Options,
&taskOptions))
+ if err != nil {
+ return err
}
- // add a new task to next stage
- if plan[j] != nil {
- j++
+ taskOptions["connectionId"] = connection.ID
+ taskOptions["transformationRules"] = transformationRules
+ taskOptions["jobFullName"] = job.FullName
+
+ op, err := tasks.DecodeTaskOptions(taskOptions)
+ if err != nil {
+ return err
}
- if j == len(plan) {
- plan = append(plan, nil)
+ _, err = tasks.ValidateTaskOptions(op)
+ if err != nil {
+ return err
}
+
+ // subtasks
+ subtasks, err :=
helper.MakePipelinePlanSubtasks(subtaskMetas, scopeElem.Entities)
if err != nil {
- return nil, err
+ return err
}
+ stage := core.PipelineStage{
+ {
+ Plugin: "jenkins",
+ Subtasks: subtasks,
+ Options: taskOptions,
+ },
+ }
+
+ plans = append(plans, stage)
+
+ return nil
+ })
+ if err != nil {
+ return nil, err
+ }
+
+ // Dora part
+ if productionPattern != nil {
doraOption := make(map[string]interface{})
doraOption["prefix"] = "jenkins"
doraRules := make(map[string]interface{})
doraRules["productionPattern"] = productionPattern
doraOption["transformationRules"] = doraRules
- plan[j] = core.PipelineStage{
+
+ stageDora := core.PipelineStage{
{
Plugin: "dora",
Subtasks: []string{"EnrichTaskEnv"},
Options: doraOption,
},
}
- // remove it from github transformationRules
- delete(transformationRules, "productionPattern")
+
+ plans = append(plans, stageDora)
}
- plan[i] = stage
}
- return plan, nil
+ return plans, nil
}
diff --git a/plugins/jenkins/api/blueprint_v100_test.go
b/plugins/jenkins/api/blueprint_v100_test.go
index 28dffa08e..acb49923e 100644
--- a/plugins/jenkins/api/blueprint_v100_test.go
+++ b/plugins/jenkins/api/blueprint_v100_test.go
@@ -18,13 +18,19 @@ limitations under the License.
package api
import (
+ "bytes"
"encoding/json"
+ "io"
+ "net/http"
+ "testing"
+
+ "github.com/apache/incubator-devlake/mocks"
"github.com/apache/incubator-devlake/models/common"
"github.com/apache/incubator-devlake/plugins/core"
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/jenkins/models"
"github.com/stretchr/testify/assert"
- "testing"
+ "github.com/stretchr/testify/mock"
)
func TestProcessScope(t *testing.T) {
@@ -48,9 +54,7 @@ func TestProcessScope(t *testing.T) {
bs := &core.BlueprintScopeV100{
Entities: []string{"CICD"},
- Options: json.RawMessage(`{
- "jobFullName": "testJob"
- }`),
+ Options: json.RawMessage(`{}`),
Transformation: json.RawMessage(`{
"productionPattern": "(?i)build-and-deploy",
"deploymentPattern": "deploy"
@@ -58,7 +62,39 @@ func TestProcessScope(t *testing.T) {
}
scopes := make([]*core.BlueprintScopeV100, 0)
scopes = append(scopes, bs)
- plan, err := makePipelinePlanV100(nil, scopes, connection)
+
+ mockApiClient := mocks.NewApiClientGetter(t)
+
+ var remoteData []*models.Job = []*models.Job{
+ {
+ Name: "devlake",
+ Color: "blue",
+ Class: "hudson.model.FreeStyleProject",
+ Base: "",
+ URL: "https://test.nddtf.com/job/devlake/";,
+ Description: "",
+ },
+ }
+
+ var data struct {
+ Jobs []json.RawMessage `json:"jobs"`
+ }
+
+ // job to apiClient
+ js, err1 := json.Marshal(remoteData[0])
+ assert.Nil(t, err1)
+ data.Jobs = append(data.Jobs, js)
+
+ js, err1 = json.Marshal(data)
+ assert.Nil(t, err1)
+
+ res := &http.Response{}
+ res.Body = io.NopCloser(bytes.NewBuffer(js))
+ res.StatusCode = http.StatusOK
+
+ mockApiClient.On("Get", mock.Anything, mock.Anything,
mock.Anything).Return(res, nil).Once()
+
+ plan, err := makePipelinePlanV100(nil, scopes, connection,
mockApiClient)
assert.Nil(t, err)
expectPlan := core.PipelinePlan{
@@ -67,7 +103,7 @@ func TestProcessScope(t *testing.T) {
Plugin: "jenkins",
Subtasks: []string{},
Options: map[string]interface{}{
- "jobFullName": "testJob",
+ "jobFullName": "devlake",
"connectionId": uint64(1),
"transformationRules":
map[string]interface{}{
"deploymentPattern": "deploy",
diff --git a/plugins/jenkins/api/jobs.go b/plugins/jenkins/api/jobs.go
new file mode 100644
index 000000000..c4e0f1b1b
--- /dev/null
+++ b/plugins/jenkins/api/jobs.go
@@ -0,0 +1,92 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "net/url"
+
+ "github.com/apache/incubator-devlake/plugins/jenkins/models"
+
+ "github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+// request all jobs
+func GetAllJobs(apiClient helper.ApiClientGetter, path string, beforename
string, pageSize int, callback func(job *models.Job, isPath bool) errors.Error)
errors.Error {
+ for i := 0; ; i += pageSize {
+ var data struct {
+ Jobs []json.RawMessage `json:"jobs"`
+ }
+
+ // set query
+ query := url.Values{}
+ treeValue :=
fmt.Sprintf("jobs[name,class,url,color,base,jobs,upstreamProjects[name]]{%d,%d}",
i, i+pageSize)
+ query.Set("tree", treeValue)
+
+ res, err := apiClient.Get(path+"/api/json", query, nil)
+ if err != nil {
+ return err
+ }
+
+ err = helper.UnmarshalResponse(res, &data)
+ if err != nil {
+ // In some directories, after testing
+ // it is found that if the second page is empty, a 500
error will be returned.
+ // So we don't think it's an error to return 500 in
this case
+ if i > 0 && res.StatusCode ==
http.StatusInternalServerError {
+ break
+ }
+ return err
+ }
+
+ for _, rawJobs := range data.Jobs {
+ job := &models.Job{}
+ err1 := json.Unmarshal(rawJobs, job)
+ if err1 != nil {
+ return errors.Convert(err1)
+ }
+
+ job.Path = path
+ job.FullName = beforename + job.Name
+
+ if job.Jobs != nil {
+ err = callback(job, true)
+ if err != nil {
+ return err
+ }
+ err = GetAllJobs(apiClient,
path+"job/"+job.Name+"/", beforename+job.Name+"/", pageSize, callback)
+ } else {
+ err = callback(job, false)
+ }
+
+ if err != nil {
+ return err
+ }
+ }
+
+ // break with empty data
+ if len(data.Jobs) < pageSize {
+ break
+ }
+ }
+
+ return nil
+}
diff --git a/plugins/jenkins/api/jobs_test.go b/plugins/jenkins/api/jobs_test.go
new file mode 100644
index 000000000..c1119eb8a
--- /dev/null
+++ b/plugins/jenkins/api/jobs_test.go
@@ -0,0 +1,218 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "bytes"
+ "encoding/json"
+ "io"
+ "net/http"
+ "testing"
+
+ "github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/helpers/unithelper"
+ "github.com/apache/incubator-devlake/mocks"
+ "github.com/apache/incubator-devlake/plugins/jenkins/models"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/mock"
+)
+
+func TestGetAllJobs(t *testing.T) {
+ const testPageSize int = 100
+
+ var remoteData []*models.Job = []*models.Job{
+ {
+ Name: "devlake",
+ Color: "blue",
+ Class: "hudson.model.FreeStyleProject",
+ Base: "",
+ URL: "https://test.nddtf.com/job/devlake/";,
+ Description: "",
+ },
+ {
+ Name: "dir-test",
+ Color: "",
+ Class: "hudson.model.FreeStyleProject",
+ Base: "",
+ URL: "https://test.nddtf.com/job/dir-test/";,
+ Description: "",
+ Jobs: &[]models.Job{{}},
+ },
+ {
+ Name: "dir-test-2",
+ Color: "",
+ Class: "hudson.model.FreeStyleProject",
+ Base: "",
+ URL:
"https://test.nddtf.com/job/dir-test/job/dir-test-2/";,
+ Description: "",
+ Jobs: &[]models.Job{{}},
+ },
+ {
+ Name: "free",
+ Color: "blue",
+ Class: "hudson.model.FreeStyleProject",
+ Base: "",
+ URL:
"https://test.nddtf.com/job/dir-test/job/dir-test-2/job/free/";,
+ Description: "",
+ },
+ {
+ Name: "free1",
+ Color: "blue",
+ Class: "hudson.model.FreeStyleProject",
+ Base: "",
+ URL:
"https://test.nddtf.com/job/dir-test/job/dir-test-2/job/free1/";,
+ Description: "",
+ },
+ }
+
+ var expectJobs []*models.Job = []*models.Job{
+ {
+ FullName: "devlake",
+ Path: "",
+ Name: "devlake",
+ Color: "blue",
+ Class: "hudson.model.FreeStyleProject",
+ Base: "",
+ URL: "https://test.nddtf.com/job/devlake/";,
+ Description: "",
+ },
+ {
+ FullName: "dir-test/dir-test-2/free",
+ Path: "job/dir-test/job/dir-test-2/",
+ Name: "free",
+ Color: "blue",
+ Class: "hudson.model.FreeStyleProject",
+ Base: "",
+ URL:
"https://test.nddtf.com/job/dir-test/job/dir-test-2/job/free/";,
+ Description: "",
+ },
+ {
+ FullName: "dir-test/dir-test-2/free1",
+ Path: "job/dir-test/job/dir-test-2/",
+ Name: "free1",
+ Color: "blue",
+ Class: "hudson.model.FreeStyleProject",
+ Base: "",
+ URL:
"https://test.nddtf.com/job/dir-test/job/dir-test-2/job/free1/";,
+ Description: "",
+ },
+ }
+
+ var expectPaths []*models.Job = []*models.Job{
+ {
+ FullName: "dir-test",
+ Path: "",
+ Name: "dir-test",
+ Color: "",
+ Class: "hudson.model.FreeStyleProject",
+ Base: "",
+ URL: "https://test.nddtf.com/job/dir-test/";,
+ Description: "",
+ Jobs: &[]models.Job{{}},
+ },
+ {
+ FullName: "dir-test/dir-test-2",
+ Path: "job/dir-test/",
+ Name: "dir-test-2",
+ Color: "",
+ Class: "hudson.model.FreeStyleProject",
+ Base: "",
+ URL:
"https://test.nddtf.com/job/dir-test/job/dir-test-2/";,
+ Description: "",
+ Jobs: &[]models.Job{{}},
+ },
+ }
+
+ mockApiClient := mocks.NewApiClientGetter(t)
+
+ var data struct {
+ Jobs []json.RawMessage `json:"jobs"`
+ }
+
+ // first path jobs
+ js, err1 := json.Marshal(remoteData[0])
+ assert.Nil(t, err1)
+ data.Jobs = append(data.Jobs, js)
+
+ js, err1 = json.Marshal(remoteData[1])
+ assert.Nil(t, err1)
+ data.Jobs = append(data.Jobs, js)
+
+ js, err1 = json.Marshal(data)
+ assert.Nil(t, err1)
+
+ res := &http.Response{}
+ res.Body = io.NopCloser(bytes.NewBuffer(js))
+ res.StatusCode = http.StatusOK
+
+ mockApiClient.On("Get", mock.Anything, mock.Anything,
mock.Anything).Return(res, nil).Once()
+
+ // second path jobs
+ data.Jobs = []json.RawMessage{}
+
+ js, err1 = json.Marshal(remoteData[2])
+ assert.Nil(t, err1)
+ data.Jobs = append(data.Jobs, js)
+
+ js, err1 = json.Marshal(data)
+ assert.Nil(t, err1)
+
+ res = &http.Response{}
+ res.Body = io.NopCloser(bytes.NewBuffer(js))
+ res.StatusCode = http.StatusOK
+
+ mockApiClient.On("Get", mock.Anything, mock.Anything,
mock.Anything).Return(res, nil).Once()
+
+ // third path jobs
+ data.Jobs = []json.RawMessage{}
+ js, err1 = json.Marshal(remoteData[3])
+ assert.Nil(t, err1)
+ data.Jobs = append(data.Jobs, js)
+
+ js, err1 = json.Marshal(remoteData[4])
+ assert.Nil(t, err1)
+ data.Jobs = append(data.Jobs, js)
+
+ js, err1 = json.Marshal(data)
+ assert.Nil(t, err1)
+
+ res = &http.Response{}
+ res.Body = io.NopCloser(bytes.NewBuffer(js))
+ res.StatusCode = http.StatusOK
+
+ mockApiClient.On("Get", mock.Anything, mock.Anything,
mock.Anything).Return(res, nil).Once()
+
+ basicRes = unithelper.DummyBasicRes(func(mockDal *mocks.Dal) {})
+
+ var jobs []*models.Job
+
+ var paths []*models.Job
+
+ err := GetAllJobs(mockApiClient, "", "", testPageSize, func(job
*models.Job, isPath bool) errors.Error {
+ if isPath {
+ paths = append(paths, job)
+ } else {
+ jobs = append(jobs, job)
+ }
+ return nil
+ })
+
+ assert.Equal(t, err, nil)
+ assert.Equal(t, expectJobs, jobs)
+ assert.Equal(t, expectPaths, paths)
+}
diff --git a/plugins/jenkins/models/response.go
b/plugins/jenkins/models/response.go
index 5974aea7d..179898d7c 100644
--- a/plugins/jenkins/models/response.go
+++ b/plugins/jenkins/models/response.go
@@ -27,6 +27,7 @@ type Job struct {
URL string `json:"url"`
Description string `json:"description"`
UpstreamProjects []Project `json:"upstreamProjects"`
+ Jobs *[]Job `json:"jobs" gorm:"-"`
*PrimaryView `json:"primaryView"`
}
