This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch fix#6075-3
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/fix#6075-3 by this push:
new 3c87e9fcd fix: incremental=true when use CollectUnfinishedDetails and
fix jenkins builds
3c87e9fcd is described below
commit 3c87e9fcd1b4c569f4d696d0b972a6f5c7e179dc
Author: abeizn <[email protected]>
AuthorDate: Wed Oct 11 15:35:52 2023 +0800
fix: incremental=true when use CollectUnfinishedDetails and fix jenkins
builds
---
.../pluginhelper/api/api_collector_with_state.go | 2 +-
backend/plugins/jenkins/tasks/build_collector.go | 76 ++++++++++++----------
2 files changed, 43 insertions(+), 35 deletions(-)
diff --git a/backend/helpers/pluginhelper/api/api_collector_with_state.go
b/backend/helpers/pluginhelper/api/api_collector_with_state.go
index 25471612b..1fac444ec 100644
--- a/backend/helpers/pluginhelper/api/api_collector_with_state.go
+++ b/backend/helpers/pluginhelper/api/api_collector_with_state.go
@@ -113,7 +113,7 @@ func NewStatefulApiCollector(args RawDataSubTaskArgs)
(*ApiCollectorStateManager
// InitCollector init the embedded collector
func (m *ApiCollectorStateManager) InitCollector(args ApiCollectorArgs)
errors.Error {
args.RawDataSubTaskArgs = m.RawDataSubTaskArgs
- args.Incremental = m.IsIncreamtal
+ args.Incremental = args.Incremental || m.IsIncreamtal
apiCollector, err := NewApiCollector(args)
if err != nil {
return err
diff --git a/backend/plugins/jenkins/tasks/build_collector.go
b/backend/plugins/jenkins/tasks/build_collector.go
index 56924f350..c29a8b901 100644
--- a/backend/plugins/jenkins/tasks/build_collector.go
+++ b/backend/plugins/jenkins/tasks/build_collector.go
@@ -20,17 +20,13 @@ package tasks
import (
"encoding/json"
"fmt"
- "io"
"net/http"
"net/url"
- "reflect"
"time"
- "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "github.com/apache/incubator-devlake/plugins/jenkins/models"
)
const RAW_BUILD_TABLE = "jenkins_api_builds"
@@ -55,7 +51,6 @@ type SimpleJenkinsApiBuild struct {
func CollectApiBuilds(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*JenkinsTaskData)
- db := taskCtx.GetDal()
collector, err :=
helper.NewStatefulApiCollectorForFinalizableEntity(helper.FinalizableApiCollectorArgs{
RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
Params: JenkinsApiParams{
@@ -87,7 +82,20 @@ func CollectApiBuilds(taskCtx plugin.SubTaskContext)
errors.Error {
if err != nil {
return nil, err
}
- return data.Builds, nil
+
+ builds := make([]json.RawMessage, 0,
len(data.Builds))
+ for _, build := range data.Builds {
+ var buildObj
map[string]interface{}
+ err := json.Unmarshal(build,
&buildObj)
+ if err != nil {
+ return nil,
errors.Convert(err)
+ }
+ if buildObj["result"] ==
"SUCCESS" || buildObj["result"] == "FAILURE" {
+ builds = append(builds,
build)
+ }
+ }
+
+ return builds, nil
},
},
GetCreated: func(item json.RawMessage) (time.Time,
errors.Error) {
@@ -101,34 +109,34 @@ func CollectApiBuilds(taskCtx plugin.SubTaskContext)
errors.Error {
return time.Unix(seconds, nanos), nil
},
},
- CollectUnfinishedDetails:
&helper.FinalizableApiCollectorDetailArgs{
- BuildInputIterator: func() (helper.Iterator,
errors.Error) {
- cursor, err := db.Cursor(
- dal.Select("number"),
- dal.From(&models.JenkinsBuild{}),
- dal.Where(
- "full_name = ? AND
connection_id = ? AND result != 'SUCCESS' AND result != 'FAILURE'",
- data.Options.JobFullName,
data.Options.ConnectionId,
- ),
- )
- if err != nil {
- return nil, err
- }
- return helper.NewDalCursorIterator(db, cursor,
reflect.TypeOf(SimpleJenkinsApiBuild{}))
- },
- FinalizableApiCollectorCommonArgs:
helper.FinalizableApiCollectorCommonArgs{
- UrlTemplate: fmt.Sprintf("%sjob/%s/{{
.Input.Number
}}/api/json?tree=number,url,result,timestamp,id,duration,estimatedDuration,building",
- data.Options.JobPath,
data.Options.JobName),
- ResponseParser: func(res *http.Response)
([]json.RawMessage, errors.Error) {
- body, err := io.ReadAll(res.Body)
- if err != nil {
- return nil, errors.Convert(err)
- }
- res.Body.Close()
- return []json.RawMessage{body}, nil
- },
- },
- },
+ // CollectUnfinishedDetails:
&helper.FinalizableApiCollectorDetailArgs{
+ // BuildInputIterator: func() (helper.Iterator,
errors.Error) {
+ // cursor, err := db.Cursor(
+ // dal.Select("number"),
+ // dal.From(&models.JenkinsBuild{}),
+ // dal.Where(
+ // "full_name = ? AND
connection_id = ? AND result != 'SUCCESS' AND result != 'FAILURE'",
+ // data.Options.JobFullName,
data.Options.ConnectionId,
+ // ),
+ // )
+ // if err != nil {
+ // return nil, err
+ // }
+ // return helper.NewDalCursorIterator(db, cursor,
reflect.TypeOf(SimpleJenkinsApiBuild{}))
+ // },
+ // FinalizableApiCollectorCommonArgs:
helper.FinalizableApiCollectorCommonArgs{
+ // UrlTemplate: fmt.Sprintf("%sjob/%s/{{
.Input.Number
}}/api/json?tree=number,url,result,timestamp,id,duration,estimatedDuration,building",
+ // data.Options.JobPath,
data.Options.JobName),
+ // ResponseParser: func(res *http.Response)
([]json.RawMessage, errors.Error) {
+ // body, err := io.ReadAll(res.Body)
+ // if err != nil {
+ // return nil, errors.Convert(err)
+ // }
+ // res.Body.Close()
+ // return []json.RawMessage{body}, nil
+ // },
+ // },
+ // },
})
if err != nil {