This is an automated email from the ASF dual-hosted git repository.

abeizn pushed a commit to branch fix#6075-3
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git


The following commit(s) were added to refs/heads/fix#6075-3 by this push:
     new 3c87e9fcd fix: incremental=true when use CollectUnfinishedDetails and 
fix jenkins builds
3c87e9fcd is described below

commit 3c87e9fcd1b4c569f4d696d0b972a6f5c7e179dc
Author: abeizn <[email protected]>
AuthorDate: Wed Oct 11 15:35:52 2023 +0800

    fix: incremental=true when use CollectUnfinishedDetails and fix jenkins 
builds
---
 .../pluginhelper/api/api_collector_with_state.go   |  2 +-
 backend/plugins/jenkins/tasks/build_collector.go   | 76 ++++++++++++----------
 2 files changed, 43 insertions(+), 35 deletions(-)

diff --git a/backend/helpers/pluginhelper/api/api_collector_with_state.go 
b/backend/helpers/pluginhelper/api/api_collector_with_state.go
index 25471612b..1fac444ec 100644
--- a/backend/helpers/pluginhelper/api/api_collector_with_state.go
+++ b/backend/helpers/pluginhelper/api/api_collector_with_state.go
@@ -113,7 +113,7 @@ func NewStatefulApiCollector(args RawDataSubTaskArgs) 
(*ApiCollectorStateManager
 // InitCollector init the embedded collector
 func (m *ApiCollectorStateManager) InitCollector(args ApiCollectorArgs) 
errors.Error {
        args.RawDataSubTaskArgs = m.RawDataSubTaskArgs
-       args.Incremental = m.IsIncreamtal
+       args.Incremental = args.Incremental || m.IsIncreamtal
        apiCollector, err := NewApiCollector(args)
        if err != nil {
                return err
diff --git a/backend/plugins/jenkins/tasks/build_collector.go 
b/backend/plugins/jenkins/tasks/build_collector.go
index 56924f350..c29a8b901 100644
--- a/backend/plugins/jenkins/tasks/build_collector.go
+++ b/backend/plugins/jenkins/tasks/build_collector.go
@@ -20,17 +20,13 @@ package tasks
 import (
        "encoding/json"
        "fmt"
-       "io"
        "net/http"
        "net/url"
-       "reflect"
        "time"
 
-       "github.com/apache/incubator-devlake/core/dal"
        "github.com/apache/incubator-devlake/core/errors"
        "github.com/apache/incubator-devlake/core/plugin"
        helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
-       "github.com/apache/incubator-devlake/plugins/jenkins/models"
 )
 
 const RAW_BUILD_TABLE = "jenkins_api_builds"
@@ -55,7 +51,6 @@ type SimpleJenkinsApiBuild struct {
 
 func CollectApiBuilds(taskCtx plugin.SubTaskContext) errors.Error {
        data := taskCtx.GetData().(*JenkinsTaskData)
-       db := taskCtx.GetDal()
        collector, err := 
helper.NewStatefulApiCollectorForFinalizableEntity(helper.FinalizableApiCollectorArgs{
                RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
                        Params: JenkinsApiParams{
@@ -87,7 +82,20 @@ func CollectApiBuilds(taskCtx plugin.SubTaskContext) 
errors.Error {
                                        if err != nil {
                                                return nil, err
                                        }
-                                       return data.Builds, nil
+
+                                       builds := make([]json.RawMessage, 0, 
len(data.Builds))
+                                       for _, build := range data.Builds {
+                                               var buildObj 
map[string]interface{}
+                                               err := json.Unmarshal(build, 
&buildObj)
+                                               if err != nil {
+                                                       return nil, 
errors.Convert(err)
+                                               }
+                                               if buildObj["result"] == 
"SUCCESS" || buildObj["result"] == "FAILURE" {
+                                                       builds = append(builds, 
build)
+                                               }
+                                       }
+
+                                       return builds, nil
                                },
                        },
                        GetCreated: func(item json.RawMessage) (time.Time, 
errors.Error) {
@@ -101,34 +109,34 @@ func CollectApiBuilds(taskCtx plugin.SubTaskContext) 
errors.Error {
                                return time.Unix(seconds, nanos), nil
                        },
                },
-               CollectUnfinishedDetails: 
&helper.FinalizableApiCollectorDetailArgs{
-                       BuildInputIterator: func() (helper.Iterator, 
errors.Error) {
-                               cursor, err := db.Cursor(
-                                       dal.Select("number"),
-                                       dal.From(&models.JenkinsBuild{}),
-                                       dal.Where(
-                                               "full_name = ? AND 
connection_id = ? AND result != 'SUCCESS' AND result != 'FAILURE'",
-                                               data.Options.JobFullName, 
data.Options.ConnectionId,
-                                       ),
-                               )
-                               if err != nil {
-                                       return nil, err
-                               }
-                               return helper.NewDalCursorIterator(db, cursor, 
reflect.TypeOf(SimpleJenkinsApiBuild{}))
-                       },
-                       FinalizableApiCollectorCommonArgs: 
helper.FinalizableApiCollectorCommonArgs{
-                               UrlTemplate: fmt.Sprintf("%sjob/%s/{{ 
.Input.Number 
}}/api/json?tree=number,url,result,timestamp,id,duration,estimatedDuration,building",
-                                       data.Options.JobPath, 
data.Options.JobName),
-                               ResponseParser: func(res *http.Response) 
([]json.RawMessage, errors.Error) {
-                                       body, err := io.ReadAll(res.Body)
-                                       if err != nil {
-                                               return nil, errors.Convert(err)
-                                       }
-                                       res.Body.Close()
-                                       return []json.RawMessage{body}, nil
-                               },
-                       },
-               },
+               // CollectUnfinishedDetails: 
&helper.FinalizableApiCollectorDetailArgs{
+               //      BuildInputIterator: func() (helper.Iterator, 
errors.Error) {
+               //              cursor, err := db.Cursor(
+               //                      dal.Select("number"),
+               //                      dal.From(&models.JenkinsBuild{}),
+               //                      dal.Where(
+               //                              "full_name = ? AND 
connection_id = ? AND result != 'SUCCESS' AND result != 'FAILURE'",
+               //                              data.Options.JobFullName, 
data.Options.ConnectionId,
+               //                      ),
+               //              )
+               //              if err != nil {
+               //                      return nil, err
+               //              }
+               //              return helper.NewDalCursorIterator(db, cursor, 
reflect.TypeOf(SimpleJenkinsApiBuild{}))
+               //      },
+               //      FinalizableApiCollectorCommonArgs: 
helper.FinalizableApiCollectorCommonArgs{
+               //              UrlTemplate: fmt.Sprintf("%sjob/%s/{{ 
.Input.Number 
}}/api/json?tree=number,url,result,timestamp,id,duration,estimatedDuration,building",
+               //                      data.Options.JobPath, 
data.Options.JobName),
+               //              ResponseParser: func(res *http.Response) 
([]json.RawMessage, errors.Error) {
+               //                      body, err := io.ReadAll(res.Body)
+               //                      if err != nil {
+               //                              return nil, errors.Convert(err)
+               //                      }
+               //                      res.Body.Close()
+               //                      return []json.RawMessage{body}, nil
+               //              },
+               //      },
+               // },
        })
 
        if err != nil {

Reply via email to