This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch option-time-after
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/option-time-after by this push:
new 1a58ce22f fix: option time after for plugins
1a58ce22f is described below
commit 1a58ce22f0028db997254f58fb79889514ee9055
Author: abeizn <[email protected]>
AuthorDate: Thu Dec 7 13:59:03 2023 +0800
fix: option time after for plugins
---
.../pluginhelper/api/api_collector_with_state.go | 27 ++++++++++++++++++++--
backend/plugins/ae/tasks/task_data.go | 1 +
backend/plugins/bamboo/models/task.go | 1 +
backend/plugins/bitbucket/tasks/task_data.go | 1 +
backend/plugins/circleci/tasks/task_data.go | 1 +
backend/plugins/customize/tasks/task_data.go | 1 +
backend/plugins/feishu/tasks/task_data.go | 1 +
backend/plugins/gitee/tasks/task_data.go | 1 +
backend/plugins/github/tasks/task_data.go | 1 +
backend/plugins/gitlab/tasks/task_data.go | 1 +
backend/plugins/icla/tasks/task_data.go | 3 ++-
backend/plugins/jenkins/tasks/task_data.go | 1 +
backend/plugins/jira/tasks/task_data.go | 1 +
backend/plugins/opsgenie/tasks/task_data.go | 1 +
backend/plugins/pagerduty/tasks/task_data.go | 1 +
backend/plugins/slack/tasks/task_data.go | 1 +
backend/plugins/sonarqube/tasks/task_data.go | 1 +
backend/plugins/tapd/tasks/task_data.go | 1 +
backend/plugins/teambition/tasks/task_data.go | 1 +
backend/plugins/trello/tasks/task_data.go | 1 +
backend/plugins/zentao/tasks/task_data.go | 1 +
21 files changed, 46 insertions(+), 3 deletions(-)
diff --git a/backend/helpers/pluginhelper/api/api_collector_with_state.go
b/backend/helpers/pluginhelper/api/api_collector_with_state.go
index 7eb54ac14..0434086d1 100644
--- a/backend/helpers/pluginhelper/api/api_collector_with_state.go
+++ b/backend/helpers/pluginhelper/api/api_collector_with_state.go
@@ -21,6 +21,7 @@ import (
"encoding/json"
"net/http"
"net/url"
+ "reflect"
"time"
"github.com/apache/incubator-devlake/core/dal"
@@ -44,12 +45,35 @@ type ApiCollectorStateManager struct {
// NewStatefulApiCollector create a new ApiCollectorStateManager
func NewStatefulApiCollector(args RawDataSubTaskArgs)
(*ApiCollectorStateManager, errors.Error) {
db := args.Ctx.GetDal()
-
+ syncPolicy := args.Ctx.TaskContext().SyncPolicy()
rawDataSubTask, err := NewRawDataSubTask(args)
if err != nil {
return nil, errors.Default.Wrap(err, "Couldn't resolve raw
subtask args")
}
+ // get optionTimeAfter from options
+ data := args.Ctx.GetData()
+ value := reflect.ValueOf(data)
+ if value.Kind() == reflect.Ptr && value.Elem().Kind() == reflect.Struct
{
+ options := value.Elem().FieldByName("Options")
+ if options.IsValid() && options.Kind() == reflect.Ptr &&
options.Elem().Kind() == reflect.Struct {
+ timeAfter := options.Elem().FieldByName("TimeAfter")
+ if timeAfter.IsValid() && timeAfter.Kind() ==
reflect.String && timeAfter.String() != "" {
+ optionTimeAfter, parseErr :=
time.Parse(time.RFC3339, timeAfter.String())
+ if parseErr != nil {
+ return nil,
errors.Default.Wrap(parseErr, "Failed to parse timeAfter!")
+ }
+ if syncPolicy != nil {
+ syncPolicy.TimeAfter = &optionTimeAfter
+ } else {
+ syncPolicy = &models.SyncPolicy{
+ TimeAfter: &optionTimeAfter,
+ }
+ }
+ }
+ }
+ }
+
// CollectorLatestState retrieves the latest collector state from the
database
oldState := models.CollectorLatestState{}
err = db.First(&oldState, dal.Where(`raw_data_table = ? AND
raw_data_params = ?`, rawDataSubTask.table, rawDataSubTask.params))
@@ -68,7 +92,6 @@ func NewStatefulApiCollector(args RawDataSubTaskArgs)
(*ApiCollectorStateManager
oldLatestSuccessStart := oldState.LatestSuccessStart
// Calculate incremental and since based on syncPolicy and old state
- syncPolicy := args.Ctx.TaskContext().SyncPolicy()
var isIncremental bool
var since *time.Time
diff --git a/backend/plugins/ae/tasks/task_data.go
b/backend/plugins/ae/tasks/task_data.go
index 3a933536c..19275ce80 100644
--- a/backend/plugins/ae/tasks/task_data.go
+++ b/backend/plugins/ae/tasks/task_data.go
@@ -24,6 +24,7 @@ import (
type AeOptions struct {
ConnectionId uint64 `json:"connectionId"`
ProjectId int
+ TimeAfter string
}
type AeTaskData struct {
diff --git a/backend/plugins/bamboo/models/task.go
b/backend/plugins/bamboo/models/task.go
index a3578cf80..64b7db30e 100644
--- a/backend/plugins/bamboo/models/task.go
+++ b/backend/plugins/bamboo/models/task.go
@@ -32,4 +32,5 @@ type BambooOptions struct {
Tasks []string `json:"tasks,omitempty"`
ScopeConfigId uint64 `mapstructure:"scopeConfigId"
json:"scopeConfigId"`
*BambooScopeConfig `mapstructure:"scopeConfig" json:"scopeConfig"`
+ TimeAfter string
}
diff --git a/backend/plugins/bitbucket/tasks/task_data.go
b/backend/plugins/bitbucket/tasks/task_data.go
index 235b05080..418e6f121 100644
--- a/backend/plugins/bitbucket/tasks/task_data.go
+++ b/backend/plugins/bitbucket/tasks/task_data.go
@@ -29,6 +29,7 @@ type BitbucketOptions struct {
FullName string `json:"fullName"
mapstructure:"fullName"`
ScopeConfigId uint64 `json:"scopeConfigId"
mapstructure:"scopeConfigId,omitempty"`
*models.BitbucketScopeConfig `mapstructure:"scopeConfig,omitempty"
json:"scopeConfig"`
+ TimeAfter string
}
type BitbucketTaskData struct {
diff --git a/backend/plugins/circleci/tasks/task_data.go
b/backend/plugins/circleci/tasks/task_data.go
index d9398e017..43d55104b 100644
--- a/backend/plugins/circleci/tasks/task_data.go
+++ b/backend/plugins/circleci/tasks/task_data.go
@@ -29,6 +29,7 @@ type CircleciOptions struct {
PageSize uint64 `mapstruct:"pageSize"`
ScopeConfigId uint64 `json:"scopeConfigId"
mapstructure:"scopeConfigId,omitempty"`
ScopeConfig *models.CircleciScopeConfig `json:"scopeConfig"
mapstructure:"scopeConfig,omitempty"`
+ TimeAfter string
}
type CircleciTaskData struct {
diff --git a/backend/plugins/customize/tasks/task_data.go
b/backend/plugins/customize/tasks/task_data.go
index 6850bd725..dc1a08e3d 100644
--- a/backend/plugins/customize/tasks/task_data.go
+++ b/backend/plugins/customize/tasks/task_data.go
@@ -26,6 +26,7 @@ type MappingRules struct {
type Options struct {
TransformationRules []MappingRules `json:"transformationRules"`
+ TimeAfter string
}
type TaskData struct {
diff --git a/backend/plugins/feishu/tasks/task_data.go
b/backend/plugins/feishu/tasks/task_data.go
index 428c06739..77b4c67d4 100644
--- a/backend/plugins/feishu/tasks/task_data.go
+++ b/backend/plugins/feishu/tasks/task_data.go
@@ -28,6 +28,7 @@ type FeishuApiParams struct {
type FeishuOptions struct {
ConnectionId uint64 `json:"connectionId"`
NumOfDaysToCollect float64 `json:"numOfDaysToCollect"`
+ TimeAfter string
}
type FeishuTaskData struct {
diff --git a/backend/plugins/gitee/tasks/task_data.go
b/backend/plugins/gitee/tasks/task_data.go
index aa551fbc3..ba834e2f0 100644
--- a/backend/plugins/gitee/tasks/task_data.go
+++ b/backend/plugins/gitee/tasks/task_data.go
@@ -30,6 +30,7 @@ type GiteeOptions struct {
Owner string
Repo string
ScopeConfig models.GiteeScopeConfig `mapstructure:"scopeConfig"
json:"scopeConfig"`
+ TimeAfter string
}
type GiteeTaskData struct {
diff --git a/backend/plugins/github/tasks/task_data.go
b/backend/plugins/github/tasks/task_data.go
index f87d57694..05c96fc91 100644
--- a/backend/plugins/github/tasks/task_data.go
+++ b/backend/plugins/github/tasks/task_data.go
@@ -34,6 +34,7 @@ type GithubOptions struct {
Repo string `json:"repo"
mapstructure:"repo,omitempty"`
Name string `json:"name"
mapstructure:"name,omitempty"`
ScopeConfig *models.GithubScopeConfig
`mapstructure:"scopeConfig,omitempty" json:"scopeConfig"`
+ TimeAfter string
}
type GithubTaskData struct {
diff --git a/backend/plugins/gitlab/tasks/task_data.go
b/backend/plugins/gitlab/tasks/task_data.go
index 0eaa36e45..8c70c2b29 100644
--- a/backend/plugins/gitlab/tasks/task_data.go
+++ b/backend/plugins/gitlab/tasks/task_data.go
@@ -28,6 +28,7 @@ type GitlabOptions struct {
ProjectId int `mapstructure:"projectId"
json:"projectId"`
ScopeConfigId uint64 `mapstructure:"scopeConfigId"
json:"scopeConfigId"`
ScopeConfig *models.GitlabScopeConfig `mapstructure:"scopeConfig"
json:"scopeConfig"`
+ TimeAfter string
}
type GitlabTaskData struct {
diff --git a/backend/plugins/icla/tasks/task_data.go
b/backend/plugins/icla/tasks/task_data.go
index dfa35a0ea..dc992b1f6 100644
--- a/backend/plugins/icla/tasks/task_data.go
+++ b/backend/plugins/icla/tasks/task_data.go
@@ -29,7 +29,8 @@ type IclaOptions struct {
// options means some custom params required by plugin running.
// Such As How many rows do your want
// You can use it in subtasks, and you need to pass it to main.go and
pipelines.
- Tasks []string `json:"tasks,omitempty"`
+ Tasks []string `json:"tasks,omitempty"`
+ TimeAfter string
}
type IclaTaskData struct {
diff --git a/backend/plugins/jenkins/tasks/task_data.go
b/backend/plugins/jenkins/tasks/task_data.go
index d34b4e6ac..f97d2bbc5 100644
--- a/backend/plugins/jenkins/tasks/task_data.go
+++ b/backend/plugins/jenkins/tasks/task_data.go
@@ -35,6 +35,7 @@ type JenkinsOptions struct {
JobPath string `json:"jobPath"` //
"job/path1/job/path2"
Tasks []string `json:"tasks,omitempty"`
ScopeConfig *models.JenkinsScopeConfig `mapstructure:"scopeConfig"
json:"scopeConfig"`
+ TimeAfter string
}
type JenkinsTaskData struct {
diff --git a/backend/plugins/jira/tasks/task_data.go
b/backend/plugins/jira/tasks/task_data.go
index 42433bf9c..cd27bb38f 100644
--- a/backend/plugins/jira/tasks/task_data.go
+++ b/backend/plugins/jira/tasks/task_data.go
@@ -31,6 +31,7 @@ type JiraOptions struct {
ScopeConfig *models.JiraScopeConfig `json:"scopeConfig"`
ScopeConfigId uint64
PageSize int
+ TimeAfter string
}
type JiraTaskData struct {
diff --git a/backend/plugins/opsgenie/tasks/task_data.go
b/backend/plugins/opsgenie/tasks/task_data.go
index 5d20fc820..02b4364a3 100644
--- a/backend/plugins/opsgenie/tasks/task_data.go
+++ b/backend/plugins/opsgenie/tasks/task_data.go
@@ -28,6 +28,7 @@ type OpsgenieOptions struct {
ServiceId string `json:"service_id,omitempty"`
ServiceName string `json:"service_name,omitempty"`
Tasks []string `json:"tasks,omitempty"`
+ TimeAfter string
*models.OpsenieScopeConfig
}
diff --git a/backend/plugins/pagerduty/tasks/task_data.go
b/backend/plugins/pagerduty/tasks/task_data.go
index 5f456f725..904c5b1f4 100644
--- a/backend/plugins/pagerduty/tasks/task_data.go
+++ b/backend/plugins/pagerduty/tasks/task_data.go
@@ -28,6 +28,7 @@ type PagerDutyOptions struct {
ServiceId string `json:"service_id,omitempty"`
ServiceName string `json:"service_name,omitempty"`
Tasks []string `json:"tasks,omitempty"`
+ TimeAfter string
*models.PagerdutyScopeConfig
}
diff --git a/backend/plugins/slack/tasks/task_data.go
b/backend/plugins/slack/tasks/task_data.go
index 41c811fb6..0f3b2f088 100644
--- a/backend/plugins/slack/tasks/task_data.go
+++ b/backend/plugins/slack/tasks/task_data.go
@@ -27,6 +27,7 @@ type SlackApiParams struct {
type SlackOptions struct {
ConnectionId uint64 `json:"connectionId"`
+ TimeAfter string
}
type SlackTaskData struct {
diff --git a/backend/plugins/sonarqube/tasks/task_data.go
b/backend/plugins/sonarqube/tasks/task_data.go
index 998054a69..1780ab918 100644
--- a/backend/plugins/sonarqube/tasks/task_data.go
+++ b/backend/plugins/sonarqube/tasks/task_data.go
@@ -31,6 +31,7 @@ type SonarqubeOptions struct {
ConnectionId uint64 `json:"connectionId"`
ProjectKey string `json:"projectKey"`
Tasks []string `json:"tasks,omitempty"`
+ TimeAfter string
}
type SonarqubeTaskData struct {
diff --git a/backend/plugins/tapd/tasks/task_data.go
b/backend/plugins/tapd/tasks/task_data.go
index 16b077bbb..676b77546 100644
--- a/backend/plugins/tapd/tasks/task_data.go
+++ b/backend/plugins/tapd/tasks/task_data.go
@@ -32,6 +32,7 @@ type TapdOptions struct {
CstZone *time.Location
ScopeConfigId uint64
ScopeConfig *models.TapdScopeConfig `json:"scopeConfig"`
+ TimeAfter string
}
type TapdTaskData struct {
diff --git a/backend/plugins/teambition/tasks/task_data.go
b/backend/plugins/teambition/tasks/task_data.go
index 961f1f33d..d112cb417 100644
--- a/backend/plugins/teambition/tasks/task_data.go
+++ b/backend/plugins/teambition/tasks/task_data.go
@@ -30,6 +30,7 @@ type TeambitionOptions struct {
PageSize uint64 `mapstruct:"pageSize"`
CstZone *time.Location
TransformationRules TransformationRules `json:"transformationRules"`
+ TimeAfter string
}
type TeambitionTaskData struct {
diff --git a/backend/plugins/trello/tasks/task_data.go
b/backend/plugins/trello/tasks/task_data.go
index 273fc3f71..169edb7d2 100644
--- a/backend/plugins/trello/tasks/task_data.go
+++ b/backend/plugins/trello/tasks/task_data.go
@@ -27,6 +27,7 @@ type TrelloOptions struct {
BoardId string `json:"boardId"`
ScopeId string
ScopeConfigId uint64
+ TimeAfter string
}
type TrelloTaskData struct {
diff --git a/backend/plugins/zentao/tasks/task_data.go
b/backend/plugins/zentao/tasks/task_data.go
index 26bc9734c..bdb0baf8a 100644
--- a/backend/plugins/zentao/tasks/task_data.go
+++ b/backend/plugins/zentao/tasks/task_data.go
@@ -38,6 +38,7 @@ type ZentaoOptions struct {
// TODO not support now
ScopeConfigId uint64 `json:"scopeConfigId"
mapstructure:"scopeConfigId,omitempty"`
ScopeConfig *models.ZentaoScopeConfig `json:"scopeConfig"
mapstructure:"scopeConfig,omitempty"`
+ TimeAfter string
}
func (o *ZentaoOptions) GetParams() any {
