This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 79f02a76 Feat: jenkins time filter (#3861)
79f02a76 is described below
commit 79f02a76839d620ff7d2b92f70405fd6288978b9
Author: Likyh <[email protected]>
AuthorDate: Tue Dec 6 16:16:30 2022 +0800
Feat: jenkins time filter (#3861)
* fix: fix some bugs #3860
* feat: add CreatedDateAfter for jenkins
---
plugins/helper/api_collector.go | 2 +-
plugins/jenkins/impl/impl.go | 24 +++++++++++++++++++++---
plugins/jenkins/jenkins.go | 2 ++
plugins/jenkins/tasks/stage_collector.go | 7 +++++--
plugins/jenkins/tasks/task_data.go | 11 +++++------
plugins/jira/impl/impl.go | 2 +-
plugins/jira/jira.go | 2 +-
7 files changed, 36 insertions(+), 14 deletions(-)
diff --git a/plugins/helper/api_collector.go b/plugins/helper/api_collector.go
index 4944b4e4..950ceaa6 100644
--- a/plugins/helper/api_collector.go
+++ b/plugins/helper/api_collector.go
@@ -283,7 +283,7 @@ func (collector *ApiCollector)
fetchPagesUndetermined(reqData *RequestData) {
return nil
}
apiClient.NextTick(func() errors.Error {
- reqDataCopy.Pager.Skip +=
collector.args.PageSize
+ reqDataCopy.Pager.Skip +=
collector.args.PageSize * concurrency
reqDataCopy.Pager.Page += concurrency
return collect()
})
diff --git a/plugins/jenkins/impl/impl.go b/plugins/jenkins/impl/impl.go
index 05f3c94c..532f6f7a 100644
--- a/plugins/jenkins/impl/impl.go
+++ b/plugins/jenkins/impl/impl.go
@@ -19,11 +19,12 @@ package impl
import (
"fmt"
- "github.com/apache/incubator-devlake/plugins/core/dal"
"strings"
+ "time"
"github.com/apache/incubator-devlake/errors"
"github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
"github.com/apache/incubator-devlake/plugins/helper"
"github.com/apache/incubator-devlake/plugins/jenkins/api"
"github.com/apache/incubator-devlake/plugins/jenkins/models"
@@ -96,6 +97,8 @@ func (plugin Jenkins) SubTaskMetas() []core.SubTaskMeta {
}
func (plugin Jenkins) PrepareTaskData(taskCtx core.TaskContext, options
map[string]interface{}) (interface{}, errors.Error) {
op, err := tasks.DecodeAndValidateTaskOptions(options)
+ logger := taskCtx.GetLogger()
+ logger.Debug("%v", options)
if err != nil {
return nil, err
}
@@ -120,9 +123,19 @@ func (plugin Jenkins) PrepareTaskData(taskCtx
core.TaskContext, options map[stri
if err != nil {
return nil, err
}
+
+ var createdDateAfter time.Time
+ if op.CreatedDateAfter != "" {
+ createdDateAfter, err =
errors.Convert01(time.Parse("2006-01-02T15:04:05Z", op.CreatedDateAfter))
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "invalid value
for `createdDateAfter`")
+ }
+ }
+
if !strings.HasSuffix(op.JobPath, "/") {
op.JobPath = fmt.Sprintf("%s/", op.JobPath)
}
+
if op.JenkinsTransformationRule == nil && op.TransformationRuleId != 0 {
var transformationRule models.JenkinsTransformationRule
err = taskCtx.GetDal().First(&transformationRule, dal.Where("id
= ?", op.TransformationRuleId))
@@ -131,11 +144,16 @@ func (plugin Jenkins) PrepareTaskData(taskCtx
core.TaskContext, options map[stri
}
op.JenkinsTransformationRule = &transformationRule
}
- return &tasks.JenkinsTaskData{
+ taskData := &tasks.JenkinsTaskData{
Options: op,
ApiClient: apiClient,
Connection: connection,
- }, nil
+ }
+ if !createdDateAfter.IsZero() {
+ taskData.CreatedDateAfter = &createdDateAfter
+ logger.Debug("collect data created from %s", createdDateAfter)
+ }
+ return taskData, nil
}
func (plugin Jenkins) RootPkgPath() string {
diff --git a/plugins/jenkins/jenkins.go b/plugins/jenkins/jenkins.go
index 692f015a..9be40b48 100644
--- a/plugins/jenkins/jenkins.go
+++ b/plugins/jenkins/jenkins.go
@@ -30,12 +30,14 @@ func main() {
connectionId := jenkinsCmd.Flags().Uint64P("connection", "c", 1,
"jenkins connection id")
jobFullName := jenkinsCmd.Flags().StringP("jobFullName", "j", "",
"jenkins job full name")
deployTagPattern := jenkinsCmd.Flags().String("deployTagPattern",
"(?i)deploy", "deploy tag name")
+ CreatedDateAfter := jenkinsCmd.Flags().StringP("createdDateAfter", "a",
"", "collect data that are updated after specified time, ie
2006-05-06T07:08:09Z")
jenkinsCmd.Run = func(cmd *cobra.Command, args []string) {
runner.DirectRun(cmd, args, PluginEntry, map[string]interface{}{
"connectionId": *connectionId,
"jobFullName": jobFullName,
"deployTagPattern": *deployTagPattern,
+ "createdDateAfter": *CreatedDateAfter,
})
}
runner.RunCmd(jenkinsCmd)
diff --git a/plugins/jenkins/tasks/stage_collector.go
b/plugins/jenkins/tasks/stage_collector.go
index 2ee02978..093532bb 100644
--- a/plugins/jenkins/tasks/stage_collector.go
+++ b/plugins/jenkins/tasks/stage_collector.go
@@ -52,9 +52,13 @@ func CollectApiStages(taskCtx core.SubTaskContext)
errors.Error {
clauses := []dal.Clause{
dal.Select("tjb.number,tjb.full_name"),
dal.From("_tool_jenkins_builds as tjb"),
- dal.Where(`tjb.connection_id = ? and tjb.job_path = ? and
tjb.job_name = ? and tjb.class = ? `,
+ dal.Where(`tjb.connection_id = ? and tjb.job_path = ? and
tjb.job_name = ? and tjb.class = ?`,
data.Options.ConnectionId, data.Options.JobPath,
data.Options.JobName, "WorkflowRun"),
}
+ createdDateAfter := data.CreatedDateAfter
+ if createdDateAfter != nil {
+ clauses = append(clauses, dal.Where(`tjb.start_time >= ?`,
createdDateAfter.Format("2006/01/02 15:04")))
+ }
cursor, err := db.Cursor(clauses...)
if err != nil {
@@ -77,7 +81,6 @@ func CollectApiStages(taskCtx core.SubTaskContext)
errors.Error {
Table: RAW_STAGE_TABLE,
},
ApiClient: data.ApiClient,
- PageSize: 100,
Input: iterator,
UrlTemplate: fmt.Sprintf("%sjob/%s/{{ .Input.Number
}}/wfapi/describe", data.Options.JobPath, data.Options.JobName),
/*
diff --git a/plugins/jenkins/tasks/task_data.go
b/plugins/jenkins/tasks/task_data.go
index adbe44db..fa91abd1 100644
--- a/plugins/jenkins/tasks/task_data.go
+++ b/plugins/jenkins/tasks/task_data.go
@@ -37,17 +37,16 @@ type JenkinsOptions struct {
JobFullName string `json:"JobFullName"` //
"path1/path2/job name"
JobName string `json:"jobName"` // "job
name"
JobPath string `json:"jobPath"` //
"job/path1/job/path2"
- Since string
+ CreatedDateAfter string
Tasks []string `json:"tasks,omitempty"`
*models.JenkinsTransformationRule `mapstructure:"transformationRules"
json:"transformationRules"`
}
type JenkinsTaskData struct {
- Options *JenkinsOptions
- ApiClient *helper.ApiAsyncClient
- Connection *models.JenkinsConnection
- Since *time.Time
- Job *models.JenkinsJob
+ Options *JenkinsOptions
+ ApiClient *helper.ApiAsyncClient
+ Connection *models.JenkinsConnection
+ CreatedDateAfter *time.Time
}
func DecodeAndValidateTaskOptions(options map[string]interface{})
(*JenkinsOptions, errors.Error) {
diff --git a/plugins/jira/impl/impl.go b/plugins/jira/impl/impl.go
index 8d02d463..e6302078 100644
--- a/plugins/jira/impl/impl.go
+++ b/plugins/jira/impl/impl.go
@@ -175,7 +175,7 @@ func (plugin Jira) PrepareTaskData(taskCtx
core.TaskContext, options map[string]
if op.CreatedDateAfter != "" {
createdDateAfter, err = time.Parse("2006-01-02T15:04:05Z",
op.CreatedDateAfter)
if err != nil {
- return nil, errors.BadInput.Wrap(err, "invalid value
for `since`")
+ return nil, errors.BadInput.Wrap(err, "invalid value
for `createdDateAfter`")
}
}
if op.BoardId == 0 && op.ScopeId != "" {
diff --git a/plugins/jira/jira.go b/plugins/jira/jira.go
index dcb50c3d..95576e7f 100644
--- a/plugins/jira/jira.go
+++ b/plugins/jira/jira.go
@@ -32,7 +32,7 @@ func main() {
boardId := cmd.Flags().Uint64P("board", "b", 0, "jira board id")
_ = cmd.MarkFlagRequired("connection")
_ = cmd.MarkFlagRequired("board")
- CreatedDateAfter := cmd.Flags().StringP("created_date_after", "a", "",
"collect data that are updated after specified time, ie 2006-05-06T07:08:09Z")
+ CreatedDateAfter := cmd.Flags().StringP("createdDateAfter", "a", "",
"collect data that are updated after specified time, ie 2006-05-06T07:08:09Z")
cmd.Run = func(c *cobra.Command, args []string) {
runner.DirectRun(c, args, PluginEntry, map[string]interface{}{
"connectionId": *connectionId,
