This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch release-v0.19
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/release-v0.19 by this push:
new 1c597520a cherry-pick fix(zentao): process tasks' children when
collecting data instead of extracting data to v0.19 (#6118)
1c597520a is described below
commit 1c597520ac79e037e997c0093ee8c783c314d7ba
Author: Lynwee <[email protected]>
AuthorDate: Wed Sep 20 14:43:40 2023 +0800
cherry-pick fix(zentao): process tasks' children when collecting data
instead of extracting data to v0.19 (#6118)
* fix(zentao): process tasks' children when collecting data instead of
extracting data
* refactor(zentao): remove unused functions
---
backend/plugins/zentao/models/task.go | 5 +
backend/plugins/zentao/tasks/task_collector.go | 38 ++++++-
.../plugins/zentao/tasks/task_collector_test.go | 117 +++++++++++++++++++++
backend/plugins/zentao/tasks/task_extractor.go | 3 -
4 files changed, 158 insertions(+), 5 deletions(-)
diff --git a/backend/plugins/zentao/models/task.go
b/backend/plugins/zentao/models/task.go
index 4e7247c25..8bbc5fe38 100644
--- a/backend/plugins/zentao/models/task.go
+++ b/backend/plugins/zentao/models/task.go
@@ -18,6 +18,7 @@ limitations under the License.
package models
import (
+ "encoding/json"
"github.com/apache/incubator-devlake/core/models/common"
)
@@ -90,6 +91,10 @@ type ZentaoTaskRes struct {
Progress float64 `json:"progress"`
}
+func (zentaoTaskRes ZentaoTaskRes) ToJsonRawMessage() (json.RawMessage, error)
{
+ return json.Marshal(zentaoTaskRes)
+}
+
type ZentaoTask struct {
common.NoPKModel
ConnectionId uint64 `gorm:"primaryKey;type:BIGINT NOT NULL"`
diff --git a/backend/plugins/zentao/tasks/task_collector.go
b/backend/plugins/zentao/tasks/task_collector.go
index 0538834e1..ea8a3976a 100644
--- a/backend/plugins/zentao/tasks/task_collector.go
+++ b/backend/plugins/zentao/tasks/task_collector.go
@@ -73,7 +73,7 @@ func CollectTask(taskCtx plugin.SubTaskContext) errors.Error {
GetTotalPages: GetTotalPagesFromResponse,
ResponseParser: func(res *http.Response) ([]json.RawMessage,
errors.Error) {
var data struct {
- Task []json.RawMessage `json:"tasks"`
+ Tasks []models.ZentaoTaskRes `json:"tasks"`
}
err := api.UnmarshalResponse(res, &data)
if errors.Is(err, api.ErrEmptyResponse) {
@@ -82,7 +82,27 @@ func CollectTask(taskCtx plugin.SubTaskContext) errors.Error
{
if err != nil {
return nil, errors.Default.Wrap(err, "error
reading endpoint response by Zentao bug collector")
}
- return data.Task, nil
+
+ allTaskRecords := make(map[int64]models.ZentaoTaskRes)
+ for _, task := range data.Tasks {
+ // extract task's children
+ childTasks, err := extractChildrenWithDFS(task)
+ if err != nil {
+ return nil,
errors.Default.New(fmt.Sprintf("extract task: %v chidren err: %v", task, err))
+ }
+ for _, task := range childTasks {
+ allTaskRecords[task.Id] = task
+ }
+ }
+ var allTask []json.RawMessage
+ for _, task := range allTaskRecords {
+ taskRawJsonMessage, err :=
task.ToJsonRawMessage()
+ if err != nil {
+ return nil,
errors.Default.New(err.Error())
+ }
+ allTask = append(allTask, taskRawJsonMessage)
+ }
+ return allTask, nil
},
})
if err != nil {
@@ -91,6 +111,20 @@ func CollectTask(taskCtx plugin.SubTaskContext)
errors.Error {
return collector.Execute()
}
+// extractChildrenWithDFS return task's child tasks and itself.
+func extractChildrenWithDFS(task models.ZentaoTaskRes)
([]models.ZentaoTaskRes, error) {
+ var tasks []models.ZentaoTaskRes
+ for _, child := range task.Children {
+ childTasks, err := extractChildrenWithDFS(*child)
+ if err != nil {
+ return nil, err
+ }
+ tasks = append(tasks, childTasks...)
+ }
+ tasks = append(tasks, task)
+ return tasks, nil
+}
+
var CollectTaskMeta = plugin.SubTaskMeta{
Name: "collectTask",
EntryPoint: CollectTask,
diff --git a/backend/plugins/zentao/tasks/task_collector_test.go
b/backend/plugins/zentao/tasks/task_collector_test.go
new file mode 100644
index 000000000..06c77b437
--- /dev/null
+++ b/backend/plugins/zentao/tasks/task_collector_test.go
@@ -0,0 +1,117 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "github.com/apache/incubator-devlake/plugins/zentao/models"
+ "reflect"
+ "testing"
+)
+
+func Test_extractChildrenWithDFS(t *testing.T) {
+ type args struct {
+ task models.ZentaoTaskRes
+ }
+
+ id1 := models.ZentaoTaskRes{Id: 1}
+ id2 := models.ZentaoTaskRes{Id: 2}
+ id3 := models.ZentaoTaskRes{Id: 3}
+ id4 := models.ZentaoTaskRes{Id: 4}
+ id5 := models.ZentaoTaskRes{Id: 5}
+ id6 := models.ZentaoTaskRes{Id: 6}
+ id7 := models.ZentaoTaskRes{Id: 7}
+ id1WithChildren3And4 := models.ZentaoTaskRes{
+ Id: 1,
+ Children: []*models.ZentaoTaskRes{&id3, &id4},
+ }
+ id1WithChildren7 := models.ZentaoTaskRes{
+ Id: 1,
+ Children: []*models.ZentaoTaskRes{&id7},
+ }
+
+ id2WithChildren3And4And5 := models.ZentaoTaskRes{
+ Id: 2,
+ Children: []*models.ZentaoTaskRes{&id3, &id4, &id5},
+ }
+
+ id3WithChildren4And6AndId1WithChildren3And4 := models.ZentaoTaskRes{
+ Id: 3,
+ Children: []*models.ZentaoTaskRes{&id4, &id6,
&id1WithChildren7},
+ }
+
+ tests := []struct {
+ name string
+ args args
+ want []models.ZentaoTaskRes
+ wantErr bool
+ }{
+ {
+ name: "0",
+ args: args{task: id2},
+ want: []models.ZentaoTaskRes{id2},
+ wantErr: false,
+ },
+ {
+ name: "1",
+ args: args{task: id1},
+ want: []models.ZentaoTaskRes{id1},
+ wantErr: false,
+ },
+ {
+ name: "2",
+ args: args{task: id1WithChildren3And4},
+ want: []models.ZentaoTaskRes{id1WithChildren3And4,
id3, id4},
+ wantErr: false,
+ },
+ {
+ name: "3",
+ args: args{task: id2WithChildren3And4And5},
+ want:
[]models.ZentaoTaskRes{id2WithChildren3And4And5, id3, id4, id5},
+ wantErr: false,
+ },
+ {
+ name: "4",
+ args: args{task:
id3WithChildren4And6AndId1WithChildren3And4},
+ want:
[]models.ZentaoTaskRes{id3WithChildren4And6AndId1WithChildren3And4, id7, id4,
id6, id1WithChildren7},
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got, err := extractChildrenWithDFS(tt.args.task)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("extractChildrenWithDFS() error = %v,
wantErr %v", err, tt.wantErr)
+ return
+ }
+ for _, v := range got {
+ var find bool
+ for _, wantv := range tt.want {
+ if v.Id == wantv.Id {
+ find = true
+ if !reflect.DeepEqual(v, wantv)
{
+ t.Errorf("got: %v,
want: %v", v, wantv)
+ }
+ }
+ }
+ if !find {
+ t.Errorf("not found: %v", v)
+ }
+ }
+ })
+ }
+}
diff --git a/backend/plugins/zentao/tasks/task_extractor.go
b/backend/plugins/zentao/tasks/task_extractor.go
index 8a1a2059e..d18701c29 100644
--- a/backend/plugins/zentao/tasks/task_extractor.go
+++ b/backend/plugins/zentao/tasks/task_extractor.go
@@ -168,7 +168,4 @@ func (c *taskExtractor) toZentaoTasks(accountCache
*AccountCache, res *models.Ze
}, task.Status)
}
*tasks = append(*tasks, task)
- for _, child := range res.Children {
- c.toZentaoTasks(accountCache, child, url, tasks)
- }
}
