This is an automated email from the ASF dual-hosted git repository.
zhangliang2022 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 2d41d1cc9 feat: zentao bug commits collector (#5334)
2d41d1cc9 is described below
commit 2d41d1cc96fcb7991f980c281e99791bd4a1fa30
Author: abeizn <[email protected]>
AuthorDate: Wed May 31 21:35:59 2023 +0800
feat: zentao bug commits collector (#5334)
---
.../e2e/snapshot_tables/_tool_zentao_bugs.csv | 10 +--
.../e2e/snapshot_tables/_tool_zentao_stories.csv | 16 ++--
.../e2e/snapshot_tables/_tool_zentao_tasks.csv | 8 +-
backend/plugins/zentao/impl/impl.go | 1 +
.../register.go => archived/bug_commits.go} | 22 ++---
backend/plugins/zentao/models/bug.go | 3 -
.../register.go => bug_commits.go} | 29 ++++---
..._to_issues.go => 20230531_add_issue_commits.go} | 43 ++--------
.../zentao/models/migrationscripts/register.go | 2 +-
backend/plugins/zentao/models/story.go | 3 -
backend/plugins/zentao/models/task.go | 13 ++-
backend/plugins/zentao/tasks/account_collector.go | 2 +-
backend/plugins/zentao/tasks/bug_collector.go | 2 +-
...{task_collector.go => bug_commits_collector.go} | 94 +++++++++++++---------
backend/plugins/zentao/tasks/bug_extractor.go | 1 -
.../plugins/zentao/tasks/department_collector.go | 2 +-
backend/plugins/zentao/tasks/story_collector.go | 2 +-
backend/plugins/zentao/tasks/story_extractor.go | 1 -
backend/plugins/zentao/tasks/task_collector.go | 2 +-
backend/plugins/zentao/tasks/task_data.go | 2 +
backend/plugins/zentao/tasks/task_extractor.go | 1 -
21 files changed, 128 insertions(+), 131 deletions(-)
diff --git a/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_bugs.csv
b/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_bugs.csv
index 17d8b8201..1f49d1a4c 100644
--- a/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_bugs.csv
+++ b/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_bugs.csv
@@ -1,13 +1,13 @@
-connection_id,id,project,product,injection,identify,branch,module,execution,plan,story,story_version,task,to_task,to_story,title,keywords,severity,pri,type,os,browser,hardware,found,steps,status,sub_status,color,confirmed,activated_count,activated_date,feedback_by,notify_email,opened_by_id,opened_by_name,opened_date,opened_build,assigned_to_id,assigned_to_name,assigned_date,deadline,resolved_by_id,resolution,resolved_build,resolved_date,closed_by_id,closed_date,duplicate_bug,link_bug,fee
[...]
+connection_id,id,project,product,injection,identify,branch,module,execution,plan,story,story_version,task,to_task,to_story,title,keywords,severity,pri,type,os,browser,hardware,found,steps,status,sub_status,color,confirmed,activated_count,activated_date,feedback_by,notify_email,opened_by_id,opened_by_name,opened_date,opened_build,assigned_to_id,assigned_to_name,assigned_date,deadline,resolved_by_id,resolution,resolved_build,resolved_date,closed_by_id,closed_date,duplicate_bug,link_bug,fee
[...]
1,1,7,3,0,0,0,8,1,0,1,1,1,0,0,首页页面问题,,3,1,codeerror,,,,,"<p>[步骤]进入首页</p>
<p>[结果]出现乱码 </p>
-<p>[期望]正常显示</p>",active,,,0,0,,,,7,测试甲,2012-06-05T02:56:11.000+00:00,主干,4,开发甲,2012-06-05T02:56:11.000+00:00,,0,,,,0,,0,,0,0,0,0,,,,,,,0,0,2021-04-28T03:09:08.000+00:00,0,1,3,0,激活,normal,
+<p>[期望]正常显示</p>",active,,,0,0,,,,7,测试甲,2012-06-05T02:56:11.000+00:00,主干,4,开发甲,2012-06-05T02:56:11.000+00:00,,0,,,,0,,0,,0,0,0,0,,,,,,,0,0,2021-04-28T03:09:08.000+00:00,0,1,3,0,激活,normal
1,2,7,3,0,0,0,9,1,1,2,1,15,0,0,新闻中心页面问题,hh,3,2,codeerror,",windows",",chrome",,,"<p>[步骤]进入新闻中心</p>
<p>[结果]页面出现乱码</p>
-<p>[期望]正常显示rew</p>",delay,,,1,1,2022-10-05T04:16:44.000+00:00,,[email protected],7,测试甲,2012-06-05T02:57:11.000+00:00,主干,0,,2022-10-05T04:19:22.000+00:00,2022-10-06,0,,,,0,,0,,0,0,0,0,,,,,,,1,1,2022-10-05T04:19:22.000+00:00,0,2,3,0,过期Bug,normal,
+<p>[期望]正常显示rew</p>",delay,,,1,1,2022-10-05T04:16:44.000+00:00,,[email protected],7,测试甲,2012-06-05T02:57:11.000+00:00,主干,0,,2022-10-05T04:19:22.000+00:00,2022-10-06,0,,,,0,,0,,0,0,0,0,,,,,,,1,1,2022-10-05T04:19:22.000+00:00,0,2,3,0,过期Bug,normal
1,3,7,3,0,0,0,10,1,0,3,2,6,0,0,成果展示页面问题,,3,1,codeerror,,,,,"<p>[步骤]进入成果展示 </p>
<p>[结果]乱码</p>
-<p>[期望]正常显示</p>",active,,,0,0,,,,8,测试乙,2012-06-05T02:58:22.000+00:00,主干,4,开发甲,2012-06-05T02:58:22.000+00:00,,0,,,,0,,0,,0,0,0,0,,,,,,,0,0,2021-04-28T03:09:08.000+00:00,0,1,3,0,激活,normal,
+<p>[期望]正常显示</p>",active,,,0,0,,,,8,测试乙,2012-06-05T02:58:22.000+00:00,主干,4,开发甲,2012-06-05T02:58:22.000+00:00,,0,,,,0,,0,,0,0,0,0,,,,,,,0,0,2021-04-28T03:09:08.000+00:00,0,1,3,0,激活,normal
1,4,7,3,0,0,0,11,1,0,4,1,9,0,0,售后服务页面问题,,3,1,codeerror,,,,,"<p>[步骤]进入售后服务</p>
<p>[结果]乱码</p>
-<p>[期望]正常显示</p>",resolved,,,1,0,,,,9,测试丙,2012-06-05T03:00:19.000+00:00,主干,9,测试丙,2022-10-05T04:10:08.000+00:00,,1,fixed,主干,2022-10-05T04:09:59.000+00:00,0,,0,,0,0,0,0,,,,,,,0,1,2022-10-05T04:10:08.000+00:00,0,1,3,0,已解决,normal,
+<p>[期望]正常显示</p>",resolved,,,1,0,,,,9,测试丙,2012-06-05T03:00:19.000+00:00,主干,9,测试丙,2022-10-05T04:10:08.000+00:00,,1,fixed,主干,2022-10-05T04:09:59.000+00:00,0,,0,,0,0,0,0,,,,,,,0,1,2022-10-05T04:10:08.000+00:00,0,1,3,0,已解决,normal
diff --git
a/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_stories.csv
b/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_stories.csv
index a556afc57..2ece4f19a 100644
--- a/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_stories.csv
+++ b/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_stories.csv
@@ -1,8 +1,8 @@
-connection_id,id,product,branch,version,order_in,vision,parent,module,plan,source,source_note,from_bug,feedback,title,keywords,type,category,pri,estimate,status,sub_status,color,stage,lib,from_story,from_version,opened_by_id,opened_by_name,opened_date,assigned_to_id,assigned_to_name,assigned_date,approved_date,last_edited_id,last_edited_date,changed_date,reviewed_by_id,reviewed_date,closed_id,closed_date,closed_reason,activated_date,to_bug,child_stories,link_stories,link_requirements,dup
[...]
-1,1,3,0,1,0,rnd,0,1,1,po,,0,0,首页设计和开发,,story,feature,1,1,active,,,developing,0,0,1,2,产品经理,2012-06-05T02:09:49.000+00:00,2,产品经理,,,2,2012-06-05T02:25:19.000+00:00,,0,2012-06-04T16:00:00.000+00:00,0,,,,0,,,,0,0,,,0,0,1,1.0版本
,
-1,2,3,0,1,0,rnd,0,2,1,po,,0,0,新闻中心的设计和开发。,,story,feature,1,1,active,,,projected,0,0,1,2,产品经理,2012-06-05T02:16:37.000+00:00,2,产品经理,2012-06-05T02:16:37.000+00:00,,2,2012-06-05T02:25:33.000+00:00,,0,2012-06-04T16:00:00.000+00:00,0,,,,0,,,,0,0,,,0,0,1,1.0版本
,
-1,3,3,0,2,0,rnd,0,3,1,po,,0,0,成果展示的设计和开发,,story,feature,1,0,active,,,developing,0,0,1,2,产品经理,2012-06-05T02:18:10.000+00:00,2,产品经理,2012-06-05T02:18:10.000+00:00,,2,2012-06-05T02:25:38.000+00:00,,0,2012-06-04T16:00:00.000+00:00,0,,,,0,,,,0,0,,,0,0,1,1.0版本
,
-1,4,3,0,1,0,rnd,0,4,1,po,,0,0,售后服务的设计和开发,,story,feature,1,1,active,,,developed,0,0,1,2,产品经理,2012-06-05T02:20:16.000+00:00,2,产品经理,2012-06-05T02:20:16.000+00:00,,2,2012-06-05T02:25:42.000+00:00,,0,2012-06-04T16:00:00.000+00:00,0,,,,0,,,,0,0,,,0,0,1,1.0版本
,
-1,5,3,0,1,0,rnd,0,5,1,po,,0,0,诚聘英才的设计和开发,,story,feature,1,1,reviewing,,,planned,0,0,1,2,产品经理,2012-06-05T02:21:39.000+00:00,2,产品经理,2012-06-05T02:21:39.000+00:00,,0,,,0,,0,,,,0,,,,0,0,,,0,0,1,1.0版本
,
-1,6,3,0,1,0,rnd,0,6,1,po,,0,0,合作洽谈的设计和开发,,story,feature,1,1,reviewing,,,planned,0,0,1,2,产品经理,2012-06-05T02:23:11.000+00:00,2,产品经理,2012-06-05T02:23:11.000+00:00,,0,,,0,,0,,,,0,,,,0,0,,,0,0,1,1.0版本
,
-1,7,3,0,1,0,rnd,0,7,1,po,,0,0,关于我们的设计和开发,,story,feature,1,1,reviewing,,,planned,0,0,1,2,产品经理,2012-06-05T02:24:19.000+00:00,2,产品经理,2012-06-05T02:24:19.000+00:00,,0,,,0,,0,,,,0,,,,0,0,,,0,0,1,1.0版本
,
+connection_id,id,product,branch,version,order_in,vision,parent,module,plan,source,source_note,from_bug,feedback,title,keywords,type,category,pri,estimate,status,sub_status,color,stage,lib,from_story,from_version,opened_by_id,opened_by_name,opened_date,assigned_to_id,assigned_to_name,assigned_date,approved_date,last_edited_id,last_edited_date,changed_date,reviewed_by_id,reviewed_date,closed_id,closed_date,closed_reason,activated_date,to_bug,child_stories,link_stories,link_requirements,dup
[...]
+1,1,3,0,1,0,rnd,0,1,1,po,,0,0,首页设计和开发,,story,feature,1,1,active,,,developing,0,0,1,2,产品经理,2012-06-05T02:09:49.000+00:00,2,产品经理,,,2,2012-06-05T02:25:19.000+00:00,,0,2012-06-04T16:00:00.000+00:00,0,,,,0,,,,0,0,,,0,0,1,1.0版本
+1,2,3,0,1,0,rnd,0,2,1,po,,0,0,新闻中心的设计和开发。,,story,feature,1,1,active,,,projected,0,0,1,2,产品经理,2012-06-05T02:16:37.000+00:00,2,产品经理,2012-06-05T02:16:37.000+00:00,,2,2012-06-05T02:25:33.000+00:00,,0,2012-06-04T16:00:00.000+00:00,0,,,,0,,,,0,0,,,0,0,1,1.0版本
+1,3,3,0,2,0,rnd,0,3,1,po,,0,0,成果展示的设计和开发,,story,feature,1,0,active,,,developing,0,0,1,2,产品经理,2012-06-05T02:18:10.000+00:00,2,产品经理,2012-06-05T02:18:10.000+00:00,,2,2012-06-05T02:25:38.000+00:00,,0,2012-06-04T16:00:00.000+00:00,0,,,,0,,,,0,0,,,0,0,1,1.0版本
+1,4,3,0,1,0,rnd,0,4,1,po,,0,0,售后服务的设计和开发,,story,feature,1,1,active,,,developed,0,0,1,2,产品经理,2012-06-05T02:20:16.000+00:00,2,产品经理,2012-06-05T02:20:16.000+00:00,,2,2012-06-05T02:25:42.000+00:00,,0,2012-06-04T16:00:00.000+00:00,0,,,,0,,,,0,0,,,0,0,1,1.0版本
+1,5,3,0,1,0,rnd,0,5,1,po,,0,0,诚聘英才的设计和开发,,story,feature,1,1,reviewing,,,planned,0,0,1,2,产品经理,2012-06-05T02:21:39.000+00:00,2,产品经理,2012-06-05T02:21:39.000+00:00,,0,,,0,,0,,,,0,,,,0,0,,,0,0,1,1.0版本
+1,6,3,0,1,0,rnd,0,6,1,po,,0,0,合作洽谈的设计和开发,,story,feature,1,1,reviewing,,,planned,0,0,1,2,产品经理,2012-06-05T02:23:11.000+00:00,2,产品经理,2012-06-05T02:23:11.000+00:00,,0,,,0,,0,,,,0,,,,0,0,,,0,0,1,1.0版本
+1,7,3,0,1,0,rnd,0,7,1,po,,0,0,关于我们的设计和开发,,story,feature,1,1,reviewing,,,planned,0,0,1,2,产品经理,2012-06-05T02:24:19.000+00:00,2,产品经理,2012-06-05T02:24:19.000+00:00,,0,,,0,,0,,,,0,,,,0,0,,,0,0,1,1.0版本
diff --git a/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_tasks.csv
b/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_tasks.csv
index 5ba09d842..40337d85f 100644
--- a/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_tasks.csv
+++ b/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_tasks.csv
@@ -1,4 +1,4 @@
-connection_id,id,project,parent,execution,module,design,story,story_version,design_version,from_bug,feedback,from_issue,name,type,mode,pri,estimate,consumed,deadline,status,sub_status,color,description,version,opened_by_id,opened_by_name,opened_date,assigned_to_id,assigned_to_name,assigned_date,est_started,real_started,finished_id,finished_date,finished_list,canceled_id,canceled_date,closed_by_id,closed_date,plan_duration,real_duration,closed_reason,last_edited_id,last_edited_date,activa
[...]
-1,1,1,0,1,0,0,0,1,0,0,0,0,任务名称,devel,,3,0,0,2022-10-01,wait,,,任务描述<span>
</span><br /><div><br
/></div>,1,1,devlake,2022-09-19T01:50:37.000+00:00,5,开发乙,2022-09-19T01:50:37.000+00:00,2022-09-20,,0,,,0,,0,,0,0,,0,,,0,0,0,,,,,0,rnd,0,,0,0,,开发乙,3,0,21.11,
-1,2,1,0,1,0,0,0,1,0,0,0,0,任务名称,devel,,3,12.1,2.1,2022-10-01,wait,,,任务描述<span>
</span><br /><div><br
/></div>,1,1,devlake,2022-09-19T01:50:37.000+00:00,5,开发乙,2022-09-19T01:50:37.000+00:00,2022-09-20,,0,,,0,,0,,0,0,,0,,,0,0,0,,,,,0,rnd,0,,0,0,,开发乙,3,0,3,
-1,3,1,-1,1,0,0,0,1,0,0,0,0,任务名称,devel,,3,11.2,0,2022-10-01,wait,,,任务描述<span>
</span><br /><div><br
/></div>,1,1,devlake,2022-09-19T01:50:37.000+00:00,5,开发乙,2022-09-19T01:50:37.000+00:00,2022-09-20,,0,,,0,,0,,0,0,,0,,,0,0,0,,,,,0,rnd,0,,0,0,,开发乙,3,0,43.22121,
+connection_id,id,project,parent,execution,module,design,story,story_version,design_version,from_bug,feedback,from_issue,name,type,mode,pri,estimate,consumed,deadline,status,sub_status,color,description,version,opened_by_id,opened_by_name,opened_date,assigned_to_id,assigned_to_name,assigned_date,est_started,real_started,finished_id,finished_date,finished_list,canceled_id,canceled_date,closed_by_id,closed_date,plan_duration,real_duration,closed_reason,last_edited_id,last_edited_date,activa
[...]
+1,1,1,0,1,0,0,0,1,0,0,0,0,任务名称,devel,,3,0,0,2022-10-01,wait,,,任务描述<span>
</span><br /><div><br
/></div>,1,1,devlake,2022-09-19T01:50:37.000+00:00,5,开发乙,2022-09-19T01:50:37.000+00:00,2022-09-20,,0,,,0,,0,,0,0,,0,,,0,0,0,,,,,0,rnd,0,,0,0,,开发乙,3,0,21.11
+1,2,1,0,1,0,0,0,1,0,0,0,0,任务名称,devel,,3,12.1,2.1,2022-10-01,wait,,,任务描述<span>
</span><br /><div><br
/></div>,1,1,devlake,2022-09-19T01:50:37.000+00:00,5,开发乙,2022-09-19T01:50:37.000+00:00,2022-09-20,,0,,,0,,0,,0,0,,0,,,0,0,0,,,,,0,rnd,0,,0,0,,开发乙,3,0,3
+1,3,1,-1,1,0,0,0,1,0,0,0,0,任务名称,devel,,3,11.2,0,2022-10-01,wait,,,任务描述<span>
</span><br /><div><br
/></div>,1,1,devlake,2022-09-19T01:50:37.000+00:00,5,开发乙,2022-09-19T01:50:37.000+00:00,2022-09-20,,0,,,0,,0,,0,0,,0,,,0,0,0,,,,,0,rnd,0,,0,0,,开发乙,3,0,43.22121
diff --git a/backend/plugins/zentao/impl/impl.go
b/backend/plugins/zentao/impl/impl.go
index 9433d0919..1a725aebb 100644
--- a/backend/plugins/zentao/impl/impl.go
+++ b/backend/plugins/zentao/impl/impl.go
@@ -76,6 +76,7 @@ func (p Zentao) SubTaskMetas() []plugin.SubTaskMeta {
tasks.CollectDepartmentMeta,
tasks.ExtractDepartmentMeta,
tasks.ConvertDepartmentMeta,
+ tasks.CollectBugCommitsMeta,
}
}
diff --git a/backend/plugins/zentao/models/migrationscripts/register.go
b/backend/plugins/zentao/models/archived/bug_commits.go
similarity index 57%
copy from backend/plugins/zentao/models/migrationscripts/register.go
copy to backend/plugins/zentao/models/archived/bug_commits.go
index 8eedfe788..21a2ad090 100644
--- a/backend/plugins/zentao/models/migrationscripts/register.go
+++ b/backend/plugins/zentao/models/archived/bug_commits.go
@@ -15,17 +15,21 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package migrationscripts
+package archived
import (
- "github.com/apache/incubator-devlake/core/plugin"
+
"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
)
-// All return all the migration scripts
-func All() []plugin.MigrationScript {
- return []plugin.MigrationScript{
- new(addInitTables),
- new(addInitChangelogTables),
- new(addActionsToIssues),
- }
+type ZentaoBugCommits struct {
+ archived.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey;type:BIGINT NOT NULL"`
+ ID int64 `json:"id" gorm:"primaryKey;type:BIGINT NOT
NULL;autoIncrement:false"`
+ Project int64 `json:"project"`
+ Product int64 `json:"product"`
+ Actions []string `gorm:"type:json;serializer:json" json:"actions"
mapstructure:"actions"`
+}
+
+func (ZentaoBugCommits) TableName() string {
+ return "_tool_zentao_bug_commits"
}
diff --git a/backend/plugins/zentao/models/bug.go
b/backend/plugins/zentao/models/bug.go
index e728152ec..664910121 100644
--- a/backend/plugins/zentao/models/bug.go
+++ b/backend/plugins/zentao/models/bug.go
@@ -20,7 +20,6 @@ package models
import (
"github.com/apache/incubator-devlake/core/models/common"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "gorm.io/datatypes"
)
type ZentaoBugRes struct {
@@ -89,7 +88,6 @@ type ZentaoBugRes struct {
Needconfirm bool `json:"needconfirm"`
StatusName string `json:"statusName"`
ProductStatus string `json:"productStatus"`
- Actions datatypes.JSON `json:"actions"`
}
type ZentaoBug struct {
@@ -162,7 +160,6 @@ type ZentaoBug struct {
Needconfirm bool `json:"needconfirm"`
StatusName string `json:"statusName"`
ProductStatus string `json:"productStatus"`
- Actions datatypes.JSON `json:"actions"`
}
func (ZentaoBug) TableName() string {
diff --git a/backend/plugins/zentao/models/migrationscripts/register.go
b/backend/plugins/zentao/models/bug_commits.go
similarity index 50%
copy from backend/plugins/zentao/models/migrationscripts/register.go
copy to backend/plugins/zentao/models/bug_commits.go
index 8eedfe788..918b8fe06 100644
--- a/backend/plugins/zentao/models/migrationscripts/register.go
+++ b/backend/plugins/zentao/models/bug_commits.go
@@ -15,17 +15,28 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package migrationscripts
+package models
import (
- "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/models/common"
)
-// All return all the migration scripts
-func All() []plugin.MigrationScript {
- return []plugin.MigrationScript{
- new(addInitTables),
- new(addInitChangelogTables),
- new(addActionsToIssues),
- }
+type ZentaoBugCommitsRes struct {
+ ID int64 `json:"id"`
+ Project int64 `json:"project"`
+ Product int64 `json:"product"`
+ Actions []string `gorm:"type:json;serializer:json" json:"actions"
mapstructure:"actions"`
+}
+
+type ZentaoBugCommits struct {
+ common.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey;type:BIGINT NOT NULL"`
+ ID int64 `json:"id" gorm:"primaryKey;type:BIGINT NOT
NULL;autoIncrement:false"`
+ Project int64 `json:"project"`
+ Product int64 `json:"product"`
+ Actions []string `gorm:"type:json;serializer:json" json:"actions"
mapstructure:"actions"`
+}
+
+func (ZentaoBugCommits) TableName() string {
+ return "_tool_zentao_bug_commits"
}
diff --git
a/backend/plugins/zentao/models/migrationscripts/20230531_add_actions_to_issues.go
b/backend/plugins/zentao/models/migrationscripts/20230531_add_issue_commits.go
similarity index 54%
rename from
backend/plugins/zentao/models/migrationscripts/20230531_add_actions_to_issues.go
rename to
backend/plugins/zentao/models/migrationscripts/20230531_add_issue_commits.go
index a9d8626fd..75b6cb606 100644
---
a/backend/plugins/zentao/models/migrationscripts/20230531_add_actions_to_issues.go
+++
b/backend/plugins/zentao/models/migrationscripts/20230531_add_issue_commits.go
@@ -20,53 +20,24 @@ package migrationscripts
import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/migrationhelper"
- "gorm.io/datatypes"
+ "github.com/apache/incubator-devlake/plugins/zentao/models/archived"
)
-var _ plugin.MigrationScript = (*addActionsToIssues)(nil)
+type addIssueCommitsTables struct{}
-type Bug20230531 struct {
- Actions datatypes.JSON `json:"actions"`
-}
-
-func (Bug20230531) TableName() string {
- return "_tool_zentao_bugs"
-}
-
-type Task20230531 struct {
- Actions datatypes.JSON `json:"actions"`
-}
-
-func (Task20230531) TableName() string {
- return "_tool_zentao_tasks"
-}
-
-type Story20230531 struct {
- Actions datatypes.JSON `json:"actions"`
-}
-
-func (Story20230531) TableName() string {
- return "_tool_zentao_stories"
-}
-
-type addActionsToIssues struct{}
-
-func (script *addActionsToIssues) Up(basicRes context.BasicRes) errors.Error {
+func (*addIssueCommitsTables) Up(basicRes context.BasicRes) errors.Error {
return migrationhelper.AutoMigrateTables(
basicRes,
- &Bug20230531{},
- &Task20230531{},
- &Story20230531{},
+ &archived.ZentaoBugCommits{},
)
}
-func (*addActionsToIssues) Version() uint64 {
+func (*addIssueCommitsTables) Version() uint64 {
return 20230531000001
}
-func (*addActionsToIssues) Name() string {
- return "add actions to bugs, stories and tasks"
+func (*addIssueCommitsTables) Name() string {
+ return "zentao add issue commits tables"
}
diff --git a/backend/plugins/zentao/models/migrationscripts/register.go
b/backend/plugins/zentao/models/migrationscripts/register.go
index 8eedfe788..e71dac9f4 100644
--- a/backend/plugins/zentao/models/migrationscripts/register.go
+++ b/backend/plugins/zentao/models/migrationscripts/register.go
@@ -26,6 +26,6 @@ func All() []plugin.MigrationScript {
return []plugin.MigrationScript{
new(addInitTables),
new(addInitChangelogTables),
- new(addActionsToIssues),
+ new(addIssueCommitsTables),
}
}
diff --git a/backend/plugins/zentao/models/story.go
b/backend/plugins/zentao/models/story.go
index e10caf88e..274fec596 100644
--- a/backend/plugins/zentao/models/story.go
+++ b/backend/plugins/zentao/models/story.go
@@ -20,7 +20,6 @@ package models
import (
"github.com/apache/incubator-devlake/core/models/common"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "gorm.io/datatypes"
)
type ZentaoStoryRes struct {
@@ -78,7 +77,6 @@ type ZentaoStoryRes struct {
PriOrder string `json:"priOrder"`
PlanTitle string `json:"planTitle"`
ProductStatus string `json:"productStatus"`
- Actions datatypes.JSON `json:"actions"`
}
type ZentaoStory struct {
@@ -139,7 +137,6 @@ type ZentaoStory struct {
Deleted bool `json:"deleted"`
PriOrder string `json:"priOrder"`
PlanTitle string `json:"planTitle"`
- Actions datatypes.JSON `json:"actions"`
}
func (ZentaoStory) TableName() string {
diff --git a/backend/plugins/zentao/models/task.go
b/backend/plugins/zentao/models/task.go
index b0a637bee..43a20be93 100644
--- a/backend/plugins/zentao/models/task.go
+++ b/backend/plugins/zentao/models/task.go
@@ -20,7 +20,6 @@ package models
import (
"github.com/apache/incubator-devlake/core/models/common"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "gorm.io/datatypes"
)
type ZentaoTaskRes struct {
@@ -85,12 +84,11 @@ type ZentaoTaskRes struct {
} `json:"latestStoryVersion"`
StoryStatus interface {
} `json:"storyStatus"`
- AssignedToRealName string `json:"assignedToRealName"`
- PriOrder string `json:"priOrder"`
- Delay int `json:"delay"`
- NeedConfirm bool `json:"needConfirm"`
- Progress float64 `json:"progress"`
- Actions datatypes.JSON `json:"actions"`
+ AssignedToRealName string `json:"assignedToRealName"`
+ PriOrder string `json:"priOrder"`
+ Delay int `json:"delay"`
+ NeedConfirm bool `json:"needConfirm"`
+ Progress float64 `json:"progress"`
}
type ZentaoTask struct {
@@ -160,7 +158,6 @@ type ZentaoTask struct {
PriOrder string `json:"priOrder"`
NeedConfirm bool `json:"needConfirm"`
Progress float64 `json:"progress"`
- Actions datatypes.JSON `json:"actions"`
}
func (ZentaoTask) TableName() string {
diff --git a/backend/plugins/zentao/tasks/account_collector.go
b/backend/plugins/zentao/tasks/account_collector.go
index 7294f449f..dd027cfa1 100644
--- a/backend/plugins/zentao/tasks/account_collector.go
+++ b/backend/plugins/zentao/tasks/account_collector.go
@@ -72,7 +72,7 @@ func CollectAccount(taskCtx plugin.SubTaskContext)
errors.Error {
}
var CollectAccountMeta = plugin.SubTaskMeta{
- Name: "CollectAccount",
+ Name: "collectAccount",
EntryPoint: CollectAccount,
EnabledByDefault: true,
Description: "Collect Account data from Zentao api",
diff --git a/backend/plugins/zentao/tasks/bug_collector.go
b/backend/plugins/zentao/tasks/bug_collector.go
index 16ffda670..7692cb596 100644
--- a/backend/plugins/zentao/tasks/bug_collector.go
+++ b/backend/plugins/zentao/tasks/bug_collector.go
@@ -79,7 +79,7 @@ func CollectBug(taskCtx plugin.SubTaskContext) errors.Error {
}
var CollectBugMeta = plugin.SubTaskMeta{
- Name: "CollectBug",
+ Name: "collectBug",
EntryPoint: CollectBug,
EnabledByDefault: true,
Description: "Collect Bug data from Zentao api",
diff --git a/backend/plugins/zentao/tasks/task_collector.go
b/backend/plugins/zentao/tasks/bug_commits_collector.go
similarity index 52%
copy from backend/plugins/zentao/tasks/task_collector.go
copy to backend/plugins/zentao/tasks/bug_commits_collector.go
index 427569c11..639885f29 100644
--- a/backend/plugins/zentao/tasks/task_collector.go
+++ b/backend/plugins/zentao/tasks/bug_commits_collector.go
@@ -20,6 +20,7 @@ package tasks
import (
"encoding/json"
"fmt"
+ "io"
"net/http"
"net/url"
"reflect"
@@ -31,79 +32,98 @@ import (
"github.com/apache/incubator-devlake/plugins/zentao/models"
)
-const RAW_TASK_TABLE = "zentao_api_tasks"
+const RAW_BUG_COMMITS_TABLE = "zentao_api_bug_commits"
-type ExecuteInput struct {
- Id int64
-}
+var _ plugin.SubTaskEntryPoint = CollectBugCommits
-var _ plugin.SubTaskEntryPoint = CollectTask
+var CollectBugCommitsMeta = plugin.SubTaskMeta{
+ Name: "collectBugCommits",
+ EntryPoint: CollectBugCommits,
+ EnabledByDefault: true,
+ Description: "Collect Bug Commits data from Zentao api",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
+}
-func CollectTask(taskCtx plugin.SubTaskContext) errors.Error {
+func CollectBugCommits(taskCtx plugin.SubTaskContext) errors.Error {
+ db := taskCtx.GetDal()
data := taskCtx.GetData().(*ZentaoTaskData)
- // this collect only work for project
- if data.Options.ProjectId == 0 {
- return nil
+ // state manager
+ collectorWithState, err :=
api.NewStatefulApiCollector(api.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: ZentaoApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ ProductId: data.Options.ProductId,
+ },
+ Table: RAW_BUG_COMMITS_TABLE,
+ }, data.TimeAfter)
+ if err != nil {
+ return err
}
- cursor, err := taskCtx.GetDal().Cursor(
- dal.Select(`id`),
- dal.From(&models.ZentaoExecution{}),
- dal.Where(`project_id = ? and connection_id = ?`,
data.Options.ProjectId, data.Options.ConnectionId),
- )
+ // load bugs id from db
+ clauses := []dal.Clause{
+ dal.Select("id"),
+ dal.From(&models.ZentaoBug{}),
+ dal.Where(
+ "product = ? AND connection_id = ?",
+ data.Options.ProductId, data.Options.ConnectionId,
+ ),
+ }
+ // incremental collection
+ incremental := collectorWithState.IsIncremental()
+ if incremental {
+ clauses = append(
+ clauses,
+ dal.Where("updated_at > ?",
collectorWithState.LatestState.LatestSuccessStart),
+ )
+ }
+ cursor, err := db.Cursor(clauses...)
if err != nil {
return err
}
- defer cursor.Close()
-
- iterator, err := api.NewDalCursorIterator(taskCtx.GetDal(), cursor,
reflect.TypeOf(ExecuteInput{}))
+ iterator, err := api.NewDalCursorIterator(db, cursor,
reflect.TypeOf(SimpleZentaoBug{}))
if err != nil {
return err
}
-
- collector, err := api.NewApiCollector(api.ApiCollectorArgs{
+ // collect bug commits
+ err = collectorWithState.InitCollector(api.ApiCollectorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
Ctx: taskCtx,
Params: ZentaoApiParams{
ConnectionId: data.Options.ConnectionId,
ProductId: data.Options.ProductId,
- ProjectId: data.Options.ProjectId,
},
- Table: RAW_TASK_TABLE,
+ Table: RAW_BUG_COMMITS_TABLE,
},
- Input: iterator,
ApiClient: data.ApiClient,
PageSize: 100,
- UrlTemplate: "/executions/{{ .Input.Id }}/tasks",
+ Input: iterator,
+ Incremental: incremental,
+ UrlTemplate: "bugs/{{ .Input.ID }}",
Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
- query.Set("limit", fmt.Sprintf("%v",
reqData.Pager.Size))
+ query.Set("per_page", fmt.Sprintf("%v",
reqData.Pager.Size))
return query, nil
},
GetTotalPages: GetTotalPagesFromResponse,
ResponseParser: func(res *http.Response) ([]json.RawMessage,
errors.Error) {
- var data struct {
- Task []json.RawMessage `json:"tasks"`
- }
- err := api.UnmarshalResponse(res, &data)
+ body, err := io.ReadAll(res.Body)
if err != nil {
- return nil, errors.Default.Wrap(err, "error
reading endpoint response by Zentao bug collector")
+ return nil, errors.Convert(err)
}
- return data.Task, nil
+ res.Body.Close()
+ return []json.RawMessage{body}, nil
},
})
if err != nil {
return err
}
- return collector.Execute()
+
+ return collectorWithState.Execute()
}
-var CollectTaskMeta = plugin.SubTaskMeta{
- Name: "CollectTask",
- EntryPoint: CollectTask,
- EnabledByDefault: true,
- Description: "Collect Task data from Zentao api",
- DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
+type SimpleZentaoBug struct {
+ ID int64 `json:"id"`
}
diff --git a/backend/plugins/zentao/tasks/bug_extractor.go
b/backend/plugins/zentao/tasks/bug_extractor.go
index 52ac50c63..eff39b734 100644
--- a/backend/plugins/zentao/tasks/bug_extractor.go
+++ b/backend/plugins/zentao/tasks/bug_extractor.go
@@ -129,7 +129,6 @@ func ExtractBug(taskCtx plugin.SubTaskContext) errors.Error
{
Needconfirm: res.Needconfirm,
StatusName: res.StatusName,
ProductStatus: res.ProductStatus,
- Actions: res.Actions,
}
results := make([]interface{}, 0)
results = append(results, bug)
diff --git a/backend/plugins/zentao/tasks/department_collector.go
b/backend/plugins/zentao/tasks/department_collector.go
index aa2fc103a..12879a41f 100644
--- a/backend/plugins/zentao/tasks/department_collector.go
+++ b/backend/plugins/zentao/tasks/department_collector.go
@@ -71,7 +71,7 @@ func CollectDepartment(taskCtx plugin.SubTaskContext)
errors.Error {
}
var CollectDepartmentMeta = plugin.SubTaskMeta{
- Name: "CollectDepartment",
+ Name: "collectDepartment",
EntryPoint: CollectDepartment,
EnabledByDefault: true,
Description: "Collect Department data from Zentao api",
diff --git a/backend/plugins/zentao/tasks/story_collector.go
b/backend/plugins/zentao/tasks/story_collector.go
index 9da194064..f11918ad8 100644
--- a/backend/plugins/zentao/tasks/story_collector.go
+++ b/backend/plugins/zentao/tasks/story_collector.go
@@ -80,7 +80,7 @@ func CollectStory(taskCtx plugin.SubTaskContext) errors.Error
{
}
var CollectStoryMeta = plugin.SubTaskMeta{
- Name: "CollectStory",
+ Name: "collectStory",
EntryPoint: CollectStory,
EnabledByDefault: true,
Description: "Collect Story data from Zentao api",
diff --git a/backend/plugins/zentao/tasks/story_extractor.go
b/backend/plugins/zentao/tasks/story_extractor.go
index 4312f90dc..e4e633fb5 100644
--- a/backend/plugins/zentao/tasks/story_extractor.go
+++ b/backend/plugins/zentao/tasks/story_extractor.go
@@ -116,7 +116,6 @@ func ExtractStory(taskCtx plugin.SubTaskContext)
errors.Error {
Deleted: res.Deleted,
PriOrder: res.PriOrder,
PlanTitle: res.PlanTitle,
- Actions: res.Actions,
}
results := make([]interface{}, 0)
results = append(results, story)
diff --git a/backend/plugins/zentao/tasks/task_collector.go
b/backend/plugins/zentao/tasks/task_collector.go
index 427569c11..ace55d330 100644
--- a/backend/plugins/zentao/tasks/task_collector.go
+++ b/backend/plugins/zentao/tasks/task_collector.go
@@ -101,7 +101,7 @@ func CollectTask(taskCtx plugin.SubTaskContext)
errors.Error {
}
var CollectTaskMeta = plugin.SubTaskMeta{
- Name: "CollectTask",
+ Name: "collectTask",
EntryPoint: CollectTask,
EnabledByDefault: true,
Description: "Collect Task data from Zentao api",
diff --git a/backend/plugins/zentao/tasks/task_data.go
b/backend/plugins/zentao/tasks/task_data.go
index 406276f2e..9632e94f6 100644
--- a/backend/plugins/zentao/tasks/task_data.go
+++ b/backend/plugins/zentao/tasks/task_data.go
@@ -19,6 +19,7 @@ package tasks
import (
"fmt"
+ "time"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
@@ -49,6 +50,7 @@ type ZentaoOptions struct {
type ZentaoTaskData struct {
Options *ZentaoOptions
RemoteDb dal.Dal
+ TimeAfter *time.Time
ApiClient *helper.ApiAsyncClient
}
diff --git a/backend/plugins/zentao/tasks/task_extractor.go
b/backend/plugins/zentao/tasks/task_extractor.go
index 79c0e16c4..7d6efc18a 100644
--- a/backend/plugins/zentao/tasks/task_extractor.go
+++ b/backend/plugins/zentao/tasks/task_extractor.go
@@ -128,7 +128,6 @@ func ExtractTask(taskCtx plugin.SubTaskContext)
errors.Error {
NeedConfirm: res.NeedConfirm,
//ProductType: res.ProductType,
Progress: res.Progress,
- Actions: res.Actions,
}
results := make([]interface{}, 0)
results = append(results, task)
