This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 322db65b2 refactor: zentao plugin (#5653)
322db65b2 is described below
commit 322db65b2538f42fa8c62f1ebdf7fd7a30ebfb08
Author: Liang Zhang <[email protected]>
AuthorDate: Tue Jul 11 10:54:47 2023 +0800
refactor: zentao plugin (#5653)
* refactor: zentao plugin
* fix: table info check
* fix: unit test and lint error
---
backend/plugins/zentao/api/blueprint_V200_test.go | 1 -
backend/plugins/zentao/api/init.go | 11 +-
backend/plugins/zentao/api/remote.go | 30 +---
backend/plugins/zentao/impl/impl.go | 23 +++-
.../plugins/zentao/models/archived/changelog.go | 8 +-
.../register.go => archived/execution_stories.go} | 24 ++--
.../register.go => archived/execution_summary.go} | 26 ++--
.../register.go => archived/product_summary.go} | 24 ++--
.../register.go => archived/project_stories.go} | 23 ++--
backend/plugins/zentao/models/bug.go | 42 +++++-
backend/plugins/zentao/models/bug_test.go | 121 ++++++++++++++++
backend/plugins/zentao/models/changelog.go | 8 +-
.../register.go => execution_stories.go} | 24 ++--
.../register.go => execution_summary.go} | 26 ++--
.../20230705_add_execution_stories.go | 45 ++++++
.../zentao/models/migrationscripts/register.go | 1 +
backend/plugins/zentao/models/product.go | 61 ---------
.../register.go => product_summary.go} | 26 ++--
backend/plugins/zentao/models/project.go | 4 +-
.../register.go => project_stories.go} | 23 ++--
backend/plugins/zentao/models/remote_db.go | 89 ++++++++----
backend/plugins/zentao/tasks/account_collector.go | 10 +-
backend/plugins/zentao/tasks/account_convertor.go | 10 +-
backend/plugins/zentao/tasks/account_extractor.go | 14 +-
backend/plugins/zentao/tasks/bug_collector.go | 26 ++--
.../plugins/zentao/tasks/bug_commits_collector.go | 37 ++---
.../plugins/zentao/tasks/bug_commits_extractor.go | 26 ++--
backend/plugins/zentao/tasks/bug_convertor.go | 34 ++---
backend/plugins/zentao/tasks/bug_extractor.go | 46 ++-----
.../zentao/tasks/bug_repo_commits_collector.go | 28 ++--
.../zentao/tasks/bug_repo_commits_convertor.go | 16 +--
.../zentao/tasks/bug_repo_commits_extractor.go | 26 ++--
.../plugins/zentao/tasks/changelog_convertor.go | 61 +++++++--
backend/plugins/zentao/tasks/changelog_dbget.go | 152 ++++++++++++---------
.../plugins/zentao/tasks/department_collector.go | 20 +--
.../plugins/zentao/tasks/department_convertor.go | 10 +-
.../plugins/zentao/tasks/department_extractor.go | 14 +-
.../plugins/zentao/tasks/execution_collector.go | 35 ++---
.../plugins/zentao/tasks/execution_convertor.go | 10 +-
.../plugins/zentao/tasks/execution_extractor.go | 28 ++--
...t_collector.go => execution_story_collector.go} | 44 +++---
...t_convertor.go => execution_story_convertor.go} | 56 +++-----
...collector.go => execution_summary_collector.go} | 32 ++---
...extractor.go => execution_summary_extractor.go} | 33 ++---
backend/plugins/zentao/tasks/iterator.go | 105 ++++++++++++++
backend/plugins/zentao/tasks/project_convertor.go | 10 +-
backend/plugins/zentao/tasks/shared.go | 68 +++++----
backend/plugins/zentao/tasks/story_collector.go | 45 ++++--
.../zentao/tasks/story_commits_collector.go | 41 ++----
.../zentao/tasks/story_commits_extractor.go | 20 +--
backend/plugins/zentao/tasks/story_convertor.go | 33 ++---
backend/plugins/zentao/tasks/story_extractor.go | 49 +++----
.../zentao/tasks/story_repo_commits_collector.go | 18 +--
.../zentao/tasks/story_repo_commits_convertor.go | 16 +--
.../zentao/tasks/story_repo_commits_extractor.go | 20 +--
backend/plugins/zentao/tasks/task_collector.go | 21 +--
.../plugins/zentao/tasks/task_commits_collector.go | 32 +----
.../plugins/zentao/tasks/task_commits_extractor.go | 16 +--
backend/plugins/zentao/tasks/task_convertor.go | 28 +---
backend/plugins/zentao/tasks/task_data.go | 15 +-
backend/plugins/zentao/tasks/task_extractor.go | 20 +--
.../zentao/tasks/task_repo_commits_collector.go | 10 +-
.../zentao/tasks/task_repo_commits_convertor.go | 10 +-
.../zentao/tasks/task_repo_commits_extractor.go | 11 +-
64 files changed, 1013 insertions(+), 983 deletions(-)
diff --git a/backend/plugins/zentao/api/blueprint_V200_test.go
b/backend/plugins/zentao/api/blueprint_V200_test.go
index 5d8cddf95..ecd98139f 100644
--- a/backend/plugins/zentao/api/blueprint_V200_test.go
+++ b/backend/plugins/zentao/api/blueprint_V200_test.go
@@ -82,7 +82,6 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
Subtasks: []string{},
Options: map[string]interface{}{
"ConnectionId": uint64(1),
- "productId": int64(0),
"projectId": int64(1),
},
},
diff --git a/backend/plugins/zentao/api/init.go
b/backend/plugins/zentao/api/init.go
index 5d2b6715a..2e7abfd49 100644
--- a/backend/plugins/zentao/api/init.go
+++ b/backend/plugins/zentao/api/init.go
@@ -34,8 +34,7 @@ var vld *validator.Validate
var connectionHelper *api.ConnectionApiHelper
var projectScopeHelper *api.ScopeApiHelper[models.ZentaoConnection,
models.ZentaoProject, models.ZentaoScopeConfig]
-var productRemoteHelper *api.RemoteApiHelper[models.ZentaoConnection,
models.ZentaoProduct, models.ZentaoProductRes, api.BaseRemoteGroupResponse]
-var projectRemoteHelper *api.RemoteApiHelper[models.ZentaoConnection,
models.ZentaoProject, models.ZentaoProject, api.NoRemoteGroupResponse]
+var projectRemoteHelper *api.RemoteApiHelper[models.ZentaoConnection,
models.ZentaoProject, models.ZentaoProject, api.BaseRemoteGroupResponse]
var basicRes context.BasicRes
var scHelper *api.ScopeConfigHelper[models.ZentaoScopeConfig]
@@ -63,12 +62,8 @@ func Init(br context.BasicRes, p plugin.PluginMeta) {
projectParams,
nil,
)
- productRemoteHelper = api.NewRemoteHelper[models.ZentaoConnection,
models.ZentaoProduct, models.ZentaoProductRes, api.BaseRemoteGroupResponse](
- basicRes,
- vld,
- connectionHelper,
- )
- projectRemoteHelper = api.NewRemoteHelper[models.ZentaoConnection,
models.ZentaoProject, models.ZentaoProject, api.NoRemoteGroupResponse](
+
+ projectRemoteHelper = api.NewRemoteHelper[models.ZentaoConnection,
models.ZentaoProject, models.ZentaoProject, api.BaseRemoteGroupResponse](
basicRes,
vld,
connectionHelper,
diff --git a/backend/plugins/zentao/api/remote.go
b/backend/plugins/zentao/api/remote.go
index 91bd3e77a..e724e42d3 100644
--- a/backend/plugins/zentao/api/remote.go
+++ b/backend/plugins/zentao/api/remote.go
@@ -74,35 +74,7 @@ func RemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, er
}
gid := groupId[0]
if gid == "" {
- return productRemoteHelper.GetScopesFromRemote(input, getGroup,
nil)
- } else if gid == `products` {
- /*
- return productRemoteHelper.GetScopesFromRemote(input,
- nil,
- func(basicRes context2.BasicRes, gid string,
queryData *api.RemoteQueryData, connection models.ZentaoConnection)
([]models.ZentaoProductRes, errors.Error) {
- query := initialQuery(queryData)
- // create api client
- apiClient, err :=
api.NewApiClientFromConnection(context.TODO(), basicRes, &connection)
- if err != nil {
- return nil, err
- }
-
- query.Set("sort", "name")
- // list projects part
- res, err := apiClient.Get("/products",
query, nil)
- if err != nil {
- return nil, err
- }
-
- resBody := &ProductResponse{}
- err = api.UnmarshalResponse(res,
resBody)
- if err != nil {
- return nil, err
- }
- return resBody.Values, nil
- })
- */
- return nil, errors.BadInput.New("products are currently
unsupported")
+ return projectRemoteHelper.GetScopesFromRemote(input, getGroup,
nil)
} else if gid == `projects` {
return projectRemoteHelper.GetScopesFromRemote(input,
nil,
diff --git a/backend/plugins/zentao/impl/impl.go
b/backend/plugins/zentao/impl/impl.go
index 1c655aae4..2da39bec2 100644
--- a/backend/plugins/zentao/impl/impl.go
+++ b/backend/plugins/zentao/impl/impl.go
@@ -86,6 +86,10 @@ func (p Zentao) GetTablesInfo() []dal.Tabler {
&models.ZentaoBugRepoCommit{},
&models.ZentaoConnection{},
&models.ZentaoScopeConfig{},
+ &models.ZentaoExecutionStory{},
+ &models.ZentaoExecutionSummary{},
+ &models.ZentaoProductSummary{},
+ &models.ZentaoProjectStory{},
}
}
@@ -103,22 +107,20 @@ func (p Zentao) ScopeConfig() dal.Tabler {
func (p Zentao) SubTaskMetas() []plugin.SubTaskMeta {
return []plugin.SubTaskMeta{
- //tasks.ConvertProductMeta,
tasks.ConvertProjectMeta,
- tasks.DBGetChangelogMeta,
- tasks.ConvertChangelogMeta,
-
// both
tasks.CollectAccountMeta,
tasks.ExtractAccountMeta,
tasks.ConvertAccountMeta,
- tasks.CollectDepartmentMeta,
- tasks.ExtractDepartmentMeta,
- tasks.ConvertDepartmentMeta,
+ //tasks.CollectDepartmentMeta,
+ //tasks.ExtractDepartmentMeta,
+ //tasks.ConvertDepartmentMeta,
// project
+ tasks.CollectExecutionSummaryMeta,
+ tasks.ExtractExecutionSummaryMeta,
tasks.CollectExecutionMeta,
tasks.ExtractExecutionMeta,
tasks.ConvertExecutionMeta,
@@ -137,6 +139,7 @@ func (p Zentao) SubTaskMetas() []plugin.SubTaskMeta {
tasks.CollectStoryMeta,
tasks.ExtractStoryMeta,
tasks.ConvertStoryMeta,
+ tasks.ConvertExecutionStoryMeta,
tasks.CollectBugMeta,
tasks.ExtractBugMeta,
@@ -153,6 +156,9 @@ func (p Zentao) SubTaskMetas() []plugin.SubTaskMeta {
tasks.CollectBugRepoCommitsMeta,
tasks.ExtractBugRepoCommitsMeta,
tasks.ConvertBugRepoCommitsMeta,
+
+ tasks.DBGetChangelogMeta,
+ tasks.ConvertChangelogMeta,
}
}
@@ -195,6 +201,9 @@ func (p Zentao) PrepareTaskData(taskCtx plugin.TaskContext,
options map[string]i
ProductList: map[int64]string{},
StoryList: map[int64]int64{},
FromBugList: map[int]bool{},
+ Stories: map[int64]struct{}{},
+ Tasks: map[int64]struct{}{},
+ Bugs: map[int64]struct{}{},
}
if connection.DbUrl != "" {
diff --git a/backend/plugins/zentao/models/archived/changelog.go
b/backend/plugins/zentao/models/archived/changelog.go
index 0e7f196b0..baaa95d08 100644
--- a/backend/plugins/zentao/models/archived/changelog.go
+++ b/backend/plugins/zentao/models/archived/changelog.go
@@ -27,14 +27,14 @@ type ZentaoChangelog struct {
archived.NoPKModel `json:"-"`
ConnectionId uint64 `json:"connectionId"
mapstructure:"connectionId" gorm:"primaryKey;type:BIGINT NOT NULL"`
Id int64 `json:"id" mapstructure:"id"
gorm:"primaryKey;type:BIGINT NOT NULL;autoIncrement:false"`
- ObjectId int `json:"objectId" mapstructure:"objectId"
gorm:"index; NOT NULL"`
- Execution int `json:"execution" mapstructure:"execution"
`
+ ObjectId int64 `json:"objectId" mapstructure:"objectId"
gorm:"index; NOT NULL"`
+ Execution int64 `json:"execution" mapstructure:"execution"
`
Actor string `json:"actor" mapstructure:"actor" `
Action string `json:"action" mapstructure:"action"`
Extra string `json:"extra" mapstructure:"extra"`
ObjectType string `json:"objectType"
mapstructure:"objectType"`
- Project int `json:"project" mapstructure:"project"`
- Product int `json:"product" mapstructure:"product"`
+ Project int64 `json:"project" mapstructure:"project"`
+ Product int64 `json:"product" mapstructure:"product"`
Vision string `json:"vision" mapstructure:"vision"`
Comment string `json:"comment" mapstructure:"comment"`
Efforted string `json:"efforted" mapstructure:"efforted"`
diff --git a/backend/plugins/zentao/models/migrationscripts/register.go
b/backend/plugins/zentao/models/archived/execution_stories.go
similarity index 65%
copy from backend/plugins/zentao/models/migrationscripts/register.go
copy to backend/plugins/zentao/models/archived/execution_stories.go
index 4d3e3dc40..8b818f0b7 100644
--- a/backend/plugins/zentao/models/migrationscripts/register.go
+++ b/backend/plugins/zentao/models/archived/execution_stories.go
@@ -15,20 +15,20 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package migrationscripts
+package archived
import (
- "github.com/apache/incubator-devlake/core/plugin"
+
"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
)
-// All return all the migration scripts
-func All() []plugin.MigrationScript {
- return []plugin.MigrationScript{
- new(addInitTables),
- new(addScopeConfigTables),
- new(addIssueRepoCommitsTables),
- new(addInitChangelogTables),
- new(addTaskLeft),
- new(addRawParamTableForScope),
- }
+type ZentaoExecutionStory struct {
+ archived.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey"`
+ ProjectId int64 `gorm:"primaryKey"`
+ ExecutionId int64 `gorm:"primaryKey"`
+ StoryId int64 `gorm:"primaryKey"`
+}
+
+func (ZentaoExecutionStory) TableName() string {
+ return "_tool_zentao_execution_stories"
}
diff --git a/backend/plugins/zentao/models/migrationscripts/register.go
b/backend/plugins/zentao/models/archived/execution_summary.go
similarity index 58%
copy from backend/plugins/zentao/models/migrationscripts/register.go
copy to backend/plugins/zentao/models/archived/execution_summary.go
index 4d3e3dc40..308085ddd 100644
--- a/backend/plugins/zentao/models/migrationscripts/register.go
+++ b/backend/plugins/zentao/models/archived/execution_summary.go
@@ -15,20 +15,22 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package migrationscripts
+package archived
import (
- "github.com/apache/incubator-devlake/core/plugin"
+
"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
)
-// All return all the migration scripts
-func All() []plugin.MigrationScript {
- return []plugin.MigrationScript{
- new(addInitTables),
- new(addScopeConfigTables),
- new(addIssueRepoCommitsTables),
- new(addInitChangelogTables),
- new(addTaskLeft),
- new(addRawParamTableForScope),
- }
+type ZentaoExecutionSummary struct {
+ ConnectionId uint64 `gorm:"primaryKey;type:BIGINT NOT NULL"`
+ Id int64 `gorm:"primaryKey;type:BIGINT NOT
NULL;autoIncrement:false"`
+ Name string `gorm:"type:varchar(255)"`
+ Project int64
+ Code string `gorm:"type:varchar(255)"`
+ Type string `gorm:"type:varchar(255)"`
+ archived.NoPKModel
+}
+
+func (ZentaoExecutionSummary) TableName() string {
+ return "_tool_zentao_execution_summary"
}
diff --git a/backend/plugins/zentao/models/migrationscripts/register.go
b/backend/plugins/zentao/models/archived/product_summary.go
similarity index 64%
copy from backend/plugins/zentao/models/migrationscripts/register.go
copy to backend/plugins/zentao/models/archived/product_summary.go
index 4d3e3dc40..6b603e947 100644
--- a/backend/plugins/zentao/models/migrationscripts/register.go
+++ b/backend/plugins/zentao/models/archived/product_summary.go
@@ -15,20 +15,20 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package migrationscripts
+package archived
import (
- "github.com/apache/incubator-devlake/core/plugin"
+
"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
)
-// All return all the migration scripts
-func All() []plugin.MigrationScript {
- return []plugin.MigrationScript{
- new(addInitTables),
- new(addScopeConfigTables),
- new(addIssueRepoCommitsTables),
- new(addInitChangelogTables),
- new(addTaskLeft),
- new(addRawParamTableForScope),
- }
+type ZentaoProductSummary struct {
+ archived.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey"`
+ ProjectId int64 `gorm:"primaryKey"`
+ Id int64 `gorm:"primaryKey;autoIncrement:false"`
+ Name string `gorm:"type:VARCHAR(255)"`
+}
+
+func (ZentaoProductSummary) TableName() string {
+ return "_tool_zentao_product_summary"
}
diff --git a/backend/plugins/zentao/models/migrationscripts/register.go
b/backend/plugins/zentao/models/archived/project_stories.go
similarity index 67%
copy from backend/plugins/zentao/models/migrationscripts/register.go
copy to backend/plugins/zentao/models/archived/project_stories.go
index 4d3e3dc40..dbeb77e24 100644
--- a/backend/plugins/zentao/models/migrationscripts/register.go
+++ b/backend/plugins/zentao/models/archived/project_stories.go
@@ -15,20 +15,19 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package migrationscripts
+package archived
import (
- "github.com/apache/incubator-devlake/core/plugin"
+
"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
)
-// All return all the migration scripts
-func All() []plugin.MigrationScript {
- return []plugin.MigrationScript{
- new(addInitTables),
- new(addScopeConfigTables),
- new(addIssueRepoCommitsTables),
- new(addInitChangelogTables),
- new(addTaskLeft),
- new(addRawParamTableForScope),
- }
+type ZentaoProjectStory struct {
+ archived.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey;type:BIGINT NOT NULL"`
+ ProjectId int64 `gorm:"primaryKey"`
+ StoryId int64 `gorm:"primaryKey"`
+}
+
+func (ZentaoProjectStory) TableName() string {
+ return "_tool_zentao_project_stories"
}
diff --git a/backend/plugins/zentao/models/bug.go
b/backend/plugins/zentao/models/bug.go
index 1a16bfa96..dd0112af5 100644
--- a/backend/plugins/zentao/models/bug.go
+++ b/backend/plugins/zentao/models/bug.go
@@ -18,10 +18,42 @@ limitations under the License.
package models
import (
+ "bytes"
+ "encoding/json"
"github.com/apache/incubator-devlake/core/models/common"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
+type ApiAccount struct {
+ ID int64 `json:"id"`
+ Account string `json:"account"`
+ Avatar string `json:"avatar"`
+ Realname string `json:"realname"`
+}
+
+func (a *ApiAccount) UnmarshalJSON(data []byte) error {
+ var dst struct {
+ ID int64 `json:"id"`
+ Account string `json:"account"`
+ Avatar string `json:"avatar"`
+ Realname string `json:"realname"`
+ }
+ data = bytes.TrimSpace(data)
+ if string(data) == "null" {
+ a = nil
+ }
+ if len(data) > 1 && data[0] == '"' && data[len(data)-1] == '"' {
+ dst.Account = string(data[1 : len(data)-1])
+ } else {
+ err := json.Unmarshal(data, &dst)
+ if err != nil {
+ return err
+ }
+ }
+ *a = dst
+ return nil
+}
+
type ZentaoBugRes struct {
ID int64 `json:"id"`
Project int64 `json:"project"`
@@ -55,17 +87,17 @@ type ZentaoBugRes struct {
ActivatedDate *helper.Iso8601Time `json:"activatedDate"`
FeedbackBy string `json:"feedbackBy"`
NotifyEmail string `json:"notifyEmail"`
- OpenedBy *ZentaoAccount `json:"openedBy"`
+ OpenedBy *ApiAccount `json:"openedBy"`
OpenedDate *helper.Iso8601Time `json:"openedDate"`
OpenedBuild string `json:"openedBuild"`
- AssignedTo *ZentaoAccount `json:"assignedTo"`
+ AssignedTo *ApiAccount `json:"assignedTo"`
AssignedDate *helper.Iso8601Time `json:"assignedDate"`
Deadline string `json:"deadline"`
- ResolvedBy *ZentaoAccount `json:"resolvedBy"`
+ ResolvedBy *ApiAccount `json:"resolvedBy"`
Resolution string `json:"resolution"`
ResolvedBuild string `json:"resolvedBuild"`
ResolvedDate *helper.Iso8601Time `json:"resolvedDate"`
- ClosedBy *ZentaoAccount `json:"closedBy"`
+ ClosedBy *ApiAccount `json:"closedBy"`
ClosedDate *helper.Iso8601Time `json:"closedDate"`
DuplicateBug int `json:"duplicateBug"`
LinkBug string `json:"linkBug"`
@@ -80,7 +112,7 @@ type ZentaoBugRes struct {
RepoType string `json:"repoType"`
IssueKey string `json:"issueKey"`
Testtask int `json:"testtask"`
- LastEditedBy *ZentaoAccount `json:"lastEditedBy"`
+ LastEditedBy *ApiAccount `json:"lastEditedBy"`
LastEditedDate *helper.Iso8601Time `json:"lastEditedDate"`
Deleted bool `json:"deleted"`
PriOrder string `json:"priOrder"`
diff --git a/backend/plugins/zentao/models/bug_test.go
b/backend/plugins/zentao/models/bug_test.go
new file mode 100644
index 000000000..ff501cef3
--- /dev/null
+++ b/backend/plugins/zentao/models/bug_test.go
@@ -0,0 +1,121 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package models
+
+import (
+ "encoding/json"
+ "reflect"
+ "testing"
+)
+
+func TestApiAccount_UnmarshalJSON(t *testing.T) {
+ type bug struct {
+ NotifyEmail string `json:"notifyEmail"`
+ OpenedBy *ApiAccount
+ }
+
+ type args struct {
+ data []byte
+ }
+ tests := []struct {
+ name string
+ args args
+ want bug
+ wantErr bool
+ }{
+ {
+ "string",
+ args{
+ data: []byte(`{
+
"notifyEmail": "[email protected]",
+ "openedBy":
"admin"
+ }`),
+ },
+ bug{
+ NotifyEmail: "[email protected]",
+ OpenedBy: &ApiAccount{Account: "admin"},
+ },
+ false,
+ },
+ {
+ "empty string",
+ args{
+ data: []byte(`{
+
"notifyEmail": "[email protected]",
+ "openedBy": ""
+ }`),
+ },
+ bug{
+ NotifyEmail: "[email protected]",
+ OpenedBy: &ApiAccount{},
+ },
+ false,
+ },
+ {
+ "struct",
+ args{
+ data: []byte(`{
+
"notifyEmail": "[email protected]",
+ "openedBy": {
+ "id": 1,
+
"account": "admin",
+
"avatar": "https://example.com/avatar.png";,
+
"realname": "root"
+ }
+ }`),
+ },
+ bug{
+ NotifyEmail: "[email protected]",
+ OpenedBy: &ApiAccount{
+ ID: 1,
+ Account: "admin",
+ Avatar:
"https://example.com/avatar.png";,
+ Realname: "root",
+ },
+ },
+ false,
+ },
+ {
+ "null",
+ args{
+ data: []byte(`{
+
"notifyEmail": "[email protected]",
+ "openedBy":
null
+ }`),
+ },
+ bug{
+ NotifyEmail: "[email protected]",
+ OpenedBy: nil,
+ },
+ false,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ var dst bug
+ if err := json.Unmarshal(tt.args.data, &dst); (err !=
nil) != tt.wantErr {
+ t.Errorf("UnmarshalJSON() error = %v, wantErr
%v", err, tt.wantErr)
+ }
+ if !reflect.DeepEqual(dst, tt.want) {
+ t.Errorf("UnmarshalJSON() got = %v, want %v",
dst, tt.want)
+ }
+ t.Logf("%+v\n", dst.OpenedBy)
+ })
+ }
+}
diff --git a/backend/plugins/zentao/models/changelog.go
b/backend/plugins/zentao/models/changelog.go
index ef7693847..91ea87a81 100644
--- a/backend/plugins/zentao/models/changelog.go
+++ b/backend/plugins/zentao/models/changelog.go
@@ -27,14 +27,14 @@ type ZentaoChangelog struct {
common.NoPKModel `json:"-"`
ConnectionId uint64 `json:"connectionId"
mapstructure:"connectionId" gorm:"primaryKey;type:BIGINT NOT NULL"`
Id int64 `json:"id" mapstructure:"id"
gorm:"primaryKey;type:BIGINT NOT NULL;autoIncrement:false"`
- ObjectId int `json:"objectId" mapstructure:"objectId"
gorm:"index; NOT NULL"`
- Execution int `json:"execution" mapstructure:"execution" `
+ ObjectId int64 `json:"objectId" mapstructure:"objectId"
gorm:"index; NOT NULL"`
+ Execution int64 `json:"execution" mapstructure:"execution" `
Actor string `json:"actor" mapstructure:"actor" `
Action string `json:"action" mapstructure:"action"`
Extra string `json:"extra" mapstructure:"extra"`
ObjectType string `json:"objectType" mapstructure:"objectType"`
- Project int `json:"project" mapstructure:"project"`
- Product int `json:"product" mapstructure:"product"`
+ Project int64 `json:"project" mapstructure:"project"`
+ Product int64 `json:"product" mapstructure:"product"`
Vision string `json:"vision" mapstructure:"vision"`
Comment string `json:"comment" mapstructure:"comment"`
Efforted string `json:"efforted" mapstructure:"efforted"`
diff --git a/backend/plugins/zentao/models/migrationscripts/register.go
b/backend/plugins/zentao/models/execution_stories.go
similarity index 65%
copy from backend/plugins/zentao/models/migrationscripts/register.go
copy to backend/plugins/zentao/models/execution_stories.go
index 4d3e3dc40..82b919c80 100644
--- a/backend/plugins/zentao/models/migrationscripts/register.go
+++ b/backend/plugins/zentao/models/execution_stories.go
@@ -15,20 +15,20 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package migrationscripts
+package models
import (
- "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/models/common"
)
-// All return all the migration scripts
-func All() []plugin.MigrationScript {
- return []plugin.MigrationScript{
- new(addInitTables),
- new(addScopeConfigTables),
- new(addIssueRepoCommitsTables),
- new(addInitChangelogTables),
- new(addTaskLeft),
- new(addRawParamTableForScope),
- }
+type ZentaoExecutionStory struct {
+ common.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey;type:BIGINT NOT NULL"`
+ ProjectId int64 `gorm:"primaryKey"`
+ ExecutionId int64 `gorm:"primaryKey"`
+ StoryId int64 `gorm:"primaryKey"`
+}
+
+func (ZentaoExecutionStory) TableName() string {
+ return "_tool_zentao_execution_stories"
}
diff --git a/backend/plugins/zentao/models/migrationscripts/register.go
b/backend/plugins/zentao/models/execution_summary.go
similarity index 59%
copy from backend/plugins/zentao/models/migrationscripts/register.go
copy to backend/plugins/zentao/models/execution_summary.go
index 4d3e3dc40..2609697aa 100644
--- a/backend/plugins/zentao/models/migrationscripts/register.go
+++ b/backend/plugins/zentao/models/execution_summary.go
@@ -15,20 +15,22 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package migrationscripts
+package models
import (
- "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/models/common"
)
-// All return all the migration scripts
-func All() []plugin.MigrationScript {
- return []plugin.MigrationScript{
- new(addInitTables),
- new(addScopeConfigTables),
- new(addIssueRepoCommitsTables),
- new(addInitChangelogTables),
- new(addTaskLeft),
- new(addRawParamTableForScope),
- }
+type ZentaoExecutionSummary struct {
+ ConnectionId uint64 `gorm:"primaryKey;type:BIGINT NOT NULL"`
+ Id int64 `gorm:"primaryKey;type:BIGINT NOT
NULL;autoIncrement:false"`
+ Name string `gorm:"type:varchar(255)"`
+ Project int64
+ Code string `gorm:"type:varchar(255)"`
+ Type string `gorm:"type:varchar(255)"`
+ common.NoPKModel
+}
+
+func (ZentaoExecutionSummary) TableName() string {
+ return "_tool_zentao_execution_summary"
}
diff --git
a/backend/plugins/zentao/models/migrationscripts/20230705_add_execution_stories.go
b/backend/plugins/zentao/models/migrationscripts/20230705_add_execution_stories.go
new file mode 100644
index 000000000..234451cf3
--- /dev/null
+++
b/backend/plugins/zentao/models/migrationscripts/20230705_add_execution_stories.go
@@ -0,0 +1,45 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+ "github.com/apache/incubator-devlake/plugins/zentao/models/archived"
+)
+
+type addExecutionStoryAndExecutionSummary struct{}
+
+func (*addExecutionStoryAndExecutionSummary) Up(basicRes context.BasicRes)
errors.Error {
+ return migrationhelper.AutoMigrateTables(
+ basicRes,
+ &archived.ZentaoExecutionStory{},
+ &archived.ZentaoExecutionSummary{},
+ &archived.ZentaoProductSummary{},
+ &archived.ZentaoProjectStory{},
+ )
+}
+
+func (*addExecutionStoryAndExecutionSummary) Version() uint64 {
+ return 20230705135809
+}
+
+func (*addExecutionStoryAndExecutionSummary) Name() string {
+ return "add table _tool_zentao_execution_stories,
_tool_zentao_execution_summary and _tool_zentao_product_summary"
+}
diff --git a/backend/plugins/zentao/models/migrationscripts/register.go
b/backend/plugins/zentao/models/migrationscripts/register.go
index 4d3e3dc40..4fdfac508 100644
--- a/backend/plugins/zentao/models/migrationscripts/register.go
+++ b/backend/plugins/zentao/models/migrationscripts/register.go
@@ -29,6 +29,7 @@ func All() []plugin.MigrationScript {
new(addIssueRepoCommitsTables),
new(addInitChangelogTables),
new(addTaskLeft),
+ new(addExecutionStoryAndExecutionSummary),
new(addRawParamTableForScope),
}
}
diff --git a/backend/plugins/zentao/models/product.go
b/backend/plugins/zentao/models/product.go
index 7790a6990..a4ec727ea 100644
--- a/backend/plugins/zentao/models/product.go
+++ b/backend/plugins/zentao/models/product.go
@@ -18,10 +18,7 @@ limitations under the License.
package models
import (
- "fmt"
-
"github.com/apache/incubator-devlake/core/models/common"
- "github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
@@ -69,49 +66,6 @@ type ZentaoProductRes struct {
CaseReview bool `json:"caseReview" mapstructure:"caseReview"`
}
-func getAccountId(account *ZentaoAccount) int64 {
- if account != nil {
- return account.ID
- }
- return 0
-}
-
-func (res ZentaoProductRes) ConvertApiScope() plugin.ToolLayerScope {
- return &ZentaoProduct{
- Id: res.ID,
- Program: res.Program,
- Name: res.Name,
- Code: res.Code,
- Bind: res.Bind,
- Line: res.Line,
- Type: `product`,
- ProductType: res.Type,
- Status: res.Status,
- SubStatus: res.SubStatus,
- Description: res.Description,
- POId: getAccountId(res.PO),
- QDId: getAccountId(res.QD),
- RDId: getAccountId(res.RD),
- Acl: res.Acl,
- Reviewer: res.Reviewer,
- CreatedById: getAccountId(res.CreatedBy),
- CreatedDate: res.CreatedDate,
- CreatedVersion: res.CreatedVersion,
- OrderIn: res.OrderIn,
- Deleted: res.Deleted,
- Plans: res.Plans,
- Releases: res.Releases,
- Builds: res.Builds,
- Cases: res.Cases,
- Projects: res.Projects,
- Executions: res.Executions,
- Bugs: res.Bugs,
- Docs: res.Docs,
- Progress: res.Progress,
- CaseReview: res.CaseReview,
- }
-}
-
type ZentaoProduct struct {
common.NoPKModel `json:"-"`
ConnectionId uint64 `json:"connectionId"
mapstructure:"connectionId" gorm:"primaryKey;type:BIGINT NOT NULL"`
@@ -152,18 +106,3 @@ type ZentaoProduct struct {
func (ZentaoProduct) TableName() string {
return "_tool_zentao_products"
}
-
-func (p ZentaoProduct) ScopeId() string {
- return fmt.Sprintf(`products/%d`, p.Id)
-}
-
-func (p ZentaoProduct) ScopeParams() interface{} {
- return &ZentaoApiParams{
- ConnectionId: p.ConnectionId,
- ZentaoId: fmt.Sprintf("products/%d", p.Id),
- }
-}
-
-func (p ZentaoProduct) ScopeName() string {
- return p.Name
-}
diff --git a/backend/plugins/zentao/models/migrationscripts/register.go
b/backend/plugins/zentao/models/product_summary.go
similarity index 64%
copy from backend/plugins/zentao/models/migrationscripts/register.go
copy to backend/plugins/zentao/models/product_summary.go
index 4d3e3dc40..f448af77b 100644
--- a/backend/plugins/zentao/models/migrationscripts/register.go
+++ b/backend/plugins/zentao/models/product_summary.go
@@ -15,20 +15,18 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package migrationscripts
+package models
-import (
- "github.com/apache/incubator-devlake/core/plugin"
-)
+import "github.com/apache/incubator-devlake/core/models/common"
-// All return all the migration scripts
-func All() []plugin.MigrationScript {
- return []plugin.MigrationScript{
- new(addInitTables),
- new(addScopeConfigTables),
- new(addIssueRepoCommitsTables),
- new(addInitChangelogTables),
- new(addTaskLeft),
- new(addRawParamTableForScope),
- }
+type ZentaoProductSummary struct {
+ common.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey"`
+ ProjectId int64 `gorm:"primaryKey"`
+ Id int64 `gorm:"primaryKey;autoIncrement:false"`
+ Name string `gorm:"type:VARCHAR(255)"`
+}
+
+func (ZentaoProductSummary) TableName() string {
+ return "_tool_zentao_product_summary"
}
diff --git a/backend/plugins/zentao/models/project.go
b/backend/plugins/zentao/models/project.go
index 42390d4f9..26402b0cc 100644
--- a/backend/plugins/zentao/models/project.go
+++ b/backend/plugins/zentao/models/project.go
@@ -157,7 +157,7 @@ func (p ZentaoProject) ScopeName() string {
func (p ZentaoProject) ScopeParams() interface{} {
return &ZentaoApiParams{
ConnectionId: p.ConnectionId,
- ZentaoId: fmt.Sprintf("projects/%d", p.Id),
+ ProjectId: p.Id,
}
}
@@ -171,5 +171,5 @@ func (p ZentaoProject) ConvertApiScope()
plugin.ToolLayerScope {
type ZentaoApiParams struct {
ConnectionId uint64
- ZentaoId string
+ ProjectId int64
}
diff --git a/backend/plugins/zentao/models/migrationscripts/register.go
b/backend/plugins/zentao/models/project_stories.go
similarity index 68%
copy from backend/plugins/zentao/models/migrationscripts/register.go
copy to backend/plugins/zentao/models/project_stories.go
index 4d3e3dc40..79a02f319 100644
--- a/backend/plugins/zentao/models/migrationscripts/register.go
+++ b/backend/plugins/zentao/models/project_stories.go
@@ -15,20 +15,19 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package migrationscripts
+package models
import (
- "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/core/models/common"
)
-// All return all the migration scripts
-func All() []plugin.MigrationScript {
- return []plugin.MigrationScript{
- new(addInitTables),
- new(addScopeConfigTables),
- new(addIssueRepoCommitsTables),
- new(addInitChangelogTables),
- new(addTaskLeft),
- new(addRawParamTableForScope),
- }
+type ZentaoProjectStory struct {
+ common.NoPKModel
+ ConnectionId uint64 `gorm:"primaryKey;type:BIGINT NOT NULL"`
+ ProjectId int64 `gorm:"primaryKey"`
+ StoryId int64 `gorm:"primaryKey"`
+}
+
+func (ZentaoProjectStory) TableName() string {
+ return "_tool_zentao_project_stories"
}
diff --git a/backend/plugins/zentao/models/remote_db.go
b/backend/plugins/zentao/models/remote_db.go
index a1b1f09fd..99233afa2 100644
--- a/backend/plugins/zentao/models/remote_db.go
+++ b/backend/plugins/zentao/models/remote_db.go
@@ -29,22 +29,34 @@ type ZentaoRemoteDbHistoryBase struct {
}
type ZentaoRemoteDbHistory struct {
- Id int `gorm:"column:id"`
- Action int `gorm:"column:action"`
+ Id int64 `gorm:"column:id"`
+ Action int64 `gorm:"column:action"`
ZentaoRemoteDbHistoryBase
}
+func (h ZentaoRemoteDbHistory) ToChangelogDetail(connectionId uint64)
*ZentaoChangelogDetail {
+ return &ZentaoChangelogDetail{
+ ConnectionId: connectionId,
+ Id: h.Id,
+ ChangelogId: h.Action,
+ Field: h.Field,
+ Old: h.Old,
+ New: h.New,
+ Diff: h.Diff,
+ }
+}
+
func (ZentaoRemoteDbHistory) TableName() string {
return "zt_history"
}
type ZentaoRemoteDbAction struct {
- Id int `gorm:"column:id"`
+ Id int64 `gorm:"column:id"`
ObjectType string `gorm:"column:objectType"`
- ObjectId int `gorm:"column:objectID"`
+ ObjectId int64 `gorm:"column:objectID"`
Product string `gorm:"column:product"`
- Project int `gorm:"column:project"`
- Execution int `gorm:"column:execution"`
+ Project int64 `gorm:"column:project"`
+ Execution int64 `gorm:"column:execution"`
Actor string `gorm:"column:actor"`
Action string `gorm:"column:action"`
Date time.Time `gorm:"column:date"`
@@ -55,6 +67,25 @@ type ZentaoRemoteDbAction struct {
Efforted string `gorm:"column:efforted"`
}
+func (a ZentaoRemoteDbAction) ToChangelog(connectionId uint64)
*ZentaoChangelog {
+ return &ZentaoChangelog{
+ ConnectionId: connectionId,
+ Id: a.Id,
+ ObjectId: a.ObjectId,
+ Execution: a.Execution,
+ Actor: a.Actor,
+ Action: a.Action,
+ Extra: a.Extra,
+ ObjectType: a.ObjectType,
+ Project: a.Project,
+ Vision: a.Vision,
+ Comment: a.Comment,
+ Efforted: a.Efforted,
+ Date: a.Date,
+ Read: a.Read,
+ }
+}
+
func (ZentaoRemoteDbAction) TableName() string {
return "zt_action"
}
@@ -63,34 +94,36 @@ type ZentaoRemoteDbActionHistory struct {
ZentaoRemoteDbAction
ZentaoRemoteDbHistoryBase
- ActionId int `gorm:"column:aid"`
- HistoryId int `gorm:"column:hid"`
+ ActionId int64 `gorm:"column:aid"`
+ HistoryId int64 `gorm:"column:hid"`
}
-func (ah *ZentaoRemoteDbActionHistory) Convert() *ZentaoChangelogCom {
+func (ah *ZentaoRemoteDbActionHistory) Convert(connectId uint64)
*ZentaoChangelogCom {
return &ZentaoChangelogCom{
&ZentaoChangelog{
- Id: int64(ah.ActionId),
- ObjectId: ah.ObjectId,
- Execution: ah.Execution,
- Actor: ah.Actor,
- Action: ah.Action,
- Extra: ah.Extra,
- ObjectType: ah.ObjectType,
- Project: ah.Project,
- Vision: ah.Vision,
- Comment: ah.Comment,
- Efforted: ah.Efforted,
- Date: ah.Date,
- Read: ah.Read,
+ ConnectionId: connectId,
+ Id: ah.ActionId,
+ ObjectId: ah.ObjectId,
+ Execution: ah.Execution,
+ Actor: ah.Actor,
+ Action: ah.Action,
+ Extra: ah.Extra,
+ ObjectType: ah.ObjectType,
+ Project: ah.Project,
+ Vision: ah.Vision,
+ Comment: ah.Comment,
+ Efforted: ah.Efforted,
+ Date: ah.Date,
+ Read: ah.Read,
},
&ZentaoChangelogDetail{
- Id: int64(ah.HistoryId),
- ChangelogId: int64(ah.ActionId),
- Field: ah.Field,
- Old: ah.Old,
- New: ah.New,
- Diff: ah.Diff,
+ ConnectionId: connectId,
+ Id: ah.HistoryId,
+ ChangelogId: ah.ActionId,
+ Field: ah.Field,
+ Old: ah.Old,
+ New: ah.New,
+ Diff: ah.Diff,
},
}
}
diff --git a/backend/plugins/zentao/tasks/account_collector.go
b/backend/plugins/zentao/tasks/account_collector.go
index 9ecd35665..dd849e638 100644
--- a/backend/plugins/zentao/tasks/account_collector.go
+++ b/backend/plugins/zentao/tasks/account_collector.go
@@ -36,13 +36,9 @@ func CollectAccount(taskCtx plugin.SubTaskContext)
errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
collector, err := api.NewApiCollector(api.ApiCollectorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_ACCOUNT_TABLE,
+ Ctx: taskCtx,
+ Table: RAW_ACCOUNT_TABLE,
+ Options: data.Options,
},
ApiClient: data.ApiClient,
PageSize: 100,
diff --git a/backend/plugins/zentao/tasks/account_convertor.go
b/backend/plugins/zentao/tasks/account_convertor.go
index 50a4aaa2b..1a07a12dd 100644
--- a/backend/plugins/zentao/tasks/account_convertor.go
+++ b/backend/plugins/zentao/tasks/account_convertor.go
@@ -57,13 +57,9 @@ func ConvertAccount(taskCtx plugin.SubTaskContext)
errors.Error {
InputRowType: reflect.TypeOf(models.ZentaoAccount{}),
Input: cursor,
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_ACCOUNT_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_ACCOUNT_TABLE,
},
Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
toolEntity := inputRow.(*models.ZentaoAccount)
diff --git a/backend/plugins/zentao/tasks/account_extractor.go
b/backend/plugins/zentao/tasks/account_extractor.go
index 9ad1b720d..eba07bea7 100644
--- a/backend/plugins/zentao/tasks/account_extractor.go
+++ b/backend/plugins/zentao/tasks/account_extractor.go
@@ -40,13 +40,9 @@ func ExtractAccount(taskCtx plugin.SubTaskContext)
errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_ACCOUNT_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_ACCOUNT_TABLE,
},
Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
account := &models.ZentaoAccount{}
@@ -55,9 +51,7 @@ func ExtractAccount(taskCtx plugin.SubTaskContext)
errors.Error {
return nil, errors.Default.WrapRaw(err)
}
account.ConnectionId = data.Options.ConnectionId
- results := make([]interface{}, 0)
- results = append(results, account)
- return results, nil
+ return []interface{}{account}, nil
},
})
diff --git a/backend/plugins/zentao/tasks/bug_collector.go
b/backend/plugins/zentao/tasks/bug_collector.go
index 700012430..2aebd07fe 100644
--- a/backend/plugins/zentao/tasks/bug_collector.go
+++ b/backend/plugins/zentao/tasks/bug_collector.go
@@ -33,30 +33,22 @@ const RAW_BUG_TABLE = "zentao_api_bugs"
var _ plugin.SubTaskEntryPoint = CollectBug
func CollectBug(taskCtx plugin.SubTaskContext) errors.Error {
- return RangeProductOneByOne(taskCtx, CollectBugForOneProduct)
-}
-
-func CollectBugForOneProduct(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
-
- // this collect only work for product
- if data.Options.ProductId == 0 {
- return nil
+ cursor, iterator, err := getProductIterator(taskCtx)
+ if err != nil {
+ return err
}
-
+ defer cursor.Close()
collector, err := api.NewApiCollector(api.ApiCollectorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_BUG_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_BUG_TABLE,
},
+ Input: iterator,
ApiClient: data.ApiClient,
PageSize: 100,
- UrlTemplate: "/{{ .Params.ZentaoId }}/bugs",
+ UrlTemplate: "/products/{{ .Input.Id }}/bugs",
Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
diff --git a/backend/plugins/zentao/tasks/bug_commits_collector.go
b/backend/plugins/zentao/tasks/bug_commits_collector.go
index 7c2b61420..70970058f 100644
--- a/backend/plugins/zentao/tasks/bug_commits_collector.go
+++ b/backend/plugins/zentao/tasks/bug_commits_collector.go
@@ -41,23 +41,20 @@ var CollectBugCommitsMeta = plugin.SubTaskMeta{
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
}
-func CollectBugCommits(taskCtx plugin.SubTaskContext) errors.Error {
- return RangeProductOneByOne(taskCtx, CollectBugCommitsForOneProduct)
+type bugInput struct {
+ BugId int64
+ ProductId int64
}
-func CollectBugCommitsForOneProduct(taskCtx plugin.SubTaskContext)
errors.Error {
+func CollectBugCommits(taskCtx plugin.SubTaskContext) errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*ZentaoTaskData)
// state manager
collectorWithState, err :=
api.NewStatefulApiCollector(api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_BUG_COMMITS_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_BUG_COMMITS_TABLE,
}, data.TimeAfter)
if err != nil {
return err
@@ -65,11 +62,11 @@ func CollectBugCommitsForOneProduct(taskCtx
plugin.SubTaskContext) errors.Error
// load bugs id from db
clauses := []dal.Clause{
- dal.Select("id, last_edited_date"),
+ dal.Select("id As bug_id, product As product_id"),
dal.From(&models.ZentaoBug{}),
dal.Where(
- "product = ? AND connection_id = ?",
- data.Options.ProductId, data.Options.ConnectionId,
+ "project = ? AND connection_id = ?",
+ data.Options.ProjectId, data.Options.ConnectionId,
),
}
// incremental collection
@@ -84,25 +81,21 @@ func CollectBugCommitsForOneProduct(taskCtx
plugin.SubTaskContext) errors.Error
if err != nil {
return err
}
- iterator, err := api.NewDalCursorIterator(db, cursor,
reflect.TypeOf(SimpleZentaoBug{}))
+ iterator, err := api.NewDalCursorIterator(db, cursor,
reflect.TypeOf(bugInput{}))
if err != nil {
return err
}
// collect bug commits
err = collectorWithState.InitCollector(api.ApiCollectorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_BUG_COMMITS_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_BUG_COMMITS_TABLE,
},
ApiClient: data.ApiClient,
Input: iterator,
Incremental: incremental,
- UrlTemplate: "bugs/{{ .Input.ID }}",
+ UrlTemplate: "bugs/{{ .Input.BugId }}",
ResponseParser: func(res *http.Response) ([]json.RawMessage,
errors.Error) {
var data struct {
Actions []json.RawMessage `json:"actions"`
diff --git a/backend/plugins/zentao/tasks/bug_commits_extractor.go
b/backend/plugins/zentao/tasks/bug_commits_extractor.go
index 23454341f..498e9bb89 100644
--- a/backend/plugins/zentao/tasks/bug_commits_extractor.go
+++ b/backend/plugins/zentao/tasks/bug_commits_extractor.go
@@ -39,27 +39,14 @@ var ExtractBugCommitsMeta = plugin.SubTaskMeta{
}
func ExtractBugCommits(taskCtx plugin.SubTaskContext) errors.Error {
- return RangeProductOneByOne(taskCtx, ExtractBugCommitsForOneProduct)
-}
-
-func ExtractBugCommitsForOneProduct(taskCtx plugin.SubTaskContext)
errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
- // this Extract only work for product
- if data.Options.ProductId == 0 {
- return nil
- }
-
re := regexp.MustCompile(`href='(.*?)'`)
extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_BUG_COMMITS_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_BUG_COMMITS_TABLE,
},
Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
res := &models.ZentaoBugCommitsRes{}
@@ -67,6 +54,11 @@ func ExtractBugCommitsForOneProduct(taskCtx
plugin.SubTaskContext) errors.Error
if err != nil {
return nil, errors.Default.WrapRaw(err)
}
+ var input bugInput
+ err = json.Unmarshal(row.Input, &input)
+ if err != nil {
+ return nil, errors.Default.WrapRaw(err)
+ }
// only linked2revision action is valid
if res.Action != "linked2revision" {
return nil, nil
@@ -77,7 +69,7 @@ func ExtractBugCommitsForOneProduct(taskCtx
plugin.SubTaskContext) errors.Error
ID: res.ID,
ObjectType: res.ObjectType,
ObjectID: res.ObjectID,
- Product: data.Options.ProductId,
+ Product: input.ProductId,
Project: data.Options.ProjectId,
Execution: res.Execution,
Actor: res.Actor,
diff --git a/backend/plugins/zentao/tasks/bug_convertor.go
b/backend/plugins/zentao/tasks/bug_convertor.go
index 945e27351..f9731cd10 100644
--- a/backend/plugins/zentao/tasks/bug_convertor.go
+++ b/backend/plugins/zentao/tasks/bug_convertor.go
@@ -42,15 +42,11 @@ var ConvertBugMeta = plugin.SubTaskMeta{
}
func ConvertBug(taskCtx plugin.SubTaskContext) errors.Error {
- return RangeProductOneByOne(taskCtx, ConvertBugForOneProduct)
-}
-
-func ConvertBugForOneProduct(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
db := taskCtx.GetDal()
bugIdGen := didgen.NewDomainIdGenerator(&models.ZentaoBug{})
accountIdGen := didgen.NewDomainIdGenerator(&models.ZentaoAccount{})
-
+ executionIdGen := didgen.NewDomainIdGenerator(&models.ZentaoExecution{})
boardIdGen := didgen.NewDomainIdGenerator(&models.ZentaoProduct{})
if data.Options.ProjectId != 0 {
boardIdGen =
didgen.NewDomainIdGenerator(&models.ZentaoProject{})
@@ -59,8 +55,8 @@ func ConvertBugForOneProduct(taskCtx plugin.SubTaskContext)
errors.Error {
storyIdGen := didgen.NewDomainIdGenerator(&models.ZentaoStory{})
cursor, err := db.Cursor(
dal.From(&models.ZentaoBug{}),
- dal.Where(`_tool_zentao_bugs.product = ? and
- _tool_zentao_bugs.connection_id = ?`,
data.Options.ProductId, data.Options.ConnectionId),
+ dal.Where(`project = ? and
+ connection_id = ?`, data.Options.ProjectId,
data.Options.ConnectionId),
)
if err != nil {
return err
@@ -70,13 +66,9 @@ func ConvertBugForOneProduct(taskCtx plugin.SubTaskContext)
errors.Error {
InputRowType: reflect.TypeOf(models.ZentaoBug{}),
Input: cursor,
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_BUG_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_BUG_TABLE,
},
Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
toolEntity := inputRow.(*models.ZentaoBug)
@@ -122,15 +114,17 @@ func ConvertBugForOneProduct(taskCtx
plugin.SubTaskContext) errors.Error {
results = append(results, issueAssignee)
}
- boardId :=
boardIdGen.Generate(data.Options.ConnectionId, data.Options.ProductId)
- if data.Options.ProjectId != 0 {
- boardId =
boardIdGen.Generate(data.Options.ConnectionId, data.Options.ProjectId)
- }
-
domainBoardIssue := &ticket.BoardIssue{
- BoardId: boardId,
+ BoardId:
boardIdGen.Generate(data.Options.ConnectionId, data.Options.ProjectId),
IssueId: domainEntity.Id,
}
+ if toolEntity.Execution != 0 {
+ sprintIssue := &ticket.SprintIssue{
+ SprintId:
executionIdGen.Generate(data.Options.ConnectionId, toolEntity.Execution),
+ IssueId: domainEntity.Id,
+ }
+ results = append(results, sprintIssue)
+ }
results = append(results, domainEntity,
domainBoardIssue)
return results, nil
},
diff --git a/backend/plugins/zentao/tasks/bug_extractor.go
b/backend/plugins/zentao/tasks/bug_extractor.go
index 9226e8847..6be9ddfbd 100644
--- a/backend/plugins/zentao/tasks/bug_extractor.go
+++ b/backend/plugins/zentao/tasks/bug_extractor.go
@@ -38,29 +38,16 @@ var ExtractBugMeta = plugin.SubTaskMeta{
}
func ExtractBug(taskCtx plugin.SubTaskContext) errors.Error {
- return RangeProductOneByOne(taskCtx, ExtractBugForOneProduct)
-}
-
-func ExtractBugForOneProduct(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
- // this Extract only work for product
- if data.Options.ProductId == 0 {
- return nil
- }
-
statusMappings := getBugStatusMapping(data)
stdTypeMappings := getStdTypeMappings(data)
-
+ cache := newAccountCache(taskCtx.GetDal(), data.Options.ConnectionId)
extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_BUG_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_BUG_TABLE,
},
Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
res := &models.ZentaoBugRes{}
@@ -68,18 +55,11 @@ func ExtractBugForOneProduct(taskCtx plugin.SubTaskContext)
errors.Error {
if err != nil {
return nil, errors.Default.WrapRaw(err)
}
-
- // project scope need filter
- if data.Options.ProjectId != 0 {
- if init, ok := data.FromBugList[int(res.ID)];
!ok || !init {
- return nil, nil
- }
- }
-
+ data.Bugs[res.ID] = struct{}{}
bug := &models.ZentaoBug{
ConnectionId: data.Options.ConnectionId,
ID: res.ID,
- Project: res.Project,
+ Project: data.Options.ProjectId,
Product: res.Product,
Injection: res.Injection,
Identify: res.Identify,
@@ -110,19 +90,19 @@ func ExtractBugForOneProduct(taskCtx
plugin.SubTaskContext) errors.Error {
ActivatedDate: res.ActivatedDate,
FeedbackBy: res.FeedbackBy,
NotifyEmail: res.NotifyEmail,
- OpenedById: getAccountId(res.OpenedBy),
- OpenedByName: getAccountName(res.OpenedBy),
+ OpenedById:
cache.getAccountIDFromApiAccount(res.OpenedBy),
+ OpenedByName:
cache.getAccountNameFromApiAccount(res.OpenedBy),
OpenedDate: res.OpenedDate,
OpenedBuild: res.OpenedBuild,
- AssignedToId: getAccountId(res.AssignedTo),
- AssignedToName: getAccountName(res.AssignedTo),
+ AssignedToId:
cache.getAccountIDFromApiAccount(res.AssignedTo),
+ AssignedToName:
cache.getAccountNameFromApiAccount(res.AssignedTo),
AssignedDate: res.AssignedDate,
Deadline: res.Deadline,
- ResolvedById: getAccountId(res.ResolvedBy),
+ ResolvedById:
cache.getAccountIDFromApiAccount(res.ResolvedBy),
Resolution: res.Resolution,
ResolvedBuild: res.ResolvedBuild,
ResolvedDate: res.ResolvedDate,
- ClosedById: getAccountId(res.ClosedBy),
+ ClosedById:
cache.getAccountIDFromApiAccount(res.ClosedBy),
ClosedDate: res.ClosedDate,
DuplicateBug: res.DuplicateBug,
LinkBug: res.LinkBug,
@@ -137,7 +117,7 @@ func ExtractBugForOneProduct(taskCtx plugin.SubTaskContext)
errors.Error {
RepoType: res.RepoType,
IssueKey: res.IssueKey,
Testtask: res.Testtask,
- LastEditedById: getAccountId(res.LastEditedBy),
+ LastEditedById:
cache.getAccountIDFromApiAccount(res.LastEditedBy),
LastEditedDate: res.LastEditedDate,
Deleted: res.Deleted,
PriOrder: res.PriOrder,
diff --git a/backend/plugins/zentao/tasks/bug_repo_commits_collector.go
b/backend/plugins/zentao/tasks/bug_repo_commits_collector.go
index 4428c042a..7d499e106 100644
--- a/backend/plugins/zentao/tasks/bug_repo_commits_collector.go
+++ b/backend/plugins/zentao/tasks/bug_repo_commits_collector.go
@@ -42,20 +42,16 @@ var CollectBugRepoCommitsMeta = plugin.SubTaskMeta{
}
func CollectBugRepoCommits(taskCtx plugin.SubTaskContext) errors.Error {
- return RangeProductOneByOne(taskCtx, CollectBugRepoCommitsForOneProduct)
-}
-
-func CollectBugRepoCommitsForOneProduct(taskCtx plugin.SubTaskContext)
errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*ZentaoTaskData)
// load bugs id from db
clauses := []dal.Clause{
- dal.Select("object_id, repo_revision"),
+ dal.Select("product, repo_revision"),
dal.From(&models.ZentaoBugCommit{}),
dal.Where(
- "product = ? AND connection_id = ?",
- data.Options.ProductId, data.Options.ConnectionId,
+ "project = ? AND connection_id = ?",
+ data.Options.ProjectId, data.Options.ConnectionId,
),
}
@@ -64,7 +60,7 @@ func CollectBugRepoCommitsForOneProduct(taskCtx
plugin.SubTaskContext) errors.Er
return err
}
- iterator, err := api.NewDalCursorIterator(db, cursor,
reflect.TypeOf(SimpleZentaoBugCommit{}))
+ iterator, err := api.NewDalCursorIterator(db, cursor,
reflect.TypeOf(bugCommitInput{}))
if err != nil {
return err
}
@@ -72,13 +68,9 @@ func CollectBugRepoCommitsForOneProduct(taskCtx
plugin.SubTaskContext) errors.Er
// collect bug repo commits
collector, err := api.NewApiCollector(api.ApiCollectorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_BUG_REPO_COMMITS_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_BUG_REPO_COMMITS_TABLE,
},
ApiClient: data.ApiClient,
Input: iterator,
@@ -102,11 +94,9 @@ func CollectBugRepoCommitsForOneProduct(taskCtx
plugin.SubTaskContext) errors.Er
return collector.Execute()
}
-type SimpleZentaoBugCommit struct {
- ObjectID int `json:"objectID"`
- Host string `json:"host"` //the host part of extra
+type bugCommitInput struct {
+ Product int64
RepoRevision string `json:"repoRevision"` // the repoRevisionJson part
of extra
-
}
type RepoRevisionResponse struct {
diff --git a/backend/plugins/zentao/tasks/bug_repo_commits_convertor.go
b/backend/plugins/zentao/tasks/bug_repo_commits_convertor.go
index 41e946bc9..cdd538ec9 100644
--- a/backend/plugins/zentao/tasks/bug_repo_commits_convertor.go
+++ b/backend/plugins/zentao/tasks/bug_repo_commits_convertor.go
@@ -40,16 +40,12 @@ var ConvertBugRepoCommitsMeta = plugin.SubTaskMeta{
}
func ConvertBugRepoCommits(taskCtx plugin.SubTaskContext) errors.Error {
- return RangeProductOneByOne(taskCtx, ConvertBugRepoCommitsForOneProduct)
-}
-
-func ConvertBugRepoCommitsForOneProduct(taskCtx plugin.SubTaskContext)
errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
db := taskCtx.GetDal()
cursor, err := db.Cursor(
dal.From(&models.ZentaoBugRepoCommit{}),
- dal.Where(`product = ? and connection_id = ?`,
data.Options.ProductId, data.Options.ConnectionId),
+ dal.Where(`project = ? and connection_id = ?`,
data.Options.ProjectId, data.Options.ConnectionId),
)
if err != nil {
return err
@@ -61,13 +57,9 @@ func ConvertBugRepoCommitsForOneProduct(taskCtx
plugin.SubTaskContext) errors.Er
InputRowType: reflect.TypeOf(models.ZentaoBugRepoCommit{}),
Input: cursor,
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_BUG_REPO_COMMITS_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_BUG_REPO_COMMITS_TABLE,
},
Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
toolEntity := inputRow.(*models.ZentaoBugRepoCommit)
diff --git a/backend/plugins/zentao/tasks/bug_repo_commits_extractor.go
b/backend/plugins/zentao/tasks/bug_repo_commits_extractor.go
index 8b49d0626..53b9215ce 100644
--- a/backend/plugins/zentao/tasks/bug_repo_commits_extractor.go
+++ b/backend/plugins/zentao/tasks/bug_repo_commits_extractor.go
@@ -38,27 +38,15 @@ var ExtractBugRepoCommitsMeta = plugin.SubTaskMeta{
}
func ExtractBugRepoCommits(taskCtx plugin.SubTaskContext) errors.Error {
- return RangeProductOneByOne(taskCtx, ExtractBugRepoCommitsForOneProduct)
-}
-
-func ExtractBugRepoCommitsForOneProduct(taskCtx plugin.SubTaskContext)
errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
- // this Extract only work for product
- if data.Options.ProductId == 0 {
- return nil
- }
re := regexp.MustCompile(`(\d+)(?:,\s*(\d+))*`)
extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_BUG_REPO_COMMITS_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_BUG_REPO_COMMITS_TABLE,
},
Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
res := &models.ZentaoBugRepoCommitsRes{}
@@ -66,14 +54,18 @@ func ExtractBugRepoCommitsForOneProduct(taskCtx
plugin.SubTaskContext) errors.Er
if err != nil {
return nil, errors.Default.WrapRaw(err)
}
-
+ var input bugCommitInput
+ err = json.Unmarshal(row.Input, &input)
+ if err != nil {
+ return nil, errors.Default.WrapRaw(err)
+ }
results := make([]interface{}, 0)
match := re.FindStringSubmatch(res.Log.Comment)
for i := 1; i < len(match); i++ {
if match[i] != "" {
bugRepoCommits :=
&models.ZentaoBugRepoCommit{
ConnectionId:
data.Options.ConnectionId,
- Product:
data.Options.ProductId,
+ Product: input.Product,
Project:
data.Options.ProjectId,
RepoUrl: res.Repo.CodePath,
CommitSha: res.Revision,
diff --git a/backend/plugins/zentao/tasks/changelog_convertor.go
b/backend/plugins/zentao/tasks/changelog_convertor.go
index 6cd43730a..2ee973655 100644
--- a/backend/plugins/zentao/tasks/changelog_convertor.go
+++ b/backend/plugins/zentao/tasks/changelog_convertor.go
@@ -79,9 +79,8 @@ func ConvertChangelog(taskCtx plugin.SubTaskContext)
errors.Error {
dal.From(&models.ZentaoChangelog{}),
dal.Join(fmt.Sprintf("LEFT JOIN %s on %s.changelog_id = %s.id",
cdn, cdn, cn)),
dal.Join(fmt.Sprintf("LEFT JOIN %s on %s.realname = %s.actor",
an, an, cn)),
- dal.Where(fmt.Sprintf(`%s.product = ? and %s.project = ? and
%s.connection_id = ?`,
- cn, cn, cn),
- data.Options.ProductId,
+ dal.Where(fmt.Sprintf(`%s.project = ? and %s.connection_id = ?`,
+ cn, cn),
data.Options.ProjectId,
data.Options.ConnectionId),
)
@@ -94,13 +93,9 @@ func ConvertChangelog(taskCtx plugin.SubTaskContext)
errors.Error {
InputRowType: reflect.TypeOf(ZentaoChangelogSelect{}),
Input: cursor,
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_ACCOUNT_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_ACCOUNT_TABLE,
},
Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
cl := inputRow.(*ZentaoChangelogSelect)
@@ -168,18 +163,18 @@ func ConvertChangelog(taskCtx plugin.SubTaskContext)
errors.Error {
// accountCache is a cache for account id
type accountCache struct {
- accounts map[string]int64
+ accounts map[string]models.ZentaoAccount
db dal.Dal
connectionId uint64
}
func newAccountCache(db dal.Dal, connectionId uint64) *accountCache {
- return &accountCache{db: db, connectionId: connectionId, accounts:
make(map[string]int64)}
+ return &accountCache{db: db, connectionId: connectionId, accounts:
make(map[string]models.ZentaoAccount)}
}
func (a *accountCache) getAccountID(account string) int64 {
- if id, ok := a.accounts[account]; ok {
- return id
+ if data, ok := a.accounts[account]; ok {
+ return data.ID
}
var zentaoAccount models.ZentaoAccount
err := a.db.First(
@@ -189,6 +184,42 @@ func (a *accountCache) getAccountID(account string) int64 {
if err != nil {
return 0
}
- a.accounts[account] = zentaoAccount.ID
+ a.accounts[account] = zentaoAccount
return zentaoAccount.ID
}
+
+func (a *accountCache) getAccountIDFromApiAccount(account *models.ApiAccount)
int64 {
+ if account == nil {
+ return 0
+ }
+ if account.ID != 0 {
+ return account.ID
+ }
+ return a.getAccountID(account.Account)
+}
+
+func (a *accountCache) getAccountName(account string) string {
+ if data, ok := a.accounts[account]; ok {
+ return data.Realname
+ }
+ var zentaoAccount models.ZentaoAccount
+ err := a.db.First(
+ &zentaoAccount,
+ dal.Where("connection_id = ? AND account = ?", a.connectionId,
account),
+ )
+ if err != nil {
+ return ""
+ }
+ a.accounts[account] = zentaoAccount
+ return zentaoAccount.Realname
+}
+
+func (a *accountCache) getAccountNameFromApiAccount(account
*models.ApiAccount) string {
+ if account == nil {
+ return ""
+ }
+ if account.Realname != "" {
+ return account.Realname
+ }
+ return a.getAccountName(account.Account)
+}
diff --git a/backend/plugins/zentao/tasks/changelog_dbget.go
b/backend/plugins/zentao/tasks/changelog_dbget.go
index 64d15630f..f192d5910 100644
--- a/backend/plugins/zentao/tasks/changelog_dbget.go
+++ b/backend/plugins/zentao/tasks/changelog_dbget.go
@@ -18,7 +18,6 @@ limitations under the License.
package tasks
import (
- "fmt"
"reflect"
"github.com/apache/incubator-devlake/core/dal"
@@ -30,93 +29,114 @@ import (
var _ plugin.SubTaskEntryPoint = DBGetActionHistory
-func DBGetActionHistory(taskCtx plugin.SubTaskContext) errors.Error {
- data := taskCtx.GetData().(*ZentaoTaskData)
+type actionHistoryHandler struct {
+ changelogBachSave *api.BatchSave
+ changelogDetailBachSave *api.BatchSave
+ stories map[int64]struct{}
+ tasks map[int64]struct{}
+ bugs map[int64]struct{}
+}
- // skip if no RemoteDb
- if data.RemoteDb == nil {
- return nil
+func newActionHistoryHandler(taskCtx plugin.SubTaskContext, divider
*api.BatchSaveDivider) (*actionHistoryHandler, errors.Error) {
+ data := taskCtx.GetData().(*ZentaoTaskData)
+ changelogBachSave, err :=
divider.ForType(reflect.TypeOf(&models.ZentaoChangelog{}))
+ if err != nil {
+ return nil, err
}
+ changelogDetailBachSave, err :=
divider.ForType(reflect.TypeOf(&models.ZentaoChangelogDetail{}))
+ if err != nil {
+ return nil, err
+ }
+ return &actionHistoryHandler{
+ changelogBachSave: changelogBachSave,
+ changelogDetailBachSave: changelogDetailBachSave,
+ stories: data.Stories,
+ tasks: data.Tasks,
+ bugs: data.Bugs,
+ }, nil
+}
- divider := api.NewBatchSaveDivider(taskCtx, 500, "", "")
- defer func() {
- err1 := divider.Close()
- if err1 != nil {
- panic(err1)
- }
- }()
+func (h actionHistoryHandler) collectActionHistory(rdb dal.Dal, connectionId
uint64) errors.Error {
+ clause := []dal.Clause{
+ dal.Select("*,zt_action.id aid,zt_history.id hid "),
+ dal.From("zt_action"),
+ dal.Join("LEFT JOIN zt_history on zt_history.action =
zt_action.id"),
+ dal.Where("zt_action.objectType IN ?", []string{"story",
"task", "bug"}),
+ }
+ cursor, err := rdb.Cursor(clause...)
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
- return dBGetActionHistory(data, func(zcc *models.ZentaoChangelogCom)
errors.Error {
- batch, err := divider.ForType(reflect.TypeOf(zcc.Changelog))
+ for cursor.Next() {
+ var ah models.ZentaoRemoteDbActionHistory
+ err = rdb.Fetch(cursor, &ah)
if err != nil {
return err
}
- zcc.Changelog.ConnectionId = data.Options.ConnectionId
- zcc.Changelog.Product = int(data.Options.ProductId)
- err = batch.Add(zcc.Changelog)
+ switch ah.ObjectType {
+ case "story":
+ if _, ok := h.stories[ah.ObjectId]; !ok {
+ continue
+ }
+ case "task":
+ if _, ok := h.tasks[ah.ObjectId]; !ok {
+ continue
+ }
+ case "bug":
+ if _, ok := h.bugs[ah.ObjectId]; !ok {
+ continue
+ }
+ default:
+ continue
+ }
+
+ zcc := ah.Convert(connectionId)
+ err = h.changelogBachSave.Add(zcc.Changelog)
if err != nil {
return err
}
if zcc.ChangelogDetail.Id != 0 {
- batch, err =
divider.ForType(reflect.TypeOf(zcc.ChangelogDetail))
- if err != nil {
- return err
- }
- zcc.ChangelogDetail.ConnectionId =
data.Options.ConnectionId
- err = batch.Add(zcc.ChangelogDetail)
+ err = h.changelogDetailBachSave.Add(zcc.ChangelogDetail)
if err != nil {
return err
}
}
- return nil
- })
-}
-
-var DBGetChangelogMeta = plugin.SubTaskMeta{
- Name: "DBGetChangelog",
- EntryPoint: DBGetActionHistory,
- EnabledByDefault: true,
- Description: "get action and history data to be changelog from
Zentao databases",
- DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
-}
-
-// it is work for zentao version 18.3
-func dBGetActionHistory(data *ZentaoTaskData, callback
func(*models.ZentaoChangelogCom) errors.Error) errors.Error {
- rdb := data.RemoteDb
- atn := (models.ZentaoRemoteDbAction{}).TableName()
- htn := (models.ZentaoRemoteDbHistory{}).TableName()
-
- clause := []dal.Clause{
- dal.Select(fmt.Sprintf("*,%s.id aid,%s.id hid ", atn, htn)),
- dal.From(atn),
}
-
- if data.Options.ProductId != 0 {
- clause = append(clause, dal.Where(fmt.Sprintf("%s.product = ?",
atn), fmt.Sprintf(",%d,", data.Options.ProductId)))
- }
- if data.Options.ProjectId != 0 {
- clause = append(clause, dal.Where(fmt.Sprintf("%s.project = ?",
atn), data.Options.ProjectId))
- }
- clause = append(clause, dal.Join(fmt.Sprintf("LEFT JOIN %s on %s.action
= %s.id", htn, htn, atn)))
-
- cursor, err := rdb.Cursor(clause...)
+ err = h.changelogBachSave.Flush()
if err != nil {
return err
}
- defer cursor.Close()
+ return h.changelogDetailBachSave.Flush()
+}
- for cursor.Next() {
- actionHistory := &models.ZentaoRemoteDbActionHistory{}
- err = rdb.Fetch(cursor, actionHistory)
- if err != nil {
- return err
- }
+func DBGetActionHistory(taskCtx plugin.SubTaskContext) errors.Error {
+ data := taskCtx.GetData().(*ZentaoTaskData)
- err = callback(actionHistory.Convert())
- if err != nil {
- return err
+ // skip if no RemoteDb
+ if data.RemoteDb == nil {
+ return nil
+ }
+
+ divider := api.NewBatchSaveDivider(taskCtx, 500, "", "")
+ defer func() {
+ err1 := divider.Close()
+ if err1 != nil {
+ panic(err1)
}
+ }()
+ handler, err := newActionHistoryHandler(taskCtx, divider)
+ if err != nil {
+ return err
}
+ return handler.collectActionHistory(data.RemoteDb,
data.Options.ConnectionId)
+}
- return nil
+var DBGetChangelogMeta = plugin.SubTaskMeta{
+ Name: "DBGetChangelog",
+ EntryPoint: DBGetActionHistory,
+ EnabledByDefault: true,
+ Description: "get action and history data to be changelog from
Zentao databases",
+ DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
}
diff --git a/backend/plugins/zentao/tasks/department_collector.go
b/backend/plugins/zentao/tasks/department_collector.go
index bdcad7ffd..8da824846 100644
--- a/backend/plugins/zentao/tasks/department_collector.go
+++ b/backend/plugins/zentao/tasks/department_collector.go
@@ -36,17 +36,13 @@ func CollectDepartment(taskCtx plugin.SubTaskContext)
errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
collector, err := api.NewApiCollector(api.ApiCollectorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_DEPARTMENT_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_DEPARTMENT_TABLE,
},
ApiClient: data.ApiClient,
PageSize: 100,
- UrlTemplate: "/users",
+ UrlTemplate: "/departments",
Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
@@ -55,14 +51,12 @@ func CollectDepartment(taskCtx plugin.SubTaskContext)
errors.Error {
},
GetTotalPages: GetTotalPagesFromResponse,
ResponseParser: func(res *http.Response) ([]json.RawMessage,
errors.Error) {
- var data struct {
- Users []json.RawMessage `json:"users"`
- }
+ var data []json.RawMessage
err := api.UnmarshalResponse(res, &data)
if err != nil {
- return nil, errors.Default.Wrap(err, "error
reading endpoint response by Zentao bug collector")
+ return nil, errors.Default.Wrap(err, "error
reading endpoint response by Zentao departments collector")
}
- return data.Users, nil
+ return data, nil
},
})
if err != nil {
diff --git a/backend/plugins/zentao/tasks/department_convertor.go
b/backend/plugins/zentao/tasks/department_convertor.go
index e5b88b917..b2c4a7020 100644
--- a/backend/plugins/zentao/tasks/department_convertor.go
+++ b/backend/plugins/zentao/tasks/department_convertor.go
@@ -56,13 +56,9 @@ func ConvertDepartment(taskCtx plugin.SubTaskContext)
errors.Error {
InputRowType: reflect.TypeOf(models.ZentaoDepartment{}),
Input: cursor,
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_DEPARTMENT_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_DEPARTMENT_TABLE,
},
Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
toolEntity := inputRow.(*models.ZentaoDepartment)
diff --git a/backend/plugins/zentao/tasks/department_extractor.go
b/backend/plugins/zentao/tasks/department_extractor.go
index 691906ac8..d6cd5ada9 100644
--- a/backend/plugins/zentao/tasks/department_extractor.go
+++ b/backend/plugins/zentao/tasks/department_extractor.go
@@ -40,13 +40,9 @@ func ExtractDepartment(taskCtx plugin.SubTaskContext)
errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_DEPARTMENT_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_DEPARTMENT_TABLE,
},
Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
department := &models.ZentaoDepartment{}
@@ -55,9 +51,7 @@ func ExtractDepartment(taskCtx plugin.SubTaskContext)
errors.Error {
return nil, errors.Default.WrapRaw(err)
}
department.ConnectionId = data.Options.ConnectionId
- results := make([]interface{}, 0)
- results = append(results, department)
- return results, nil
+ return []interface{}{department}, nil
},
})
diff --git a/backend/plugins/zentao/tasks/execution_collector.go
b/backend/plugins/zentao/tasks/execution_collector.go
index 2dcc54441..ee04a3dcd 100644
--- a/backend/plugins/zentao/tasks/execution_collector.go
+++ b/backend/plugins/zentao/tasks/execution_collector.go
@@ -20,12 +20,11 @@ package tasks
import (
"encoding/json"
"fmt"
- "net/http"
- "net/url"
-
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "net/http"
+ "net/url"
)
const RAW_EXECUTION_TABLE = "zentao_api_executions"
@@ -34,24 +33,20 @@ var _ plugin.SubTaskEntryPoint = CollectExecutions
func CollectExecutions(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
-
- // this collect only work for project
- if data.Options.ProjectId == 0 {
- return nil
+ cursor, iterator, err := getExecutionIterator(taskCtx)
+ if err != nil {
+ return err
}
-
+ defer cursor.Close()
collector, err := api.NewApiCollector(api.ApiCollectorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_EXECUTION_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_EXECUTION_TABLE,
},
+ Input: iterator,
ApiClient: data.ApiClient,
- UrlTemplate: "/{{ .Params.ZentaoId }}/executions",
+ UrlTemplate: "/executions/{{ .Input.Id }}",
Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
@@ -59,14 +54,12 @@ func CollectExecutions(taskCtx plugin.SubTaskContext)
errors.Error {
return query, nil
},
ResponseParser: func(res *http.Response) ([]json.RawMessage,
errors.Error) {
- var data struct {
- Executions []json.RawMessage `json:"executions"`
- }
+ var data json.RawMessage
err := api.UnmarshalResponse(res, &data)
if err != nil {
- return nil, errors.Default.Wrap(err, "error
reading endpoint response by Zentao bug collector")
+ return nil, err
}
- return data.Executions, nil
+ return []json.RawMessage{data}, nil
},
})
if err != nil {
diff --git a/backend/plugins/zentao/tasks/execution_convertor.go
b/backend/plugins/zentao/tasks/execution_convertor.go
index 470609ca5..46804813f 100644
--- a/backend/plugins/zentao/tasks/execution_convertor.go
+++ b/backend/plugins/zentao/tasks/execution_convertor.go
@@ -57,13 +57,9 @@ func ConvertExecutions(taskCtx plugin.SubTaskContext)
errors.Error {
InputRowType: reflect.TypeOf(models.ZentaoExecution{}),
Input: cursor,
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_EXECUTION_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_EXECUTION_TABLE,
},
Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
toolExecution := inputRow.(*models.ZentaoExecution)
diff --git a/backend/plugins/zentao/tasks/execution_extractor.go
b/backend/plugins/zentao/tasks/execution_extractor.go
index 27b3b4316..1b94772d9 100644
--- a/backend/plugins/zentao/tasks/execution_extractor.go
+++ b/backend/plugins/zentao/tasks/execution_extractor.go
@@ -19,7 +19,6 @@ package tasks
import (
"encoding/json"
-
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
@@ -39,20 +38,11 @@ var ExtractExecutionMeta = plugin.SubTaskMeta{
func ExtractExecutions(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
- // this Extract only work for project
- if data.Options.ProjectId == 0 {
- return nil
- }
-
extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_EXECUTION_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_EXECUTION_TABLE,
},
Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
res := &models.ZentaoExecutionRes{}
@@ -60,10 +50,15 @@ func ExtractExecutions(taskCtx plugin.SubTaskContext)
errors.Error {
if err != nil {
return nil, errors.Default.WrapRaw(err)
}
-
- // append product to taskdata
+ var results []interface{}
for _, product := range res.Products {
- data.ProductList[product.ID] = product.Name
+ p := &models.ZentaoProductSummary{
+ ConnectionId: data.Options.ConnectionId,
+ ProjectId: data.Options.ProjectId,
+ Id: product.ID,
+ Name: product.Name,
+ }
+ results = append(results, p)
}
execution := &models.ZentaoExecution{
@@ -126,7 +121,6 @@ func ExtractExecutions(taskCtx plugin.SubTaskContext)
errors.Error {
Progress: res.Progress,
CaseReview: res.CaseReview,
}
- results := make([]interface{}, 0)
results = append(results, execution)
return results, nil
},
diff --git a/backend/plugins/zentao/tasks/account_collector.go
b/backend/plugins/zentao/tasks/execution_story_collector.go
similarity index 69%
copy from backend/plugins/zentao/tasks/account_collector.go
copy to backend/plugins/zentao/tasks/execution_story_collector.go
index 9ecd35665..b1fd01c5c 100644
--- a/backend/plugins/zentao/tasks/account_collector.go
+++ b/backend/plugins/zentao/tasks/execution_story_collector.go
@@ -20,49 +20,49 @@ package tasks
import (
"encoding/json"
"fmt"
- "net/http"
- "net/url"
-
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "net/http"
+ "net/url"
)
-const RAW_ACCOUNT_TABLE = "zentao_api_accounts"
-
-var _ plugin.SubTaskEntryPoint = CollectAccount
+var _ plugin.SubTaskEntryPoint = CollectExecutionStory
-func CollectAccount(taskCtx plugin.SubTaskContext) errors.Error {
+func CollectExecutionStory(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
+ cursor, iterator, err := getExecutionIterator(taskCtx)
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
collector, err := api.NewApiCollector(api.ApiCollectorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_ACCOUNT_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_STORY_TABLE,
},
ApiClient: data.ApiClient,
+ Input: iterator,
PageSize: 100,
- UrlTemplate: "/users",
+ UrlTemplate: "/executions/{{ .Input.Id }}/stories",
Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
query.Set("limit", fmt.Sprintf("%v",
reqData.Pager.Size))
+ query.Set("status", "allstory")
return query, nil
},
GetTotalPages: GetTotalPagesFromResponse,
ResponseParser: func(res *http.Response) ([]json.RawMessage,
errors.Error) {
var data struct {
- Users []json.RawMessage `json:"users"`
+ Story []json.RawMessage `json:"stories"`
}
err := api.UnmarshalResponse(res, &data)
if err != nil {
- return nil, errors.Default.Wrap(err, "error
reading endpoint response by Zentao bug collector")
+ return nil, err
}
- return data.Users, nil
+ return data.Story, nil
},
})
if err != nil {
@@ -71,11 +71,3 @@ func CollectAccount(taskCtx plugin.SubTaskContext)
errors.Error {
return collector.Execute()
}
-
-var CollectAccountMeta = plugin.SubTaskMeta{
- Name: "collectAccount",
- EntryPoint: CollectAccount,
- EnabledByDefault: true,
- Description: "Collect Account data from Zentao api",
- DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
-}
diff --git a/backend/plugins/zentao/tasks/product_convertor.go
b/backend/plugins/zentao/tasks/execution_story_convertor.go
similarity index 54%
rename from backend/plugins/zentao/tasks/product_convertor.go
rename to backend/plugins/zentao/tasks/execution_story_convertor.go
index 0030f12ed..01536807a 100644
--- a/backend/plugins/zentao/tasks/product_convertor.go
+++ b/backend/plugins/zentao/tasks/execution_story_convertor.go
@@ -18,12 +18,10 @@ limitations under the License.
package tasks
import (
- "fmt"
"reflect"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/core/plugin"
@@ -31,60 +29,44 @@ import (
"github.com/apache/incubator-devlake/plugins/zentao/models"
)
-const RAW_PRODUCT_TABLE = "zentao_api_products"
+var _ plugin.SubTaskEntryPoint = ConvertExecutionStory
-var _ plugin.SubTaskEntryPoint = ConvertProducts
-
-var ConvertProductMeta = plugin.SubTaskMeta{
- Name: "convertProducts",
- EntryPoint: ConvertProducts,
+var ConvertExecutionStoryMeta = plugin.SubTaskMeta{
+ Name: "convertExecutionStory",
+ EntryPoint: ConvertExecutionStory,
EnabledByDefault: true,
- Description: "convert Zentao products",
+ Description: "convert Zentao execution_stories",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
}
-func ConvertProducts(taskCtx plugin.SubTaskContext) errors.Error {
+func ConvertExecutionStory(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
db := taskCtx.GetDal()
- boardIdGen := didgen.NewDomainIdGenerator(&models.ZentaoProduct{})
+ executionIdGen := didgen.NewDomainIdGenerator(&models.ZentaoExecution{})
+ storyIdGen := didgen.NewDomainIdGenerator(&models.ZentaoStory{})
cursor, err := db.Cursor(
- dal.From(&models.ZentaoProduct{}),
- dal.Where(`id = ? and connection_id = ?`,
data.Options.ProductId, data.Options.ConnectionId),
+ dal.From(&models.ZentaoExecutionStory{}),
+ dal.Where(`project_id = ? and connection_id = ?`,
data.Options.ProjectId, data.Options.ConnectionId),
)
if err != nil {
return err
}
defer cursor.Close()
convertor, err := api.NewDataConverter(api.DataConverterArgs{
- InputRowType: reflect.TypeOf(models.ZentaoProduct{}),
+ InputRowType: reflect.TypeOf(models.ZentaoExecutionStory{}),
Input: cursor,
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_PRODUCT_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_EXECUTION_TABLE,
},
Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
- toolProduct := inputRow.(*models.ZentaoProduct)
-
- data.ProductList[toolProduct.Id] = toolProduct.Name
-
- domainBoard := &ticket.Board{
- DomainEntity: domainlayer.DomainEntity{
- Id:
boardIdGen.Generate(toolProduct.ConnectionId, toolProduct.Id),
- },
- Name: toolProduct.Name,
- Description: toolProduct.Description,
- CreatedDate:
toolProduct.CreatedDate.ToNullableTime(),
- Type: toolProduct.Type + "/" +
toolProduct.ProductType,
- Url:
fmt.Sprintf("/product-index-%d.html", data.Options.ProductId),
+ executionStory :=
inputRow.(*models.ZentaoExecutionStory)
+ sprintIssue := &ticket.SprintIssue{
+ SprintId:
executionIdGen.Generate(data.Options.ConnectionId, executionStory.ExecutionId),
+ IssueId:
storyIdGen.Generate(data.Options.ConnectionId, executionStory.StoryId),
}
- results := make([]interface{}, 0)
- results = append(results, domainBoard)
- return results, nil
+ return []interface{}{sprintIssue}, nil
},
})
diff --git a/backend/plugins/zentao/tasks/execution_collector.go
b/backend/plugins/zentao/tasks/execution_summary_collector.go
similarity index 74%
copy from backend/plugins/zentao/tasks/execution_collector.go
copy to backend/plugins/zentao/tasks/execution_summary_collector.go
index 2dcc54441..3688b3826 100644
--- a/backend/plugins/zentao/tasks/execution_collector.go
+++ b/backend/plugins/zentao/tasks/execution_summary_collector.go
@@ -28,30 +28,20 @@ import (
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
-const RAW_EXECUTION_TABLE = "zentao_api_executions"
+const RAW_EXECUTION_SUMMARY_TABLE = "zentao_api_execution_summary"
-var _ plugin.SubTaskEntryPoint = CollectExecutions
+var _ plugin.SubTaskEntryPoint = CollectExecutionSummary
-func CollectExecutions(taskCtx plugin.SubTaskContext) errors.Error {
+func CollectExecutionSummary(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
-
- // this collect only work for project
- if data.Options.ProjectId == 0 {
- return nil
- }
-
collector, err := api.NewApiCollector(api.ApiCollectorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_EXECUTION_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_EXECUTION_SUMMARY_TABLE,
},
ApiClient: data.ApiClient,
- UrlTemplate: "/{{ .Params.ZentaoId }}/executions",
+ UrlTemplate: fmt.Sprintf("/projects/%d/executions",
data.Options.ProjectId),
Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
@@ -76,10 +66,10 @@ func CollectExecutions(taskCtx plugin.SubTaskContext)
errors.Error {
return collector.Execute()
}
-var CollectExecutionMeta = plugin.SubTaskMeta{
- Name: "collectExecutions",
- EntryPoint: CollectExecutions,
+var CollectExecutionSummaryMeta = plugin.SubTaskMeta{
+ Name: "collectExecutionSummary",
+ EntryPoint: CollectExecutionSummary,
EnabledByDefault: true,
- Description: "Collect Execution data from Zentao api",
+ Description: "Collect Execution summary data from Zentao api",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
}
diff --git a/backend/plugins/zentao/tasks/account_extractor.go
b/backend/plugins/zentao/tasks/execution_summary_extractor.go
similarity index 68%
copy from backend/plugins/zentao/tasks/account_extractor.go
copy to backend/plugins/zentao/tasks/execution_summary_extractor.go
index 9ad1b720d..3aecf9cfa 100644
--- a/backend/plugins/zentao/tasks/account_extractor.go
+++ b/backend/plugins/zentao/tasks/execution_summary_extractor.go
@@ -26,38 +26,33 @@ import (
"github.com/apache/incubator-devlake/plugins/zentao/models"
)
-var _ plugin.SubTaskEntryPoint = ExtractAccount
+var _ plugin.SubTaskEntryPoint = ExtractExecutionSummary
-var ExtractAccountMeta = plugin.SubTaskMeta{
- Name: "extractAccount",
- EntryPoint: ExtractAccount,
+var ExtractExecutionSummaryMeta = plugin.SubTaskMeta{
+ Name: "extractExecutionSummary",
+ EntryPoint: ExtractExecutionSummary,
EnabledByDefault: true,
- Description: "extract Zentao account",
+ Description: "extract Zentao execution summary",
DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET},
}
-func ExtractAccount(taskCtx plugin.SubTaskContext) errors.Error {
+func ExtractExecutionSummary(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
+
extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_ACCOUNT_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_EXECUTION_SUMMARY_TABLE,
},
Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
- account := &models.ZentaoAccount{}
- err := json.Unmarshal(row.Data, account)
+ executionSummary := &models.ZentaoExecutionSummary{}
+ err := json.Unmarshal(row.Data, executionSummary)
if err != nil {
return nil, errors.Default.WrapRaw(err)
}
- account.ConnectionId = data.Options.ConnectionId
- results := make([]interface{}, 0)
- results = append(results, account)
- return results, nil
+ executionSummary.ConnectionId =
data.Options.ConnectionId
+ return []interface{}{executionSummary}, nil
},
})
diff --git a/backend/plugins/zentao/tasks/iterator.go
b/backend/plugins/zentao/tasks/iterator.go
new file mode 100644
index 000000000..0c3400a72
--- /dev/null
+++ b/backend/plugins/zentao/tasks/iterator.go
@@ -0,0 +1,105 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+)
+
+type iteratorConcator struct {
+ index int
+ iterators []api.Iterator
+}
+
+func newIteratorConcator(iterators ...api.Iterator) *iteratorConcator {
+ return &iteratorConcator{iterators: iterators}
+}
+
+func (w *iteratorConcator) HasNext() bool {
+ for w.index < len(w.iterators) {
+ if w.iterators[w.index].HasNext() {
+ return true
+ }
+ w.index++
+ }
+ return false
+}
+
+func (w *iteratorConcator) Fetch() (interface{}, errors.Error) {
+ if w.index >= len(w.iterators) {
+ return nil, errors.Default.New("index out of range")
+ }
+ return w.iterators[w.index].Fetch()
+}
+
+func (w *iteratorConcator) Close() errors.Error {
+ for _, iterator := range w.iterators {
+ iterator.Close()
+ }
+ return nil
+}
+
+type iteratorWrapper struct {
+ original api.Iterator
+ wrapperFunc func(interface{}) interface{}
+}
+
+func newIteratorWrapper(original api.Iterator, wrapperFunc func(interface{})
interface{}) *iteratorWrapper {
+ return &iteratorWrapper{original: original, wrapperFunc: wrapperFunc}
+}
+
+func (w *iteratorWrapper) HasNext() bool {
+ return w.original.HasNext()
+}
+
+func (w *iteratorWrapper) Fetch() (interface{}, errors.Error) {
+ data, err := w.original.Fetch()
+ if err != nil {
+ return nil, err
+ }
+ return w.wrapperFunc(data), nil
+}
+
+func (w *iteratorWrapper) Close() errors.Error {
+ return w.original.Close()
+}
+
+type iteratorFromSlice struct {
+ index int
+ data []interface{}
+}
+
+func newIteratorFromSlice(data []interface{}) *iteratorFromSlice {
+ return &iteratorFromSlice{data: data}
+}
+
+func (i *iteratorFromSlice) HasNext() bool {
+ return i.index < len(i.data)
+}
+
+func (i *iteratorFromSlice) Fetch() (interface{}, errors.Error) {
+ data := i.data[i.index]
+ i.index++
+ return data, nil
+}
+
+func (i *iteratorFromSlice) Close() errors.Error {
+ i.index = len(i.data) - 1
+ return nil
+}
diff --git a/backend/plugins/zentao/tasks/project_convertor.go
b/backend/plugins/zentao/tasks/project_convertor.go
index 60d458c63..7beb16d5e 100644
--- a/backend/plugins/zentao/tasks/project_convertor.go
+++ b/backend/plugins/zentao/tasks/project_convertor.go
@@ -59,13 +59,9 @@ func ConvertProjects(taskCtx plugin.SubTaskContext)
errors.Error {
InputRowType: reflect.TypeOf(models.ZentaoProject{}),
Input: cursor,
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_PROJECT_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_PROJECT_TABLE,
},
Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
toolProject := inputRow.(*models.ZentaoProject)
diff --git a/backend/plugins/zentao/tasks/shared.go
b/backend/plugins/zentao/tasks/shared.go
index 6f029ed09..93d749f2b 100644
--- a/backend/plugins/zentao/tasks/shared.go
+++ b/backend/plugins/zentao/tasks/shared.go
@@ -19,16 +19,22 @@ package tasks
import (
"fmt"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/plugin"
"net/http"
"net/url"
+ "reflect"
"strings"
"github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/zentao/models"
)
+type input struct {
+ Id int64
+}
+
func GetTotalPagesFromResponse(res *http.Response, args *api.ApiCollectorArgs)
(int, errors.Error) {
body := &ZentaoPagination{}
err := api.UnmarshalResponse(res, body)
@@ -83,9 +89,6 @@ func getOriginalProject(data *ZentaoTaskData) string {
if data.Options.ProjectId != 0 {
return data.ProjectName
}
- if data.Options.ProductId != 0 {
- return data.ProductName
- }
return ""
}
@@ -182,30 +185,49 @@ func ignoreHTTPStatus404(res *http.Response) errors.Error
{
return nil
}
-func RangeProductOneByOne(taskCtx plugin.SubTaskContext, callback func(taskCtx
plugin.SubTaskContext) errors.Error) errors.Error {
+func getProductIterator(taskCtx plugin.SubTaskContext) (dal.Rows,
*api.DalCursorIterator, errors.Error) {
data := taskCtx.GetData().(*ZentaoTaskData)
- for id, name := range data.ProductList {
- data.Options.ProductId = id
- data.ProductName = name
-
- err := callback(taskCtx)
- if err != nil {
- return err
- }
+ db := taskCtx.GetDal()
+ clauses := []dal.Clause{
+ dal.Select("id"),
+ dal.From(&models.ZentaoProductSummary{}),
+ dal.Where(
+ "project_id = ? AND connection_id = ?",
+ data.Options.ProjectId, data.Options.ConnectionId,
+ ),
}
- return nil
+ cursor, err := db.Cursor(clauses...)
+ if err != nil {
+ return nil, nil, err
+ }
+ iterator, err := api.NewDalCursorIterator(db, cursor,
reflect.TypeOf(input{}))
+ if err != nil {
+ cursor.Close()
+ return nil, nil, err
+ }
+ return cursor, iterator, nil
}
-func ScopeParams(cid uint64, projectId int64, productId int64) ZentaoApiParams
{
- param := ZentaoApiParams{
- ConnectionId: cid,
+func getExecutionIterator(taskCtx plugin.SubTaskContext) (dal.Rows,
*api.DalCursorIterator, errors.Error) {
+ data := taskCtx.GetData().(*ZentaoTaskData)
+ db := taskCtx.GetDal()
+ clauses := []dal.Clause{
+ dal.Select("id"),
+ dal.From(&models.ZentaoExecutionSummary{}),
+ dal.Where(
+ "project = ? AND connection_id = ?",
+ data.Options.ProjectId, data.Options.ConnectionId,
+ ),
+ }
+ cursor, err := db.Cursor(clauses...)
+ if err != nil {
+ return nil, nil, err
}
- if projectId != 0 {
- param.ZentaoId = fmt.Sprintf("projects/%d", projectId)
- } else {
- param.ZentaoId = fmt.Sprintf("products/%d", productId)
+ iterator, err := api.NewDalCursorIterator(db, cursor,
reflect.TypeOf(input{}))
+ if err != nil {
+ cursor.Close()
+ return nil, nil, err
}
-
- return param
+ return cursor, iterator, nil
}
diff --git a/backend/plugins/zentao/tasks/story_collector.go
b/backend/plugins/zentao/tasks/story_collector.go
index 02dd21af2..342840f98 100644
--- a/backend/plugins/zentao/tasks/story_collector.go
+++ b/backend/plugins/zentao/tasks/story_collector.go
@@ -32,31 +32,48 @@ const RAW_STORY_TABLE = "zentao_api_stories"
var _ plugin.SubTaskEntryPoint = CollectStory
-func CollectStory(taskCtx plugin.SubTaskContext) errors.Error {
- return RangeProductOneByOne(taskCtx, CollectStoryForOneProduct)
+type storyInput struct {
+ Path string
+ ProjectId int64
+ ProductId int64
+ ExecutionId int64
}
-func CollectStoryForOneProduct(taskCtx plugin.SubTaskContext) errors.Error {
+func CollectStory(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
+ // project iterator
+ iter0 := newIteratorFromSlice([]interface{}{&storyInput{ProjectId:
data.Options.ProjectId, Path: fmt.Sprintf("/projects/%d",
data.Options.ProjectId)}})
+
+ // product iterator
+ productCursor, productIterator, err := getProductIterator(taskCtx)
+ if err != nil {
+ return err
+ }
+ defer productCursor.Close()
+ iter1 := newIteratorWrapper(productIterator, func(arg interface{})
interface{} {
+ return &storyInput{ProductId: arg.(*input).Id, Path:
fmt.Sprintf("/products/%d", arg.(*input).Id)}
+ })
- // this collect only work for product
- if data.Options.ProductId == 0 {
- return nil
+ // execution iterator
+ executionCursor, executionIterator, err := getExecutionIterator(taskCtx)
+ if err != nil {
+ return err
}
+ defer executionCursor.Close()
+ iter2 := newIteratorWrapper(executionIterator, func(arg interface{})
interface{} {
+ return &storyInput{ExecutionId: arg.(*input).Id, Path:
fmt.Sprintf("/executions/%d", arg.(*input).Id)}
+ })
collector, err := api.NewApiCollector(api.ApiCollectorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_STORY_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_STORY_TABLE,
},
+ Input: newIteratorConcator(iter0, iter1, iter2),
ApiClient: data.ApiClient,
PageSize: 100,
- UrlTemplate: "/{{ .Params.ZentaoId }}/stories",
+ UrlTemplate: "{{ .Input.Path }}/stories",
Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page))
diff --git a/backend/plugins/zentao/tasks/story_commits_collector.go
b/backend/plugins/zentao/tasks/story_commits_collector.go
index ec780c76f..07fea3cfe 100644
--- a/backend/plugins/zentao/tasks/story_commits_collector.go
+++ b/backend/plugins/zentao/tasks/story_commits_collector.go
@@ -42,22 +42,14 @@ var CollectStoryCommitsMeta = plugin.SubTaskMeta{
}
func CollectStoryCommits(taskCtx plugin.SubTaskContext) errors.Error {
- return RangeProductOneByOne(taskCtx, CollectStoryCommitsForOneProduct)
-}
-
-func CollectStoryCommitsForOneProduct(taskCtx plugin.SubTaskContext)
errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*ZentaoTaskData)
// state manager
collectorWithState, err :=
api.NewStatefulApiCollector(api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_STORY_COMMITS_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_STORY_COMMITS_TABLE,
}, data.TimeAfter)
if err != nil {
return err
@@ -65,12 +57,13 @@ func CollectStoryCommitsForOneProduct(taskCtx
plugin.SubTaskContext) errors.Erro
// load stories id from db
clauses := []dal.Clause{
- dal.Select("id, last_edited_date"),
- dal.From(&models.ZentaoStory{}),
- dal.Where(
- "product = ? AND connection_id = ?",
- data.Options.ProductId, data.Options.ConnectionId,
- ),
+ dal.Select("_tool_zentao_project_stories.story_id AS id,
last_edited_date"),
+ dal.From(&models.ZentaoProjectStory{}),
+ dal.Join(`LEFT JOIN _tool_zentao_stories ON
+
_tool_zentao_project_stories.story_id = _tool_zentao_stories.id
+ AND
_tool_zentao_project_stories.connection_id =
_tool_zentao_stories.connection_id`),
+ dal.Where(`_tool_zentao_project_stories.project_id = ? and
+ _tool_zentao_project_stories.connection_id = ?`,
data.Options.ProjectId, data.Options.ConnectionId),
}
// incremental collection
incremental := collectorWithState.IsIncremental()
@@ -85,7 +78,7 @@ func CollectStoryCommitsForOneProduct(taskCtx
plugin.SubTaskContext) errors.Erro
return err
}
- iterator, err := api.NewDalCursorIterator(db, cursor,
reflect.TypeOf(SimpleZentaoStory{}))
+ iterator, err := api.NewDalCursorIterator(db, cursor,
reflect.TypeOf(inputWithLastEditedDate{}))
if err != nil {
return err
}
@@ -93,13 +86,9 @@ func CollectStoryCommitsForOneProduct(taskCtx
plugin.SubTaskContext) errors.Erro
// collect story commits
err = collectorWithState.InitCollector(api.ApiCollectorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_STORY_COMMITS_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_STORY_COMMITS_TABLE,
},
ApiClient: data.ApiClient,
Input: iterator,
@@ -125,7 +114,7 @@ func CollectStoryCommitsForOneProduct(taskCtx
plugin.SubTaskContext) errors.Erro
return collectorWithState.Execute()
}
-type SimpleZentaoStory struct {
+type inputWithLastEditedDate struct {
ID int64 `json:"id"`
LastEditedDate *api.Iso8601Time `json:"lastEditedDate"`
}
diff --git a/backend/plugins/zentao/tasks/story_commits_extractor.go
b/backend/plugins/zentao/tasks/story_commits_extractor.go
index c883e3beb..97243db25 100644
--- a/backend/plugins/zentao/tasks/story_commits_extractor.go
+++ b/backend/plugins/zentao/tasks/story_commits_extractor.go
@@ -39,27 +39,14 @@ var ExtractStoryCommitsMeta = plugin.SubTaskMeta{
}
func ExtractStoryCommits(taskCtx plugin.SubTaskContext) errors.Error {
- return RangeProductOneByOne(taskCtx, ExtractStoryCommitsForOneProduct)
-}
-
-func ExtractStoryCommitsForOneProduct(taskCtx plugin.SubTaskContext)
errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
- // this Extract only work for product
- if data.Options.ProductId == 0 {
- return nil
- }
-
re := regexp.MustCompile(`href='(.*?)'`)
extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_STORY_COMMITS_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_STORY_COMMITS_TABLE,
},
Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
res := &models.ZentaoStoryCommitsRes{}
@@ -78,7 +65,6 @@ func ExtractStoryCommitsForOneProduct(taskCtx
plugin.SubTaskContext) errors.Erro
ID: res.ID,
ObjectType: res.ObjectType,
ObjectID: res.ObjectID,
- Product: data.Options.ProductId,
Project: data.Options.ProjectId,
Execution: res.Execution,
Actor: res.Actor,
diff --git a/backend/plugins/zentao/tasks/story_convertor.go
b/backend/plugins/zentao/tasks/story_convertor.go
index bd7eca35e..66b73dd2b 100644
--- a/backend/plugins/zentao/tasks/story_convertor.go
+++ b/backend/plugins/zentao/tasks/story_convertor.go
@@ -42,23 +42,19 @@ var ConvertStoryMeta = plugin.SubTaskMeta{
}
func ConvertStory(taskCtx plugin.SubTaskContext) errors.Error {
- return RangeProductOneByOne(taskCtx, ConvertStoryForOneProduct)
-}
-
-func ConvertStoryForOneProduct(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
db := taskCtx.GetDal()
storyIdGen := didgen.NewDomainIdGenerator(&models.ZentaoStory{})
- boardIdGen := didgen.NewDomainIdGenerator(&models.ZentaoProduct{})
+ boardIdGen := didgen.NewDomainIdGenerator(&models.ZentaoProject{})
accountIdGen := didgen.NewDomainIdGenerator(&models.ZentaoAccount{})
- if data.Options.ProjectId != 0 {
- boardIdGen =
didgen.NewDomainIdGenerator(&models.ZentaoProject{})
- }
cursor, err := db.Cursor(
dal.From(&models.ZentaoStory{}),
- dal.Where(`_tool_zentao_stories.product = ? and
- _tool_zentao_stories.connection_id = ?`,
data.Options.ProductId, data.Options.ConnectionId),
+ dal.Join(`LEFT JOIN _tool_zentao_project_stories ON
+
_tool_zentao_project_stories.story_id = _tool_zentao_stories.id
+ AND
_tool_zentao_project_stories.connection_id =
_tool_zentao_stories.connection_id`),
+ dal.Where(`_tool_zentao_project_stories.project_id = ? and
+ _tool_zentao_project_stories.connection_id = ?`,
data.Options.ProjectId, data.Options.ConnectionId),
)
if err != nil {
return err
@@ -68,13 +64,9 @@ func ConvertStoryForOneProduct(taskCtx
plugin.SubTaskContext) errors.Error {
InputRowType: reflect.TypeOf(models.ZentaoStory{}),
Input: cursor,
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_STORY_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_STORY_TABLE,
},
Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
toolEntity := inputRow.(*models.ZentaoStory)
@@ -124,13 +116,8 @@ func ConvertStoryForOneProduct(taskCtx
plugin.SubTaskContext) errors.Error {
domainEntity.LeadTimeMinutes =
int64(toolEntity.ClosedDate.ToNullableTime().Sub(toolEntity.OpenedDate.ToTime()).Minutes())
}
- boardId :=
boardIdGen.Generate(data.Options.ConnectionId, data.Options.ProductId)
- if data.Options.ProjectId != 0 {
- boardId =
boardIdGen.Generate(data.Options.ConnectionId, data.Options.ProjectId)
- }
-
domainBoardIssue := &ticket.BoardIssue{
- BoardId: boardId,
+ BoardId:
boardIdGen.Generate(data.Options.ConnectionId, data.Options.ProjectId),
IssueId: domainEntity.Id,
}
results = append(results, domainEntity,
domainBoardIssue)
diff --git a/backend/plugins/zentao/tasks/story_extractor.go
b/backend/plugins/zentao/tasks/story_extractor.go
index 6495e7bf6..88c1ba324 100644
--- a/backend/plugins/zentao/tasks/story_extractor.go
+++ b/backend/plugins/zentao/tasks/story_extractor.go
@@ -38,44 +38,37 @@ var ExtractStoryMeta = plugin.SubTaskMeta{
}
func ExtractStory(taskCtx plugin.SubTaskContext) errors.Error {
- return RangeProductOneByOne(taskCtx, ExtractStoryForOneProduct)
-}
-
-func ExtractStoryForOneProduct(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
- // this collect only work for product
- if data.Options.ProductId == 0 {
- return nil
- }
-
statusMappings := getStoryStatusMapping(data)
stdTypeMappings := getStdTypeMappings(data)
extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_STORY_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_STORY_TABLE,
},
Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
+ var inputParams storyInput
+ err := json.Unmarshal(row.Input, &inputParams)
+ if err != nil {
+ return nil, errors.Default.WrapRaw(err)
+ }
res := &models.ZentaoStoryRes{}
- err := json.Unmarshal(row.Data, res)
+ err = json.Unmarshal(row.Data, res)
if err != nil {
return nil, errors.Default.WrapRaw(err)
}
- // project scope need filter
- if data.Options.ProjectId != 0 {
- if _, ok := data.StoryList[res.ID]; !ok {
- return nil, nil
- }
+ data.Stories[res.ID] = struct{}{}
+ var results []interface{}
+ projectStory := &models.ZentaoProjectStory{
+ ConnectionId: data.Options.ConnectionId,
+ ProjectId: data.Options.ProjectId,
+ StoryId: res.ID,
}
-
+ results = append(results, projectStory)
story := &models.ZentaoStory{
ConnectionId: data.Options.ConnectionId,
ID: res.ID,
@@ -150,8 +143,16 @@ func ExtractStoryForOneProduct(taskCtx
plugin.SubTaskContext) errors.Error {
}, story.Stage)
}
- results := make([]interface{}, 0)
results = append(results, story)
+ if inputParams.ExecutionId != 0 {
+ executionStory := &models.ZentaoExecutionStory{
+ ConnectionId: data.Options.ConnectionId,
+ ProjectId: data.Options.ProjectId,
+ ExecutionId: inputParams.ExecutionId,
+ StoryId: story.ID,
+ }
+ results = append(results, executionStory)
+ }
return results, nil
},
})
diff --git a/backend/plugins/zentao/tasks/story_repo_commits_collector.go
b/backend/plugins/zentao/tasks/story_repo_commits_collector.go
index 371e39d50..49af17b0f 100644
--- a/backend/plugins/zentao/tasks/story_repo_commits_collector.go
+++ b/backend/plugins/zentao/tasks/story_repo_commits_collector.go
@@ -42,10 +42,6 @@ var CollectStoryRepoCommitsMeta = plugin.SubTaskMeta{
}
func CollectStoryRepoCommits(taskCtx plugin.SubTaskContext) errors.Error {
- return RangeProductOneByOne(taskCtx,
CollectStoryRepoCommitsForOneProduct)
-}
-
-func CollectStoryRepoCommitsForOneProduct(taskCtx plugin.SubTaskContext)
errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*ZentaoTaskData)
@@ -54,8 +50,8 @@ func CollectStoryRepoCommitsForOneProduct(taskCtx
plugin.SubTaskContext) errors.
dal.Select("object_id, repo_revision"),
dal.From(&models.ZentaoStoryCommit{}),
dal.Where(
- "product = ? AND connection_id = ?",
- data.Options.ProductId, data.Options.ConnectionId,
+ "project = ? AND connection_id = ?",
+ data.Options.ProjectId, data.Options.ConnectionId,
),
}
@@ -72,13 +68,9 @@ func CollectStoryRepoCommitsForOneProduct(taskCtx
plugin.SubTaskContext) errors.
// collect story repo commits
collector, err := api.NewApiCollector(api.ApiCollectorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_STORY_REPO_COMMITS_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_STORY_REPO_COMMITS_TABLE,
},
ApiClient: data.ApiClient,
Input: iterator,
diff --git a/backend/plugins/zentao/tasks/story_repo_commits_convertor.go
b/backend/plugins/zentao/tasks/story_repo_commits_convertor.go
index 8c5126396..2fb5654a0 100644
--- a/backend/plugins/zentao/tasks/story_repo_commits_convertor.go
+++ b/backend/plugins/zentao/tasks/story_repo_commits_convertor.go
@@ -40,16 +40,12 @@ var ConvertStoryRepoCommitsMeta = plugin.SubTaskMeta{
}
func ConvertStoryRepoCommits(taskCtx plugin.SubTaskContext) errors.Error {
- return RangeProductOneByOne(taskCtx,
ConvertStoryRepoCommitsForOneProduct)
-}
-
-func ConvertStoryRepoCommitsForOneProduct(taskCtx plugin.SubTaskContext)
errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
db := taskCtx.GetDal()
cursor, err := db.Cursor(
dal.From(&models.ZentaoStoryRepoCommit{}),
- dal.Where(`product = ? and connection_id = ?`,
data.Options.ProductId, data.Options.ConnectionId),
+ dal.Where(`project = ? and connection_id = ?`,
data.Options.ProjectId, data.Options.ConnectionId),
)
if err != nil {
return err
@@ -61,13 +57,9 @@ func ConvertStoryRepoCommitsForOneProduct(taskCtx
plugin.SubTaskContext) errors.
InputRowType: reflect.TypeOf(models.ZentaoStoryRepoCommit{}),
Input: cursor,
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_STORY_REPO_COMMITS_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_STORY_REPO_COMMITS_TABLE,
},
Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
toolEntity := inputRow.(*models.ZentaoStoryRepoCommit)
diff --git a/backend/plugins/zentao/tasks/story_repo_commits_extractor.go
b/backend/plugins/zentao/tasks/story_repo_commits_extractor.go
index 6bcea26ac..3176d8ea3 100644
--- a/backend/plugins/zentao/tasks/story_repo_commits_extractor.go
+++ b/backend/plugins/zentao/tasks/story_repo_commits_extractor.go
@@ -38,27 +38,14 @@ var ExtractStoryRepoCommitsMeta = plugin.SubTaskMeta{
}
func ExtractStoryRepoCommits(taskCtx plugin.SubTaskContext) errors.Error {
- return RangeProductOneByOne(taskCtx,
ExtractStoryRepoCommitsForOneProduct)
-}
-
-func ExtractStoryRepoCommitsForOneProduct(taskCtx plugin.SubTaskContext)
errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
- // this Extract only work for product
- if data.Options.ProductId == 0 {
- return nil
- }
-
re := regexp.MustCompile(`(\d+)(?:,\s*(\d+))*`)
extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_STORY_REPO_COMMITS_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_STORY_REPO_COMMITS_TABLE,
},
Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
res := &models.ZentaoStoryRepoCommitsRes{}
@@ -73,7 +60,6 @@ func ExtractStoryRepoCommitsForOneProduct(taskCtx
plugin.SubTaskContext) errors.
if match[i] != "" {
storyRepoCommits :=
&models.ZentaoStoryRepoCommit{
ConnectionId:
data.Options.ConnectionId,
- Product:
data.Options.ProductId,
Project:
data.Options.ProjectId,
RepoUrl: res.Repo.CodePath,
CommitSha: res.Revision,
diff --git a/backend/plugins/zentao/tasks/task_collector.go
b/backend/plugins/zentao/tasks/task_collector.go
index 854cc970b..0ac2332a7 100644
--- a/backend/plugins/zentao/tasks/task_collector.go
+++ b/backend/plugins/zentao/tasks/task_collector.go
@@ -33,20 +33,11 @@ import (
const RAW_TASK_TABLE = "zentao_api_tasks"
-type ExecuteInput struct {
- Id int64
-}
-
var _ plugin.SubTaskEntryPoint = CollectTask
func CollectTask(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
- // this collect only work for project
- if data.Options.ProjectId == 0 {
- return nil
- }
-
cursor, err := taskCtx.GetDal().Cursor(
dal.Select(`id`),
dal.From(&models.ZentaoExecution{}),
@@ -57,20 +48,16 @@ func CollectTask(taskCtx plugin.SubTaskContext)
errors.Error {
}
defer cursor.Close()
- iterator, err := api.NewDalCursorIterator(taskCtx.GetDal(), cursor,
reflect.TypeOf(ExecuteInput{}))
+ iterator, err := api.NewDalCursorIterator(taskCtx.GetDal(), cursor,
reflect.TypeOf(input{}))
if err != nil {
return err
}
collector, err := api.NewApiCollector(api.ApiCollectorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_TASK_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_TASK_TABLE,
},
Input: iterator,
ApiClient: data.ApiClient,
diff --git a/backend/plugins/zentao/tasks/task_commits_collector.go
b/backend/plugins/zentao/tasks/task_commits_collector.go
index f4ee06783..bfb91099d 100644
--- a/backend/plugins/zentao/tasks/task_commits_collector.go
+++ b/backend/plugins/zentao/tasks/task_commits_collector.go
@@ -45,20 +45,11 @@ func CollectTaskCommits(taskCtx plugin.SubTaskContext)
errors.Error {
db := taskCtx.GetDal()
data := taskCtx.GetData().(*ZentaoTaskData)
- // this collect only work for project
- if data.Options.ProjectId == 0 {
- return nil
- }
-
// state manager
collectorWithState, err :=
api.NewStatefulApiCollector(api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_TASK_COMMITS_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_TASK_COMMITS_TABLE,
}, data.TimeAfter)
if err != nil {
return err
@@ -86,7 +77,7 @@ func CollectTaskCommits(taskCtx plugin.SubTaskContext)
errors.Error {
return err
}
- iterator, err := api.NewDalCursorIterator(db, cursor,
reflect.TypeOf(SimpleZentaoTask{}))
+ iterator, err := api.NewDalCursorIterator(db, cursor,
reflect.TypeOf(inputWithLastEditedDate{}))
if err != nil {
return err
}
@@ -94,13 +85,9 @@ func CollectTaskCommits(taskCtx plugin.SubTaskContext)
errors.Error {
// collect task commits
err = collectorWithState.InitCollector(api.ApiCollectorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_TASK_COMMITS_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_TASK_COMMITS_TABLE,
},
ApiClient: data.ApiClient,
Input: iterator,
@@ -125,8 +112,3 @@ func CollectTaskCommits(taskCtx plugin.SubTaskContext)
errors.Error {
return collectorWithState.Execute()
}
-
-type SimpleZentaoTask struct {
- ID int64 `json:"id"`
- LastEditedDate *api.Iso8601Time `json:"lastEditedDate"`
-}
diff --git a/backend/plugins/zentao/tasks/task_commits_extractor.go
b/backend/plugins/zentao/tasks/task_commits_extractor.go
index 09113504c..533df08e7 100644
--- a/backend/plugins/zentao/tasks/task_commits_extractor.go
+++ b/backend/plugins/zentao/tasks/task_commits_extractor.go
@@ -41,21 +41,12 @@ var ExtractTaskCommitsMeta = plugin.SubTaskMeta{
func ExtractTaskCommits(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
- // this Extract only work for project
- if data.Options.ProjectId == 0 {
- return nil
- }
-
re := regexp.MustCompile(`href='(.*?)'`)
extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_TASK_COMMITS_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_TASK_COMMITS_TABLE,
},
Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
res := &models.ZentaoTaskCommitsRes{}
@@ -74,7 +65,6 @@ func ExtractTaskCommits(taskCtx plugin.SubTaskContext)
errors.Error {
ID: res.ID,
ObjectType: res.ObjectType,
ObjectID: res.ObjectID,
- Product: data.Options.ProductId,
Project: data.Options.ProjectId,
Execution: res.Execution,
Actor: res.Actor,
diff --git a/backend/plugins/zentao/tasks/task_convertor.go
b/backend/plugins/zentao/tasks/task_convertor.go
index 7d39b7d2b..916a9c609 100644
--- a/backend/plugins/zentao/tasks/task_convertor.go
+++ b/backend/plugins/zentao/tasks/task_convertor.go
@@ -45,7 +45,7 @@ func ConvertTask(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
db := taskCtx.GetDal()
storyIdGen := didgen.NewDomainIdGenerator(&models.ZentaoStory{})
- bugIdGen := didgen.NewDomainIdGenerator(&models.ZentaoBug{})
+ //bugIdGen := didgen.NewDomainIdGenerator(&models.ZentaoBug{})
boardIdGen := didgen.NewDomainIdGenerator(&models.ZentaoProject{})
executionIdGen := didgen.NewDomainIdGenerator(&models.ZentaoExecution{})
taskIdGen := didgen.NewDomainIdGenerator(&models.ZentaoTask{})
@@ -62,13 +62,9 @@ func ConvertTask(taskCtx plugin.SubTaskContext) errors.Error
{
InputRowType: reflect.TypeOf(models.ZentaoTask{}),
Input: cursor,
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_TASK_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_TASK_TABLE,
},
Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
toolEntity := inputRow.(*models.ZentaoTask)
@@ -127,22 +123,6 @@ func ConvertTask(taskCtx plugin.SubTaskContext)
errors.Error {
results = append(results, domainEntity,
domainBoardIssue, sprintIssueTask)
- if toolEntity.StoryID != 0 {
- sprintIssueStory := &ticket.SprintIssue{
- SprintId: sprintId,
- IssueId:
storyIdGen.Generate(data.Options.ConnectionId, toolEntity.StoryID),
- }
- results = append(results, sprintIssueStory)
- }
-
- if toolEntity.FromBug != 0 {
- sprintIssueBug := &ticket.SprintIssue{
- SprintId: sprintId,
- IssueId:
bugIdGen.Generate(data.Options.ConnectionId, int64(toolEntity.FromBug)),
- }
- results = append(results, sprintIssueBug)
- }
-
return results, nil
},
})
diff --git a/backend/plugins/zentao/tasks/task_data.go
b/backend/plugins/zentao/tasks/task_data.go
index 7bf510976..c453ea1d7 100644
--- a/backend/plugins/zentao/tasks/task_data.go
+++ b/backend/plugins/zentao/tasks/task_data.go
@@ -36,7 +36,6 @@ type ZentaoOptions struct {
// Such As How many rows do your want
// You can use it in subtasks, and you need to pass it to main.go and
pipelines.
ConnectionId uint64 `json:"connectionId"`
- ProductId int64 `json:"productId" mapstructure:"productId"`
ProjectId int64 `json:"projectId" mapstructure:"projectId"`
// TODO not support now
TimeAfter string `json:"timeAfter"
mapstructure:"timeAfter,omitempty"`
@@ -44,6 +43,13 @@ type ZentaoOptions struct {
ScopeConfigs *ZentaoScopeConfigs `json:"scopeConfigs"
mapstructure:"scopeConfigs,omitempty"`
}
+func (o *ZentaoOptions) GetParams() any {
+ return models.ZentaoApiParams{
+ ConnectionId: o.ConnectionId,
+ ProjectId: o.ProjectId,
+ }
+}
+
type TypeMappings map[string]string
type StatusMappings map[string]string
@@ -104,6 +110,9 @@ type ZentaoTaskData struct {
ProductList map[int64]string // set if it is setting project id, it is
map[id]name
StoryList map[int64]int64 // set if it is run the task_extractor
FromBugList map[int]bool // set if it is run the task_extracor
+ Stories map[int64]struct{}
+ Tasks map[int64]struct{}
+ Bugs map[int64]struct{}
ApiClient *helper.ApiAsyncClient
}
@@ -117,8 +126,8 @@ func DecodeAndValidateTaskOptions(options
map[string]interface{}) (*ZentaoOption
if op.ConnectionId == 0 {
return nil, fmt.Errorf("connectionId is invalid")
}
- if op.ProductId == 0 && op.ProjectId == 0 {
- return nil, fmt.Errorf("please set productId")
+ if op.ProjectId == 0 {
+ return nil, fmt.Errorf("please set projectId")
}
return &op, nil
}
diff --git a/backend/plugins/zentao/tasks/task_extractor.go
b/backend/plugins/zentao/tasks/task_extractor.go
index ba52d0dc1..cb1d1248e 100644
--- a/backend/plugins/zentao/tasks/task_extractor.go
+++ b/backend/plugins/zentao/tasks/task_extractor.go
@@ -40,23 +40,14 @@ var ExtractTaskMeta = plugin.SubTaskMeta{
func ExtractTask(taskCtx plugin.SubTaskContext) errors.Error {
data := taskCtx.GetData().(*ZentaoTaskData)
- // this collect only work for project
- if data.Options.ProjectId == 0 {
- return nil
- }
-
statusMappings := getTaskStatusMapping(data)
stdTypeMappings := getStdTypeMappings(data)
extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_TASK_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_TASK_TABLE,
},
Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
res := &models.ZentaoTaskRes{}
@@ -65,10 +56,7 @@ func ExtractTask(taskCtx plugin.SubTaskContext) errors.Error
{
return nil, errors.Default.WrapRaw(err)
}
- // set storyList and FromBugList
- data.StoryList[res.StoryID] = res.Story
- data.FromBugList[res.FromBug] = true
-
+ data.Tasks[res.Id] = struct{}{}
task := &models.ZentaoTask{
ConnectionId: data.Options.ConnectionId,
ID: res.Id,
diff --git a/backend/plugins/zentao/tasks/task_repo_commits_collector.go
b/backend/plugins/zentao/tasks/task_repo_commits_collector.go
index bcfaabf4c..03b09055d 100644
--- a/backend/plugins/zentao/tasks/task_repo_commits_collector.go
+++ b/backend/plugins/zentao/tasks/task_repo_commits_collector.go
@@ -68,13 +68,9 @@ func CollectTaskRepoCommits(taskCtx plugin.SubTaskContext)
errors.Error {
// collect task repo commits
collector, err := api.NewApiCollector(api.ApiCollectorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_TASK_REPO_COMMITS_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_TASK_REPO_COMMITS_TABLE,
},
ApiClient: data.ApiClient,
Input: iterator,
diff --git a/backend/plugins/zentao/tasks/task_repo_commits_convertor.go
b/backend/plugins/zentao/tasks/task_repo_commits_convertor.go
index 24f446e09..c52326fa3 100644
--- a/backend/plugins/zentao/tasks/task_repo_commits_convertor.go
+++ b/backend/plugins/zentao/tasks/task_repo_commits_convertor.go
@@ -57,13 +57,9 @@ func ConvertTaskRepoCommits(taskCtx plugin.SubTaskContext)
errors.Error {
InputRowType: reflect.TypeOf(models.ZentaoTaskRepoCommit{}),
Input: cursor,
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_TASK_REPO_COMMITS_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_TASK_REPO_COMMITS_TABLE,
},
Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
toolEntity := inputRow.(*models.ZentaoTaskRepoCommit)
diff --git a/backend/plugins/zentao/tasks/task_repo_commits_extractor.go
b/backend/plugins/zentao/tasks/task_repo_commits_extractor.go
index 2ff135b54..da670f2b0 100644
--- a/backend/plugins/zentao/tasks/task_repo_commits_extractor.go
+++ b/backend/plugins/zentao/tasks/task_repo_commits_extractor.go
@@ -48,13 +48,9 @@ func ExtractTaskRepoCommits(taskCtx plugin.SubTaskContext)
errors.Error {
re := regexp.MustCompile(`(\d+)(?:,\s*(\d+))*`)
extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{
RawDataSubTaskArgs: api.RawDataSubTaskArgs{
- Ctx: taskCtx,
- Params: ScopeParams(
- data.Options.ConnectionId,
- data.Options.ProjectId,
- data.Options.ProductId,
- ),
- Table: RAW_TASK_REPO_COMMITS_TABLE,
+ Ctx: taskCtx,
+ Options: data.Options,
+ Table: RAW_TASK_REPO_COMMITS_TABLE,
},
Extract: func(row *api.RawData) ([]interface{}, errors.Error) {
res := &models.ZentaoTaskRepoCommitsRes{}
@@ -69,7 +65,6 @@ func ExtractTaskRepoCommits(taskCtx plugin.SubTaskContext)
errors.Error {
if match[i] != "" {
taskRepoCommits :=
&models.ZentaoTaskRepoCommit{
ConnectionId:
data.Options.ConnectionId,
- Product:
data.Options.ProductId,
Project:
data.Options.ProjectId,
RepoUrl: res.Repo.CodePath,
CommitSha: res.Revision,
![https://example.com/avatar.png"](https://example.com/avatar.png"){kind=link}