This is an automated email from the ASF dual-hosted git repository. warren pushed a commit to branch feat-plugin-zentao in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 47f3ecd6f1e500fa84a2aa8b6b818e891a236d07 Author: Yingchu Chen <[email protected]> AuthorDate: Mon Sep 19 10:38:53 2022 +0800 feat(zentao): add execution --- plugins/zentao/impl/impl.go | 2 + plugins/zentao/models/archived/execution.go | 151 +++++++++++++++++++++ plugins/zentao/models/execution.go | 151 +++++++++++++++++++++ .../migrationscripts/20220906_add_init_tables.go | 5 +- plugins/zentao/tasks/execution_collector.go | 79 +++++++++++ plugins/zentao/tasks/execution_extractor.go | 67 +++++++++ 6 files changed, 453 insertions(+), 2 deletions(-) diff --git a/plugins/zentao/impl/impl.go b/plugins/zentao/impl/impl.go index f24f6afe..920bced0 100644 --- a/plugins/zentao/impl/impl.go +++ b/plugins/zentao/impl/impl.go @@ -54,6 +54,8 @@ func (plugin Zentao) SubTaskMetas() []core.SubTaskMeta { return []core.SubTaskMeta{ tasks.CollectProjectMeta, tasks.ExtractProjectsMeta, + tasks.CollectExecutionMeta, + tasks.ExtractExecutionsMeta, } } diff --git a/plugins/zentao/models/archived/execution.go b/plugins/zentao/models/archived/execution.go new file mode 100644 index 00000000..787527c1 --- /dev/null +++ b/plugins/zentao/models/archived/execution.go @@ -0,0 +1,151 @@ +package archived + +import ( + "github.com/apache/incubator-devlake/models/migrationscripts/archived" + "time" +) + +type ZentaoExecution struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id uint64 `json:"id"` + Project uint64 `json:"project"` + Model string `json:"model"` + Type string `json:"type"` + Lifetime string `json:"lifetime"` + Budget string `json:"budget"` + BudgetUnit string `json:"budgetUnit"` + Attribute string `json:"attribute"` + Percent int `json:"percent"` + Milestone string `json:"milestone"` + Output string `json:"output"` + Auth string `json:"auth"` + Parent int `json:"parent"` + Path string `json:"path"` + Grade int `json:"grade"` + Name string `json:"name"` + Code string `json:"code"` + Begin string `json:"begin"` + End string `json:"end"` + RealBegan string `json:"realBegan"` + RealEnd string `json:"realEnd"` + Days int `json:"days"` + Status string `json:"status"` + SubStatus string `json:"subStatus"` + Pri string `json:"pri"` + Desc string `json:"desc"` + Version int `json:"version"` + ParentVersion int `json:"parentVersion"` + PlanDuration int `json:"planDuration"` + RealDuration int `json:"realDuration"` + OpenedBy `json:"openedBy"` + OpenedDate time.Time `json:"openedDate"` + OpenedVersion string `json:"openedVersion"` + LastEditedBy `json:"lastEditedBy"` + LastEditedDate time.Time `json:"lastEditedDate"` + ClosedBy `json:"closedBy"` + ClosedDate time.Time `json:"closedDate"` + CanceledBy `json:"canceledBy"` + CanceledDate time.Time `json:"canceledDate"` + SuspendedDate string `json:"suspendedDate"` + PO `json:"PO"` + PM `json:"PM"` + QD `json:"QD"` + RD `json:"RD"` + Team string `json:"team"` + Acl string `json:"acl"` + //Whitelist []Whitelist `json:"whitelist" gorm:"-:all"` + Order int `json:"order"` + Vision string `json:"vision"` + DisplayCards int `json:"displayCards"` + FluidBoard string `json:"fluidBoard"` + Deleted bool `json:"deleted"` + TotalHours int `json:"totalHours"` + TotalEstimate int `json:"totalEstimate"` + TotalConsumed int `json:"totalConsumed"` + TotalLeft int `json:"totalLeft"` + ProjectInfo bool `json:"projectInfo"` + Progress int `json:"progress"` + TeamMembers []TeamMember `json:"teamMembers" gorm:"-:all"` + Products []Product `json:"products" gorm:"-:all"` + CaseReview bool `json:"caseReview"` + archived.NoPKModel +} + +type OpenedBy struct { + OpenedByID int `json:"id"` + OpenedByAccount string `json:"account"` + OpenedByAvatar string `json:"avatar"` + OpenedByRealname string `json:"realname"` +} + +type LastEditedBy struct { + LastEditedByID int `json:"id"` + LastEditedByAccount string `json:"account"` + LastEditedByAvatar string `json:"avatar"` + LastEditedByRealname string `json:"realname"` +} + +type ClosedBy struct { + ClosedByID int `json:"id"` + ClosedByAccount string `json:"account"` + ClosedByAvatar string `json:"avatar"` + ClosedByRealname string `json:"realname"` +} + +type CanceledBy struct { + CanceledByID int `json:"id"` + CanceledByAccount string `json:"account"` + CanceledByAvatar string `json:"avatar"` + CanceledByRealname string `json:"realname"` +} + +type PO struct { + PoID int `json:"id"` + PoAccount string `json:"account"` + PoAvatar string `json:"avatar"` + PoRealname string `json:"realname"` +} + +type QD struct { + ID int `json:"id"` + Account string `json:"account"` + Avatar string `json:"avatar"` + Realname string `json:"realname"` +} + +type RD struct { + ID int `json:"id"` + Account string `json:"account"` + Avatar string `json:"avatar"` + Realname string `json:"realname"` +} + +type Product struct { + ID int `json:"id"` + Name string `json:"name"` + Plans []interface{} `json:"plans"` +} + +type TeamMember struct { + ID int `json:"id"` + Root int `json:"root"` + Type string `json:"type"` + Account string `json:"account"` + Role string `json:"role"` + Position string `json:"position"` + Limited string `json:"limited"` + Join string `json:"join"` + Days int `json:"days"` + Hours int `json:"hours"` + Estimate string `json:"estimate"` + Consumed string `json:"consumed"` + Left string `json:"left"` + Order int `json:"order"` + TotalHours int `json:"totalHours"` + UserID int `json:"userID"` + Realname string `json:"realname"` +} + +func (ZentaoExecution) TableName() string { + return "_tool_zentao_execution" +} diff --git a/plugins/zentao/models/execution.go b/plugins/zentao/models/execution.go new file mode 100644 index 00000000..e51102cf --- /dev/null +++ b/plugins/zentao/models/execution.go @@ -0,0 +1,151 @@ +package models + +import ( + "github.com/apache/incubator-devlake/models/common" + "time" +) + +type ZentaoExecution struct { + ConnectionId uint64 `gorm:"primaryKey"` + Id uint64 `json:"id"` + Project uint64 `json:"project"` + Model string `json:"model"` + Type string `json:"type"` + Lifetime string `json:"lifetime"` + Budget string `json:"budget"` + BudgetUnit string `json:"budgetUnit"` + Attribute string `json:"attribute"` + Percent int `json:"percent"` + Milestone string `json:"milestone"` + Output string `json:"output"` + Auth string `json:"auth"` + Parent int `json:"parent"` + Path string `json:"path"` + Grade int `json:"grade"` + Name string `json:"name"` + Code string `json:"code"` + Begin string `json:"begin"` + End string `json:"end"` + RealBegan string `json:"realBegan"` + RealEnd string `json:"realEnd"` + Days int `json:"days"` + Status string `json:"status"` + SubStatus string `json:"subStatus"` + Pri string `json:"pri"` + Desc string `json:"desc"` + Version int `json:"version"` + ParentVersion int `json:"parentVersion"` + PlanDuration int `json:"planDuration"` + RealDuration int `json:"realDuration"` + OpenedBy `json:"openedBy"` + OpenedDate time.Time `json:"openedDate"` + OpenedVersion string `json:"openedVersion"` + LastEditedBy `json:"lastEditedBy"` + LastEditedDate time.Time `json:"lastEditedDate"` + ClosedBy `json:"closedBy"` + ClosedDate time.Time `json:"closedDate"` + CanceledBy `json:"canceledBy"` + CanceledDate time.Time `json:"canceledDate"` + SuspendedDate string `json:"suspendedDate"` + PO `json:"PO"` + PM `json:"PM"` + QD `json:"QD"` + RD `json:"RD"` + Team string `json:"team"` + Acl string `json:"acl"` + //Whitelist []Whitelist `json:"whitelist" gorm:"-:all"` + Order int `json:"order"` + Vision string `json:"vision"` + DisplayCards int `json:"displayCards"` + FluidBoard string `json:"fluidBoard"` + Deleted bool `json:"deleted"` + TotalHours int `json:"totalHours"` + TotalEstimate int `json:"totalEstimate"` + TotalConsumed int `json:"totalConsumed"` + TotalLeft int `json:"totalLeft"` + ProjectInfo bool `json:"projectInfo"` + Progress int `json:"progress"` + TeamMembers []TeamMember `json:"teamMembers" gorm:"-:all"` + Products []Product `json:"products" gorm:"-:all"` + CaseReview bool `json:"caseReview"` + common.NoPKModel +} + +func (ZentaoExecution) TableName() string { + return "_tool_zentao_execution" +} + +type OpenedBy struct { + OpenedByID int `json:"id"` + OpenedByAccount string `json:"account"` + OpenedByAvatar string `json:"avatar"` + OpenedByRealname string `json:"realname"` +} + +type LastEditedBy struct { + LastEditedByID int `json:"id"` + LastEditedByAccount string `json:"account"` + LastEditedByAvatar string `json:"avatar"` + LastEditedByRealname string `json:"realname"` +} + +type ClosedBy struct { + ClosedByID int `json:"id"` + ClosedByAccount string `json:"account"` + ClosedByAvatar string `json:"avatar"` + ClosedByRealname string `json:"realname"` +} + +type CanceledBy struct { + CanceledByID int `json:"id"` + CanceledByAccount string `json:"account"` + CanceledByAvatar string `json:"avatar"` + CanceledByRealname string `json:"realname"` +} + +type PO struct { + PoID int `json:"id"` + PoAccount string `json:"account"` + PoAvatar string `json:"avatar"` + PoRealname string `json:"realname"` +} + +type QD struct { + ID int `json:"id"` + Account string `json:"account"` + Avatar string `json:"avatar"` + Realname string `json:"realname"` +} + +type RD struct { + ID int `json:"id"` + Account string `json:"account"` + Avatar string `json:"avatar"` + Realname string `json:"realname"` +} + +type Product struct { + ID int `json:"id"` + Name string `json:"name"` + Plans []interface{} `json:"plans"` +} + +type TeamMember struct { + ID int `json:"id"` + Root int `json:"root"` + Type string `json:"type"` + Account string `json:"account"` + Role string `json:"role"` + Position string `json:"position"` + Limited string `json:"limited"` + Join string `json:"join"` + Days int `json:"days"` + Hours int `json:"hours"` + Estimate string `json:"estimate"` + Consumed string `json:"consumed"` + Left string `json:"left"` + Order int `json:"order"` + TotalHours int `json:"totalHours"` + UserID int `json:"userID"` + Realname string `json:"realname"` +} diff --git a/plugins/zentao/models/migrationscripts/20220906_add_init_tables.go b/plugins/zentao/models/migrationscripts/20220906_add_init_tables.go index 1a743c46..6635dda3 100644 --- a/plugins/zentao/models/migrationscripts/20220906_add_init_tables.go +++ b/plugins/zentao/models/migrationscripts/20220906_add_init_tables.go @@ -27,8 +27,9 @@ type addInitTables struct{} func (u *addInitTables) Up(ctx context.Context, db *gorm.DB) error { return db.Migrator().AutoMigrate( - archived.ZentaoConnection{}, - archived.ZentaoProject{}, + //archived.ZentaoConnection{}, + //archived.ZentaoProject{}, + archived.ZentaoExecution{}, ) } diff --git a/plugins/zentao/tasks/execution_collector.go b/plugins/zentao/tasks/execution_collector.go new file mode 100644 index 00000000..b012084f --- /dev/null +++ b/plugins/zentao/tasks/execution_collector.go @@ -0,0 +1,79 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "fmt" + "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/helper" + "io" + "net/http" + "net/url" +) + +const RAW_EXECUTION_TABLE = "zentao_execution" + +var _ core.SubTaskEntryPoint = CollectExecution + +func CollectExecution(taskCtx core.SubTaskContext) error { + data := taskCtx.GetData().(*ZentaoTaskData) + collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: ZentaoApiParams{ + ProductId: data.Options.ProductId, + ExecutionId: data.Options.ExecutionId, + ProjectId: data.Options.ProjectId, + }, + Table: RAW_EXECUTION_TABLE, + }, + ApiClient: data.ApiClient, + Incremental: false, + PageSize: 100, + // TODO write which api would you want request + UrlTemplate: "executions/{{ .Params.ExecutionId }}", + Query: func(reqData *helper.RequestData) (url.Values, error) { + query := url.Values{} + query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page)) + query.Set("limit", fmt.Sprintf("%v", reqData.Pager.Size)) + return query, nil + }, + GetTotalPages: GetTotalPagesFromResponse, + ResponseParser: func(res *http.Response) ([]json.RawMessage, error) { + body, err := io.ReadAll(res.Body) + res.Body.Close() + if err != nil { + return nil, err + } + return []json.RawMessage{body}, nil + }, + }) + if err != nil { + return err + } + + return collector.Execute() +} + +var CollectExecutionMeta = core.SubTaskMeta{ + Name: "CollectExecution", + EntryPoint: CollectExecution, + EnabledByDefault: true, + Description: "Collect Execution data from Zentao api", +} diff --git a/plugins/zentao/tasks/execution_extractor.go b/plugins/zentao/tasks/execution_extractor.go new file mode 100644 index 00000000..f8b42226 --- /dev/null +++ b/plugins/zentao/tasks/execution_extractor.go @@ -0,0 +1,67 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "github.com/apache/incubator-devlake/plugins/core" + "github.com/apache/incubator-devlake/plugins/helper" + "github.com/apache/incubator-devlake/plugins/zentao/models" +) + +var _ core.SubTaskEntryPoint = ExtractExecutions + +var ExtractExecutionsMeta = core.SubTaskMeta{ + Name: "extractExecutions", + EntryPoint: ExtractExecutions, + EnabledByDefault: true, + Description: "extract Zentao executions", + DomainTypes: []string{core.DOMAIN_TYPE_TICKET}, +} + +func ExtractExecutions(taskCtx core.SubTaskContext) error { + data := taskCtx.GetData().(*ZentaoTaskData) + extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: ZentaoApiParams{ + ProductId: data.Options.ProductId, + ExecutionId: data.Options.ExecutionId, + ProjectId: data.Options.ProjectId, + }, + Table: RAW_EXECUTION_TABLE, + }, + Extract: func(row *helper.RawData) ([]interface{}, error) { + execution := &models.ZentaoExecution{} + err := json.Unmarshal(row.Data, execution) + if err != nil { + return nil, err + } + execution.ConnectionId = data.Options.ConnectionId + results := make([]interface{}, 0) + results = append(results, execution) + return results, nil + }, + }) + + if err != nil { + return err + } + + return extractor.Execute() +}
