This is an automated email from the ASF dual-hosted git repository. lynwee pushed a commit to branch cp-to-v1.0 in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit a1f82d6655a5c190312fdca9581943694fe88eaa Author: Chaojie Yan <[email protected]> AuthorDate: Fri Feb 21 17:06:21 2025 +0800 feat: support Zentao task worklogs (#8298) * feat: support Zentao task worklogs * feat: Zentao worklog tool layer migration * fix: dict as raw data instead of slice * test: e2e for zentao task worklogs * fix: compatibility when no account available --------- Co-authored-by: Lynwee <[email protected]> --- .../raw_tables/_raw_zentao_api_task_worklogs.csv | 3 + .../e2e/snapshot_tables/_tool_zentao_worklogs.csv | 3 + .../zentao/e2e/snapshot_tables/issue_worklogs.csv | 3 + backend/plugins/zentao/e2e/task_worklogs_test.go | 65 +++++++++++ backend/plugins/zentao/impl/impl.go | 5 + .../{register.go => 20250219_add_worklogs.go} | 32 +++--- .../models/migrationscripts/archived/worklog.go | 48 ++++++++ .../zentao/models/migrationscripts/register.go | 1 + backend/plugins/zentao/models/worklog.go | 48 ++++++++ .../plugins/zentao/tasks/task_worklog_collector.go | 121 ++++++++++++++++++++ .../plugins/zentao/tasks/task_worklog_convertor.go | 124 +++++++++++++++++++++ .../plugins/zentao/tasks/task_worklog_extractor.go | 100 +++++++++++++++++ 12 files changed, 540 insertions(+), 13 deletions(-) diff --git a/backend/plugins/zentao/e2e/raw_tables/_raw_zentao_api_task_worklogs.csv b/backend/plugins/zentao/e2e/raw_tables/_raw_zentao_api_task_worklogs.csv new file mode 100644 index 000000000..0d4bed0d5 --- /dev/null +++ b/backend/plugins/zentao/e2e/raw_tables/_raw_zentao_api_task_worklogs.csv @@ -0,0 +1,3 @@ +id,params,data,url,input,created_at +1,"{""ConnectionId"":1,""ProjectId"":48}","{""id"":106,""objectType"":""task"",""objectID"":135,""product"":"""",""project"":48,""execution"":49,""account"":""devlake"",""work"":""sample worklog"",""vision"":""rnd"",""date":""2025-02-20"",""left"":5,""consumed"":11.3,""begin"":0,""end"":0,""extra"":null,""order"":0,""deleted"":""0""}",http://iwater.red:8000/api.php/v1/tasks/135/estimate,{"id":135},2025-02-21 06:28:36.902 +2,"{""ConnectionId"":1,""ProjectId"":48}","{""id"":107,""objectType"":""task"",""objectID"":135,""product"":"""",""project"":48,""execution"":49,""account"":""devlake"",""work"":"""",""vision"":""rnd"",""date":""2025-02-20"",""left"":1,""consumed"":4,""begin"":0,""end"":0,""extra"":null,""order"":0,""deleted"":""0""}",http://iwater.red:8000/api.php/v1/tasks/135/estimate,{"id":135},2025-02-21 06:28:37.001 diff --git a/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_worklogs.csv b/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_worklogs.csv new file mode 100644 index 000000000..889258d6c --- /dev/null +++ b/backend/plugins/zentao/e2e/snapshot_tables/_tool_zentao_worklogs.csv @@ -0,0 +1,3 @@ +connection_id,id,object_id,object_type,project,execution,product,account,work,vision,date,left,consumed,begin,end,extra,order,deleted +1,106,135,task,48,49,,devlake,sample worklog,rnd,2025-02-20,5,11.3,0,0,NULL,0,0 +1,107,135,task,48,49,,devlake,,rnd,2025-02-20,1,4,0,0,NULL,0,0 diff --git a/backend/plugins/zentao/e2e/snapshot_tables/issue_worklogs.csv b/backend/plugins/zentao/e2e/snapshot_tables/issue_worklogs.csv new file mode 100644 index 000000000..f6c818e57 --- /dev/null +++ b/backend/plugins/zentao/e2e/snapshot_tables/issue_worklogs.csv @@ -0,0 +1,3 @@ +id,author_id,comment,time_spent_minutes,logged_date,started_date,issue_id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark +zentao:ZentaoWorklog:1:106,zentao:ZentaoAccount:1:1,sample worklog,678,2025-02-20T08:00:00.000+00:00,2025-02-20T08:00:00.000+00:00,zentao:ZentaoTask:1:135,"{""ConnectionId"":1,""ProjectId"":48}",_raw_zentao_api_task_worklogs,1, +zentao:ZentaoWorklog:1:107,zentao:ZentaoAccount:1:1,,240,2025-02-20T08:00:00.000+00:00,2025-02-20T08:00:00.000+00:00,zentao:ZentaoTask:1:135,"{""ConnectionId"":1,""ProjectId"":48}",_raw_zentao_api_task_worklogs,2, diff --git a/backend/plugins/zentao/e2e/task_worklogs_test.go b/backend/plugins/zentao/e2e/task_worklogs_test.go new file mode 100644 index 000000000..4002e58ad --- /dev/null +++ b/backend/plugins/zentao/e2e/task_worklogs_test.go @@ -0,0 +1,65 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "testing" + + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain" + "github.com/apache/incubator-devlake/helpers/e2ehelper" + "github.com/apache/incubator-devlake/plugins/zentao/impl" + "github.com/apache/incubator-devlake/plugins/zentao/models" + "github.com/apache/incubator-devlake/plugins/zentao/tasks" +) + +func TestZentaoTaskWorklogDataFlow(t *testing.T) { + + var zentao impl.Zentao + dataflowTester := e2ehelper.NewDataFlowTester(t, "zentao", zentao) + + taskData := &tasks.ZentaoTaskData{ + Options: &tasks.ZentaoOptions{ + ConnectionId: 1, + ProjectId: 48, + }, + ApiClient: getFakeAPIClient(), + HomePageURL: getFakeHomepage(), + } + + // import _raw_zentao_api_task_worklogs raw data table + dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_zentao_api_task_worklogs.csv", + "_raw_zentao_api_task_worklogs") + dataflowTester.ImportCsvIntoTabler("./snapshot_tables/_tool_zentao_accounts.csv", &models.ZentaoAccount{}) + + // verify worklogs extraction + dataflowTester.FlushTabler(&models.ZentaoWorklog{}) + dataflowTester.Subtask(tasks.ExtractTaskWorklogsMeta, taskData) + dataflowTester.VerifyTableWithOptions(&models.ZentaoWorklog{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/_tool_zentao_worklogs.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) + + // verify task repo commit conversion + dataflowTester.FlushTabler(&crossdomain.IssueWorklog{}) + dataflowTester.Subtask(tasks.ConvertTaskWorklogsMeta, taskData) + dataflowTester.VerifyTableWithOptions(&crossdomain.IssueWorklog{}, e2ehelper.TableOptions{ + CSVRelPath: "./snapshot_tables/issue_worklogs.csv", + IgnoreTypes: []interface{}{common.NoPKModel{}}, + }) +} diff --git a/backend/plugins/zentao/impl/impl.go b/backend/plugins/zentao/impl/impl.go index 2c24a235d..2111426b0 100644 --- a/backend/plugins/zentao/impl/impl.go +++ b/backend/plugins/zentao/impl/impl.go @@ -91,6 +91,7 @@ func (p Zentao) GetTablesInfo() []dal.Tabler { &models.ZentaoExecutionSummary{}, &models.ZentaoProductSummary{}, &models.ZentaoProjectStory{}, + &models.ZentaoWorklog{}, } } @@ -160,6 +161,10 @@ func (p Zentao) SubTaskMetas() []plugin.SubTaskMeta { tasks.DBGetChangelogMeta, tasks.ConvertChangelogMeta, + + tasks.CollectTaskWorklogsMeta, + tasks.ExtractTaskWorklogsMeta, + tasks.ConvertTaskWorklogsMeta, } } diff --git a/backend/plugins/zentao/models/migrationscripts/register.go b/backend/plugins/zentao/models/migrationscripts/20250219_add_worklogs.go similarity index 58% copy from backend/plugins/zentao/models/migrationscripts/register.go copy to backend/plugins/zentao/models/migrationscripts/20250219_add_worklogs.go index 914eb1f11..0b11778a4 100644 --- a/backend/plugins/zentao/models/migrationscripts/register.go +++ b/backend/plugins/zentao/models/migrationscripts/20250219_add_worklogs.go @@ -18,19 +18,25 @@ limitations under the License. package migrationscripts import ( - "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/helpers/migrationhelper" + "github.com/apache/incubator-devlake/plugins/zentao/models/migrationscripts/archived" ) -// All return all the migration scripts -func All() []plugin.MigrationScript { - return []plugin.MigrationScript{ - new(addInitTables), - new(addScopeConfigTables), - new(addIssueRepoCommitsTables), - new(addInitChangelogTables), - new(addTaskLeft), - new(addExecutionStoryAndExecutionSummary), - new(addRawParamTableForScope), - new(dropTotalReal), - } +type addWorklogs struct{} + +func (*addWorklogs) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables( + basicRes, + &archived.ZentaoWorklog{}, + ) +} + +func (*addWorklogs) Version() uint64 { + return 20250219153329 +} + +func (*addWorklogs) Name() string { + return "add table _tool_zentao_worklogs" } diff --git a/backend/plugins/zentao/models/migrationscripts/archived/worklog.go b/backend/plugins/zentao/models/migrationscripts/archived/worklog.go new file mode 100644 index 000000000..86fdca66f --- /dev/null +++ b/backend/plugins/zentao/models/migrationscripts/archived/worklog.go @@ -0,0 +1,48 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package archived + +import ( + "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" +) + +type ZentaoWorklog struct { + archived.NoPKModel + ConnectionId uint64 `gorm:"primaryKey;type:BIGINT NOT NULL"` + Id uint64 `gorm:"primaryKey;type:BIGINT NOT NULL;autoIncrement:false" json:"id"` + ObjectId uint64 `json:"objectID"` + ObjectType string `json:"objectType"` + Project uint64 `json:"project"` + Execution uint64 `json:"execution"` + Product string `json:"product"` + Account string `json:"account"` + Work string `json:"work"` + Vision string `json:"vision"` + Date string `json:"date"` + Left float32 `json:"left"` + Consumed float32 `json:"consumed"` + Begin uint64 `json:"begin"` + End uint64 `json:"end"` + Extra *string `json:"extra"` + Order uint64 `json:"order"` + Deleted string `json:"deleted"` +} + +func (ZentaoWorklog) TableName() string { + return "_tool_zentao_worklogs" +} diff --git a/backend/plugins/zentao/models/migrationscripts/register.go b/backend/plugins/zentao/models/migrationscripts/register.go index 914eb1f11..582d5432c 100644 --- a/backend/plugins/zentao/models/migrationscripts/register.go +++ b/backend/plugins/zentao/models/migrationscripts/register.go @@ -32,5 +32,6 @@ func All() []plugin.MigrationScript { new(addExecutionStoryAndExecutionSummary), new(addRawParamTableForScope), new(dropTotalReal), + new(addWorklogs), } } diff --git a/backend/plugins/zentao/models/worklog.go b/backend/plugins/zentao/models/worklog.go new file mode 100644 index 000000000..24ec38249 --- /dev/null +++ b/backend/plugins/zentao/models/worklog.go @@ -0,0 +1,48 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import ( + "github.com/apache/incubator-devlake/core/models/common" +) + +type ZentaoWorklog struct { + ConnectionId uint64 `gorm:"primaryKey;type:BIGINT NOT NULL"` + Id int64 `gorm:"primaryKey;type:BIGINT NOT NULL;autoIncrement:false" json:"id"` + ObjectId int64 `json:"objectID"` + ObjectType string `json:"objectType"` + Project int64 `json:"project"` + Execution int64 `json:"execution"` + Product string `json:"product"` + Account string `json:"account"` + Work string `json:"work"` + Vision string `json:"vision"` + Date string `json:"date"` + Left float32 `json:"left"` + Consumed float32 `json:"consumed"` + Begin int64 `json:"begin"` + End int64 `json:"end"` + Extra *string `json:"extra"` + Order int64 `json:"order"` + Deleted string `json:"deleted"` + common.NoPKModel +} + +func (ZentaoWorklog) TableName() string { + return "_tool_zentao_worklogs" +} diff --git a/backend/plugins/zentao/tasks/task_worklog_collector.go b/backend/plugins/zentao/tasks/task_worklog_collector.go new file mode 100644 index 000000000..4e40cdd01 --- /dev/null +++ b/backend/plugins/zentao/tasks/task_worklog_collector.go @@ -0,0 +1,121 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "net/http" + "net/url" + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/zentao/models" +) + +const RAW_TASK_WORKLOGS_TABLE = "zentao_api_task_worklogs" + +var CollectTaskWorklogsMeta = plugin.SubTaskMeta{ + Name: "collectTaskWorklogs", + EntryPoint: CollectTaskWorklogs, + EnabledByDefault: true, + Description: "collect Zentao task work logs, supports both timeFilter and diffSync.", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +type Input struct { + Id uint64 `json:"id"` +} + +func CollectTaskWorklogs(taskCtx plugin.SubTaskContext) errors.Error { + db := taskCtx.GetDal() + data := taskCtx.GetData().(*ZentaoTaskData) + + logger := taskCtx.GetLogger() + + apiCollector, err := api.NewStatefulApiCollector(api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Options: data.Options, + Table: RAW_TASK_WORKLOGS_TABLE, + }) + if err != nil { + return err + } + + // load task IDs from db + clauses := []dal.Clause{ + dal.Select("id"), + dal.From(&models.ZentaoTask{}), + dal.Where( + "project = ? AND connection_id = ?", + data.Options.ProjectId, data.Options.ConnectionId, + ), + } + if apiCollector.IsIncremental() && apiCollector.GetSince() != nil { + clauses = append(clauses, dal.Where("last_edited_date IS NOT NULL AND last_edited_date > ?", apiCollector.GetSince())) + } + + // construct the input iterator + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + iterator, err := api.NewDalCursorIterator(db, cursor, reflect.TypeOf(Input{})) + if err != nil { + return err + } + + // collect task worklogs + err = apiCollector.InitCollector(api.ApiCollectorArgs{ + Input: iterator, + ApiClient: data.ApiClient, + UrlTemplate: "tasks/{{ .Input.Id }}/estimate", + Query: func(reqData *api.RequestData) (url.Values, errors.Error) { + return nil, nil + }, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var data struct { + Effort json.RawMessage `json:"effort"` + } + err := api.UnmarshalResponse(res, &data) + if err != nil { + return nil, err + } + + if string(data.Effort) == "{}" || string(data.Effort) == "null" { + return nil, nil + } + + var efforts []json.RawMessage + jsonErr := json.Unmarshal(data.Effort, &efforts) + if jsonErr != nil { + return nil, errors.Default.Wrap(jsonErr, "failed to unmarshal efforts") + } + return efforts, nil + }, + AfterResponse: ignoreHTTPStatus404, + }) + if err != nil { + logger.Error(err, "collect Zentao task worklogs error") + return err + } + + return apiCollector.Execute() +} diff --git a/backend/plugins/zentao/tasks/task_worklog_convertor.go b/backend/plugins/zentao/tasks/task_worklog_convertor.go new file mode 100644 index 000000000..4a2a1fb72 --- /dev/null +++ b/backend/plugins/zentao/tasks/task_worklog_convertor.go @@ -0,0 +1,124 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "reflect" + + "github.com/apache/incubator-devlake/core/dal" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/models/domainlayer" + "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" + "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" + "github.com/apache/incubator-devlake/core/plugin" + helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/zentao/models" +) + +var ConvertTaskWorklogsMeta = plugin.SubTaskMeta{ + Name: "convertTaskWorklogs", + EntryPoint: ConvertTaskWorklogs, + EnabledByDefault: true, + Description: "convert Zentao task worklogs", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func ConvertTaskWorklogs(taskCtx plugin.SubTaskContext) errors.Error { + logger := taskCtx.GetLogger() + db := taskCtx.GetDal() + data := taskCtx.GetData().(*ZentaoTaskData) + logger.Info( + "convert Zentao task worklogs of %d in %d", + data.Options.ProjectId, + data.Options.ConnectionId, + ) + worklogIdGen := didgen.NewDomainIdGenerator(&models.ZentaoWorklog{}) + clauses := []dal.Clause{ + dal.From(&models.ZentaoWorklog{}), + dal.Where( + "connection_id = ? AND project = ? AND object_type = ?", + data.Options.ConnectionId, + data.Options.ProjectId, + "task", + ), + } + + cursor, err := db.Cursor(clauses...) + if err != nil { + return err + } + defer cursor.Close() + + taskIdGen := didgen.NewDomainIdGenerator(&models.ZentaoTask{}) + + converter, err := helper.NewDataConverter(helper.DataConverterArgs{ + RawDataSubTaskArgs: helper.RawDataSubTaskArgs{ + Ctx: taskCtx, + Options: data.Options, + Table: RAW_TASK_WORKLOGS_TABLE, + }, + InputRowType: reflect.TypeOf(models.ZentaoWorklog{}), + Input: cursor, + Convert: func(inputRow interface{}) ([]interface{}, errors.Error) { + toolL := inputRow.(*models.ZentaoWorklog) + domainL := &ticket.IssueWorklog{ + DomainEntity: domainlayer.DomainEntity{ + Id: worklogIdGen.Generate(data.Options.ConnectionId, toolL.Id), + }, + Comment: toolL.Work, + TimeSpentMinutes: int(toolL.Consumed * 60), + } + timeData, err := common.ConvertStringToTime(toolL.Date) + if err != nil { + return nil, errors.Default.Wrap(err, "failed to convert zentao task worklog date") + } + // zentao task only has one field as date type for worklog creation + domainL.StartedDate = &timeData + domainL.LoggedDate = &timeData + + domainL.IssueId = taskIdGen.Generate(data.Options.ConnectionId, toolL.ObjectId) + + // get ID of account by username + var account models.ZentaoAccount + err = db.First(&account, dal.Where("connection_id = ? AND account = ?", + data.Options.ConnectionId, toolL.Account)) + if err != nil { + // if account isn't available, giving empty string as ID + if db.IsErrorNotFound(err) { + logger.Warn(nil, "cannot find zentao account by account: %s", toolL.Account) + domainL.AuthorId = "" + } else { + return nil, errors.Default.Wrap(err, "failed to get zentao account by account") + } + } else { + accountIdGen := didgen.NewDomainIdGenerator(&models.ZentaoAccount{}) + domainL.AuthorId = accountIdGen.Generate(account.ConnectionId, account.ID) + } + + return []interface{}{ + domainL, + }, nil + }, + }) + if err != nil { + return err + } + + return converter.Execute() +} diff --git a/backend/plugins/zentao/tasks/task_worklog_extractor.go b/backend/plugins/zentao/tasks/task_worklog_extractor.go new file mode 100644 index 000000000..928890675 --- /dev/null +++ b/backend/plugins/zentao/tasks/task_worklog_extractor.go @@ -0,0 +1,100 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/zentao/models" +) + +var _ plugin.SubTaskEntryPoint = ExtractTaskWorklogs + +var ExtractTaskWorklogsMeta = plugin.SubTaskMeta{ + Name: "extractTaskWorklogs", + EntryPoint: ExtractTaskWorklogs, + EnabledByDefault: true, + Description: "Extract raw zentao task worklog data into tool layer table _tool_zentao_worklogs", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func ExtractTaskWorklogs(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*ZentaoTaskData) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Options: data.Options, + Table: RAW_TASK_WORKLOGS_TABLE, + }, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + var input struct { + Id int64 `json:"id"` + ObjectType string `json:"objectType"` + ObjectId int64 `json:"objectID"` + Product string `json:"product"` + Project int64 `json:"project"` + Execution int64 `json:"Execution"` + Account string `json:"account"` + Work string `json:"work"` + Vision string `json:"vision"` + Date string `json:"date"` + Left float32 `json:"left"` + Consumed float32 `json:"consumed"` + Begin int64 `json:"begin"` + End int64 `json:"end"` + Extra *string `json:"extra"` + Order int64 `json:"order"` + Deleted string `json:"deleted"` + } + + err := errors.Convert(json.Unmarshal(row.Data, &input)) + if err != nil { + return nil, err + } + worklog := &models.ZentaoWorklog{ + ConnectionId: data.Options.ConnectionId, + Id: input.Id, + ObjectId: input.ObjectId, + ObjectType: input.ObjectType, + Project: input.Project, + Execution: input.Execution, + Product: input.Product, + Account: input.Account, + Work: input.Work, + Vision: input.Vision, + Date: input.Date, + Left: input.Left, + Consumed: input.Consumed, + Begin: input.Begin, + End: input.End, + Extra: input.Extra, + Order: input.Order, + Deleted: input.Deleted, + } + return []interface{}{worklog}, nil + }, + }) + + if err != nil { + return err + } + + return extractor.Execute() +}
