This is an automated email from the ASF dual-hosted git repository. lynwee pushed a commit to branch 10-cp in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 076ae9b511846bb0fa61791ee4da323373596898 Author: d4x1 <[email protected]> AuthorDate: Thu Jul 11 15:09:22 2024 +0800 feat(jira): add `_tool_jira_issue_fields`, collect account field from the new table --- backend/plugins/jira/impl/impl.go | 3 + backend/plugins/jira/models/issue_field.go | 43 +++++++++ .../20240710_add_issue_field_table.go | 41 +++++++++ .../migrationscripts/archived/issue_field.go | 42 +++++++++ .../jira/models/migrationscripts/register.go | 1 + .../jira/tasks/issue_changelog_convertor.go | 34 ++++++- .../plugins/jira/tasks/issue_field_collector.go | 73 +++++++++++++++ .../plugins/jira/tasks/issue_field_extractor.go | 100 +++++++++++++++++++++ 8 files changed, 333 insertions(+), 4 deletions(-) diff --git a/backend/plugins/jira/impl/impl.go b/backend/plugins/jira/impl/impl.go index 6eeb146ce..3c8c284fe 100644 --- a/backend/plugins/jira/impl/impl.go +++ b/backend/plugins/jira/impl/impl.go @@ -103,6 +103,9 @@ func (p Jira) Name() string { func (p Jira) SubTaskMetas() []plugin.SubTaskMeta { return []plugin.SubTaskMeta{ + tasks.CollectIssueFieldsMeta, + tasks.ExtractIssueFieldsMeta, + tasks.CollectBoardFilterBeginMeta, tasks.CollectStatusMeta, diff --git a/backend/plugins/jira/models/issue_field.go b/backend/plugins/jira/models/issue_field.go new file mode 100644 index 000000000..3ca31b186 --- /dev/null +++ b/backend/plugins/jira/models/issue_field.go @@ -0,0 +1,43 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package models + +import "github.com/apache/incubator-devlake/core/models/common" + +type JiraIssueField struct { + common.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + BoardId uint64 `gorm:"primaryKey"` + + ID string `json:"id" gorm:"primaryKey"` + Name string `json:"name"` + Custom bool `json:"custom"` + Orderable bool `json:"orderable"` + Navigable bool `json:"navigable"` + Searchable bool `json:"searchable"` + //ClauseNames []string `json:"clauseNames"` + SchemaType string `json:"schema_type"` + SchemaItems string `json:"schema_items"` + SchemaCustom string `json:"schema_custom"` + SchemaCustomID int `json:"schema_custom_id"` + ScheCustomSystem string `json:"sche_custom_system"` +} + +func (JiraIssueField) TableName() string { + return "_tool_jira_issue_fields" +} diff --git a/backend/plugins/jira/models/migrationscripts/20240710_add_issue_field_table.go b/backend/plugins/jira/models/migrationscripts/20240710_add_issue_field_table.go new file mode 100644 index 000000000..05ba584d0 --- /dev/null +++ b/backend/plugins/jira/models/migrationscripts/20240710_add_issue_field_table.go @@ -0,0 +1,41 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package migrationscripts + +import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/plugins/jira/models/migrationscripts/archived" +) + +var _ plugin.MigrationScript = (*addIssueFieldTable)(nil) + +type addIssueFieldTable struct{} + +func (script *addIssueFieldTable) Up(basicRes context.BasicRes) errors.Error { + return basicRes.GetDal().AutoMigrate(&archived.JiraIssueField{}) +} + +func (*addIssueFieldTable) Version() uint64 { + return 20240710100000 +} + +func (*addIssueFieldTable) Name() string { + return "init table _tool_jira_issue_fields" +} diff --git a/backend/plugins/jira/models/migrationscripts/archived/issue_field.go b/backend/plugins/jira/models/migrationscripts/archived/issue_field.go new file mode 100644 index 000000000..865aea15d --- /dev/null +++ b/backend/plugins/jira/models/migrationscripts/archived/issue_field.go @@ -0,0 +1,42 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package archived + +import "github.com/apache/incubator-devlake/core/models/migrationscripts/archived" + +type JiraIssueField struct { + archived.NoPKModel + ConnectionId uint64 `gorm:"primaryKey"` + BoardId uint64 `gorm:"primaryKey"` + + ID string `json:"id" gorm:"primaryKey"` + Name string `json:"name"` + Custom bool `json:"custom"` + Orderable bool `json:"orderable"` + Navigable bool `json:"navigable"` + Searchable bool `json:"searchable"` + SchemaType string `json:"schema_type"` + SchemaItems string `json:"schema_items"` + SchemaCustom string `json:"schema_custom"` + SchemaCustomID int `json:"schema_custom_id"` + ScheCustomSystem string `json:"sche_custom_system"` +} + +func (JiraIssueField) TableName() string { + return "_tool_jira_issue_fields" +} diff --git a/backend/plugins/jira/models/migrationscripts/register.go b/backend/plugins/jira/models/migrationscripts/register.go index 4507c5841..f2764ef42 100644 --- a/backend/plugins/jira/models/migrationscripts/register.go +++ b/backend/plugins/jira/models/migrationscripts/register.go @@ -50,5 +50,6 @@ func All() []plugin.MigrationScript { new(addWorklogToIssue), new(addSubtaskToIssue), new(addTmpAccountIdToJiraIssueChangelogItem), + new(addIssueFieldTable), } } diff --git a/backend/plugins/jira/tasks/issue_changelog_convertor.go b/backend/plugins/jira/tasks/issue_changelog_convertor.go index e575c2356..2d3e01a82 100644 --- a/backend/plugins/jira/tasks/issue_changelog_convertor.go +++ b/backend/plugins/jira/tasks/issue_changelog_convertor.go @@ -54,6 +54,7 @@ type IssueChangelogItemResult struct { func ConvertIssueChangelogs(subtaskCtx plugin.SubTaskContext) errors.Error { data := subtaskCtx.GetData().(*JiraTaskData) db := subtaskCtx.GetDal() + logger := subtaskCtx.GetLogger() connectionId := data.Options.ConnectionId boardId := data.Options.BoardId @@ -67,6 +68,18 @@ func ConvertIssueChangelogs(subtaskCtx plugin.SubTaskContext) errors.Error { statusMap[v.ID] = v } + var allIssueFields []models.JiraIssueField + if err := db.All(&allIssueFields, dal.Where("connection_id = ?", connectionId)); err != nil { + return err + } + issueFieldMap := make(map[string]models.JiraIssueField) + for _, v := range allIssueFields { + if _, ok := issueFieldMap[v.Name]; ok { + logger.Warn(nil, "filed name %s is duplicated", v.Name) + } + issueFieldMap[v.Name] = v + } + issueIdGenerator := didgen.NewDomainIdGenerator(&models.JiraIssue{}) sprintIdGenerator := didgen.NewDomainIdGenerator(&models.JiraSprint{}) changelogIdGenerator := didgen.NewDomainIdGenerator(&models.JiraIssueChangelogItems{}) @@ -121,12 +134,12 @@ func ConvertIssueChangelogs(subtaskCtx plugin.SubTaskContext) errors.Error { } switch row.Field { case "assignee": - if row.ToValue != "" { - changelog.OriginalToValue = accountIdGen.Generate(connectionId, row.ToValue) - } if row.FromValue != "" { changelog.OriginalFromValue = accountIdGen.Generate(connectionId, row.FromValue) } + if row.ToValue != "" { + changelog.OriginalToValue = accountIdGen.Generate(connectionId, row.ToValue) + } case "Sprint": changelog.OriginalFromValue, err = convertIds(row.FromValue, connectionId, sprintIdGenerator) if err != nil { @@ -146,7 +159,7 @@ func ConvertIssueChangelogs(subtaskCtx plugin.SubTaskContext) errors.Error { changelog.ToValue = getStdStatus(toStatus.StatusCategory) } default: - // process other account-like fields + // process other account-like fields, it works on jira9 and jira cloud. if row.TmpFromAccountId != "" { if row.FromValue != "" { changelog.OriginalFromValue = accountIdGen.Generate(connectionId, row.FromValue) @@ -161,6 +174,19 @@ func ConvertIssueChangelogs(subtaskCtx plugin.SubTaskContext) errors.Error { changelog.OriginalToValue = accountIdGen.Generate(connectionId, row.TmpToAccountId) } } + if row.TmpFromAccountId == "" && row.TmpToAccountId == "" { + // it works on jira8 + // notice: field name is not unique, but we cannot fetch field id here. + if v, ok := issueFieldMap[row.Field]; ok && v.SchemaType == "user" { + // field type is account + if row.FromValue != "" { + changelog.OriginalFromValue = accountIdGen.Generate(connectionId, row.FromValue) + } + if row.ToValue != "" { + changelog.OriginalToValue = accountIdGen.Generate(connectionId, row.ToValue) + } + } + } } return []interface{}{changelog}, nil diff --git a/backend/plugins/jira/tasks/issue_field_collector.go b/backend/plugins/jira/tasks/issue_field_collector.go new file mode 100644 index 000000000..63b535d2d --- /dev/null +++ b/backend/plugins/jira/tasks/issue_field_collector.go @@ -0,0 +1,73 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "net/http" +) + +const RAW_ISSUE_FIELDS_TABLE = "jira_api_issue_fields" + +var _ plugin.SubTaskEntryPoint = CollectIssueField + +var CollectIssueFieldsMeta = plugin.SubTaskMeta{ + Name: "collectIssuleField", + EntryPoint: CollectIssueField, + EnabledByDefault: true, + Description: "collect Jira issue field, does not support either timeFilter or diffSync.", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +func CollectIssueField(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*JiraTaskData) + logger := taskCtx.GetLogger() + logger.Info("collect issue fields") + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: JiraApiParams{ + ConnectionId: data.Options.ConnectionId, + BoardId: data.Options.BoardId, + }, + Table: RAW_ISSUE_FIELDS_TABLE, + }, + ApiClient: data.ApiClient, + PageSize: 0, + UrlTemplate: "api/2/field", + Query: nil, + ResponseParser: func(res *http.Response) ([]json.RawMessage, errors.Error) { + var data []json.RawMessage + err := api.UnmarshalResponse(res, &data) + if err != nil { + return nil, err + } + return data, nil + }, + AfterResponse: ignoreHTTPStatus400, + }) + + if err != nil { + return err + } + + return collector.Execute() +} diff --git a/backend/plugins/jira/tasks/issue_field_extractor.go b/backend/plugins/jira/tasks/issue_field_extractor.go new file mode 100644 index 000000000..13fae198a --- /dev/null +++ b/backend/plugins/jira/tasks/issue_field_extractor.go @@ -0,0 +1,100 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package tasks + +import ( + "encoding/json" + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/models/common" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/plugins/jira/models" +) + +var _ plugin.SubTaskEntryPoint = ExtractIssueFields + +var ExtractIssueFieldsMeta = plugin.SubTaskMeta{ + Name: "extractIssueFields", + EntryPoint: ExtractIssueFields, + EnabledByDefault: true, + Description: "extract Jira issue fields", + DomainTypes: []string{plugin.DOMAIN_TYPE_TICKET}, +} + +type JiraIssueField struct { + ID string `json:"id"` + Name string `json:"name"` + Custom bool `json:"custom"` + Orderable bool `json:"orderable"` + Navigable bool `json:"navigable"` + Searchable bool `json:"searchable"` + ClauseNames []string `json:"clauseNames"` + Schema struct { + Type string `json:"type"` + Items string `json:"items"` + Custom string `json:"custom"` + System string `json:"system"` + CustomID int `json:"customId"` + } `json:"schema"` +} + +func ExtractIssueFields(taskCtx plugin.SubTaskContext) errors.Error { + data := taskCtx.GetData().(*JiraTaskData) + extractor, err := api.NewApiExtractor(api.ApiExtractorArgs{ + RawDataSubTaskArgs: api.RawDataSubTaskArgs{ + Ctx: taskCtx, + Params: JiraApiParams{ + ConnectionId: data.Options.ConnectionId, + BoardId: data.Options.BoardId, + }, + Table: RAW_ISSUE_FIELDS_TABLE, + }, + Extract: func(row *api.RawData) ([]interface{}, errors.Error) { + var issueField JiraIssueField + err := errors.Convert(json.Unmarshal(row.Data, &issueField)) + if err != nil { + return nil, err + } + jiraIssueField := &models.JiraIssueField{ + NoPKModel: common.NewNoPKModel(), + ConnectionId: data.Options.ConnectionId, + BoardId: data.Options.BoardId, + + ID: issueField.ID, + Name: issueField.Name, + Custom: issueField.Custom, + Orderable: issueField.Orderable, + Navigable: issueField.Navigable, + Searchable: issueField.Searchable, + //ClauseNames: issueField.ClauseNames, + SchemaType: issueField.Schema.Type, + SchemaItems: issueField.Schema.Items, + SchemaCustom: issueField.Schema.Custom, + SchemaCustomID: issueField.Schema.CustomID, + ScheCustomSystem: issueField.Schema.System, + } + return []interface{}{jiraIssueField}, nil + }, + }) + + if err != nil { + return err + } + + return extractor.Execute() +}
