This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 8040e4ec2 feat: support extracting array data from Jira's raw layer
(#6134)
8040e4ec2 is described below
commit 8040e4ec2ff14f363d63cf473bed24911de983d4
Author: abeizn <[email protected]>
AuthorDate: Thu Sep 21 19:42:02 2023 +0800
feat: support extracting array data from Jira's raw layer (#6134)
* feat: add issue custom array field
* feat: add issue custom array field
* feat: support extracting array data from Jira's raw layer
* feat: add delete issue_custom_array_fields function and fix ci
* fix: use delete+create instead of create+update
* fix: use dal.delete instead of dal.exec
---
.../models/domainlayer/domaininfo/domaininfo.go | 1 +
.../ticket/issue_custom_array_fields.go | 32 ++++++++++++++++
.../20230921_add_issue_custom_array_fields.go | 43 ++++++++++++++++++++++
.../archived/issue_custom_array_fields.go | 30 +++++++++++++++
backend/core/models/migrationscripts/register.go | 1 +
.../customize/tasks/customized_fields_extractor.go | 42 ++++++++++++++++++++-
6 files changed, 148 insertions(+), 1 deletion(-)
diff --git a/backend/core/models/domainlayer/domaininfo/domaininfo.go
b/backend/core/models/domainlayer/domaininfo/domaininfo.go
index 45f16135f..9f67181eb 100644
--- a/backend/core/models/domainlayer/domaininfo/domaininfo.go
+++ b/backend/core/models/domainlayer/domaininfo/domaininfo.go
@@ -87,5 +87,6 @@ func GetDomainTablesInfo() []dal.Tabler {
&ticket.SprintIssue{},
&ticket.IssueAssignee{},
&ticket.IssueRelationship{},
+ &ticket.IssueCustomArrayField{},
}
}
diff --git
a/backend/core/models/domainlayer/ticket/issue_custom_array_fields.go
b/backend/core/models/domainlayer/ticket/issue_custom_array_fields.go
new file mode 100644
index 000000000..16b7844dd
--- /dev/null
+++ b/backend/core/models/domainlayer/ticket/issue_custom_array_fields.go
@@ -0,0 +1,32 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ticket
+
+import "github.com/apache/incubator-devlake/core/models/common"
+
+type IssueCustomArrayField struct {
+ IssueId string `gorm:"primaryKey;type:varchar(255)"`
+ FieldId string `gorm:"primaryKey;type:varchar(255)"`
+ FieldValue string `gorm:"primaryKey;type:varchar(255)"`
+
+ common.NoPKModel
+}
+
+func (IssueCustomArrayField) TableName() string {
+ return "issue_custom_array_fields"
+}
diff --git
a/backend/core/models/migrationscripts/20230921_add_issue_custom_array_fields.go
b/backend/core/models/migrationscripts/20230921_add_issue_custom_array_fields.go
new file mode 100644
index 000000000..378f76e8d
--- /dev/null
+++
b/backend/core/models/migrationscripts/20230921_add_issue_custom_array_fields.go
@@ -0,0 +1,43 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+
"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+type addIssueCustomArrayField struct{}
+
+func (u *addIssueCustomArrayField) Up(basicRes context.BasicRes) errors.Error {
+
+ return migrationhelper.AutoMigrateTables(
+ basicRes,
+ &archived.IssueCustomArrayField{},
+ )
+}
+
+func (*addIssueCustomArrayField) Version() uint64 {
+ return 20230921000083
+}
+
+func (*addIssueCustomArrayField) Name() string {
+ return "add issue custom array field"
+}
diff --git
a/backend/core/models/migrationscripts/archived/issue_custom_array_fields.go
b/backend/core/models/migrationscripts/archived/issue_custom_array_fields.go
new file mode 100644
index 000000000..9328ecafa
--- /dev/null
+++ b/backend/core/models/migrationscripts/archived/issue_custom_array_fields.go
@@ -0,0 +1,30 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package archived
+
+type IssueCustomArrayField struct {
+ IssueId string `gorm:"primaryKey;type:varchar(255)"`
+ FieldId string `gorm:"primaryKey;type:varchar(255)"`
+ FieldValue string `gorm:"primaryKey;type:varchar(255)"`
+
+ NoPKModel
+}
+
+func (IssueCustomArrayField) TableName() string {
+ return "issue_custom_array_fields"
+}
diff --git a/backend/core/models/migrationscripts/register.go
b/backend/core/models/migrationscripts/register.go
index 3874e9d91..647a27020 100644
--- a/backend/core/models/migrationscripts/register.go
+++ b/backend/core/models/migrationscripts/register.go
@@ -94,5 +94,6 @@ func All() []plugin.MigrationScript {
new(addCICDDeploymentsTable),
new(normalizeBpSettings),
new(addSyncPolicy),
+ new(addIssueCustomArrayField),
}
}
diff --git a/backend/plugins/customize/tasks/customized_fields_extractor.go
b/backend/plugins/customize/tasks/customized_fields_extractor.go
index 8930745dd..0351d01ba 100644
--- a/backend/plugins/customize/tasks/customized_fields_extractor.go
+++ b/backend/plugins/customize/tasks/customized_fields_extractor.go
@@ -24,6 +24,8 @@ import (
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models/common"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/plugins/customize/models"
"github.com/tidwall/gjson"
@@ -99,7 +101,45 @@ func extractCustomizedFields(ctx context.Context, d
dal.Dal, table, rawTable, ra
case string:
for field, path := range extractor {
result := gjson.Get(blob, path)
- fillInUpdates(result, field, updates)
+ // special case for issues custom_fields
+ rawDataId, ok := row["_raw_data_id"].(int64)
+ if !ok {
+ return errors.Default.New("_raw_data_id
is not int64")
+ }
+ if table == "issues" && result.IsArray() {
+ issueId := row["id"].(string)
+ fieldId := field
+ // Delete existing records for the
given issue and field
+ err = d.Delete(
+ &ticket.IssueCustomArrayField{},
+ dal.Where("issue_id = ? AND
field_id = ?", issueId, fieldId),
+ )
+ if err != nil {
+ return err
+ }
+
+ result.ForEach(func(_, v gjson.Result)
bool {
+ err1 :=
d.Create(&ticket.IssueCustomArrayField{
+ IssueId: issueId,
+ FieldId: fieldId,
+ FieldValue: v.String(),
+ NoPKModel:
common.NoPKModel{
+ RawDataOrigin:
common.RawDataOrigin{
+
RawDataParams: rawDataParams,
+
RawDataTable: rawTable,
+
RawDataId: uint64(rawDataId),
+ },
+ },
+ })
+ if err1 != nil {
+ err = err1
+ return false
+ }
+ return true
+ })
+ } else {
+ fillInUpdates(result, field, updates)
+ }
}
default:
return nil
