This is an automated email from the ASF dual-hosted git repository.
zhangliang2022 pushed a commit to branch release-v0.16
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/release-v0.16 by this push:
new afde33f43 feat: customize plugin support float and integer extraction
(#4833) (#4849)
afde33f43 is described below
commit afde33f4353db3a1bae5619e12fcc57c25e37542
Author: Liang Zhang <[email protected]>
AuthorDate: Mon Apr 3 15:07:49 2023 +0800
feat: customize plugin support float and integer extraction (#4833) (#4849)
---
.../plugins/customize/e2e/extract_fields_test.go | 34 ++++++++++++++++++----
.../customize/e2e/import_issue_commits_test.go | 5 ++--
.../plugins/customize/e2e/import_issues_test.go | 5 ++--
.../e2e/raw_tables/_raw_jira_api_issues.csv | 14 ++++-----
.../snapshot_tables/issues_for_data_extraction.csv | 31 ++++++++++++++++++++
.../customize/tasks/customized_fields_extractor.go | 24 +++++++++++----
6 files changed, 91 insertions(+), 22 deletions(-)
diff --git a/backend/plugins/customize/e2e/extract_fields_test.go
b/backend/plugins/customize/e2e/extract_fields_test.go
index 23d297bcc..6f018c811 100644
--- a/backend/plugins/customize/e2e/extract_fields_test.go
+++ b/backend/plugins/customize/e2e/extract_fields_test.go
@@ -18,17 +18,17 @@ limitations under the License.
package e2e
import (
- "github.com/apache/incubator-devlake/plugins/customize/models"
"testing"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
"github.com/apache/incubator-devlake/plugins/customize/impl"
+ "github.com/apache/incubator-devlake/plugins/customize/models"
"github.com/apache/incubator-devlake/plugins/customize/service"
"github.com/apache/incubator-devlake/plugins/customize/tasks"
)
-func TestBoardDataFlow(t *testing.T) {
+func TestExtractFieldDataFlow(t *testing.T) {
var plugin impl.Customize
dataflowTester := e2ehelper.NewDataFlowTester(t, "customize", plugin)
@@ -37,8 +37,12 @@ func TestBoardDataFlow(t *testing.T) {
TransformationRules: []tasks.MappingRules{{
Table: "issues",
RawDataTable: "_raw_jira_api_issues",
- RawDataParams:
"{\"ConnectionId\":1,\"BoardId\":8}",
- Mapping: map[string]string{"x_test":
"fields.created"},
+ RawDataParams: `{"ConnectionId":1,"BoardId":8}`,
+ Mapping: map[string]string{
+ "x_test": "fields.created",
+ "x_float": "fields.customfield_10024",
+ "x_int": "fields.customfield_10146",
+ },
}}}}
// import raw data table
@@ -55,14 +59,34 @@ func TestBoardDataFlow(t *testing.T) {
if err != nil {
t.Fatal(err)
}
+ err = svc.CreateField(&models.CustomizedField{
+ TbName: "issues",
+ ColumnName: "x_float",
+ DisplayName: "test column x_float",
+ DataType: "float",
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ err = svc.CreateField(&models.CustomizedField{
+ TbName: "issues",
+ ColumnName: "x_int",
+ DisplayName: "test column x_int",
+ DataType: "bigint",
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
// verify extension fields extraction
dataflowTester.Subtask(tasks.ExtractCustomizedFieldsMeta, taskData)
dataflowTester.VerifyTable(
ticket.Issue{},
- "./snapshot_tables/issues.csv",
+ "./snapshot_tables/issues_for_data_extraction.csv",
e2ehelper.ColumnWithRawData(
"id",
"x_test",
+ "x_float",
+ "x_int",
),
)
}
diff --git a/backend/plugins/customize/e2e/import_issue_commits_test.go
b/backend/plugins/customize/e2e/import_issue_commits_test.go
index 3b98f6b86..160c33f73 100644
--- a/backend/plugins/customize/e2e/import_issue_commits_test.go
+++ b/backend/plugins/customize/e2e/import_issue_commits_test.go
@@ -18,12 +18,13 @@ limitations under the License.
package e2e
import (
+ "os"
+ "testing"
+
"github.com/apache/incubator-devlake/core/models/domainlayer/crossdomain"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
"github.com/apache/incubator-devlake/plugins/customize/impl"
"github.com/apache/incubator-devlake/plugins/customize/service"
- "os"
- "testing"
)
func TestImportIssueCommitDataFlow(t *testing.T) {
diff --git a/backend/plugins/customize/e2e/import_issues_test.go
b/backend/plugins/customize/e2e/import_issues_test.go
index 81a0db4e9..849a0cfe2 100644
--- a/backend/plugins/customize/e2e/import_issues_test.go
+++ b/backend/plugins/customize/e2e/import_issues_test.go
@@ -18,13 +18,14 @@ limitations under the License.
package e2e
import (
+ "os"
+ "testing"
+
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
"github.com/apache/incubator-devlake/plugins/customize/impl"
"github.com/apache/incubator-devlake/plugins/customize/models"
"github.com/apache/incubator-devlake/plugins/customize/service"
- "os"
- "testing"
)
func TestImportIssueDataFlow(t *testing.T) {
diff --git a/backend/plugins/customize/e2e/raw_tables/_raw_jira_api_issues.csv
b/backend/plugins/customize/e2e/raw_tables/_raw_jira_api_issues.csv
index 8b546b688..5e01fe08f 100644
--- a/backend/plugins/customize/e2e/raw_tables/_raw_jira_api_issues.csv
+++ b/backend/plugins/customize/e2e/raw_tables/_raw_jira_api_issues.csv
@@ -1,17 +1,17 @@
"id","params","data","url","input","created_at"
1701,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10063"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10063"",""key"":""EE-1"",""changelog"":{""startAt"":0,""maxResults"":25,""total"":25,""histories"":[{""id"":""90706"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{""4
[...]
1702,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10064"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10064"",""key"":""EE-2"",""changelog"":{""startAt"":0,""maxResults"":24,""total"":24,""histories"":[{""id"":""90429"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{""4
[...]
-1703,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10065"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10065"",""key"":""EE-3"",""changelog"":{""startAt"":0,""maxResults"":26,""total"":26,""histories"":[{""id"":""90430"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{""4
[...]
-1704,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10066"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10066"",""key"":""EE-4"",""changelog"":{""startAt"":0,""maxResults"":23,""total"":23,""histories"":[{""id"":""90722"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{""4
[...]
-1705,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10067"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10067"",""key"":""EE-5"",""changelog"":{""startAt"":0,""maxResults"":18,""total"":18,""histories"":[{""id"":""295895"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5ecfbd0c2490cf0c09e2e598"",""accountId"":""5ecfbd0c2490cf0c09e2e598"",""avatarUrls"":{"";
[...]
-1706,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10068"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10068"",""key"":""EE-6"",""changelog"":{""startAt"":0,""maxResults"":17,""total"":17,""histories"":[{""id"":""90473"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{""4
[...]
+1703,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10065"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10065"",""key"":""EE-3"",""changelog"":{""startAt"":0,""maxResults"":26,""total"":26,""histories"":[{""id"":""90430"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{""4
[...]
+1704,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10066"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10066"",""key"":""EE-4"",""changelog"":{""startAt"":0,""maxResults"":23,""total"":23,""histories"":[{""id"":""90722"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{""4
[...]
+1705,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10067"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10067"",""key"":""EE-5"",""changelog"":{""startAt"":0,""maxResults"":18,""total"":18,""histories"":[{""id"":""295895"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5ecfbd0c2490cf0c09e2e598"",""accountId"":""5ecfbd0c2490cf0c09e2e598"",""avatarUrls"":{"";
[...]
+1706,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10068"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10068"",""key"":""EE-6"",""changelog"":{""startAt"":0,""maxResults"":17,""total"":17,""histories"":[{""id"":""90473"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{""4
[...]
1707,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10070"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10070"",""key"":""EE-8"",""changelog"":{""startAt"":0,""maxResults"":17,""total"":17,""histories"":[{""id"":""295892"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5ecfbd0c2490cf0c09e2e598"",""accountId"":""5ecfbd0c2490cf0c09e2e598"",""avatarUrls"":{"";
[...]
1708,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10071"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10071"",""key"":""EE-9"",""changelog"":{""startAt"":0,""maxResults"":18,""total"":18,""histories"":[{""id"":""296002"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5ecfbd0c2490cf0c09e2e598"",""accountId"":""5ecfbd0c2490cf0c09e2e598"",""avatarUrls"":{"";
[...]
1709,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10072"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10072"",""key"":""EE-10"",""changelog"":{""startAt"":0,""maxResults"":17,""total"":17,""histories"":[{""id"":""295894"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5ecfbd0c2490cf0c09e2e598"",""accountId"":""5ecfbd0c2490cf0c09e2e598"",""avatarUrls"":{";
[...]
1710,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10076"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10076"",""key"":""EE-14"",""changelog"":{""startAt"":0,""maxResults"":18,""total"":18,""histories"":[{""id"":""295897"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5ecfbd0c2490cf0c09e2e598"",""accountId"":""5ecfbd0c2490cf0c09e2e598"",""avatarUrls"":{";
[...]
-1711,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10077"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10077"",""key"":""EE-15"",""changelog"":{""startAt"":0,""maxResults"":20,""total"":20,""histories"":[{""id"":""296102"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5ecfbd0c2490cf0c09e2e598"",""accountId"":""5ecfbd0c2490cf0c09e2e598"",""avatarUrls"":{";
[...]
-1712,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10078"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10078"",""key"":""EE-16"",""changelog"":{""startAt"":0,""maxResults"":18,""total"":18,""histories"":[{""id"":""295898"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5ecfbd0c2490cf0c09e2e598"",""accountId"":""5ecfbd0c2490cf0c09e2e598"",""avatarUrls"":{";
[...]
-1713,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10079"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10079"",""key"":""EE-17"",""changelog"":{""startAt"":0,""maxResults"":31,""total"":31,""histories"":[{""id"":""296582"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5ecfbd0c2490cf0c09e2e598"",""accountId"":""5ecfbd0c2490cf0c09e2e598"",""avatarUrls"":{";
[...]
+1711,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10077"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10077"",""key"":""EE-15"",""changelog"":{""startAt"":0,""maxResults"":20,""total"":20,""histories"":[{""id"":""296102"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5ecfbd0c2490cf0c09e2e598"",""accountId"":""5ecfbd0c2490cf0c09e2e598"",""avatarUrls"":{";
[...]
+1712,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10078"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10078"",""key"":""EE-16"",""changelog"":{""startAt"":0,""maxResults"":18,""total"":18,""histories"":[{""id"":""295898"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5ecfbd0c2490cf0c09e2e598"",""accountId"":""5ecfbd0c2490cf0c09e2e598"",""avatarUrls"":{";
[...]
+1713,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10079"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10079"",""key"":""EE-17"",""changelog"":{""startAt"":0,""maxResults"":31,""total"":31,""histories"":[{""id"":""296582"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5ecfbd0c2490cf0c09e2e598"",""accountId"":""5ecfbd0c2490cf0c09e2e598"",""avatarUrls"":{";
[...]
1714,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10081"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10081"",""key"":""EE-19"",""changelog"":{""startAt"":0,""maxResults"":17,""total"":17,""histories"":[{""id"":""90485"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{"";
[...]
1715,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10082"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10082"",""key"":""EE-20"",""changelog"":{""startAt"":0,""maxResults"":14,""total"":14,""histories"":[{""id"":""142717"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5ecfbd0984083c0c12e5af8f"",""accountId"":""5ecfbd0984083c0c12e5af8f"",""avatarUrls"":{";
[...]
1716,"{""ConnectionId"":1,""BoardId"":8}","{""expand"":""operations,versionedRepresentations,editmeta,changelog,renderedFields"",""id"":""10085"",""self"":""https://merico.atlassian.net/rest/agile/1.0/issue/10085"",""key"":""EE-23"",""changelog"":{""startAt"":0,""maxResults"":20,""total"":20,""histories"":[{""id"":""123813"",""author"":{""self"":""https://merico.atlassian.net/rest/api/2/user?accountId=5e9711ba34f7b90c0fbc37d3"",""accountId"":""5e9711ba34f7b90c0fbc37d3"",""avatarUrls"":{";
[...]
diff --git
a/backend/plugins/customize/e2e/snapshot_tables/issues_for_data_extraction.csv
b/backend/plugins/customize/e2e/snapshot_tables/issues_for_data_extraction.csv
new file mode 100644
index 000000000..ef4a6bf93
--- /dev/null
+++
b/backend/plugins/customize/e2e/snapshot_tables/issues_for_data_extraction.csv
@@ -0,0 +1,31 @@
+id,x_test,x_float,x_int,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+jira:JiraIssue:1:10063,2020-06-12T08:13:13.360+0800,,,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1701,
+jira:JiraIssue:1:10064,2020-06-12T08:15:36.123+0800,,,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1702,
+jira:JiraIssue:1:10065,2020-06-12T08:15:41.600+0800,10,,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1703,
+jira:JiraIssue:1:10066,2020-06-12T08:15:46.144+0800,15,,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1704,
+jira:JiraIssue:1:10067,2020-06-12T08:16:44.157+0800,0,,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1705,
+jira:JiraIssue:1:10068,2020-06-12T08:17:26.986+0800,-0.5,,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1706,
+jira:JiraIssue:1:10070,2020-06-12T08:18:58.050+0800,,,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1707,
+jira:JiraIssue:1:10071,2020-06-12T08:19:17.336+0800,,,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1708,
+jira:JiraIssue:1:10072,2020-06-12T08:19:24.637+0800,,,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1709,
+jira:JiraIssue:1:10076,2020-06-12T08:24:25.922+0800,,,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1710,
+jira:JiraIssue:1:10077,2020-06-12T08:24:39.624+0800,,42,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1711,
+jira:JiraIssue:1:10078,2020-06-12T08:24:49.017+0800,,0,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1712,
+jira:JiraIssue:1:10079,2020-06-12T08:24:56.048+0800,,-789,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1713,
+jira:JiraIssue:1:10081,2020-06-12T08:28:00.241+0800,,,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1714,
+jira:JiraIssue:1:10082,2020-06-12T08:29:43.677+0800,,,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1715,
+jira:JiraIssue:1:10085,2020-06-12T08:33:57.204+0800,,,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1716,
+jira:JiraIssue:1:10086,2020-06-12T08:35:15.489+0800,,,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1717,
+jira:JiraIssue:1:10087,2020-06-12T08:40:54.210+0800,,,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1718,
+jira:JiraIssue:1:10088,2020-06-12T08:41:01.683+0800,,,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1719,
+jira:JiraIssue:1:10089,2020-06-12T08:41:16.622+0800,,,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1720,
+jira:JiraIssue:1:10090,2020-06-12T08:41:36.317+0800,,,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1721,
+jira:JiraIssue:1:10091,2020-06-12T08:48:29.035+0800,,,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1722,
+jira:JiraIssue:1:10092,2020-06-12T08:48:39.803+0800,,,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1723,
+jira:JiraIssue:1:10093,2020-06-12T08:48:46.751+0800,,,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1724,
+jira:JiraIssue:1:10094,2020-06-12T08:48:53.279+0800,,,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1725,
+jira:JiraIssue:1:10095,2020-06-12T08:50:45.674+0800,,,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1726,
+jira:JiraIssue:1:10096,2020-06-12T08:51:57.807+0800,,,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1727,
+jira:JiraIssue:1:10097,2020-06-12T08:52:04.767+0800,,,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1728,
+jira:JiraIssue:1:10098,2020-06-12T08:52:12.678+0800,,,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1729,
+jira:JiraIssue:1:10099,2020-06-12T08:52:23.456+0800,,,"{""ConnectionId"":1,""BoardId"":8}",_raw_jira_api_issues,1730,
diff --git a/backend/plugins/customize/tasks/customized_fields_extractor.go
b/backend/plugins/customize/tasks/customized_fields_extractor.go
index e64156582..8930745dd 100644
--- a/backend/plugins/customize/tasks/customized_fields_extractor.go
+++ b/backend/plugins/customize/tasks/customized_fields_extractor.go
@@ -20,12 +20,12 @@ package tasks
import (
"context"
"fmt"
+ "strings"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/plugins/customize/models"
- "strings"
-
"github.com/tidwall/gjson"
)
@@ -85,7 +85,7 @@ func extractCustomizedFields(ctx context.Context, d dal.Dal,
table, rawTable, ra
default:
}
row := make(map[string]interface{})
- updates := make(map[string]string)
+ updates := make(map[string]interface{})
err = d.Fetch(rows, &row)
if err != nil {
return err
@@ -93,11 +93,13 @@ func extractCustomizedFields(ctx context.Context, d
dal.Dal, table, rawTable, ra
switch blob := row["data"].(type) {
case []byte:
for field, path := range extractor {
- updates[field] = gjson.GetBytes(blob,
path).String()
+ result := gjson.GetBytes(blob, path)
+ fillInUpdates(result, field, updates)
}
case string:
for field, path := range extractor {
- updates[field] = gjson.Get(blob, path).String()
+ result := gjson.Get(blob, path)
+ fillInUpdates(result, field, updates)
}
default:
return nil
@@ -117,7 +119,17 @@ func extractCustomizedFields(ctx context.Context, d
dal.Dal, table, rawTable, ra
return nil
}
-func mkUpdate(table string, updates map[string]string, pk
map[string]interface{}) (string, []interface{}) {
+// fillInUpdates fills in the updates map with the result of the gjson query
+func fillInUpdates(result gjson.Result, field string, updates
map[string]interface{}) {
+ if result.Type == gjson.Null {
+ updates[field] = nil
+ } else {
+ updates[field] = result.String()
+ }
+}
+
+// mkUpdate generates SQL statement and parameters for updating a record
+func mkUpdate(table string, updates map[string]interface{}, pk
map[string]interface{}) (string, []interface{}) {
var params []interface{}
stat := fmt.Sprintf("UPDATE %s SET ", table)
var uu []string
