This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 040b88da0 feat(gitlab): add author date (#4978)
040b88da0 is described below
commit 040b88da06f2ee52dcee0a024595e51d8b27466e
Author: Warren Chen <[email protected]>
AuthorDate: Thu Apr 20 14:00:57 2023 +0800
feat(gitlab): add author date (#4978)
---
.../20230420_add_pr_commit_author_date.go} | 45 +++++++++----
backend/plugins/gitlab/e2e/mr_commits_test.go | 26 +++-----
.../snapshot_tables/_tool_gitlab_mr_commits.csv | 76 +++++++++++-----------
.../e2e/snapshot_tables/pull_request_commits.csv | 70 ++++++++++----------
...er.go => 20230420_add_mr_commit_author_date.go} | 45 +++++++++----
.../gitlab/models/migrationscripts/register.go | 1 +
backend/plugins/gitlab/models/mr_commit.go | 8 ++-
.../plugins/gitlab/tasks/mr_commit_convertor.go | 5 +-
.../plugins/gitlab/tasks/mr_commit_extractor.go | 7 +-
9 files changed, 157 insertions(+), 126 deletions(-)
diff --git a/backend/plugins/gitlab/models/migrationscripts/register.go
b/backend/plugins/bitbucket/models/migrationscripts/20230420_add_pr_commit_author_date.go
similarity index 50%
copy from backend/plugins/gitlab/models/migrationscripts/register.go
copy to
backend/plugins/bitbucket/models/migrationscripts/20230420_add_pr_commit_author_date.go
index 53823fae4..b4f77fef0 100644
--- a/backend/plugins/gitlab/models/migrationscripts/register.go
+++
b/backend/plugins/bitbucket/models/migrationscripts/20230420_add_pr_commit_author_date.go
@@ -18,21 +18,38 @@ limitations under the License.
package migrationscripts
import (
+ "time"
+
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
)
-// All return all the migration scripts
-func All() []plugin.MigrationScript {
- return []plugin.MigrationScript{
- new(addInitTables),
- new(addGitlabCI),
- new(addPipelineID),
- new(addPipelineProjects),
- new(fixDurationToFloat8),
- new(addTransformationRule20221125),
- new(addStdTypeToIssue221230),
- new(addIsDetailRequired20230210),
- new(addConnectionIdToTransformationRule),
- new(addTypeEnvToPipeline),
- }
+var _ plugin.MigrationScript = (*addGitlabCommitAuthoredDate)(nil)
+
+type GitlabMrCommit20230420 struct {
+ CommitAuthoredDate time.Time
+}
+
+func (GitlabMrCommit20230420) TableName() string {
+ return "_tool_gitlab_mr_commits"
+}
+
+type addGitlabCommitAuthoredDate struct{}
+
+func (script *addGitlabCommitAuthoredDate) Up(basicRes context.BasicRes)
errors.Error {
+
+ return migrationhelper.AutoMigrateTables(
+ basicRes,
+ &GitlabMrCommit20230420{},
+ )
+}
+
+func (*addGitlabCommitAuthoredDate) Version() uint64 {
+ return 20230420135127
+}
+
+func (*addGitlabCommitAuthoredDate) Name() string {
+ return "add commit_authored_date to _tool_gitlab_mr_commits table"
}
diff --git a/backend/plugins/gitlab/e2e/mr_commits_test.go
b/backend/plugins/gitlab/e2e/mr_commits_test.go
index 6c65dfb8f..521eb207e 100644
--- a/backend/plugins/gitlab/e2e/mr_commits_test.go
+++ b/backend/plugins/gitlab/e2e/mr_commits_test.go
@@ -18,6 +18,7 @@ limitations under the License.
package e2e
import (
+ "github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
"github.com/apache/incubator-devlake/helpers/e2ehelper"
"github.com/apache/incubator-devlake/plugins/gitlab/impl"
@@ -117,27 +118,18 @@ func TestGitlabMrCommitDataFlow(t *testing.T) {
),
)
- dataflowTester.VerifyTable(
- models.GitlabMrCommit{},
- "./snapshot_tables/_tool_gitlab_mr_commits.csv",
- e2ehelper.ColumnWithRawData(
- "connection_id",
- "merge_request_id",
- "commit_sha",
- ),
- )
+ dataflowTester.VerifyTableWithOptions(models.GitlabMrCommit{},
e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/_tool_gitlab_mr_commits.csv",
+ IgnoreTypes: []interface{}{common.Model{}},
+ })
// verify conversion
dataflowTester.FlushTabler(&code.PullRequestCommit{})
dataflowTester.Subtask(tasks.ConvertApiMrCommitsMeta, taskData)
- dataflowTester.VerifyTable(
- code.PullRequestCommit{},
- "./snapshot_tables/pull_request_commits.csv",
- e2ehelper.ColumnWithRawData(
- "commit_sha",
- "pull_request_id",
- ),
- )
+ dataflowTester.VerifyTableWithOptions(code.PullRequestCommit{},
e2ehelper.TableOptions{
+ CSVRelPath: "./snapshot_tables/pull_request_commits.csv",
+ IgnoreTypes: []interface{}{common.Model{}},
+ })
// verify conversion
dataflowTester.FlushTabler(&code.Commit{})
diff --git
a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_mr_commits.csv
b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_mr_commits.csv
index f7f613899..db5d31faf 100644
--- a/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_mr_commits.csv
+++ b/backend/plugins/gitlab/e2e/snapshot_tables/_tool_gitlab_mr_commits.csv
@@ -1,38 +1,38 @@
-connection_id,commit_sha,merge_request_id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,05976c113b02fb71d5c43682bd2f9d96a3454efa,53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,34,
-1,083864201849f6707a34d24e61c43638696f95a6,35064956,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,6,
-1,08973065d33c6fd2fb1f592b7a1e0fbe2ee72edd,34300715,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,11,
-1,0a4574d52b5964c37bfcb2952f6c6a8a9c6d5bcf,34300715,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,7,
-1,0dbb8bfdbbf7373656fa3eb036bdcbb25ae162da,53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,26,
-1,1cba71b355d2bd4e1350602cfb9b98cce5a5cb4d,34300715,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,8,
-1,1f16955d45cec904c8968bb27689d91a71e9a2f2,34300715,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,12,
-1,211795deaaaaddb23b797261cdcd148e6083d5b2,53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,21,
-1,25f2b34099a3e1867fcf73a54e50f78f3c8d8f75,34300715,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,9,
-1,382084b42697577d3a6adf71ce73d4b5ddd22977,32348491,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,2,
-1,3f04e0a61d0c4d2dd736a6bcaa3a06826269a533,32348491,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,3,
-1,45bed79fcdc6ba3444245f55660f839db50d79c8,53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,33,
-1,48295660f8bd0c0a024849d4407830face57d4bc,53640548,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,37,
-1,4f0d2f7b1a47d21b01a68e59c12d6f8885ab230e,53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,24,
-1,553f7613b178301da4075202afe91ebda5d97ab5,53627854,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,16,
-1,5a141ad0c28960b02c40e6c476d8df3cc76f58c4,53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,31,
-1,5b6a2be437e0c66ba6276030a61a10583c3f83fa,53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,22,
-1,5d6c369200c633b06d85c9375f61fef297761ef1,53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,19,
-1,645a724b558a7fa3885fa233d06f68f833d893fc,53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,27,
-1,64abdcdd9ee9c821f56faf93c5bb0ebc57f575e4,53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,28,
-1,6ebf519f25328e49a004f5a574c8b69ba7420be1,34300715,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,15,
-1,81b12c995b2fc12e20fd05113cf36dbd57b6e7b0,53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,20,
-1,846deac12a1263e4631ced45abbc5e157b23c1be,53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,30,
-1,8a40765e605966842b05b5d8cdf2dd6890556d09,34300715,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,14,
-1,8a96911c5350f7575215f02f0242f7e27ad3d1f4,34300715,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,10,
-1,91e5666bdfe90e5ada3e65aaa9240b2b0b1a072f,35841926,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,17,
-1,93d13cf14a470995ec35186292bf165e1152dc2a,53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,32,
-1,9a80506e158547d87e7ad2902c0196deff5b5c07,53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,35,
-1,9e5ec04b0a8273b73936ea150275919b0602c260,34300715,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,13,
-1,9ef8183140bae8c39a440d9c7eb47f3bc7dcab75,53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,18,
-1,a09f2d4ae88e78d7424fba1ae6d0c036b677271d,53640548,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,36,
-1,abbe0ab2c7bb1dc2cfaa3ef3062f378fb908ba71,32348491,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,5,
-1,ad25fcda15e310a7eb34b3e38ff9b962b54f0e03,32348491,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,1,
-1,bb6dfec8dcdacb5b6b37dbd8eb1aa6f9e8c06138,53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,23,
-1,c4d406c077af582837d3611a46eded70151f93fb,53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,29,
-1,e01d4f03811cd0da9949848731236e0aa261cf54,32348491,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,4,
-1,e24fecacb1a2c85f904bef57ffd5598ca2df023c,53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,25,
+connection_id,commit_sha,merge_request_id,commit_authored_date,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+1,05976c113b02fb71d5c43682bd2f9d96a3454efa,53445063,2020-03-24T18:14:03.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,34,
+1,083864201849f6707a34d24e61c43638696f95a6,35064956,2019-08-15T19:32:19.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,6,
+1,08973065d33c6fd2fb1f592b7a1e0fbe2ee72edd,34300715,2019-08-02T18:35:08.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,11,
+1,0a4574d52b5964c37bfcb2952f6c6a8a9c6d5bcf,34300715,2019-08-02T19:00:50.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,7,
+1,0dbb8bfdbbf7373656fa3eb036bdcbb25ae162da,53445063,2020-03-24T22:39:40.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,26,
+1,1cba71b355d2bd4e1350602cfb9b98cce5a5cb4d,34300715,2019-08-02T18:53:44.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,8,
+1,1f16955d45cec904c8968bb27689d91a71e9a2f2,34300715,2019-08-02T17:28:44.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,12,
+1,211795deaaaaddb23b797261cdcd148e6083d5b2,53445063,2020-03-24T22:44:07.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,21,
+1,25f2b34099a3e1867fcf73a54e50f78f3c8d8f75,34300715,2019-08-02T18:47:28.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,9,
+1,382084b42697577d3a6adf71ce73d4b5ddd22977,32348491,2019-06-28T10:54:26.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,2,
+1,3f04e0a61d0c4d2dd736a6bcaa3a06826269a533,32348491,2019-06-28T10:48:49.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,3,
+1,45bed79fcdc6ba3444245f55660f839db50d79c8,53445063,2020-03-24T18:15:29.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,33,
+1,48295660f8bd0c0a024849d4407830face57d4bc,53640548,2020-04-05T14:39:43.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,37,
+1,4f0d2f7b1a47d21b01a68e59c12d6f8885ab230e,53445063,2020-03-24T22:40:36.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,24,
+1,553f7613b178301da4075202afe91ebda5d97ab5,53627854,2020-03-25T19:01:57.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,16,
+1,5a141ad0c28960b02c40e6c476d8df3cc76f58c4,53445063,2020-03-24T21:18:16.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,31,
+1,5b6a2be437e0c66ba6276030a61a10583c3f83fa,53445063,2020-03-24T22:42:54.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,22,
+1,5d6c369200c633b06d85c9375f61fef297761ef1,53445063,2020-03-25T13:22:42.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,19,
+1,645a724b558a7fa3885fa233d06f68f833d893fc,53445063,2020-03-24T22:39:36.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,27,
+1,64abdcdd9ee9c821f56faf93c5bb0ebc57f575e4,53445063,2020-03-24T22:39:31.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,28,
+1,6ebf519f25328e49a004f5a574c8b69ba7420be1,34300715,2019-08-02T17:18:15.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,15,
+1,81b12c995b2fc12e20fd05113cf36dbd57b6e7b0,53445063,2020-03-25T13:21:04.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,20,
+1,846deac12a1263e4631ced45abbc5e157b23c1be,53445063,2020-03-24T21:27:46.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,30,
+1,8a40765e605966842b05b5d8cdf2dd6890556d09,34300715,2019-08-02T17:22:09.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,14,
+1,8a96911c5350f7575215f02f0242f7e27ad3d1f4,34300715,2019-08-02T18:46:42.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,10,
+1,91e5666bdfe90e5ada3e65aaa9240b2b0b1a072f,35841926,2019-08-26T15:34:33.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,17,
+1,93d13cf14a470995ec35186292bf165e1152dc2a,53445063,2020-03-24T18:55:11.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,32,
+1,9a80506e158547d87e7ad2902c0196deff5b5c07,53445063,2020-03-24T12:52:28.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,35,
+1,9e5ec04b0a8273b73936ea150275919b0602c260,34300715,2019-08-02T17:24:35.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,13,
+1,9ef8183140bae8c39a440d9c7eb47f3bc7dcab75,53445063,2020-03-25T13:22:47.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,18,
+1,a09f2d4ae88e78d7424fba1ae6d0c036b677271d,53640548,2020-04-05T14:43:47.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,36,
+1,abbe0ab2c7bb1dc2cfaa3ef3062f378fb908ba71,32348491,2019-06-28T05:40:36.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,5,
+1,ad25fcda15e310a7eb34b3e38ff9b962b54f0e03,32348491,2019-06-28T11:03:52.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,1,
+1,bb6dfec8dcdacb5b6b37dbd8eb1aa6f9e8c06138,53445063,2020-03-24T22:41:35.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,23,
+1,c4d406c077af582837d3611a46eded70151f93fb,53445063,2020-03-24T22:37:23.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,29,
+1,e01d4f03811cd0da9949848731236e0aa261cf54,32348491,2019-06-28T10:43:17.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,4,
+1,e24fecacb1a2c85f904bef57ffd5598ca2df023c,53445063,2020-03-24T22:39:42.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,25,
diff --git
a/backend/plugins/gitlab/e2e/snapshot_tables/pull_request_commits.csv
b/backend/plugins/gitlab/e2e/snapshot_tables/pull_request_commits.csv
index 3309a5093..308127b39 100644
--- a/backend/plugins/gitlab/e2e/snapshot_tables/pull_request_commits.csv
+++ b/backend/plugins/gitlab/e2e/snapshot_tables/pull_request_commits.csv
@@ -1,35 +1,35 @@
-commit_sha,pull_request_id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-05976c113b02fb71d5c43682bd2f9d96a3454efa,gitlab:GitlabMergeRequest:1:53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,34,
-083864201849f6707a34d24e61c43638696f95a6,gitlab:GitlabMergeRequest:1:35064956,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,6,
-08973065d33c6fd2fb1f592b7a1e0fbe2ee72edd,gitlab:GitlabMergeRequest:1:34300715,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,11,
-0a4574d52b5964c37bfcb2952f6c6a8a9c6d5bcf,gitlab:GitlabMergeRequest:1:34300715,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,7,
-0dbb8bfdbbf7373656fa3eb036bdcbb25ae162da,gitlab:GitlabMergeRequest:1:53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,26,
-1cba71b355d2bd4e1350602cfb9b98cce5a5cb4d,gitlab:GitlabMergeRequest:1:34300715,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,8,
-1f16955d45cec904c8968bb27689d91a71e9a2f2,gitlab:GitlabMergeRequest:1:34300715,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,12,
-211795deaaaaddb23b797261cdcd148e6083d5b2,gitlab:GitlabMergeRequest:1:53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,21,
-25f2b34099a3e1867fcf73a54e50f78f3c8d8f75,gitlab:GitlabMergeRequest:1:34300715,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,9,
-382084b42697577d3a6adf71ce73d4b5ddd22977,gitlab:GitlabMergeRequest:1:32348491,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,2,
-3f04e0a61d0c4d2dd736a6bcaa3a06826269a533,gitlab:GitlabMergeRequest:1:32348491,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,3,
-45bed79fcdc6ba3444245f55660f839db50d79c8,gitlab:GitlabMergeRequest:1:53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,33,
-4f0d2f7b1a47d21b01a68e59c12d6f8885ab230e,gitlab:GitlabMergeRequest:1:53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,24,
-5a141ad0c28960b02c40e6c476d8df3cc76f58c4,gitlab:GitlabMergeRequest:1:53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,31,
-5b6a2be437e0c66ba6276030a61a10583c3f83fa,gitlab:GitlabMergeRequest:1:53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,22,
-5d6c369200c633b06d85c9375f61fef297761ef1,gitlab:GitlabMergeRequest:1:53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,19,
-645a724b558a7fa3885fa233d06f68f833d893fc,gitlab:GitlabMergeRequest:1:53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,27,
-64abdcdd9ee9c821f56faf93c5bb0ebc57f575e4,gitlab:GitlabMergeRequest:1:53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,28,
-6ebf519f25328e49a004f5a574c8b69ba7420be1,gitlab:GitlabMergeRequest:1:34300715,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,15,
-81b12c995b2fc12e20fd05113cf36dbd57b6e7b0,gitlab:GitlabMergeRequest:1:53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,20,
-846deac12a1263e4631ced45abbc5e157b23c1be,gitlab:GitlabMergeRequest:1:53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,30,
-8a40765e605966842b05b5d8cdf2dd6890556d09,gitlab:GitlabMergeRequest:1:34300715,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,14,
-8a96911c5350f7575215f02f0242f7e27ad3d1f4,gitlab:GitlabMergeRequest:1:34300715,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,10,
-91e5666bdfe90e5ada3e65aaa9240b2b0b1a072f,gitlab:GitlabMergeRequest:1:35841926,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,17,
-93d13cf14a470995ec35186292bf165e1152dc2a,gitlab:GitlabMergeRequest:1:53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,32,
-9a80506e158547d87e7ad2902c0196deff5b5c07,gitlab:GitlabMergeRequest:1:53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,35,
-9e5ec04b0a8273b73936ea150275919b0602c260,gitlab:GitlabMergeRequest:1:34300715,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,13,
-9ef8183140bae8c39a440d9c7eb47f3bc7dcab75,gitlab:GitlabMergeRequest:1:53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,18,
-abbe0ab2c7bb1dc2cfaa3ef3062f378fb908ba71,gitlab:GitlabMergeRequest:1:32348491,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,5,
-ad25fcda15e310a7eb34b3e38ff9b962b54f0e03,gitlab:GitlabMergeRequest:1:32348491,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,1,
-bb6dfec8dcdacb5b6b37dbd8eb1aa6f9e8c06138,gitlab:GitlabMergeRequest:1:53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,23,
-c4d406c077af582837d3611a46eded70151f93fb,gitlab:GitlabMergeRequest:1:53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,29,
-e01d4f03811cd0da9949848731236e0aa261cf54,gitlab:GitlabMergeRequest:1:32348491,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,4,
-e24fecacb1a2c85f904bef57ffd5598ca2df023c,gitlab:GitlabMergeRequest:1:53445063,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,25,
+commit_sha,pull_request_id,commit_authored_date,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
+05976c113b02fb71d5c43682bd2f9d96a3454efa,gitlab:GitlabMergeRequest:1:53445063,2020-03-24T18:14:03.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,34,
+083864201849f6707a34d24e61c43638696f95a6,gitlab:GitlabMergeRequest:1:35064956,2019-08-15T19:32:19.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,6,
+08973065d33c6fd2fb1f592b7a1e0fbe2ee72edd,gitlab:GitlabMergeRequest:1:34300715,2019-08-02T18:35:08.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,11,
+0a4574d52b5964c37bfcb2952f6c6a8a9c6d5bcf,gitlab:GitlabMergeRequest:1:34300715,2019-08-02T19:00:50.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,7,
+0dbb8bfdbbf7373656fa3eb036bdcbb25ae162da,gitlab:GitlabMergeRequest:1:53445063,2020-03-24T22:39:40.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,26,
+1cba71b355d2bd4e1350602cfb9b98cce5a5cb4d,gitlab:GitlabMergeRequest:1:34300715,2019-08-02T18:53:44.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,8,
+1f16955d45cec904c8968bb27689d91a71e9a2f2,gitlab:GitlabMergeRequest:1:34300715,2019-08-02T17:28:44.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,12,
+211795deaaaaddb23b797261cdcd148e6083d5b2,gitlab:GitlabMergeRequest:1:53445063,2020-03-24T22:44:07.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,21,
+25f2b34099a3e1867fcf73a54e50f78f3c8d8f75,gitlab:GitlabMergeRequest:1:34300715,2019-08-02T18:47:28.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,9,
+382084b42697577d3a6adf71ce73d4b5ddd22977,gitlab:GitlabMergeRequest:1:32348491,2019-06-28T10:54:26.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,2,
+3f04e0a61d0c4d2dd736a6bcaa3a06826269a533,gitlab:GitlabMergeRequest:1:32348491,2019-06-28T10:48:49.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,3,
+45bed79fcdc6ba3444245f55660f839db50d79c8,gitlab:GitlabMergeRequest:1:53445063,2020-03-24T18:15:29.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,33,
+4f0d2f7b1a47d21b01a68e59c12d6f8885ab230e,gitlab:GitlabMergeRequest:1:53445063,2020-03-24T22:40:36.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,24,
+5a141ad0c28960b02c40e6c476d8df3cc76f58c4,gitlab:GitlabMergeRequest:1:53445063,2020-03-24T21:18:16.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,31,
+5b6a2be437e0c66ba6276030a61a10583c3f83fa,gitlab:GitlabMergeRequest:1:53445063,2020-03-24T22:42:54.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,22,
+5d6c369200c633b06d85c9375f61fef297761ef1,gitlab:GitlabMergeRequest:1:53445063,2020-03-25T13:22:42.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,19,
+645a724b558a7fa3885fa233d06f68f833d893fc,gitlab:GitlabMergeRequest:1:53445063,2020-03-24T22:39:36.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,27,
+64abdcdd9ee9c821f56faf93c5bb0ebc57f575e4,gitlab:GitlabMergeRequest:1:53445063,2020-03-24T22:39:31.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,28,
+6ebf519f25328e49a004f5a574c8b69ba7420be1,gitlab:GitlabMergeRequest:1:34300715,2019-08-02T17:18:15.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,15,
+81b12c995b2fc12e20fd05113cf36dbd57b6e7b0,gitlab:GitlabMergeRequest:1:53445063,2020-03-25T13:21:04.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,20,
+846deac12a1263e4631ced45abbc5e157b23c1be,gitlab:GitlabMergeRequest:1:53445063,2020-03-24T21:27:46.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,30,
+8a40765e605966842b05b5d8cdf2dd6890556d09,gitlab:GitlabMergeRequest:1:34300715,2019-08-02T17:22:09.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,14,
+8a96911c5350f7575215f02f0242f7e27ad3d1f4,gitlab:GitlabMergeRequest:1:34300715,2019-08-02T18:46:42.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,10,
+91e5666bdfe90e5ada3e65aaa9240b2b0b1a072f,gitlab:GitlabMergeRequest:1:35841926,2019-08-26T15:34:33.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,17,
+93d13cf14a470995ec35186292bf165e1152dc2a,gitlab:GitlabMergeRequest:1:53445063,2020-03-24T18:55:11.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,32,
+9a80506e158547d87e7ad2902c0196deff5b5c07,gitlab:GitlabMergeRequest:1:53445063,2020-03-24T12:52:28.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,35,
+9e5ec04b0a8273b73936ea150275919b0602c260,gitlab:GitlabMergeRequest:1:34300715,2019-08-02T17:24:35.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,13,
+9ef8183140bae8c39a440d9c7eb47f3bc7dcab75,gitlab:GitlabMergeRequest:1:53445063,2020-03-25T13:22:47.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,18,
+abbe0ab2c7bb1dc2cfaa3ef3062f378fb908ba71,gitlab:GitlabMergeRequest:1:32348491,2019-06-28T05:40:36.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,5,
+ad25fcda15e310a7eb34b3e38ff9b962b54f0e03,gitlab:GitlabMergeRequest:1:32348491,2019-06-28T11:03:52.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,1,
+bb6dfec8dcdacb5b6b37dbd8eb1aa6f9e8c06138,gitlab:GitlabMergeRequest:1:53445063,2020-03-24T22:41:35.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,23,
+c4d406c077af582837d3611a46eded70151f93fb,gitlab:GitlabMergeRequest:1:53445063,2020-03-24T22:37:23.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,29,
+e01d4f03811cd0da9949848731236e0aa261cf54,gitlab:GitlabMergeRequest:1:32348491,2019-06-28T10:43:17.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,4,
+e24fecacb1a2c85f904bef57ffd5598ca2df023c,gitlab:GitlabMergeRequest:1:53445063,2020-03-24T22:39:42.000+00:00,"{""ConnectionId"":1,""ProjectId"":12345678}",_raw_gitlab_api_merge_request_commits,25,
diff --git a/backend/plugins/gitlab/models/migrationscripts/register.go
b/backend/plugins/gitlab/models/migrationscripts/20230420_add_mr_commit_author_date.go
similarity index 50%
copy from backend/plugins/gitlab/models/migrationscripts/register.go
copy to
backend/plugins/gitlab/models/migrationscripts/20230420_add_mr_commit_author_date.go
index 53823fae4..b4f77fef0 100644
--- a/backend/plugins/gitlab/models/migrationscripts/register.go
+++
b/backend/plugins/gitlab/models/migrationscripts/20230420_add_mr_commit_author_date.go
@@ -18,21 +18,38 @@ limitations under the License.
package migrationscripts
import (
+ "time"
+
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
)
-// All return all the migration scripts
-func All() []plugin.MigrationScript {
- return []plugin.MigrationScript{
- new(addInitTables),
- new(addGitlabCI),
- new(addPipelineID),
- new(addPipelineProjects),
- new(fixDurationToFloat8),
- new(addTransformationRule20221125),
- new(addStdTypeToIssue221230),
- new(addIsDetailRequired20230210),
- new(addConnectionIdToTransformationRule),
- new(addTypeEnvToPipeline),
- }
+var _ plugin.MigrationScript = (*addGitlabCommitAuthoredDate)(nil)
+
+type GitlabMrCommit20230420 struct {
+ CommitAuthoredDate time.Time
+}
+
+func (GitlabMrCommit20230420) TableName() string {
+ return "_tool_gitlab_mr_commits"
+}
+
+type addGitlabCommitAuthoredDate struct{}
+
+func (script *addGitlabCommitAuthoredDate) Up(basicRes context.BasicRes)
errors.Error {
+
+ return migrationhelper.AutoMigrateTables(
+ basicRes,
+ &GitlabMrCommit20230420{},
+ )
+}
+
+func (*addGitlabCommitAuthoredDate) Version() uint64 {
+ return 20230420135127
+}
+
+func (*addGitlabCommitAuthoredDate) Name() string {
+ return "add commit_authored_date to _tool_gitlab_mr_commits table"
}
diff --git a/backend/plugins/gitlab/models/migrationscripts/register.go
b/backend/plugins/gitlab/models/migrationscripts/register.go
index 53823fae4..4f65f5ea3 100644
--- a/backend/plugins/gitlab/models/migrationscripts/register.go
+++ b/backend/plugins/gitlab/models/migrationscripts/register.go
@@ -33,6 +33,7 @@ func All() []plugin.MigrationScript {
new(addStdTypeToIssue221230),
new(addIsDetailRequired20230210),
new(addConnectionIdToTransformationRule),
+ new(addGitlabCommitAuthoredDate),
new(addTypeEnvToPipeline),
}
}
diff --git a/backend/plugins/gitlab/models/mr_commit.go
b/backend/plugins/gitlab/models/mr_commit.go
index 1c6cd6ee4..3db711f23 100644
--- a/backend/plugins/gitlab/models/mr_commit.go
+++ b/backend/plugins/gitlab/models/mr_commit.go
@@ -19,6 +19,7 @@ package models
import (
"github.com/apache/incubator-devlake/core/models/common"
+ "time"
)
// This Model is intended to save commits that are associated to a merge
request
@@ -28,9 +29,10 @@ import (
// Thus a "Merge Request Commit" needs to be considered as distinct from a
"Commit"
type GitlabMrCommit struct {
- ConnectionId uint64 `gorm:"primaryKey"`
- CommitSha string `gorm:"primaryKey;type:varchar(40)"`
- MergeRequestId int `gorm:"primaryKey;autoIncrement:false"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ CommitSha string `gorm:"primaryKey;type:varchar(40)"`
+ MergeRequestId int `gorm:"primaryKey;autoIncrement:false"`
+ CommitAuthoredDate *time.Time
common.NoPKModel
}
diff --git a/backend/plugins/gitlab/tasks/mr_commit_convertor.go
b/backend/plugins/gitlab/tasks/mr_commit_convertor.go
index d0c3b7444..4b20ae600 100644
--- a/backend/plugins/gitlab/tasks/mr_commit_convertor.go
+++ b/backend/plugins/gitlab/tasks/mr_commit_convertor.go
@@ -67,8 +67,9 @@ func ConvertApiMergeRequestsCommits(taskCtx
plugin.SubTaskContext) errors.Error
Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
GitlabMrCommit := inputRow.(*models.GitlabMrCommit)
domainPrcommit := &code.PullRequestCommit{
- CommitSha: GitlabMrCommit.CommitSha,
- PullRequestId:
domainIdGenerator.Generate(data.Options.ConnectionId,
GitlabMrCommit.MergeRequestId),
+ CommitSha: GitlabMrCommit.CommitSha,
+ PullRequestId:
domainIdGenerator.Generate(data.Options.ConnectionId,
GitlabMrCommit.MergeRequestId),
+ CommitAuthoredDate:
*GitlabMrCommit.CommitAuthoredDate,
}
return []interface{}{
domainPrcommit,
diff --git a/backend/plugins/gitlab/tasks/mr_commit_extractor.go
b/backend/plugins/gitlab/tasks/mr_commit_extractor.go
index 83ad97ec8..b0bb1d362 100644
--- a/backend/plugins/gitlab/tasks/mr_commit_extractor.go
+++ b/backend/plugins/gitlab/tasks/mr_commit_extractor.go
@@ -57,9 +57,10 @@ func ExtractApiMergeRequestsCommits(taskCtx
plugin.SubTaskContext) errors.Error
}
gitlabMrCommit := &models.GitlabMrCommit{
- CommitSha: gitlabApiCommit.GitlabId,
- MergeRequestId: input.GitlabId,
- ConnectionId: data.Options.ConnectionId,
+ CommitSha: gitlabApiCommit.GitlabId,
+ MergeRequestId: input.GitlabId,
+ ConnectionId: data.Options.ConnectionId,
+ CommitAuthoredDate:
api.Iso8601TimeToTime(&gitlabApiCommit.AuthoredDate),
}
gitlabProjectCommit := &models.GitlabProjectCommit{
ConnectionId: data.Options.ConnectionId,
