This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 26facdfc5 Feat(bitbucket): add author info to pr commit (#4980)
26facdfc5 is described below
commit 26facdfc52aa6902436c004863fa132d1ada7b06
Author: Warren Chen <[email protected]>
AuthorDate: Thu Apr 20 15:39:28 2023 +0800
Feat(bitbucket): add author info to pr commit (#4980)
* feat(bitbucket): add authordate author name and author email
---
backend/plugins/bitbucket/e2e/pr_commit_test.go | 66 ++++++++++++++++++++++
.../_raw_bitbucket_api_pull_request_commits.csv | 9 +++
.../_tool_bitbucket_pull_request_commits.csv | 9 +++
.../e2e/snapshot_tables/pull_request_commits.csv | 9 +++
.../20230420_add_pr_commit_author_date.go | 26 +++++----
.../bitbucket/models/migrationscripts/register.go | 1 +
backend/plugins/bitbucket/models/pr_commit.go | 12 ++--
.../plugins/bitbucket/tasks/pr_commit_convertor.go | 7 ++-
.../plugins/bitbucket/tasks/pr_commit_extractor.go | 15 +++--
9 files changed, 132 insertions(+), 22 deletions(-)
diff --git a/backend/plugins/bitbucket/e2e/pr_commit_test.go
b/backend/plugins/bitbucket/e2e/pr_commit_test.go
new file mode 100644
index 000000000..8ab43f70e
--- /dev/null
+++ b/backend/plugins/bitbucket/e2e/pr_commit_test.go
@@ -0,0 +1,66 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+ "github.com/apache/incubator-devlake/core/models/common"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/helpers/e2ehelper"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/impl"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/models"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/tasks"
+ "testing"
+)
+
+func TestPrCommitDataFlow(t *testing.T) {
+ var plugin impl.Bitbucket
+ dataflowTester := e2ehelper.NewDataFlowTester(t, "bitbucket", plugin)
+
+ taskData := &tasks.BitbucketTaskData{
+ Options: &tasks.BitbucketOptions{
+ ConnectionId: 1,
+ FullName: "likyh/likyhphp",
+ },
+ }
+
+ // import raw data table
+
dataflowTester.ImportCsvIntoRawTable("./raw_tables/_raw_bitbucket_api_pull_request_commits.csv",
"_raw_bitbucket_api_pull_request_commits")
+
+ // verify pr extraction
+ dataflowTester.FlushTabler(&models.BitbucketPrCommit{})
+ dataflowTester.FlushTabler(&models.BitbucketCommit{})
+ dataflowTester.FlushTabler(&models.BitbucketRepoCommit{})
+ dataflowTester.Subtask(tasks.ExtractApiPrCommitsMeta, taskData)
+ dataflowTester.VerifyTableWithOptions(
+ models.BitbucketPrCommit{}, e2ehelper.TableOptions{
+ CSVRelPath:
"./snapshot_tables/_tool_bitbucket_pull_request_commits.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ },
+ )
+
+ // verify pr conversion
+ dataflowTester.FlushTabler(&code.PullRequestCommit{})
+ dataflowTester.Subtask(tasks.ConvertPrCommitsMeta, taskData)
+ dataflowTester.VerifyTableWithOptions(
+ code.PullRequestCommit{},
+ e2ehelper.TableOptions{
+ CSVRelPath:
"./snapshot_tables/pull_request_commits.csv",
+ IgnoreTypes: []interface{}{common.NoPKModel{}},
+ },
+ )
+}
diff --git
a/backend/plugins/bitbucket/e2e/raw_tables/_raw_bitbucket_api_pull_request_commits.csv
b/backend/plugins/bitbucket/e2e/raw_tables/_raw_bitbucket_api_pull_request_commits.csv
new file mode 100644
index 000000000..0fd5b7d14
--- /dev/null
+++
b/backend/plugins/bitbucket/e2e/raw_tables/_raw_bitbucket_api_pull_request_commits.csv
@@ -0,0 +1,9 @@
+"id","params","data","url","input","created_at"
+1,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}","{""hash"":
""11115d276440bc1263c194fa164c061b91df9144"", ""date"":
""2015-09-12T08:53:47+00:00"", ""author"": {""raw"": ""孙婉译 <[email protected]>"",
""user"": {""account_id"": ""557058:de34a04f-99e2-4e9d-a952-c9be660a2379""}},
""message"": ""修改密码验证\n"", ""links"": {""self"": {""href"":
""https://api.bitbucket.org/2.0/repositories/likyh/likyhphp/commit/11115d276440bc1263c194fa164c061b91df9144""}}}",https://api.bitbucket.org/2.0/reposit
[...]
+2,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}","{""hash"":
""cc5d32ab6b04f0211d3384df9912cb2654ae918f"", ""date"":
""2015-08-15T13:05:13+00:00"", ""author"": {""raw"": ""linyh
<[email protected]>""}, ""message"": ""修改两处细节bug\n"", ""links"": {""self"":
{""href"":
""https://api.bitbucket.org/2.0/repositories/likyh/likyhphp/commit/cc5d32ab6b04f0211d3384df9912cb2654ae918f""}}}",https://api.bitbucket.org/2.0/repositories/likyh/likyhphp/pullrequests/3/commits?fields=values.hash%2Cvalues.da
[...]
+3,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}","{""hash"":
""af22d6a98faeccd9cd7f015c7e2f9bd52f493854"", ""date"":
""2015-08-11T16:01:06+00:00"", ""author"": {""raw"": ""linyh
<[email protected]>""}, ""message"":
""codeCms增加了文件方式,GD库增加了按比例缩放。Uploader实在太难用了,得赶紧改进,至少用DataMessage返回信息\n"",
""links"": {""self"": {""href"":
""https://api.bitbucket.org/2.0/repositories/likyh/likyhphp/commit/af22d6a98faeccd9cd7f015c7e2f9bd52f493854""}}}",https://api.bitbucket.org/2.0/repositories/likyh/likyh
[...]
+4,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}","{""hash"":
""894d55d8edd7ac0079b2752d261196b295754b29"", ""date"":
""2015-08-10T16:13:54+00:00"", ""author"": {""raw"": ""linyh
<[email protected]>""}, ""message"":
""修改CodeCms,增加外键类型,增加Form_Select对二维数组的支持。\n"", ""links"": {""self"": {""href"":
""https://api.bitbucket.org/2.0/repositories/likyh/likyhphp/commit/894d55d8edd7ac0079b2752d261196b295754b29""}}}",https://api.bitbucket.org/2.0/repositories/likyh/likyhphp/pullrequests/3/commits?
[...]
+5,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}","{""hash"":
""726dedad7bc4abf901670fc9c17bbd571bb607f8"", ""date"":
""2015-08-10T12:41:47+00:00"", ""author"": {""raw"": ""al <[email protected]>""},
""message"": ""单选复选开关其它优化\n"", ""links"": {""self"": {""href"":
""https://api.bitbucket.org/2.0/repositories/likyh/likyhphp/commit/726dedad7bc4abf901670fc9c17bbd571bb607f8""}}}",https://api.bitbucket.org/2.0/repositories/likyh/likyhphp/pullrequests/3/commits?fields=values.hash%2Cvalues.dat
[...]
+6,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}","{""hash"":
""32ef6538e466e4e8b773b32c04ca983866eb51ca"", ""date"":
""2015-08-09T09:07:00+00:00"", ""author"": {""raw"": ""linyh
<[email protected]>""}, ""message"": ""增加cms对ueditor的支持(js部分)\n"", ""links"":
{""self"": {""href"":
""https://api.bitbucket.org/2.0/repositories/likyh/likyhphp/commit/32ef6538e466e4e8b773b32c04ca983866eb51ca""}}}",https://api.bitbucket.org/2.0/repositories/likyh/likyhphp/pullrequests/3/commits?fields=values.has
[...]
+7,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}","{""hash"":
""3bf102513ac2949a1584b70b8649dfd7807332ec"", ""date"":
""2015-08-09T08:59:59+00:00"", ""author"": {""raw"": ""al <[email protected]>""},
""message"": ""Merge branch '0.5-alpha' of https://bitbucket.org/likyh/likyhphp
into 0.5-alpha\n"", ""links"": {""self"": {""href"":
""https://api.bitbucket.org/2.0/repositories/likyh/likyhphp/commit/3bf102513ac2949a1584b70b8649dfd7807332ec""}}}",https://api.bitbucket.org/2.0/repositories/
[...]
+8,"{""ConnectionId"":1,""FullName"":""likyh/likyhphp""}","{""hash"":
""65d491b85aa2266bd59e89b34b32c908b9109082"", ""date"":
""2015-08-09T08:58:59+00:00"", ""author"": {""raw"": ""al <[email protected]>""},
""message"": ""css：form，page，item\n"", ""links"": {""self"": {""href"":
""https://api.bitbucket.org/2.0/repositories/likyh/likyhphp/commit/65d491b85aa2266bd59e89b34b32c908b9109082""}}}",https://api.bitbucket.org/2.0/repositories/likyh/likyhphp/pullrequests/3/commits?fields=values.hash%2Cva
[...]
diff --git
a/backend/plugins/bitbucket/e2e/snapshot_tables/_tool_bitbucket_pull_request_commits.csv
b/backend/plugins/bitbucket/e2e/snapshot_tables/_tool_bitbucket_pull_request_commits.csv
new file mode 100644
index 000000000..94989f1b2
--- /dev/null
+++
b/backend/plugins/bitbucket/e2e/snapshot_tables/_tool_bitbucket_pull_request_commits.csv
@@ -0,0 +1,9 @@
+connection_id,repo_id,pull_request_id,commit_sha,commit_authored_name,commit_authored_email,commit_authored_date
+1,likyh/likyhphp,3,11115d276440bc1263c194fa164c061b91df9144,孙婉译,[email protected],2015-09-12T08:53:47.000+00:00
+1,likyh/likyhphp,3,32ef6538e466e4e8b773b32c04ca983866eb51ca,linyh,[email protected],2015-08-09T09:07:00.000+00:00
+1,likyh/likyhphp,3,3bf102513ac2949a1584b70b8649dfd7807332ec,al,[email protected],2015-08-09T08:59:59.000+00:00
+1,likyh/likyhphp,3,65d491b85aa2266bd59e89b34b32c908b9109082,al,[email protected],2015-08-09T08:58:59.000+00:00
+1,likyh/likyhphp,3,726dedad7bc4abf901670fc9c17bbd571bb607f8,al,[email protected],2015-08-10T12:41:47.000+00:00
+1,likyh/likyhphp,3,894d55d8edd7ac0079b2752d261196b295754b29,linyh,[email protected],2015-08-10T16:13:54.000+00:00
+1,likyh/likyhphp,3,af22d6a98faeccd9cd7f015c7e2f9bd52f493854,linyh,[email protected],2015-08-11T16:01:06.000+00:00
+1,likyh/likyhphp,3,cc5d32ab6b04f0211d3384df9912cb2654ae918f,linyh,[email protected],2015-08-15T13:05:13.000+00:00
diff --git
a/backend/plugins/bitbucket/e2e/snapshot_tables/pull_request_commits.csv
b/backend/plugins/bitbucket/e2e/snapshot_tables/pull_request_commits.csv
new file mode 100644
index 000000000..c6da2235e
--- /dev/null
+++ b/backend/plugins/bitbucket/e2e/snapshot_tables/pull_request_commits.csv
@@ -0,0 +1,9 @@
+commit_sha,pull_request_id,commit_authored_name,commit_authored_email,commit_authored_date
+11115d276440bc1263c194fa164c061b91df9144,bitbucket:BitbucketPullRequest:1:likyh/likyhphp:3,孙婉译,[email protected],2015-09-12T08:53:47.000+00:00
+32ef6538e466e4e8b773b32c04ca983866eb51ca,bitbucket:BitbucketPullRequest:1:likyh/likyhphp:3,linyh,[email protected],2015-08-09T09:07:00.000+00:00
+3bf102513ac2949a1584b70b8649dfd7807332ec,bitbucket:BitbucketPullRequest:1:likyh/likyhphp:3,al,[email protected],2015-08-09T08:59:59.000+00:00
+65d491b85aa2266bd59e89b34b32c908b9109082,bitbucket:BitbucketPullRequest:1:likyh/likyhphp:3,al,[email protected],2015-08-09T08:58:59.000+00:00
+726dedad7bc4abf901670fc9c17bbd571bb607f8,bitbucket:BitbucketPullRequest:1:likyh/likyhphp:3,al,[email protected],2015-08-10T12:41:47.000+00:00
+894d55d8edd7ac0079b2752d261196b295754b29,bitbucket:BitbucketPullRequest:1:likyh/likyhphp:3,linyh,[email protected],2015-08-10T16:13:54.000+00:00
+af22d6a98faeccd9cd7f015c7e2f9bd52f493854,bitbucket:BitbucketPullRequest:1:likyh/likyhphp:3,linyh,[email protected],2015-08-11T16:01:06.000+00:00
+cc5d32ab6b04f0211d3384df9912cb2654ae918f,bitbucket:BitbucketPullRequest:1:likyh/likyhphp:3,linyh,[email protected],2015-08-15T13:05:13.000+00:00
diff --git
a/backend/plugins/bitbucket/models/migrationscripts/20230420_add_pr_commit_author_date.go
b/backend/plugins/bitbucket/models/migrationscripts/20230420_add_pr_commit_author_date.go
index b4f77fef0..7f39fd1cd 100644
---
a/backend/plugins/bitbucket/models/migrationscripts/20230420_add_pr_commit_author_date.go
+++
b/backend/plugins/bitbucket/models/migrationscripts/20230420_add_pr_commit_author_date.go
@@ -26,30 +26,32 @@ import (
"github.com/apache/incubator-devlake/helpers/migrationhelper"
)
-var _ plugin.MigrationScript = (*addGitlabCommitAuthoredDate)(nil)
+var _ plugin.MigrationScript = (*addBitbucketCommitAuthoredDate)(nil)
-type GitlabMrCommit20230420 struct {
- CommitAuthoredDate time.Time
+type BitbucketPrCommit20230420 struct {
+ CommitAuthoredName string `gorm:"type:varchar(255)"`
+ CommitAuthoredEmail string `gorm:"type:varchar(255)"`
+ CommitAuthoredDate time.Time
}
-func (GitlabMrCommit20230420) TableName() string {
- return "_tool_gitlab_mr_commits"
+func (BitbucketPrCommit20230420) TableName() string {
+ return "_tool_bitbucket_pull_request_commits"
}
-type addGitlabCommitAuthoredDate struct{}
+type addBitbucketCommitAuthoredDate struct{}
-func (script *addGitlabCommitAuthoredDate) Up(basicRes context.BasicRes)
errors.Error {
+func (script *addBitbucketCommitAuthoredDate) Up(basicRes context.BasicRes)
errors.Error {
return migrationhelper.AutoMigrateTables(
basicRes,
- &GitlabMrCommit20230420{},
+ &BitbucketPrCommit20230420{},
)
}
-func (*addGitlabCommitAuthoredDate) Version() uint64 {
- return 20230420135127
+func (*addBitbucketCommitAuthoredDate) Version() uint64 {
+ return 20230420135129
}
-func (*addGitlabCommitAuthoredDate) Name() string {
- return "add commit_authored_date to _tool_gitlab_mr_commits table"
+func (*addBitbucketCommitAuthoredDate) Name() string {
+ return "add commit_authored_date to
_tool_bitbucket_pull_request_commits table"
}
diff --git a/backend/plugins/bitbucket/models/migrationscripts/register.go
b/backend/plugins/bitbucket/models/migrationscripts/register.go
index 854e23f79..8cec7a7f0 100644
--- a/backend/plugins/bitbucket/models/migrationscripts/register.go
+++ b/backend/plugins/bitbucket/models/migrationscripts/register.go
@@ -35,5 +35,6 @@ func All() []plugin.MigrationScript {
new(addTypeEnvToPipelineAndStep),
new(addRepoIdField20230411),
new(addRepoIdToPr),
+ new(addBitbucketCommitAuthoredDate),
}
}
diff --git a/backend/plugins/bitbucket/models/pr_commit.go
b/backend/plugins/bitbucket/models/pr_commit.go
index 74327a32f..fc0f5e592 100644
--- a/backend/plugins/bitbucket/models/pr_commit.go
+++ b/backend/plugins/bitbucket/models/pr_commit.go
@@ -19,13 +19,17 @@ package models
import (
"github.com/apache/incubator-devlake/core/models/common"
+ "time"
)
type BitbucketPrCommit struct {
- ConnectionId uint64 `gorm:"primaryKey"`
- RepoId string `gorm:"primaryKey"` // PullRequestId is not unique
across multiple repos of a connection
- PullRequestId int `gorm:"primaryKey;autoIncrement:false"`
- CommitSha string `gorm:"primaryKey;type:varchar(40)"`
+ ConnectionId uint64 `gorm:"primaryKey"`
+ RepoId string `gorm:"primaryKey"` // PullRequestId is not
unique across multiple repos of a connection
+ PullRequestId int `gorm:"primaryKey;autoIncrement:false"`
+ CommitSha string `gorm:"primaryKey;type:varchar(40)"`
+ CommitAuthoredName string
+ CommitAuthoredEmail string
+ CommitAuthoredDate time.Time
common.NoPKModel
}
diff --git a/backend/plugins/bitbucket/tasks/pr_commit_convertor.go
b/backend/plugins/bitbucket/tasks/pr_commit_convertor.go
index dfde587ff..2112c1887 100644
--- a/backend/plugins/bitbucket/tasks/pr_commit_convertor.go
+++ b/backend/plugins/bitbucket/tasks/pr_commit_convertor.go
@@ -60,8 +60,11 @@ func ConvertPullRequestCommits(taskCtx
plugin.SubTaskContext) (err errors.Error)
Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
prCommit :=
inputRow.(*bitbucketModels.BitbucketPrCommit)
domainPrCommit := &code.PullRequestCommit{
- CommitSha: prCommit.CommitSha,
- PullRequestId:
pullIdGen.Generate(prCommit.ConnectionId, prCommit.RepoId,
prCommit.PullRequestId),
+ CommitSha: prCommit.CommitSha,
+ PullRequestId:
pullIdGen.Generate(prCommit.ConnectionId, prCommit.RepoId,
prCommit.PullRequestId),
+ CommitAuthoredEmail:
prCommit.CommitAuthoredEmail,
+ CommitAuthoredName:
prCommit.CommitAuthoredName,
+ CommitAuthoredDate:
prCommit.CommitAuthoredDate,
}
return []interface{}{
domainPrCommit,
diff --git a/backend/plugins/bitbucket/tasks/pr_commit_extractor.go
b/backend/plugins/bitbucket/tasks/pr_commit_extractor.go
index 7c6f6de7d..0f81a5e82 100644
--- a/backend/plugins/bitbucket/tasks/pr_commit_extractor.go
+++ b/backend/plugins/bitbucket/tasks/pr_commit_extractor.go
@@ -88,11 +88,18 @@ func ExtractApiPullRequestCommits(taskCtx
plugin.SubTaskContext) errors.Error {
}
results = append(results, bitbucketCommit)
+ authorInfo := apiPullRequestCommit.Author.Raw
+ authorName :=
strings.TrimRight(strings.Split(authorInfo, "<")[0], " ")
+ authorEmail := strings.Trim(strings.Split(authorInfo,
"<")[1], ">")
+
bitbucketPullRequestCommit := &models.BitbucketPrCommit{
- ConnectionId: data.Options.ConnectionId,
- RepoId: repoId,
- PullRequestId: pull.BitbucketId,
- CommitSha: apiPullRequestCommit.Hash,
+ ConnectionId: data.Options.ConnectionId,
+ RepoId: repoId,
+ PullRequestId: pull.BitbucketId,
+ CommitSha: apiPullRequestCommit.Hash,
+ CommitAuthoredName: authorName,
+ CommitAuthoredEmail: authorEmail,
+ CommitAuthoredDate: apiPullRequestCommit.Date,
}
if err != nil {
return nil, err
