This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch fix-bitbucket
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/fix-bitbucket by this push:
new 4384091df feat: add bitbucket pullrequest_commits
collector&extractor&convertor (#3337)
4384091df is described below
commit 4384091df63a7cb6e6914e5ca350fa672b242502
Author: tsoc <[email protected]>
AuthorDate: Fri Jan 6 13:54:30 2023 +0800
feat: add bitbucket pullrequest_commits collector&extractor&convertor
(#3337)
* feat: add bitbucket pullrequest_commits collector&extractor&convertor
* fix: bitbucket commit response do not have committer
Co-authored-by: Klesh Wong <[email protected]>
---
plugins/bitbucket/impl/impl.go | 15 +-
...egister.go => 20221008_add_prcommits_tables.go} | 27 ++--
.../{register.go => archived/pr_commit.go} | 21 +--
.../bitbucket/models/migrationscripts/register.go | 1 +
.../{migrationscripts/register.go => pr_commit.go} | 21 +--
plugins/bitbucket/tasks/pr_commit_collector.go | 69 +++++++++
plugins/bitbucket/tasks/pr_commit_convertor.go | 86 +++++++++++
plugins/bitbucket/tasks/pr_commit_extractor.go | 159 +++++++++++++++++++++
8 files changed, 369 insertions(+), 30 deletions(-)
diff --git a/plugins/bitbucket/impl/impl.go b/plugins/bitbucket/impl/impl.go
index 54da3c035..95c41518d 100644
--- a/plugins/bitbucket/impl/impl.go
+++ b/plugins/bitbucket/impl/impl.go
@@ -69,22 +69,33 @@ func (plugin Bitbucket) SubTaskMetas() []core.SubTaskMeta {
return []core.SubTaskMeta{
tasks.CollectApiRepoMeta,
tasks.ExtractApiRepoMeta,
+
tasks.CollectApiPullRequestsMeta,
tasks.ExtractApiPullRequestsMeta,
- tasks.CollectApiIssuesMeta,
- tasks.ExtractApiIssuesMeta,
+
tasks.CollectApiPrCommentsMeta,
tasks.ExtractApiPrCommentsMeta,
+
+ tasks.CollectApiPrCommitsMeta,
+ tasks.ExtractApiPrCommitsMeta,
+
+ tasks.CollectApiIssuesMeta,
+ tasks.ExtractApiIssuesMeta,
+
tasks.CollectApiIssueCommentsMeta,
tasks.ExtractApiIssueCommentsMeta,
+
tasks.CollectApiPipelinesMeta,
tasks.ExtractApiPipelinesMeta,
+
tasks.CollectApiDeploymentsMeta,
tasks.ExtractApiDeploymentsMeta,
+
tasks.ConvertRepoMeta,
tasks.ConvertAccountsMeta,
tasks.ConvertPullRequestsMeta,
tasks.ConvertPrCommentsMeta,
+ tasks.ConvertPrCommitsMeta,
tasks.ConvertIssuesMeta,
tasks.ConvertIssueCommentsMeta,
tasks.ConvertPipelineMeta,
diff --git a/plugins/bitbucket/models/migrationscripts/register.go
b/plugins/bitbucket/models/migrationscripts/20221008_add_prcommits_tables.go
similarity index 57%
copy from plugins/bitbucket/models/migrationscripts/register.go
copy to
plugins/bitbucket/models/migrationscripts/20221008_add_prcommits_tables.go
index d6d7fc8eb..d3a0f0888 100644
--- a/plugins/bitbucket/models/migrationscripts/register.go
+++ b/plugins/bitbucket/models/migrationscripts/20221008_add_prcommits_tables.go
@@ -18,15 +18,26 @@ limitations under the License.
package migrationscripts
import (
- "github.com/apache/incubator-devlake/plugins/core"
+ "context"
+ "github.com/apache/incubator-devlake/errors"
+
"github.com/apache/incubator-devlake/plugins/bitbucket/models/migrationscripts/archived"
+ "gorm.io/gorm"
)
-// All return all the migration scripts
-func All() []core.MigrationScript {
- return []core.MigrationScript{
- new(addInitTables20220803),
- new(addPipeline20220914),
- new(addDeployment20221013),
- new(addRepoIdAndCommitShaField20221014),
+type addPrCommits20221008 struct{}
+
+func (*addPrCommits20221008) Up(ctx context.Context, db *gorm.DB) errors.Error
{
+ err := db.Migrator().AutoMigrate(&archived.BitbucketPrCommit{})
+ if err != nil {
+ return errors.Convert(err)
}
+ return nil
+}
+
+func (*addPrCommits20221008) Version() uint64 {
+ return 20221008182354
+}
+
+func (*addPrCommits20221008) Name() string {
+ return "bitbucket add _tool_bitbucket_pull_requests_commits table"
}
diff --git a/plugins/bitbucket/models/migrationscripts/register.go
b/plugins/bitbucket/models/migrationscripts/archived/pr_commit.go
similarity index 66%
copy from plugins/bitbucket/models/migrationscripts/register.go
copy to plugins/bitbucket/models/migrationscripts/archived/pr_commit.go
index d6d7fc8eb..0721a9606 100644
--- a/plugins/bitbucket/models/migrationscripts/register.go
+++ b/plugins/bitbucket/models/migrationscripts/archived/pr_commit.go
@@ -15,18 +15,19 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package migrationscripts
+package archived
import (
- "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
)
-// All return all the migration scripts
-func All() []core.MigrationScript {
- return []core.MigrationScript{
- new(addInitTables20220803),
- new(addPipeline20220914),
- new(addDeployment20221013),
- new(addRepoIdAndCommitShaField20221014),
- }
+type BitbucketPrCommit struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ CommitSha string `gorm:"primaryKey;type:varchar(40)"`
+ PullRequestId int `gorm:"primaryKey;autoIncrement:false"`
+ archived.NoPKModel
+}
+
+func (BitbucketPrCommit) TableName() string {
+ return "_tool_bitbucket_pull_request_commits"
}
diff --git a/plugins/bitbucket/models/migrationscripts/register.go
b/plugins/bitbucket/models/migrationscripts/register.go
index d6d7fc8eb..d5f5374b1 100644
--- a/plugins/bitbucket/models/migrationscripts/register.go
+++ b/plugins/bitbucket/models/migrationscripts/register.go
@@ -26,6 +26,7 @@ func All() []core.MigrationScript {
return []core.MigrationScript{
new(addInitTables20220803),
new(addPipeline20220914),
+ new(addPrCommits20221008),
new(addDeployment20221013),
new(addRepoIdAndCommitShaField20221014),
}
diff --git a/plugins/bitbucket/models/migrationscripts/register.go
b/plugins/bitbucket/models/pr_commit.go
similarity index 67%
copy from plugins/bitbucket/models/migrationscripts/register.go
copy to plugins/bitbucket/models/pr_commit.go
index d6d7fc8eb..262093242 100644
--- a/plugins/bitbucket/models/migrationscripts/register.go
+++ b/plugins/bitbucket/models/pr_commit.go
@@ -15,18 +15,19 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package migrationscripts
+package models
import (
- "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/models/common"
)
-// All return all the migration scripts
-func All() []core.MigrationScript {
- return []core.MigrationScript{
- new(addInitTables20220803),
- new(addPipeline20220914),
- new(addDeployment20221013),
- new(addRepoIdAndCommitShaField20221014),
- }
+type BitbucketPrCommit struct {
+ ConnectionId uint64 `gorm:"primaryKey"`
+ CommitSha string `gorm:"primaryKey;type:varchar(40)"`
+ PullRequestId int `gorm:"primaryKey;autoIncrement:false"`
+ common.NoPKModel
+}
+
+func (BitbucketPrCommit) TableName() string {
+ return "_tool_bitbucket_pull_request_commits"
}
diff --git a/plugins/bitbucket/tasks/pr_commit_collector.go
b/plugins/bitbucket/tasks/pr_commit_collector.go
new file mode 100644
index 000000000..f3464c8e3
--- /dev/null
+++ b/plugins/bitbucket/tasks/pr_commit_collector.go
@@ -0,0 +1,69 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "fmt"
+ "net/url"
+
+ "github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+const RAW_PULL_REQUEST_COMMITS_TABLE = "bitbucket_api_pull_request_commits"
+
+var CollectApiPrCommitsMeta = core.SubTaskMeta{
+ Name: "collectApiPullRequestCommits",
+ EntryPoint: CollectApiPullRequestCommits,
+ EnabledByDefault: true,
+ Description: "Collect PullRequestCommits data from Bitbucket api",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE_REVIEW},
+}
+
+func CollectApiPullRequestCommits(taskCtx core.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_PULL_REQUEST_COMMITS_TABLE)
+
+ iterator, err := GetPullRequestsIterator(taskCtx)
+ if err != nil {
+ return err
+ }
+ defer iterator.Close()
+
+ collector, err := helper.NewApiCollector(helper.ApiCollectorArgs{
+ RawDataSubTaskArgs: *rawDataSubTaskArgs,
+ ApiClient: data.ApiClient,
+ PageSize: 100,
+ Incremental: false,
+ Input: iterator,
+ UrlTemplate: "repositories/{{ .Params.Owner }}/{{
.Params.Repo }}/pullrequests/{{ .Input.BitbucketId }}/commits",
+ Query: func(reqData *helper.RequestData) (url.Values,
errors.Error) {
+ query := url.Values{}
+ query.Set("state", "all")
+ query.Set("pagelen", fmt.Sprintf("%v",
reqData.Pager.Size))
+
+ return query, nil
+ },
+ ResponseParser: GetRawMessageFromResponse,
+ })
+
+ if err != nil {
+ return err
+ }
+ return collector.Execute()
+}
diff --git a/plugins/bitbucket/tasks/pr_commit_convertor.go
b/plugins/bitbucket/tasks/pr_commit_convertor.go
new file mode 100644
index 000000000..5e8f58951
--- /dev/null
+++ b/plugins/bitbucket/tasks/pr_commit_convertor.go
@@ -0,0 +1,86 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "reflect"
+
+ "github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/models/domainlayer/code"
+ "github.com/apache/incubator-devlake/models/domainlayer/didgen"
+ bitbucketModels
"github.com/apache/incubator-devlake/plugins/bitbucket/models"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ConvertPrCommitsMeta = core.SubTaskMeta{
+ Name: "convertPullRequestCommits",
+ EntryPoint: ConvertPullRequestCommits,
+ EnabledByDefault: true,
+ Description: "Convert tool layer table
bitbucket_pull_request_commits into domain layer table pull_request_commits",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE_REVIEW},
+}
+
+func ConvertPullRequestCommits(taskCtx core.SubTaskContext) (err errors.Error)
{
+ db := taskCtx.GetDal()
+ data := taskCtx.GetData().(*BitbucketTaskData)
+ repoId := data.Repo.BitbucketId
+
+ pullIdGen :=
didgen.NewDomainIdGenerator(&bitbucketModels.BitbucketPullRequest{})
+
+ cursor, err := db.Cursor(
+ dal.From(&bitbucketModels.BitbucketPrCommit{}),
+ dal.Join(`left join _tool_bitbucket_pull_requests on
_tool_bitbucket_pull_requests.bitbucket_id =
_tool_bitbucket_pull_request_commits.pull_request_id`),
+ dal.Where("_tool_bitbucket_pull_requests.repo_id = ? and
_tool_bitbucket_pull_requests.connection_id = ?", repoId,
data.Options.ConnectionId),
+ dal.Orderby("pull_request_id ASC"),
+ )
+ if err != nil {
+ return err
+ }
+ defer cursor.Close()
+
+ converter, err := helper.NewDataConverter(helper.DataConverterArgs{
+ InputRowType:
reflect.TypeOf(bitbucketModels.BitbucketPrCommit{}),
+ Input: cursor,
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ Params: BitbucketApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ Owner: data.Options.Owner,
+ Repo: data.Options.Repo,
+ },
+ Table: RAW_PULL_REQUEST_COMMITS_TABLE,
+ },
+ Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
+ bitbucketPullRequestCommit :=
inputRow.(*bitbucketModels.BitbucketPrCommit)
+ domainPrCommit := &code.PullRequestCommit{
+ CommitSha:
bitbucketPullRequestCommit.CommitSha,
+ PullRequestId:
pullIdGen.Generate(data.Options.ConnectionId,
bitbucketPullRequestCommit.PullRequestId),
+ }
+ return []interface{}{
+ domainPrCommit,
+ }, nil
+ },
+ })
+ if err != nil {
+ return err
+ }
+
+ return converter.Execute()
+}
diff --git a/plugins/bitbucket/tasks/pr_commit_extractor.go
b/plugins/bitbucket/tasks/pr_commit_extractor.go
new file mode 100644
index 000000000..38552d996
--- /dev/null
+++ b/plugins/bitbucket/tasks/pr_commit_extractor.go
@@ -0,0 +1,159 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package tasks
+
+import (
+ "encoding/json"
+ "strings"
+ "time"
+
+ "github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/models"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
+
+var ExtractApiPrCommitsMeta = core.SubTaskMeta{
+ Name: "extractApiPullRequestCommits",
+ EntryPoint: ExtractApiPullRequestCommits,
+ EnabledByDefault: true,
+ Description: "Extract raw PullRequestCommits data into tool layer
table bitbucket_commits",
+ DomainTypes: []string{core.DOMAIN_TYPE_CODE_REVIEW},
+}
+
+type ApiPrCommitsResponse struct {
+ Type string `json:"type"`
+ Hash string `json:"hash"`
+ Date time.Time `json:"date"`
+ Author struct {
+ Type string `json:"type"`
+ Raw string `json:"raw"`
+ User BitbucketAccountResponse `json:"user"`
+ } `json:"author"`
+ Message string `json:"message"`
+ Summary struct {
+ Type string `json:"type"`
+ Raw string `json:"raw"`
+ Markup string `json:"markup"`
+ HTML string `json:"html"`
+ } `json:"summary"`
+ Links struct {
+ Self struct {
+ Href string `json:"href"`
+ } `json:"self"`
+ HTML struct {
+ Href string `json:"href"`
+ } `json:"html"`
+ } `json:"links"`
+ Parents []struct {
+ Type string `json:"type"`
+ Hash string `json:"hash"`
+ Links struct {
+ Self struct {
+ Href string `json:"href"`
+ } `json:"self"`
+ HTML struct {
+ Href string `json:"href"`
+ } `json:"html"`
+ } `json:"links"`
+ } `json:"parents"`
+ Repository BitbucketApiRepo `json:"repository"`
+}
+
+func ExtractApiPullRequestCommits(taskCtx core.SubTaskContext) errors.Error {
+ data := taskCtx.GetData().(*BitbucketTaskData)
+ repoId := data.Repo.BitbucketId
+ extractor, err := helper.NewApiExtractor(helper.ApiExtractorArgs{
+ RawDataSubTaskArgs: helper.RawDataSubTaskArgs{
+ Ctx: taskCtx,
+ /*
+ This struct will be JSONEncoded and stored into
database along with raw data itself, to identity minimal
+ set of data to be process, for example, we
process JiraIssues by Board
+ */
+ Params: BitbucketApiParams{
+ ConnectionId: data.Options.ConnectionId,
+ Owner: data.Options.Owner,
+ Repo: data.Options.Repo,
+ },
+ /*
+ Table store raw data
+ */
+ Table: RAW_PULL_REQUEST_COMMITS_TABLE,
+ },
+ Extract: func(row *helper.RawData) ([]interface{},
errors.Error) {
+ apiPullRequestCommit := &ApiPrCommitsResponse{}
+ if strings.HasPrefix(string(row.Data), "Not Found") {
+ return nil, nil
+ }
+ err := errors.Convert(json.Unmarshal(row.Data,
apiPullRequestCommit))
+ if err != nil {
+ return nil, err
+ }
+ pull := &BitbucketInput{}
+ err = errors.Convert(json.Unmarshal(row.Input, pull))
+ if err != nil {
+ return nil, err
+ }
+ // need to extract 2 kinds of entities here
+ results := make([]interface{}, 0, 3)
+ bitbucketRepoCommit := &models.BitbucketRepoCommit{
+ ConnectionId: data.Options.ConnectionId,
+ RepoId: repoId,
+ CommitSha: apiPullRequestCommit.Hash,
+ }
+ results = append(results, bitbucketRepoCommit)
+
+ bitbucketCommit, err :=
convertPullRequestCommit(apiPullRequestCommit, data.Options.ConnectionId)
+ if err != nil {
+ return nil, err
+ }
+ results = append(results, bitbucketCommit)
+
+ bitbucketPullRequestCommit := &models.BitbucketPrCommit{
+ ConnectionId: data.Options.ConnectionId,
+ CommitSha: apiPullRequestCommit.Hash,
+ PullRequestId: pull.BitbucketId,
+ }
+ if err != nil {
+ return nil, err
+ }
+ results = append(results, bitbucketPullRequestCommit)
+ return results, nil
+ },
+ })
+
+ if err != nil {
+ return err
+ }
+
+ return extractor.Execute()
+}
+
+func convertPullRequestCommit(prCommit *ApiPrCommitsResponse, connId uint64)
(*models.BitbucketCommit, errors.Error) {
+ bitbucketCommit := &models.BitbucketCommit{
+ Sha: prCommit.Hash,
+ Message: prCommit.Message,
+ AuthorId: prCommit.Author.User.AccountId,
+ AuthorName: prCommit.Author.User.UserName,
+ AuthorEmail: prCommit.Author.Raw,
+ AuthoredDate: prCommit.Date,
+ CommittedDate: prCommit.Date,
+ Url: prCommit.Links.Self.Href,
+ }
+ return bitbucketCommit, nil
+}
