This is an automated email from the ASF dual-hosted git repository.
zhangliang2022 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new bd5ce8df fix: bitbucket generated wrong clone url (#3315)
bd5ce8df is described below
commit bd5ce8dfa7beb05f06fa14917058dcc0d9d57d0c
Author: mindlesscloud <[email protected]>
AuthorDate: Sun Oct 9 20:32:02 2022 +0800
fix: bitbucket generated wrong clone url (#3315)
* fix: bitbucket generated wrong clone url
* fix: e2e test
* refactor: unit test instead of e2e
---
plugins/bitbucket/api/blueprint.go | 111 +++++++++++++--------
plugins/bitbucket/api/blueprint_test.go | 57 +++++++++++
.../e2e/snapshot_tables/_tool_bitbucket_repos.csv | 2 +-
plugins/bitbucket/e2e/snapshot_tables/boards.csv | 2 +-
plugins/bitbucket/e2e/snapshot_tables/repos.csv | 2 +-
plugins/bitbucket/tasks/api_common.go | 4 +-
plugins/bitbucket/tasks/repo_extractor.go | 2 +-
7 files changed, 131 insertions(+), 49 deletions(-)
diff --git a/plugins/bitbucket/api/blueprint.go
b/plugins/bitbucket/api/blueprint.go
index c0f8ebe5..825670e6 100644
--- a/plugins/bitbucket/api/blueprint.go
+++ b/plugins/bitbucket/api/blueprint.go
@@ -21,13 +21,14 @@ import (
"context"
"encoding/json"
"fmt"
- "github.com/apache/incubator-devlake/errors"
"io"
"net/http"
"net/url"
+ "path"
"strings"
"time"
+ "github.com/apache/incubator-devlake/errors"
"github.com/apache/incubator-devlake/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/plugins/bitbucket/models"
"github.com/apache/incubator-devlake/plugins/bitbucket/tasks"
@@ -36,7 +37,66 @@ import (
"github.com/apache/incubator-devlake/utils"
)
+type repoGetter func(connectionId uint64, owner, repo string) (string, string,
errors.Error)
+
func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64,
scope []*core.BlueprintScopeV100) (core.PipelinePlan, errors.Error) {
+ return makePipelinePlan(subtaskMetas, connectionId,
getBitbucketApiRepo, scope)
+}
+func getBitbucketApiRepo(connectionId uint64, owner, repo string) (string,
string, errors.Error) {
+ // here is the tricky part, we have to obtain the repo id beforehand
+ connection := new(models.BitbucketConnection)
+ err := connectionHelper.FirstById(connection, connectionId)
+ if err != nil {
+ return "", "", err
+ }
+ tokens := strings.Split(connection.GetEncodedToken(), ",")
+ if len(tokens) == 0 {
+ return "", "", errors.Default.New("no token")
+ }
+ token := tokens[0]
+ apiClient, err := helper.NewApiClient(
+ context.TODO(),
+ connection.Endpoint,
+ map[string]string{
+ "Authorization": fmt.Sprintf("Basic %s", token),
+ },
+ 10*time.Second,
+ connection.Proxy,
+ basicRes,
+ )
+ if err != nil {
+ return "", "", err
+ }
+
+ res, err := apiClient.Get(path.Join("repositories", owner, repo), nil,
nil)
+ if err != nil {
+ return "", "", err
+ }
+ defer res.Body.Close()
+ if res.StatusCode != http.StatusOK {
+ return "", "", errors.Default.New(fmt.Sprintf(
+ "unexpected status code when requesting repo detail %d
%s",
+ res.StatusCode, res.Request.URL.String(),
+ ))
+ }
+ body, err := errors.Convert01(io.ReadAll(res.Body))
+ if err != nil {
+ return "", "", err
+ }
+ apiRepo := new(tasks.BitbucketApiRepo)
+ err = errors.Convert(json.Unmarshal(body, apiRepo))
+ if err != nil {
+ return "", "", err
+ }
+ for _, u := range apiRepo.Links.Clone {
+ if u.Name == "https" {
+ return u.Href, connection.Password, nil
+ }
+ }
+ return "", "", errors.Default.New("no clone url")
+}
+
+func makePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64,
getter repoGetter, scope []*core.BlueprintScopeV100) (core.PipelinePlan,
errors.Error) {
var err errors.Error
plan := make(core.PipelinePlan, len(scope))
for i, scopeElem := range scope {
@@ -93,58 +153,23 @@ func MakePipelinePlan(subtaskMetas []core.SubTaskMeta,
connectionId uint64, scop
})
// collect git data by gitextractor if CODE was requested
if utils.StringsContains(scopeElem.Entities,
core.DOMAIN_TYPE_CODE) {
- // here is the tricky part, we have to obtain the repo
id beforehand
- connection := new(models.BitbucketConnection)
- err = connectionHelper.FirstById(connection,
connectionId)
- if err != nil {
- return nil, err
- }
- token := strings.Split(connection.GetEncodedToken(),
",")[0]
- apiClient, err := helper.NewApiClient(
- context.TODO(),
- connection.Endpoint,
- map[string]string{
- "Authorization": fmt.Sprintf("Basic
%s", token),
- },
- 10*time.Second,
- connection.Proxy,
- basicRes,
- )
- if err != nil {
- return nil, err
- }
- res, err :=
apiClient.Get(fmt.Sprintf("repositories/%s/%s", op.Owner, op.Repo), nil, nil)
- if err != nil {
- return nil, err
+ original, password, err1 := getter(connectionId,
op.Owner, op.Repo)
+ if err1 != nil {
+ return nil, err1
}
- defer res.Body.Close()
- if res.StatusCode != http.StatusOK {
- return nil, errors.Default.New(fmt.Sprintf(
- "unexpected status code when requesting
repo detail %d %s",
- res.StatusCode,
res.Request.URL.String(),
- ))
- }
- body, err := errors.Convert01(io.ReadAll(res.Body))
- if err != nil {
- return nil, err
- }
- apiRepo := new(tasks.BitbucketApiRepo)
- err = errors.Convert(json.Unmarshal(body, apiRepo))
+ cloneUrl, err := errors.Convert01(url.Parse(original))
if err != nil {
return nil, err
}
- cloneUrl, err :=
errors.Convert01(url.Parse(apiRepo.Links.Clone[0].Href))
- if err != nil {
- return nil, err
- }
- cloneUrl.User = url.UserPassword("git", token)
+ cloneUrl.User = url.UserPassword(op.Owner, password)
stage = append(stage, &core.PipelineTask{
Plugin: "gitextractor",
Options: map[string]interface{}{
"url": cloneUrl.String(),
- "repoId":
didgen.NewDomainIdGenerator(&models.BitbucketRepo{}).Generate(connectionId,
apiRepo.BitbucketId),
+ "repoId":
didgen.NewDomainIdGenerator(&models.BitbucketRepo{}).Generate(connectionId,
fmt.Sprintf("%s/%s", op.Owner, op.Repo)),
},
})
+
}
plan[i] = stage
}
diff --git a/plugins/bitbucket/api/blueprint_test.go
b/plugins/bitbucket/api/blueprint_test.go
new file mode 100644
index 00000000..14e284a3
--- /dev/null
+++ b/plugins/bitbucket/api/blueprint_test.go
@@ -0,0 +1,57 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "github.com/apache/incubator-devlake/mocks"
+ "testing"
+
+ "github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/stretchr/testify/assert"
+)
+
+func TestMakePipelinePlan(t *testing.T) {
+ var mockGetter repoGetter
+ mockGetter = func(connectionId uint64, owner, repo string) (string,
string, errors.Error) {
+ return "https://[email protected]/thenicetgp/lake.git";,
"secret", nil
+ }
+ scope := &core.BlueprintScopeV100{
+ Entities: []string{core.DOMAIN_TYPE_CODE,
core.DOMAIN_TYPE_TICKET, core.DOMAIN_TYPE_CODE_REVIEW, core.DOMAIN_TYPE_CROSS},
+ Options: []byte(`{
+ "owner": "thenicetgp",
+ "repo": "lake"
+ }`),
+ Transformation: nil,
+ }
+ mockMeta := mocks.NewPluginMeta(t)
+
mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/bitbucket")
+ err := core.RegisterPlugin("bitbucket", mockMeta)
+ assert.Nil(t, err)
+ plan, err := makePipelinePlan(nil, 1, mockGetter,
[]*core.BlueprintScopeV100{scope})
+ assert.Nil(t, err)
+ for _, stage := range plan {
+ for _, task := range stage {
+ if task.Plugin == "gitextractor" {
+ assert.Equal(t, task.Options["url"],
"https://thenicetgp:[email protected]/thenicetgp/lake.git";)
+ return
+ }
+ }
+ }
+ t.Fatal("no gitextractor plugin")
+}
diff --git a/plugins/bitbucket/e2e/snapshot_tables/_tool_bitbucket_repos.csv
b/plugins/bitbucket/e2e/snapshot_tables/_tool_bitbucket_repos.csv
index a3883fbb..0a13fe85 100644
--- a/plugins/bitbucket/e2e/snapshot_tables/_tool_bitbucket_repos.csv
+++ b/plugins/bitbucket/e2e/snapshot_tables/_tool_bitbucket_repos.csv
@@ -1,2 +1,2 @@
connection_id,bitbucket_id,name,html_url,description,owner_id,language,created_date,updated_date,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark
-1,repositories/panjf2000/ants,panjf2000/ants,https://bitbucket.org/panjf2000/ants,,,,2022-06-17T03:27:18.865+00:00,2022-08-12T15:40:12.409+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_bitbucket_api_repositories,2,
+1,panjf2000/ants,panjf2000/ants,https://bitbucket.org/panjf2000/ants,,,,2022-06-17T03:27:18.865+00:00,2022-08-12T15:40:12.409+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_bitbucket_api_repositories,2,
diff --git a/plugins/bitbucket/e2e/snapshot_tables/boards.csv
b/plugins/bitbucket/e2e/snapshot_tables/boards.csv
index d4cbb227..99f1824d 100644
--- a/plugins/bitbucket/e2e/snapshot_tables/boards.csv
+++ b/plugins/bitbucket/e2e/snapshot_tables/boards.csv
@@ -1,2 +1,2 @@
id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,name,description,url,created_date
-bitbucket:BitbucketRepo:1:repositories/panjf2000/ants,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_bitbucket_api_repositories,2,,panjf2000/ants,,https://bitbucket.org/panjf2000/ants/issues,2022-06-17T03:27:18.865+00:00
+bitbucket:BitbucketRepo:1:panjf2000/ants,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_bitbucket_api_repositories,2,,panjf2000/ants,,https://bitbucket.org/panjf2000/ants/issues,2022-06-17T03:27:18.865+00:00
diff --git a/plugins/bitbucket/e2e/snapshot_tables/repos.csv
b/plugins/bitbucket/e2e/snapshot_tables/repos.csv
index 32c0ecc0..646a4390 100644
--- a/plugins/bitbucket/e2e/snapshot_tables/repos.csv
+++ b/plugins/bitbucket/e2e/snapshot_tables/repos.csv
@@ -1,2 +1,2 @@
id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,name,url,description,owner_id,language,forked_from,deleted
-bitbucket:BitbucketRepo:1:repositories/panjf2000/ants,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_bitbucket_api_repositories,2,,panjf2000/ants,https://bitbucket.org/panjf2000/ants,,,,,0
+bitbucket:BitbucketRepo:1:panjf2000/ants,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_bitbucket_api_repositories,2,,panjf2000/ants,https://bitbucket.org/panjf2000/ants,,,,,0
diff --git a/plugins/bitbucket/tasks/api_common.go
b/plugins/bitbucket/tasks/api_common.go
index 000089da..b931a8a4 100644
--- a/plugins/bitbucket/tasks/api_common.go
+++ b/plugins/bitbucket/tasks/api_common.go
@@ -113,7 +113,7 @@ func GetPullRequestsIterator(taskCtx core.SubTaskContext)
(*helper.DalCursorIter
dal.From("_tool_bitbucket_pull_requests bpr"),
dal.Where(
`bpr.repo_id = ? and bpr.connection_id = ?`,
-
"repositories/"+data.Options.Owner+"/"+data.Options.Repo,
data.Options.ConnectionId,
+ fmt.Sprintf("%s/%s", data.Options.Owner,
data.Options.Repo), data.Options.ConnectionId,
),
}
// construct the input iterator
@@ -133,7 +133,7 @@ func GetIssuesIterator(taskCtx core.SubTaskContext)
(*helper.DalCursorIterator,
dal.From("_tool_bitbucket_issues bpr"),
dal.Where(
`bpr.repo_id = ? and bpr.connection_id = ?`,
-
"repositories/"+data.Options.Owner+"/"+data.Options.Repo,
data.Options.ConnectionId,
+ fmt.Sprintf("%s/%s", data.Options.Owner,
data.Options.Repo), data.Options.ConnectionId,
),
}
// construct the input iterator
diff --git a/plugins/bitbucket/tasks/repo_extractor.go
b/plugins/bitbucket/tasks/repo_extractor.go
index d79f5c36..c39e19c6 100644
--- a/plugins/bitbucket/tasks/repo_extractor.go
+++ b/plugins/bitbucket/tasks/repo_extractor.go
@@ -97,7 +97,7 @@ func ExtractApiRepositories(taskCtx core.SubTaskContext)
errors.Error {
results := make([]interface{}, 0, 1)
bitbucketRepository := &models.BitbucketRepo{
ConnectionId: data.Options.ConnectionId,
- BitbucketId: "repositories/" + body.FullName,
+ BitbucketId: body.FullName,
Name: body.FullName,
HTMLUrl: body.Links.Html.Href,
Description: body.Description,
