This is an automated email from the ASF dual-hosted git repository. zhangliang2022 pushed a commit to tag v0.14.1 in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit 214ad97e4ab084ab46ef1248c65b21274dbe7e7a Author: mindlesscloud <[email protected]> AuthorDate: Sun Oct 9 20:32:02 2022 +0800 fix: bitbucket generated wrong clone url (#3315) * fix: bitbucket generated wrong clone url * fix: e2e test * refactor: unit test instead of e2e --- plugins/bitbucket/api/blueprint.go | 111 +++++++++++++-------- plugins/bitbucket/api/blueprint_test.go | 57 +++++++++++ .../e2e/snapshot_tables/_tool_bitbucket_repos.csv | 2 +- plugins/bitbucket/e2e/snapshot_tables/boards.csv | 2 +- plugins/bitbucket/e2e/snapshot_tables/repos.csv | 2 +- plugins/bitbucket/tasks/api_common.go | 4 +- plugins/bitbucket/tasks/repo_extractor.go | 2 +- 7 files changed, 131 insertions(+), 49 deletions(-) diff --git a/plugins/bitbucket/api/blueprint.go b/plugins/bitbucket/api/blueprint.go index c0f8ebe5..825670e6 100644 --- a/plugins/bitbucket/api/blueprint.go +++ b/plugins/bitbucket/api/blueprint.go @@ -21,13 +21,14 @@ import ( "context" "encoding/json" "fmt" - "github.com/apache/incubator-devlake/errors" "io" "net/http" "net/url" + "path" "strings" "time" + "github.com/apache/incubator-devlake/errors" "github.com/apache/incubator-devlake/models/domainlayer/didgen" "github.com/apache/incubator-devlake/plugins/bitbucket/models" "github.com/apache/incubator-devlake/plugins/bitbucket/tasks" @@ -36,7 +37,66 @@ import ( "github.com/apache/incubator-devlake/utils" ) +type repoGetter func(connectionId uint64, owner, repo string) (string, string, errors.Error) + func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scope []*core.BlueprintScopeV100) (core.PipelinePlan, errors.Error) { + return makePipelinePlan(subtaskMetas, connectionId, getBitbucketApiRepo, scope) +} +func getBitbucketApiRepo(connectionId uint64, owner, repo string) (string, string, errors.Error) { + // here is the tricky part, we have to obtain the repo id beforehand + connection := new(models.BitbucketConnection) + err := connectionHelper.FirstById(connection, connectionId) + if err != nil { + return "", "", err + } + tokens := strings.Split(connection.GetEncodedToken(), ",") + if len(tokens) == 0 { + return "", "", errors.Default.New("no token") + } + token := tokens[0] + apiClient, err := helper.NewApiClient( + context.TODO(), + connection.Endpoint, + map[string]string{ + "Authorization": fmt.Sprintf("Basic %s", token), + }, + 10*time.Second, + connection.Proxy, + basicRes, + ) + if err != nil { + return "", "", err + } + + res, err := apiClient.Get(path.Join("repositories", owner, repo), nil, nil) + if err != nil { + return "", "", err + } + defer res.Body.Close() + if res.StatusCode != http.StatusOK { + return "", "", errors.Default.New(fmt.Sprintf( + "unexpected status code when requesting repo detail %d %s", + res.StatusCode, res.Request.URL.String(), + )) + } + body, err := errors.Convert01(io.ReadAll(res.Body)) + if err != nil { + return "", "", err + } + apiRepo := new(tasks.BitbucketApiRepo) + err = errors.Convert(json.Unmarshal(body, apiRepo)) + if err != nil { + return "", "", err + } + for _, u := range apiRepo.Links.Clone { + if u.Name == "https" { + return u.Href, connection.Password, nil + } + } + return "", "", errors.Default.New("no clone url") +} + +func makePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, getter repoGetter, scope []*core.BlueprintScopeV100) (core.PipelinePlan, errors.Error) { var err errors.Error plan := make(core.PipelinePlan, len(scope)) for i, scopeElem := range scope { @@ -93,58 +153,23 @@ func MakePipelinePlan(subtaskMetas []core.SubTaskMeta, connectionId uint64, scop }) // collect git data by gitextractor if CODE was requested if utils.StringsContains(scopeElem.Entities, core.DOMAIN_TYPE_CODE) { - // here is the tricky part, we have to obtain the repo id beforehand - connection := new(models.BitbucketConnection) - err = connectionHelper.FirstById(connection, connectionId) - if err != nil { - return nil, err - } - token := strings.Split(connection.GetEncodedToken(), ",")[0] - apiClient, err := helper.NewApiClient( - context.TODO(), - connection.Endpoint, - map[string]string{ - "Authorization": fmt.Sprintf("Basic %s", token), - }, - 10*time.Second, - connection.Proxy, - basicRes, - ) - if err != nil { - return nil, err - } - res, err := apiClient.Get(fmt.Sprintf("repositories/%s/%s", op.Owner, op.Repo), nil, nil) - if err != nil { - return nil, err + original, password, err1 := getter(connectionId, op.Owner, op.Repo) + if err1 != nil { + return nil, err1 } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - return nil, errors.Default.New(fmt.Sprintf( - "unexpected status code when requesting repo detail %d %s", - res.StatusCode, res.Request.URL.String(), - )) - } - body, err := errors.Convert01(io.ReadAll(res.Body)) - if err != nil { - return nil, err - } - apiRepo := new(tasks.BitbucketApiRepo) - err = errors.Convert(json.Unmarshal(body, apiRepo)) + cloneUrl, err := errors.Convert01(url.Parse(original)) if err != nil { return nil, err } - cloneUrl, err := errors.Convert01(url.Parse(apiRepo.Links.Clone[0].Href)) - if err != nil { - return nil, err - } - cloneUrl.User = url.UserPassword("git", token) + cloneUrl.User = url.UserPassword(op.Owner, password) stage = append(stage, &core.PipelineTask{ Plugin: "gitextractor", Options: map[string]interface{}{ "url": cloneUrl.String(), - "repoId": didgen.NewDomainIdGenerator(&models.BitbucketRepo{}).Generate(connectionId, apiRepo.BitbucketId), + "repoId": didgen.NewDomainIdGenerator(&models.BitbucketRepo{}).Generate(connectionId, fmt.Sprintf("%s/%s", op.Owner, op.Repo)), }, }) + } plan[i] = stage } diff --git a/plugins/bitbucket/api/blueprint_test.go b/plugins/bitbucket/api/blueprint_test.go new file mode 100644 index 00000000..14e284a3 --- /dev/null +++ b/plugins/bitbucket/api/blueprint_test.go @@ -0,0 +1,57 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "github.com/apache/incubator-devlake/mocks" + "testing" + + "github.com/apache/incubator-devlake/errors" + "github.com/apache/incubator-devlake/plugins/core" + "github.com/stretchr/testify/assert" +) + +func TestMakePipelinePlan(t *testing.T) { + var mockGetter repoGetter + mockGetter = func(connectionId uint64, owner, repo string) (string, string, errors.Error) { + return "https://[email protected]/thenicetgp/lake.git";, "secret", nil + } + scope := &core.BlueprintScopeV100{ + Entities: []string{core.DOMAIN_TYPE_CODE, core.DOMAIN_TYPE_TICKET, core.DOMAIN_TYPE_CODE_REVIEW, core.DOMAIN_TYPE_CROSS}, + Options: []byte(`{ + "owner": "thenicetgp", + "repo": "lake" + }`), + Transformation: nil, + } + mockMeta := mocks.NewPluginMeta(t) + mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/bitbucket") + err := core.RegisterPlugin("bitbucket", mockMeta) + assert.Nil(t, err) + plan, err := makePipelinePlan(nil, 1, mockGetter, []*core.BlueprintScopeV100{scope}) + assert.Nil(t, err) + for _, stage := range plan { + for _, task := range stage { + if task.Plugin == "gitextractor" { + assert.Equal(t, task.Options["url"], "https://thenicetgp:[email protected]/thenicetgp/lake.git";) + return + } + } + } + t.Fatal("no gitextractor plugin") +} diff --git a/plugins/bitbucket/e2e/snapshot_tables/_tool_bitbucket_repos.csv b/plugins/bitbucket/e2e/snapshot_tables/_tool_bitbucket_repos.csv index a3883fbb..0a13fe85 100644 --- a/plugins/bitbucket/e2e/snapshot_tables/_tool_bitbucket_repos.csv +++ b/plugins/bitbucket/e2e/snapshot_tables/_tool_bitbucket_repos.csv @@ -1,2 +1,2 @@ connection_id,bitbucket_id,name,html_url,description,owner_id,language,created_date,updated_date,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark -1,repositories/panjf2000/ants,panjf2000/ants,https://bitbucket.org/panjf2000/ants,,,,2022-06-17T03:27:18.865+00:00,2022-08-12T15:40:12.409+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_bitbucket_api_repositories,2, +1,panjf2000/ants,panjf2000/ants,https://bitbucket.org/panjf2000/ants,,,,2022-06-17T03:27:18.865+00:00,2022-08-12T15:40:12.409+00:00,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_bitbucket_api_repositories,2, diff --git a/plugins/bitbucket/e2e/snapshot_tables/boards.csv b/plugins/bitbucket/e2e/snapshot_tables/boards.csv index d4cbb227..99f1824d 100644 --- a/plugins/bitbucket/e2e/snapshot_tables/boards.csv +++ b/plugins/bitbucket/e2e/snapshot_tables/boards.csv @@ -1,2 +1,2 @@ id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,name,description,url,created_date -bitbucket:BitbucketRepo:1:repositories/panjf2000/ants,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_bitbucket_api_repositories,2,,panjf2000/ants,,https://bitbucket.org/panjf2000/ants/issues,2022-06-17T03:27:18.865+00:00 +bitbucket:BitbucketRepo:1:panjf2000/ants,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_bitbucket_api_repositories,2,,panjf2000/ants,,https://bitbucket.org/panjf2000/ants/issues,2022-06-17T03:27:18.865+00:00 diff --git a/plugins/bitbucket/e2e/snapshot_tables/repos.csv b/plugins/bitbucket/e2e/snapshot_tables/repos.csv index 32c0ecc0..646a4390 100644 --- a/plugins/bitbucket/e2e/snapshot_tables/repos.csv +++ b/plugins/bitbucket/e2e/snapshot_tables/repos.csv @@ -1,2 +1,2 @@ id,_raw_data_params,_raw_data_table,_raw_data_id,_raw_data_remark,name,url,description,owner_id,language,forked_from,deleted -bitbucket:BitbucketRepo:1:repositories/panjf2000/ants,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_bitbucket_api_repositories,2,,panjf2000/ants,https://bitbucket.org/panjf2000/ants,,,,,0 +bitbucket:BitbucketRepo:1:panjf2000/ants,"{""ConnectionId"":1,""Owner"":""panjf2000"",""Repo"":""ants""}",_raw_bitbucket_api_repositories,2,,panjf2000/ants,https://bitbucket.org/panjf2000/ants,,,,,0 diff --git a/plugins/bitbucket/tasks/api_common.go b/plugins/bitbucket/tasks/api_common.go index 000089da..b931a8a4 100644 --- a/plugins/bitbucket/tasks/api_common.go +++ b/plugins/bitbucket/tasks/api_common.go @@ -113,7 +113,7 @@ func GetPullRequestsIterator(taskCtx core.SubTaskContext) (*helper.DalCursorIter dal.From("_tool_bitbucket_pull_requests bpr"), dal.Where( `bpr.repo_id = ? and bpr.connection_id = ?`, - "repositories/"+data.Options.Owner+"/"+data.Options.Repo, data.Options.ConnectionId, + fmt.Sprintf("%s/%s", data.Options.Owner, data.Options.Repo), data.Options.ConnectionId, ), } // construct the input iterator @@ -133,7 +133,7 @@ func GetIssuesIterator(taskCtx core.SubTaskContext) (*helper.DalCursorIterator, dal.From("_tool_bitbucket_issues bpr"), dal.Where( `bpr.repo_id = ? and bpr.connection_id = ?`, - "repositories/"+data.Options.Owner+"/"+data.Options.Repo, data.Options.ConnectionId, + fmt.Sprintf("%s/%s", data.Options.Owner, data.Options.Repo), data.Options.ConnectionId, ), } // construct the input iterator diff --git a/plugins/bitbucket/tasks/repo_extractor.go b/plugins/bitbucket/tasks/repo_extractor.go index d79f5c36..c39e19c6 100644 --- a/plugins/bitbucket/tasks/repo_extractor.go +++ b/plugins/bitbucket/tasks/repo_extractor.go @@ -97,7 +97,7 @@ func ExtractApiRepositories(taskCtx core.SubTaskContext) errors.Error { results := make([]interface{}, 0, 1) bitbucketRepository := &models.BitbucketRepo{ ConnectionId: data.Options.ConnectionId, - BitbucketId: "repositories/" + body.FullName, + BitbucketId: body.FullName, Name: body.FullName, HTMLUrl: body.Links.Html.Href, Description: body.Description,
