likyh commented on code in PR #4373:
URL:
https://github.com/apache/incubator-devlake/pull/4373#discussion_r1106609648
##########
backend/plugins/bitbucket/tasks/account_convertor.go:
##########
@@ -53,17 +53,9 @@ func ConvertAccounts(taskCtx plugin.SubTaskContext)
errors.Error {
accountIdGen :=
didgen.NewDomainIdGenerator(&bitbucketModels.BitbucketAccount{})
converter, err := api.NewDataConverter(api.DataConverterArgs{
- InputRowType:
reflect.TypeOf(bitbucketModels.BitbucketAccount{}),
- Input: cursor,
- RawDataSubTaskArgs: api.RawDataSubTaskArgs{
Review Comment:
replace with CreateRawDataSubTaskArgs
##########
backend/plugins/bitbucket/models/migrationscripts/20230206_add_scope_and_transformation.go:
##########
@@ -0,0 +1,68 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+
"github.com/apache/incubator-devlake/plugins/bitbucket/models/migrationscripts/archived"
+)
+
+type BitbucketRepo20230206 struct {
+ TransformationRuleId uint64 `json:"transformationRuleId,omitempty"
mapstructure:"transformationRuleId,omitempty"`
+ CloneUrl string `json:"cloneUrl" gorm:"type:varchar(255)"
mapstructure:"cloneUrl,omitempty"`
+ Owner string `json:"owner"
mapstructure:"owner,omitempty"`
Review Comment:
rename owner_id to owner. because other `XXX_id` are uuid.
##########
backend/plugins/bitbucket/tasks/deployment_extractor.go:
##########
@@ -27,19 +27,18 @@ import (
)
type bitbucketApiDeploymentsResponse struct {
- Type string `json:"type"`
- Number int `json:"number"`
- UUID string `json:"uuid"`
- Key string `json:"key"`
- Step struct {
- UUID string `json:"uuid"`
- } `json:"step"`
- Environment struct {
- UUID string `json:"uuid"`
- } `json:"environment"`
+ Type string `json:"type"`
+ UUID string `json:"uuid"`
+ //Key string `json:"key"`
Review Comment:
delete because the fields column in collector
##########
backend/plugins/bitbucket/tasks/deployment_collector.go:
##########
@@ -41,12 +41,14 @@ func CollectApiDeployments(taskCtx plugin.SubTaskContext)
errors.Error {
ApiClient: data.ApiClient,
PageSize: 50,
Incremental: false,
- UrlTemplate: "repositories/{{ .Params.Owner }}/{{
.Params.Repo }}/deployments/",
- Query: GetQuery,
- ResponseParser: GetRawMessageFromResponse,
- GetTotalPages: GetTotalPagesFromResponse,
+ UrlTemplate: "repositories/{{ .Params.FullName
}}/deployments/",
+ Query: GetQueryFields(`values.type,values.uuid,` +
+
`values.release.pipeline,values.release.key,values.release.name,values.release.url,values.release.created_on,`
+
+
`values.release.commit.hash,values.release.commit.links.html,` +
+
`values.state.name,values.state.url,values.state.started_on,values.state.completed_on,values.last_update_time`),
Review Comment:
add fields to reduce response body to 30%. But it's no use in collect speed.
##########
backend/plugins/gitlab/api/remote.go:
##########
@@ -250,7 +250,7 @@ func RemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, er
// @Tags plugins/gitlab
// @Accept application/json
// @Param connectionId path int false "connection ID"
-// @Param search query string false "group ID"
+// @Param search query string false "search"
Review Comment:
typo
##########
backend/plugins/bitbucket/models/migrationscripts/20230206_add_scope_and_transformation.go:
##########
@@ -0,0 +1,68 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+
"github.com/apache/incubator-devlake/plugins/bitbucket/models/migrationscripts/archived"
+)
+
+type BitbucketRepo20230206 struct {
+ TransformationRuleId uint64 `json:"transformationRuleId,omitempty"
mapstructure:"transformationRuleId,omitempty"`
+ CloneUrl string `json:"cloneUrl" gorm:"type:varchar(255)"
mapstructure:"cloneUrl,omitempty"`
Review Comment:
record clone url to add gitextractor easily.
##########
backend/plugins/bitbucket/api/remote.go:
##########
@@ -0,0 +1,371 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "fmt"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/models"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/tasks"
+ "net/http"
+ "net/url"
+ "strconv"
+ "strings"
+)
+
+type RemoteScopesChild struct {
+ Type string `json:"type"`
+ ParentId *string `json:"parentId"`
+ Id string `json:"id"`
+ Name string `json:"name"`
+ Data interface{} `json:"data"`
+}
+
+type RemoteScopesOutput struct {
+ Children []RemoteScopesChild `json:"children"`
+ NextPageToken string `json:"nextPageToken"`
+}
+
+type SearchRemoteScopesOutput struct {
+ Children []RemoteScopesChild `json:"children"`
+ Page int `json:"page"`
+ PageSize int `json:"pageSize"`
+}
+
+type PageData struct {
+ Page int `json:"page"`
+ PerPage int `json:"per_page"`
+}
+
+type WorkspaceResponse struct {
+ Pagelen int `json:"pagelen"`
+ Page int `json:"page"`
+ Size int `json:"size"`
+ Values []struct {
+ //Type string `json:"type"`
+ //Permission string `json:"permission"`
+ //LastAccessed time.Time `json:"last_accessed"`
+ //AddedOn time.Time `json:"added_on"`
+ Workspace WorkspaceItem `json:"workspace"`
+ } `json:"values"`
+}
+
+type WorkspaceItem struct {
+ //Type string `json:"type"`
+ //Uuid string `json:"uuid"`
+ Slug string `json:"slug"`
+ Name string `json:"name"`
+}
+
+type ReposResponse struct {
+ Pagelen int `json:"pagelen"`
+ Page int `json:"page"`
+ Size int `json:"size"`
+ Values []tasks.BitbucketApiRepo `json:"values"`
+}
+
+const RemoteScopesPerPage int = 100
+const TypeScope string = "scope"
+const TypeGroup string = "group"
+
+// RemoteScopes list all available scope for users
+// @Summary list all available scope for users
+// @Description list all available scope for users
+// @Tags plugins/bitbucket
+// @Accept application/json
+// @Param connectionId path int false "connection ID"
+// @Param groupId query string false "group ID"
+// @Param pageToken query string false "page Token"
+// @Success 200 {object} RemoteScopesOutput
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/bitbucket/connections/{connectionId}/remote-scopes [GET]
+func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
+ connectionId, _ := extractParam(input.Params)
+ if connectionId == 0 {
+ return nil, errors.BadInput.New("invalid connectionId")
+ }
+
+ connection := &models.BitbucketConnection{}
+ err := connectionHelper.First(connection, input.Params)
+ if err != nil {
+ return nil, err
+ }
+
+ groupId, ok := input.Query["groupId"]
+ if !ok || len(groupId) == 0 {
+ groupId = []string{""}
+ }
+
+ pageToken, ok := input.Query["pageToken"]
+ if !ok || len(pageToken) == 0 {
+ pageToken = []string{""}
+ }
+
+ // get gid and pageData
+ gid := groupId[0]
+ pageData, err := GetPageDataFromPageToken(pageToken[0])
+ if err != nil {
+ return nil, errors.BadInput.New("failed to get page token")
+ }
+
+ // create api client
+ apiClient, err := api.NewApiClientFromConnection(context.TODO(),
basicRes, connection)
+ if err != nil {
+ return nil, err
+ }
+
+ query, err := GetQueryFromPageData(pageData)
+ if err != nil {
+ return nil, err
+ }
+
+ var res *http.Response
+ outputBody := &RemoteScopesOutput{}
+
+ // list groups part
+ if gid == "" {
+ query.Set("sort", "workspace.slug")
+ query.Set("fields",
"values.workspace.slug,values.workspace.name,pagelen,page,size")
+ res, err = apiClient.Get("/user/permissions/workspaces", query,
nil)
+ if err != nil {
+ return nil, err
+ }
+
+ resBody := &WorkspaceResponse{}
+ err = api.UnmarshalResponse(res, resBody)
+ if err != nil {
+ return nil, err
+ }
+
+ // append group to output
+ for _, group := range resBody.Values {
+ child := RemoteScopesChild{
+ Type: TypeGroup,
+ Id: group.Workspace.Slug,
+ Name: group.Workspace.Name,
+ // don't need to save group into data
+ Data: nil,
+ }
+ if gid != "" {
Review Comment:
fixed
##########
backend/plugins/bitbucket/api/remote.go:
##########
@@ -0,0 +1,371 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "fmt"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/models"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/tasks"
+ "net/http"
+ "net/url"
+ "strconv"
+ "strings"
+)
+
+type RemoteScopesChild struct {
+ Type string `json:"type"`
+ ParentId *string `json:"parentId"`
+ Id string `json:"id"`
+ Name string `json:"name"`
+ Data interface{} `json:"data"`
+}
+
+type RemoteScopesOutput struct {
+ Children []RemoteScopesChild `json:"children"`
+ NextPageToken string `json:"nextPageToken"`
+}
+
+type SearchRemoteScopesOutput struct {
+ Children []RemoteScopesChild `json:"children"`
+ Page int `json:"page"`
+ PageSize int `json:"pageSize"`
+}
+
+type PageData struct {
+ Page int `json:"page"`
+ PerPage int `json:"per_page"`
+}
+
+type WorkspaceResponse struct {
+ Pagelen int `json:"pagelen"`
+ Page int `json:"page"`
+ Size int `json:"size"`
+ Values []struct {
+ //Type string `json:"type"`
+ //Permission string `json:"permission"`
+ //LastAccessed time.Time `json:"last_accessed"`
+ //AddedOn time.Time `json:"added_on"`
+ Workspace WorkspaceItem `json:"workspace"`
+ } `json:"values"`
+}
+
+type WorkspaceItem struct {
+ //Type string `json:"type"`
+ //Uuid string `json:"uuid"`
+ Slug string `json:"slug"`
+ Name string `json:"name"`
+}
+
+type ReposResponse struct {
+ Pagelen int `json:"pagelen"`
+ Page int `json:"page"`
+ Size int `json:"size"`
+ Values []tasks.BitbucketApiRepo `json:"values"`
+}
+
+const RemoteScopesPerPage int = 100
+const TypeScope string = "scope"
+const TypeGroup string = "group"
+
+// RemoteScopes list all available scope for users
+// @Summary list all available scope for users
+// @Description list all available scope for users
+// @Tags plugins/bitbucket
+// @Accept application/json
+// @Param connectionId path int false "connection ID"
+// @Param groupId query string false "group ID"
+// @Param pageToken query string false "page Token"
+// @Success 200 {object} RemoteScopesOutput
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/bitbucket/connections/{connectionId}/remote-scopes [GET]
+func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
+ connectionId, _ := extractParam(input.Params)
+ if connectionId == 0 {
+ return nil, errors.BadInput.New("invalid connectionId")
+ }
+
+ connection := &models.BitbucketConnection{}
+ err := connectionHelper.First(connection, input.Params)
+ if err != nil {
+ return nil, err
+ }
+
+ groupId, ok := input.Query["groupId"]
+ if !ok || len(groupId) == 0 {
+ groupId = []string{""}
+ }
+
+ pageToken, ok := input.Query["pageToken"]
+ if !ok || len(pageToken) == 0 {
+ pageToken = []string{""}
+ }
+
+ // get gid and pageData
+ gid := groupId[0]
+ pageData, err := GetPageDataFromPageToken(pageToken[0])
+ if err != nil {
+ return nil, errors.BadInput.New("failed to get page token")
+ }
+
+ // create api client
+ apiClient, err := api.NewApiClientFromConnection(context.TODO(),
basicRes, connection)
+ if err != nil {
+ return nil, err
+ }
+
+ query, err := GetQueryFromPageData(pageData)
+ if err != nil {
+ return nil, err
+ }
+
+ var res *http.Response
+ outputBody := &RemoteScopesOutput{}
+
+ // list groups part
+ if gid == "" {
+ query.Set("sort", "workspace.slug")
+ query.Set("fields",
"values.workspace.slug,values.workspace.name,pagelen,page,size")
+ res, err = apiClient.Get("/user/permissions/workspaces", query,
nil)
+ if err != nil {
+ return nil, err
+ }
+
+ resBody := &WorkspaceResponse{}
+ err = api.UnmarshalResponse(res, resBody)
+ if err != nil {
+ return nil, err
+ }
+
+ // append group to output
+ for _, group := range resBody.Values {
+ child := RemoteScopesChild{
+ Type: TypeGroup,
+ Id: group.Workspace.Slug,
+ Name: group.Workspace.Name,
+ // don't need to save group into data
+ Data: nil,
+ }
+ if gid != "" {
+ child.ParentId = &gid
+ }
+
+ outputBody.Children = append(outputBody.Children, child)
+ }
+
+ // check groups count
+ if resBody.Size < pageData.PerPage {
Review Comment:
resBody are not the same type
##########
backend/plugins/bitbucket/api/remote.go:
##########
@@ -0,0 +1,371 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "fmt"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/models"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/tasks"
+ "net/http"
+ "net/url"
+ "strconv"
+ "strings"
+)
+
+type RemoteScopesChild struct {
+ Type string `json:"type"`
+ ParentId *string `json:"parentId"`
+ Id string `json:"id"`
+ Name string `json:"name"`
+ Data interface{} `json:"data"`
+}
+
+type RemoteScopesOutput struct {
+ Children []RemoteScopesChild `json:"children"`
+ NextPageToken string `json:"nextPageToken"`
+}
+
+type SearchRemoteScopesOutput struct {
+ Children []RemoteScopesChild `json:"children"`
+ Page int `json:"page"`
+ PageSize int `json:"pageSize"`
+}
+
+type PageData struct {
+ Page int `json:"page"`
+ PerPage int `json:"per_page"`
+}
+
+type WorkspaceResponse struct {
+ Pagelen int `json:"pagelen"`
+ Page int `json:"page"`
+ Size int `json:"size"`
+ Values []struct {
+ //Type string `json:"type"`
+ //Permission string `json:"permission"`
+ //LastAccessed time.Time `json:"last_accessed"`
+ //AddedOn time.Time `json:"added_on"`
+ Workspace WorkspaceItem `json:"workspace"`
+ } `json:"values"`
+}
+
+type WorkspaceItem struct {
+ //Type string `json:"type"`
+ //Uuid string `json:"uuid"`
+ Slug string `json:"slug"`
+ Name string `json:"name"`
+}
+
+type ReposResponse struct {
+ Pagelen int `json:"pagelen"`
+ Page int `json:"page"`
+ Size int `json:"size"`
+ Values []tasks.BitbucketApiRepo `json:"values"`
+}
+
+const RemoteScopesPerPage int = 100
+const TypeScope string = "scope"
+const TypeGroup string = "group"
+
+// RemoteScopes list all available scope for users
+// @Summary list all available scope for users
+// @Description list all available scope for users
+// @Tags plugins/bitbucket
+// @Accept application/json
+// @Param connectionId path int false "connection ID"
+// @Param groupId query string false "group ID"
+// @Param pageToken query string false "page Token"
+// @Success 200 {object} RemoteScopesOutput
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/bitbucket/connections/{connectionId}/remote-scopes [GET]
+func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
+ connectionId, _ := extractParam(input.Params)
+ if connectionId == 0 {
+ return nil, errors.BadInput.New("invalid connectionId")
+ }
+
+ connection := &models.BitbucketConnection{}
+ err := connectionHelper.First(connection, input.Params)
+ if err != nil {
+ return nil, err
+ }
+
+ groupId, ok := input.Query["groupId"]
+ if !ok || len(groupId) == 0 {
+ groupId = []string{""}
+ }
+
+ pageToken, ok := input.Query["pageToken"]
+ if !ok || len(pageToken) == 0 {
+ pageToken = []string{""}
+ }
+
+ // get gid and pageData
+ gid := groupId[0]
+ pageData, err := GetPageDataFromPageToken(pageToken[0])
+ if err != nil {
+ return nil, errors.BadInput.New("failed to get page token")
+ }
+
+ // create api client
+ apiClient, err := api.NewApiClientFromConnection(context.TODO(),
basicRes, connection)
+ if err != nil {
+ return nil, err
+ }
+
+ query, err := GetQueryFromPageData(pageData)
+ if err != nil {
+ return nil, err
+ }
+
+ var res *http.Response
+ outputBody := &RemoteScopesOutput{}
+
+ // list groups part
+ if gid == "" {
+ query.Set("sort", "workspace.slug")
+ query.Set("fields",
"values.workspace.slug,values.workspace.name,pagelen,page,size")
+ res, err = apiClient.Get("/user/permissions/workspaces", query,
nil)
+ if err != nil {
+ return nil, err
+ }
+
+ resBody := &WorkspaceResponse{}
+ err = api.UnmarshalResponse(res, resBody)
+ if err != nil {
+ return nil, err
+ }
+
+ // append group to output
+ for _, group := range resBody.Values {
+ child := RemoteScopesChild{
+ Type: TypeGroup,
+ Id: group.Workspace.Slug,
+ Name: group.Workspace.Name,
+ // don't need to save group into data
+ Data: nil,
+ }
+ if gid != "" {
+ child.ParentId = &gid
+ }
+
+ outputBody.Children = append(outputBody.Children, child)
+ }
+
+ // check groups count
+ if resBody.Size < pageData.PerPage {
+ pageData = nil
+ }
+ } else {
+ query.Set("sort", "name")
+ query.Set("fields",
"values.name,values.full_name,values.language,values.description,values.owner.username,values.created_on,values.updated_on,values.links.clone,values.links.self,pagelen,page,size")
+ // list projects part
+ res, err = apiClient.Get(fmt.Sprintf("/repositories/%s", gid),
query, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ resBody := &ReposResponse{}
+ err = api.UnmarshalResponse(res, resBody)
+ if err != nil {
+ return nil, err
+ }
+
+ // append repo to output
+ for _, repo := range resBody.Values {
+ child := RemoteScopesChild{
+ Type: TypeScope,
+ Id: repo.FullName,
+ Name: repo.Name,
+ Data: tasks.ConvertApiRepoToScope(&repo,
connection.ID),
+ }
+ child.ParentId = &gid
+ if *child.ParentId == "" {
+ child.ParentId = nil
+ }
+
+ outputBody.Children = append(outputBody.Children, child)
+ }
+
+ // check repo count
+ if resBody.Size < pageData.PerPage {
+ pageData = nil
+ }
+ }
+
+ // get the next page token
+ outputBody.NextPageToken = ""
+ if pageData != nil {
+ pageData.Page += 1
+ outputBody.NextPageToken, err =
GetPageTokenFromPageData(pageData)
+ if err != nil {
+ return nil, err
+ }
+ }
+
+ return &plugin.ApiResourceOutput{Body: outputBody, Status:
http.StatusOK}, nil
+}
+
+// SearchRemoteScopes use the Search API and only return project
+// @Summary use the Search API and only return project
+// @Description use the Search API and only return project
+// @Tags plugins/bitbucket
+// @Accept application/json
+// @Param connectionId path int false "connection ID"
+// @Param search query string false "search"
+// @Param page query int false "page number"
+// @Param pageSize query int false "page size per page"
+// @Success 200 {object} SearchRemoteScopesOutput
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/bitbucket/connections/{connectionId}/search-remote-scopes
[GET]
+func SearchRemoteScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ connectionId, _ := extractParam(input.Params)
+ if connectionId == 0 {
+ return nil, errors.BadInput.New("invalid connectionId")
+ }
+
+ connection := &models.BitbucketConnection{}
+ err := connectionHelper.First(connection, input.Params)
+ if err != nil {
+ return nil, err
+ }
+
+ search, ok := input.Query["search"]
+ if !ok || len(search) == 0 {
+ search = []string{""}
+ }
+ s := search[0]
+
+ p := 1
+ page, ok := input.Query["page"]
+ if ok && len(page) != 0 {
+ p, err = errors.Convert01(strconv.Atoi(page[0]))
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err,
fmt.Sprintf("failed to Atoi page:%s", page[0]))
+ }
+ }
+
+ ps := RemoteScopesPerPage
+ pageSize, ok := input.Query["pageSize"]
+ if ok && len(pageSize) != 0 {
+ ps, err = errors.Convert01(strconv.Atoi(pageSize[0]))
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err,
fmt.Sprintf("failed to Atoi pageSize:%s", pageSize[0]))
+ }
+ }
+
+ // create api client
+ apiClient, err := api.NewApiClientFromConnection(context.TODO(),
basicRes, connection)
+ if err != nil {
+ return nil, err
+ }
+
+ // set query
+ query, err := GetQueryFromPageData(&PageData{p, ps})
+ if err != nil {
+ return nil, err
+ }
+
+ // request search
+ query.Set("sort", "name")
+ query.Set("fields",
"values.name,values.full_name,values.language,values.description,values.owner.username,values.created_on,values.updated_on,values.links.clone,values.links.self,pagelen,page,size")
+
+ gid := ``
+ if strings.Contains(s, `/`) {
+ gid = strings.Split(s, `/`)[0]
+ s = strings.Split(s, `/`)[0]
Review Comment:
Bitbucket must search in a workspace(group). So here search text may like
`aaa/searchText`
##########
backend/plugins/bitbucket/tasks/pr_extractor.go:
##########
@@ -138,6 +111,7 @@ func ExtractApiPullRequests(taskCtx plugin.SubTaskContext)
errors.Error {
}
if rawL.MergeCommit != nil {
bitbucketPr.MergeCommitSha =
rawL.MergeCommit.Hash
+ bitbucketPr.MergedAt =
rawL.MergeCommit.Date.ToNullableTime()
Review Comment:
fill here instead of in converter.
##########
backend/plugins/bitbucket/tasks/commit_convertor.go:
##########
@@ -38,17 +38,17 @@ var ConvertCommitsMeta = plugin.SubTaskMeta{
}
func ConvertCommits(taskCtx plugin.SubTaskContext) errors.Error {
+ rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_COMMIT_TABLE)
db := taskCtx.GetDal()
- data := taskCtx.GetData().(*BitbucketTaskData)
- repoId := data.Repo.BitbucketId
+ repoId := data.Options.FullName
cursor, err := db.Cursor(
- dal.From("_tool_bitbucket_commits gc"),
Review Comment:
g means `github`, so delete them.
##########
backend/plugins/bitbucket/tasks/pr_comment_extractor.go:
##########
@@ -94,15 +74,14 @@ func ExtractApiPullRequestsComments(taskCtx
plugin.SubTaskContext) errors.Error
return nil, err
}
toolprComment.AuthorId = bitbucketUser.AccountId
- toolprComment.AuthorName =
bitbucketUser.UserName
+ toolprComment.AuthorName =
bitbucketUser.DisplayName
Review Comment:
Other AuthorName is filled by DisplayName, so here changed
##########
backend/plugins/bitbucket/tasks/issue_extractor.go:
##########
@@ -132,8 +109,9 @@ func ExtractApiIssues(taskCtx plugin.SubTaskContext)
errors.Error {
results = append(results, relatedUser)
}
if status, ok := issueStatusMap[bitbucketIssue.State];
ok {
- bitbucketIssue.State = status
+ bitbucketIssue.StdState = status
Review Comment:
add a new column to save state after transformation rule
##########
backend/plugins/bitbucket/api/remote.go:
##########
@@ -0,0 +1,371 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "context"
+ "encoding/base64"
+ "encoding/json"
+ "fmt"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/models"
+ "github.com/apache/incubator-devlake/plugins/bitbucket/tasks"
+ "net/http"
+ "net/url"
+ "strconv"
+ "strings"
+)
+
+type RemoteScopesChild struct {
+ Type string `json:"type"`
+ ParentId *string `json:"parentId"`
+ Id string `json:"id"`
+ Name string `json:"name"`
+ Data interface{} `json:"data"`
+}
+
+type RemoteScopesOutput struct {
+ Children []RemoteScopesChild `json:"children"`
+ NextPageToken string `json:"nextPageToken"`
+}
+
+type SearchRemoteScopesOutput struct {
+ Children []RemoteScopesChild `json:"children"`
+ Page int `json:"page"`
+ PageSize int `json:"pageSize"`
+}
+
+type PageData struct {
+ Page int `json:"page"`
+ PerPage int `json:"per_page"`
+}
+
+type WorkspaceResponse struct {
+ Pagelen int `json:"pagelen"`
+ Page int `json:"page"`
+ Size int `json:"size"`
+ Values []struct {
+ //Type string `json:"type"`
+ //Permission string `json:"permission"`
+ //LastAccessed time.Time `json:"last_accessed"`
+ //AddedOn time.Time `json:"added_on"`
+ Workspace WorkspaceItem `json:"workspace"`
+ } `json:"values"`
+}
+
+type WorkspaceItem struct {
+ //Type string `json:"type"`
+ //Uuid string `json:"uuid"`
+ Slug string `json:"slug"`
+ Name string `json:"name"`
+}
+
+type ReposResponse struct {
+ Pagelen int `json:"pagelen"`
+ Page int `json:"page"`
+ Size int `json:"size"`
+ Values []tasks.BitbucketApiRepo `json:"values"`
+}
+
+const RemoteScopesPerPage int = 100
+const TypeScope string = "scope"
+const TypeGroup string = "group"
+
+// RemoteScopes list all available scope for users
+// @Summary list all available scope for users
+// @Description list all available scope for users
+// @Tags plugins/bitbucket
+// @Accept application/json
+// @Param connectionId path int false "connection ID"
+// @Param groupId query string false "group ID"
+// @Param pageToken query string false "page Token"
+// @Success 200 {object} RemoteScopesOutput
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/bitbucket/connections/{connectionId}/remote-scopes [GET]
+func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
+ connectionId, _ := extractParam(input.Params)
+ if connectionId == 0 {
+ return nil, errors.BadInput.New("invalid connectionId")
+ }
+
+ connection := &models.BitbucketConnection{}
+ err := connectionHelper.First(connection, input.Params)
+ if err != nil {
+ return nil, err
+ }
+
+ groupId, ok := input.Query["groupId"]
+ if !ok || len(groupId) == 0 {
+ groupId = []string{""}
+ }
+
+ pageToken, ok := input.Query["pageToken"]
+ if !ok || len(pageToken) == 0 {
+ pageToken = []string{""}
+ }
+
+ // get gid and pageData
+ gid := groupId[0]
+ pageData, err := GetPageDataFromPageToken(pageToken[0])
+ if err != nil {
+ return nil, errors.BadInput.New("failed to get page token")
+ }
+
+ // create api client
+ apiClient, err := api.NewApiClientFromConnection(context.TODO(),
basicRes, connection)
+ if err != nil {
+ return nil, err
+ }
+
+ query, err := GetQueryFromPageData(pageData)
+ if err != nil {
+ return nil, err
+ }
+
+ var res *http.Response
+ outputBody := &RemoteScopesOutput{}
+
+ // list groups part
+ if gid == "" {
+ query.Set("sort", "workspace.slug")
+ query.Set("fields",
"values.workspace.slug,values.workspace.name,pagelen,page,size")
+ res, err = apiClient.Get("/user/permissions/workspaces", query,
nil)
+ if err != nil {
+ return nil, err
+ }
+
+ resBody := &WorkspaceResponse{}
+ err = api.UnmarshalResponse(res, resBody)
+ if err != nil {
+ return nil, err
+ }
+
+ // append group to output
+ for _, group := range resBody.Values {
+ child := RemoteScopesChild{
+ Type: TypeGroup,
+ Id: group.Workspace.Slug,
+ Name: group.Workspace.Name,
+ // don't need to save group into data
+ Data: nil,
+ }
+ if gid != "" {
+ child.ParentId = &gid
+ }
+
+ outputBody.Children = append(outputBody.Children, child)
+ }
+
+ // check groups count
+ if resBody.Size < pageData.PerPage {
+ pageData = nil
+ }
+ } else {
+ query.Set("sort", "name")
+ query.Set("fields",
"values.name,values.full_name,values.language,values.description,values.owner.username,values.created_on,values.updated_on,values.links.clone,values.links.self,pagelen,page,size")
+ // list projects part
+ res, err = apiClient.Get(fmt.Sprintf("/repositories/%s", gid),
query, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ resBody := &ReposResponse{}
+ err = api.UnmarshalResponse(res, resBody)
+ if err != nil {
+ return nil, err
+ }
+
+ // append repo to output
+ for _, repo := range resBody.Values {
+ child := RemoteScopesChild{
+ Type: TypeScope,
+ Id: repo.FullName,
+ Name: repo.Name,
+ Data: tasks.ConvertApiRepoToScope(&repo,
connection.ID),
+ }
+ child.ParentId = &gid
+ if *child.ParentId == "" {
Review Comment:
fixed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
