This is an automated email from the ASF dual-hosted git repository. klesh pushed a commit to branch kw-5519-dshelper-refactor in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
commit f8344cfe24b4e0b98d84c38544d35c82523c9b06 Author: Klesh Wong <[email protected]> AuthorDate: Tue Apr 16 19:54:08 2024 +0800 refactor: tapd adopts dshelper and move the company_id to connection table --- backend/plugins/tapd/api/blueprint_v200.go | 90 +++++------ backend/plugins/tapd/api/blueprint_v200_test.go | 129 ---------------- .../tapd/api/{connection.go => connection_api.go} | 52 +------ backend/plugins/tapd/api/init.go | 45 +----- backend/plugins/tapd/api/proxy.go | 72 --------- backend/plugins/tapd/api/remote.go | 168 --------------------- backend/plugins/tapd/api/remote_api.go | 164 ++++++++++++++++++++ .../plugins/tapd/api/{scope.go => scope_api.go} | 26 ++-- .../api/{scope_config.go => scope_config_api.go} | 14 +- backend/plugins/tapd/impl/impl.go | 23 +-- backend/plugins/tapd/models/connection.go | 1 + ...go => 20240415_add_company_id_to_connection.go} | 38 +++-- .../tapd/models/migrationscripts/register.go | 1 + backend/plugins/tapd/tasks/task_data.go | 12 +- 14 files changed, 272 insertions(+), 563 deletions(-) diff --git a/backend/plugins/tapd/api/blueprint_v200.go b/backend/plugins/tapd/api/blueprint_v200.go index 8ab9cb0cb..159c1ec3a 100644 --- a/backend/plugins/tapd/api/blueprint_v200.go +++ b/backend/plugins/tapd/api/blueprint_v200.go @@ -18,17 +18,16 @@ limitations under the License. package api import ( - "strconv" - "github.com/apache/incubator-devlake/core/errors" coreModels "github.com/apache/incubator-devlake/core/models" - "github.com/apache/incubator-devlake/core/models/domainlayer" "github.com/apache/incubator-devlake/core/models/domainlayer/didgen" "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/core/utils" helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" "github.com/apache/incubator-devlake/plugins/tapd/models" + "github.com/apache/incubator-devlake/plugins/tapd/tasks" ) func MakeDataSourcePipelinePlanV200( @@ -36,53 +35,50 @@ func MakeDataSourcePipelinePlanV200( connectionId uint64, bpScopes []*coreModels.BlueprintScope, ) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) { - plan := make(coreModels.PipelinePlan, len(bpScopes)) - plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, plan, bpScopes, connectionId) + connection, err := dsHelper.ConnSrv.FindByPk(connectionId) if err != nil { return nil, nil, err } - scopes, err := makeScopesV200(bpScopes, connectionId) + scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes) if err != nil { return nil, nil, err } - - return plan, scopes, nil + plan, err := makePipelinePlanV200(subtaskMetas, scopeDetails, connection) + if err != nil { + return nil, nil, err + } + scopes, err := makeScopesV200(scopeDetails, connection) + return plan, scopes, err } -func makeDataSourcePipelinePlanV200( +func makePipelinePlanV200( subtaskMetas []plugin.SubTaskMeta, - plan coreModels.PipelinePlan, - bpScopes []*coreModels.BlueprintScope, - connectionId uint64, + scopeDetails []*srvhelper.ScopeDetail[models.TapdWorkspace, models.TapdScopeConfig], + connection *models.TapdConnection, ) (coreModels.PipelinePlan, errors.Error) { - for i, bpScope := range bpScopes { + + plan := make(coreModels.PipelinePlan, len(scopeDetails)) + for i, scopeDetail := range scopeDetails { stage := plan[i] if stage == nil { stage = coreModels.PipelineStage{} } - // construct task options for tapd - options := make(map[string]interface{}) - intNum, err := errors.Convert01(strconv.Atoi(bpScope.ScopeId)) - if err != nil { - return nil, err - } - options["workspaceId"] = intNum - options["connectionId"] = connectionId - _, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connectionId, bpScope.ScopeId) + scope, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig + // construct task options for circleci + task, err := helper.MakePipelinePlanTask( + "tapd", + subtaskMetas, + scopeConfig.Entities, + tasks.TapdOptions{ + ConnectionId: connection.ID, + WorkspaceId: scope.Id, + }, + ) if err != nil { return nil, err } - - subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas, scopeConfig.Entities) - if err != nil { - return nil, err - } - stage = append(stage, &coreModels.PipelineTask{ - Plugin: "tapd", - Subtasks: subtasks, - Options: options, - }) + stage = append(stage, task) plan[i] = stage } @@ -90,28 +86,22 @@ func makeDataSourcePipelinePlanV200( } func makeScopesV200( - bpScopes []*coreModels.BlueprintScope, - connectionId uint64) ([]plugin.Scope, errors.Error, -) { - scopes := make([]plugin.Scope, 0) - for _, bpScope := range bpScopes { - // get workspace and scope config from db + scopeDetails []*srvhelper.ScopeDetail[models.TapdWorkspace, models.TapdScopeConfig], + connection *models.TapdConnection, +) ([]plugin.Scope, errors.Error) { + scopes := make([]plugin.Scope, len(scopeDetails)) - tapdWorkspace, scopeConfig, err := scopeHelper.DbHelper().GetScopeAndConfig(connectionId, bpScope.ScopeId) - if err != nil { - return nil, err - } + idgen := didgen.NewDomainIdGenerator(&models.TapdWorkspace{}) + for _, scopeDetail := range scopeDetails { + // get workspace and scope config from db + tapdWorkspace, scopeConfig := scopeDetail.Scope, scopeDetail.ScopeConfig // add wrokspace to scopes if utils.StringsContains(scopeConfig.Entities, plugin.DOMAIN_TYPE_TICKET) { - domainBoard := &ticket.Board{ - DomainEntity: domainlayer.DomainEntity{ - Id: didgen.NewDomainIdGenerator(&models.TapdWorkspace{}).Generate(tapdWorkspace.ConnectionId, tapdWorkspace.Id), - }, - Name: tapdWorkspace.Name, - Type: "scrum", - } - scopes = append(scopes, domainBoard) + id := idgen.Generate(connection.ID, tapdWorkspace) + board := ticket.NewBoard(id, tapdWorkspace.Name) + board.Type = "scrum" + scopes = append(scopes, board) } } return scopes, nil diff --git a/backend/plugins/tapd/api/blueprint_v200_test.go b/backend/plugins/tapd/api/blueprint_v200_test.go deleted file mode 100644 index 9b050470c..000000000 --- a/backend/plugins/tapd/api/blueprint_v200_test.go +++ /dev/null @@ -1,129 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package api - -import ( - "database/sql" - "github.com/apache/incubator-devlake/core/dal" - "gorm.io/gorm/migrator" - "testing" - - coreModels "github.com/apache/incubator-devlake/core/models" - "github.com/apache/incubator-devlake/core/models/common" - "github.com/apache/incubator-devlake/core/models/domainlayer" - "github.com/apache/incubator-devlake/core/models/domainlayer/ticket" - "github.com/apache/incubator-devlake/core/plugin" - "github.com/apache/incubator-devlake/helpers/unithelper" - mockdal "github.com/apache/incubator-devlake/mocks/core/dal" - mockplugin "github.com/apache/incubator-devlake/mocks/core/plugin" - "github.com/apache/incubator-devlake/plugins/tapd/models" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/mock" -) - -func TestMakeDataSourcePipelinePlanV200(t *testing.T) { - mockMeta := mockplugin.NewPluginMeta(t) - mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/tapd") - mockMeta.On("Name").Return("dummy").Maybe() - err := plugin.RegisterPlugin("tapd", mockMeta) - assert.Nil(t, err) - bs := &coreModels.BlueprintScope{ - ScopeId: "10", - } - bpScopes := make([]*coreModels.BlueprintScope, 0) - bpScopes = append(bpScopes, bs) - plan := make(coreModels.PipelinePlan, len(bpScopes)) - mockBasicRes(t) - - plan, err = makeDataSourcePipelinePlanV200(nil, plan, bpScopes, uint64(1)) - assert.Nil(t, err) - scopes, err := makeScopesV200(bpScopes, uint64(1)) - assert.Nil(t, err) - - expectPlan := coreModels.PipelinePlan{ - coreModels.PipelineStage{ - { - Plugin: "tapd", - Subtasks: []string{}, - Options: map[string]interface{}{ - "connectionId": uint64(1), - "workspaceId": 10, - }, - }, - }, - } - assert.Equal(t, expectPlan, plan) - - expectScopes := make([]plugin.Scope, 0) - tapdBoard := &ticket.Board{ - DomainEntity: domainlayer.DomainEntity{ - Id: "tapd:TapdWorkspace:1:10", - }, - Name: "a", - Type: "scrum", - } - - expectScopes = append(expectScopes, tapdBoard) - assert.Equal(t, expectScopes, scopes) -} - -func mockBasicRes(t *testing.T) { - tapdWorkspace := &models.TapdWorkspace{ - Scope: common.Scope{ - ConnectionId: 1, - }, - Id: 10, - Name: "a", - } - scopeConfig := &models.TapdScopeConfig{ - ScopeConfig: common.ScopeConfig{ - Entities: []string{plugin.DOMAIN_TYPE_TICKET}, - }, - } - var testColumTypes = []dal.ColumnMeta{ - migrator.ColumnType{ - NameValue: sql.NullString{ - String: "abc", - Valid: true, - }, - }, - } - - // Refresh Global Variables and set the sql mock - mockRes := unithelper.DummyBasicRes(func(mockDal *mockdal.Dal) { - mockDal.On("First", mock.AnythingOfType("*models.TapdScopeConfig"), mock.Anything).Run(func(args mock.Arguments) { - dst := args.Get(0).(*models.TapdScopeConfig) - *dst = *scopeConfig - }).Return(nil) - mockDal.On("First", mock.AnythingOfType("*models.TapdWorkspace"), mock.Anything).Run(func(args mock.Arguments) { - dst := args.Get(0).(*models.TapdWorkspace) - *dst = *tapdWorkspace - }).Return(nil) - mockDal.On("GetPrimarykeyColumns", mock.AnythingOfType("*models.TapdConnection"), mock.Anything).Run(nil).Return( - testColumTypes, nil) - mockDal.On("GetColumns", mock.AnythingOfType("models.TapdConnection"), mock.Anything).Run(nil).Return( - testColumTypes, nil) - mockDal.On("GetColumns", mock.AnythingOfType("models.TapdWorkspace"), mock.Anything).Run(nil).Return( - testColumTypes, nil) - mockDal.On("GetColumns", mock.AnythingOfType("models.TapdScopeConfig"), mock.Anything).Run(nil).Return( - testColumTypes, nil) - }) - p := mockplugin.NewPluginMeta(t) - p.On("Name").Return("dummy").Maybe() - Init(mockRes, p) -} diff --git a/backend/plugins/tapd/api/connection.go b/backend/plugins/tapd/api/connection_api.go similarity index 80% rename from backend/plugins/tapd/api/connection.go rename to backend/plugins/tapd/api/connection_api.go index e7e9733ab..97224d75d 100644 --- a/backend/plugins/tapd/api/connection.go +++ b/backend/plugins/tapd/api/connection_api.go @@ -100,8 +100,7 @@ func TestConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, // @Failure 500 {string} errcode.Error "Internal Error" // @Router /plugins/tapd/connections/{connectionId}/test [POST] func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { - connection := &models.TapdConnection{} - err := connectionHelper.First(connection, input.Params) + connection, err := dsHelper.ConnApi.GetMergedConnection(input) if err != nil { return nil, errors.BadInput.Wrap(err, "find connection from db") } @@ -125,17 +124,7 @@ func TestExistingConnection(input *plugin.ApiResourceInput) (*plugin.ApiResource // @Failure 500 {string} errcode.Error "Internal Error" // @Router /plugins/tapd/connections [POST] func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { - // create a new connections - connection := &models.TapdConnection{} - - // update from request and save to database - //err := refreshAndSaveTapdConnection(tapdConnection, input.Body) - err := connectionHelper.Create(connection, input) - if err != nil { - return nil, err - } - - return &plugin.ApiResourceOutput{Body: connection.Sanitize(), Status: http.StatusOK}, nil + return dsHelper.ConnApi.Post(input) } // @Summary patch tapd connection @@ -147,17 +136,7 @@ func PostConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, // @Failure 500 {string} errcode.Error "Internal Error" // @Router /plugins/tapd/connections/{connectionId} [PATCH] func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { - connection := &models.TapdConnection{} - if err := connectionHelper.First(&connection, input.Params); err != nil { - return nil, err - } - if err := (&models.TapdConnection{}).MergeFromRequest(connection, input.Body); err != nil { - return nil, errors.Convert(err) - } - if err := connectionHelper.SaveWithCreateOrUpdate(connection); err != nil { - return nil, err - } - return &plugin.ApiResourceOutput{Body: connection.Sanitize()}, nil + return dsHelper.ConnApi.Patch(input) } // @Summary delete a tapd connection @@ -169,13 +148,7 @@ func PatchConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, // @Failure 500 {string} errcode.Error "Internal Error" // @Router /plugins/tapd/connections/{connectionId} [DELETE] func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { - conn := &models.TapdConnection{} - output, err := connectionHelper.Delete(conn, input) - if err != nil { - return output, err - } - output.Body = conn.Sanitize() - return output, nil + return dsHelper.ConnApi.Delete(input) } // @Summary get all tapd connections @@ -186,15 +159,7 @@ func DeleteConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput // @Failure 500 {string} errcode.Error "Internal Error" // @Router /plugins/tapd/connections [GET] func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { - var connections []models.TapdConnection - err := connectionHelper.List(&connections) - if err != nil { - return nil, err - } - for idx, c := range connections { - connections[idx] = c.Sanitize() - } - return &plugin.ApiResourceOutput{Body: connections, Status: http.StatusOK}, nil + return dsHelper.ConnApi.GetAll(input) } // @Summary get tapd connection detail @@ -205,10 +170,5 @@ func ListConnections(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, // @Failure 500 {string} errcode.Error "Internal Error" // @Router /plugins/tapd/connections/{connectionId} [GET] func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { - connection := &models.TapdConnection{} - err := connectionHelper.First(connection, input.Params) - if err != nil { - return nil, err - } - return &plugin.ApiResourceOutput{Body: connection.Sanitize()}, nil + return dsHelper.ConnApi.GetDetail(input) } diff --git a/backend/plugins/tapd/api/init.go b/backend/plugins/tapd/api/init.go index 9a387f616..029f53a8a 100644 --- a/backend/plugins/tapd/api/init.go +++ b/backend/plugins/tapd/api/init.go @@ -21,53 +21,22 @@ import ( "github.com/apache/incubator-devlake/core/context" "github.com/apache/incubator-devlake/core/plugin" "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + "github.com/apache/incubator-devlake/helpers/srvhelper" "github.com/apache/incubator-devlake/plugins/tapd/models" "github.com/go-playground/validator/v10" ) var vld *validator.Validate -var connectionHelper *api.ConnectionApiHelper var basicRes context.BasicRes -var scopeHelper *api.ScopeApiHelper[models.TapdConnection, models.TapdWorkspace, models.TapdScopeConfig] -var remoteHelper *api.RemoteApiHelper[models.TapdConnection, models.TapdWorkspace, models.TapdWorkspace, api.BaseRemoteGroupResponse] -var scHelper *api.ScopeConfigHelper[models.TapdScopeConfig, *models.TapdScopeConfig] var dsHelper *api.DsHelper[models.TapdConnection, models.TapdWorkspace, models.TapdScopeConfig] +var raProxy *api.DsRemoteApiProxyHelper[models.TapdConnection] +var raScopeList *api.DsRemoteApiScopeListHelper[models.TapdConnection, models.TapdWorkspace, srvhelper.NoPagintation] -func Init(br context.BasicRes, p plugin.PluginMeta) { +// var raScopeSearch *api.DsRemoteApiScopeSearchHelper[models.TapdConnection, models.TapdWorkspace] +func Init(br context.BasicRes, p plugin.PluginMeta) { basicRes = br vld = validator.New() - connectionHelper = api.NewConnectionHelper( - basicRes, - vld, - p.Name(), - ) - params := &api.ReflectionParameters{ - ScopeIdFieldName: "Id", - ScopeIdColumnName: "id", - RawScopeParamName: "WorkSpaceId", - SearchScopeParamName: "name", - } - scopeHelper = api.NewScopeHelper[models.TapdConnection, models.TapdWorkspace, models.TapdScopeConfig]( - basicRes, - vld, - connectionHelper, - api.NewScopeDatabaseHelperImpl[models.TapdConnection, models.TapdWorkspace, models.TapdScopeConfig]( - basicRes, connectionHelper, params), - params, - nil, - ) - remoteHelper = api.NewRemoteHelper[models.TapdConnection, models.TapdWorkspace, models.TapdWorkspace, api.BaseRemoteGroupResponse]( - basicRes, - vld, - connectionHelper, - ) - scHelper = api.NewScopeConfigHelper[models.TapdScopeConfig, *models.TapdScopeConfig]( - basicRes, - vld, - p.Name(), - ) - dsHelper = api.NewDataSourceHelper[ models.TapdConnection, models.TapdWorkspace, models.TapdScopeConfig, ]( @@ -80,5 +49,7 @@ func Init(br context.BasicRes, p plugin.PluginMeta) { nil, nil, ) - + raProxy = api.NewDsRemoteApiProxyHelper(dsHelper.ConnApi.ModelApiHelper) + raScopeList = api.NewDsRemoteApiScopeListHelper(raProxy, listTapdRemoteScopes) + // raScopeSearch = api.NewDsRemoteApiScopeSearchHelper[models.TapdConnection, models.TapdWorkspace](raProxy, searchTapdRepos) } diff --git a/backend/plugins/tapd/api/proxy.go b/backend/plugins/tapd/api/proxy.go deleted file mode 100644 index 0fe2c478c..000000000 --- a/backend/plugins/tapd/api/proxy.go +++ /dev/null @@ -1,72 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package api - -import ( - "context" - "encoding/json" - "fmt" - "github.com/apache/incubator-devlake/core/errors" - "github.com/apache/incubator-devlake/core/plugin" - helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api" - "github.com/apache/incubator-devlake/plugins/tapd/models" - "io" - "time" -) - -const ( - TimeOut = 10 * time.Second -) - -func Proxy(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { - connection := &models.TapdConnection{} - err := connectionHelper.First(connection, input.Params) - if err != nil { - return nil, err - } - apiClient, err := helper.NewApiClient( - context.TODO(), - connection.Endpoint, - map[string]string{ - "Authorization": fmt.Sprintf("Basic %v", connection.GetEncodedToken()), - }, - 30*time.Second, - connection.Proxy, - basicRes, - ) - if err != nil { - return nil, err - } - resp, err := apiClient.Get(input.Params["path"], input.Query, nil) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - body, err := errors.Convert01(io.ReadAll(resp.Body)) - if err != nil { - return nil, err - } - // verify response body is json - var tmp interface{} - err = errors.Convert(json.Unmarshal(body, &tmp)) - if err != nil { - return nil, err - } - return &plugin.ApiResourceOutput{Status: resp.StatusCode, Body: json.RawMessage(body)}, nil -} diff --git a/backend/plugins/tapd/api/remote.go b/backend/plugins/tapd/api/remote.go deleted file mode 100644 index edc403538..000000000 --- a/backend/plugins/tapd/api/remote.go +++ /dev/null @@ -1,168 +0,0 @@ -/* -Licensed to the Apache Software Foundation (ASF) under one or more -contributor license agreements. See the NOTICE file distributed with -this work for additional information regarding copyright ownership. -The ASF licenses this file to You under the Apache License, Version 2.0 -(the "License"); you may not use this file except in compliance with -the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package api - -import ( - gocontext "context" - "encoding/json" - "fmt" - "io" - "net/http" - "net/url" - - "github.com/apache/incubator-devlake/core/context" - "github.com/apache/incubator-devlake/core/errors" - "github.com/apache/incubator-devlake/core/plugin" - "github.com/apache/incubator-devlake/helpers/pluginhelper/api" - "github.com/apache/incubator-devlake/plugins/tapd/models" - "github.com/apache/incubator-devlake/plugins/tapd/tasks" -) - -// PrepareFirstPageToken prepare first page token -// @Summary prepare first page token -// @Description prepare first page token -// @Tags plugins/tapd -// @Accept application/json -// @Param connectionId path int false "connection ID" -// @Param companyId query string false "company ID" -// @Success 200 {object} api.RemoteScopesOutput -// @Failure 400 {object} shared.ApiBody "Bad Request" -// @Failure 500 {object} shared.ApiBody "Internal Error" -// @Router /plugins/tapd/connections/{connectionId}/remote-scopes-prepare-token [GET] -func PrepareFirstPageToken(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { - return remoteHelper.PrepareFirstPageToken(input.Query[`companyId`][0]) -} - -// RemoteScopes list all available scope for users -// @Summary list all available scope for users -// @Description list all available scope for users -// @Tags plugins/tapd -// @Accept application/json -// @Param connectionId path int false "connection ID" -// @Param groupId query string false "group ID" -// @Param pageToken query string false "page Token" -// @Success 200 {object} api.RemoteScopesOutput -// @Failure 400 {object} shared.ApiBody "Bad Request" -// @Failure 500 {object} shared.ApiBody "Internal Error" -// @Router /plugins/tapd/connections/{connectionId}/remote-scopes [GET] -func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { - return remoteHelper.GetScopesFromRemote(input, - func(basicRes context.BasicRes, gid string, queryData *api.RemoteQueryData, connection models.TapdConnection) ([]api.BaseRemoteGroupResponse, errors.Error) { - if gid == "" { - // if gid is empty, it means we need to query company - gid = "1" - } - apiClient, err := api.NewApiClientFromConnection(gocontext.TODO(), basicRes, &connection) - if err != nil { - return nil, errors.BadInput.Wrap(err, "failed to get create apiClient") - } - var res *http.Response - query := url.Values{} - query.Set("company_id", queryData.CustomInfo) - res, err = apiClient.Get("/workspaces/projects", query, nil) - if err != nil { - return nil, err - } - - var resBody models.WorkspacesResponse - err = api.UnmarshalResponse(res, &resBody) - if err != nil { - return nil, err - } - if resBody.Status != 1 { - return nil, errors.BadInput.Wrap(err, "failed to get workspaces") - } - - // check if workspace is a group - isGroupMap := map[uint64]bool{} - for _, workspace := range resBody.Data { - isGroupMap[workspace.TapdWorkspace.ParentId] = true - } - - groups := []api.BaseRemoteGroupResponse{} - for _, workspace := range resBody.Data { - if fmt.Sprintf(`%d`, workspace.TapdWorkspace.ParentId) == gid && - isGroupMap[workspace.TapdWorkspace.Id] { - groups = append(groups, api.BaseRemoteGroupResponse{ - Id: fmt.Sprintf(`%d`, workspace.TapdWorkspace.Id), - Name: workspace.TapdWorkspace.Name, - }) - } - } - - return groups, err - }, - func(basicRes context.BasicRes, gid string, queryData *api.RemoteQueryData, connection models.TapdConnection) ([]models.TapdWorkspace, errors.Error) { - if gid == "" { - return nil, nil - } - - apiClient, err := api.NewApiClientFromConnection(gocontext.TODO(), basicRes, &connection) - if err != nil { - return nil, errors.BadInput.Wrap(err, "failed to get create apiClient") - } - var res *http.Response - query := url.Values{} - query.Set("company_id", queryData.CustomInfo) - res, err = apiClient.Get("/workspaces/projects", query, nil) - if err != nil { - return nil, err - } - var resBody models.WorkspacesResponse - err = api.UnmarshalResponse(res, &resBody) - if err != nil { - return nil, err - } - workspaces := []models.TapdWorkspace{} - for _, workspace := range resBody.Data { - if fmt.Sprintf(`%d`, workspace.TapdWorkspace.ParentId) == gid { - // filter from all project to query what we need... - workspaces = append(workspaces, models.TapdWorkspace(workspace.TapdWorkspace)) - } - - } - return workspaces, err - }, - ) -} - -func GetApiWorkspace(op *tasks.TapdOptions, apiClient plugin.ApiClient) (*models.TapdWorkspace, errors.Error) { - query := url.Values{} - query.Set("workspace_id", fmt.Sprintf("%v", op.WorkspaceId)) - res, err := apiClient.Get("workspaces/get_workspace_info", query, nil) - if err != nil { - return nil, err - } - defer res.Body.Close() - if res.StatusCode != http.StatusOK { - return nil, errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("unexpected status code when requesting repo detail from %s", res.Request.URL.String())) - } - body, err := errors.Convert01(io.ReadAll(res.Body)) - if err != nil { - return nil, err - } - - var resBody models.WorkspaceResponse - err = errors.Convert(json.Unmarshal(body, &resBody)) - if err != nil { - return nil, err - } - workspace := models.TapdWorkspace(resBody.Data.TapdWorkspace) - workspace.ConnectionId = op.ConnectionId - return &workspace, nil -} diff --git a/backend/plugins/tapd/api/remote_api.go b/backend/plugins/tapd/api/remote_api.go new file mode 100644 index 000000000..a22c2fd94 --- /dev/null +++ b/backend/plugins/tapd/api/remote_api.go @@ -0,0 +1,164 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one or more +contributor license agreements. See the NOTICE file distributed with +this work for additional information regarding copyright ownership. +The ASF licenses this file to You under the Apache License, Version 2.0 +(the "License"); you may not use this file except in compliance with +the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package api + +import ( + "fmt" + "net/url" + "sort" + + "github.com/apache/incubator-devlake/core/errors" + "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/pluginhelper/api" + dsmodels "github.com/apache/incubator-devlake/helpers/pluginhelper/api/models" + "github.com/apache/incubator-devlake/helpers/srvhelper" + "github.com/apache/incubator-devlake/plugins/tapd/models" +) + +func listTapdRemoteScopes( + connection *models.TapdConnection, + apiClient plugin.ApiClient, + groupId string, + page srvhelper.NoPagintation, +) ( + children []dsmodels.DsRemoteApiScopeListEntry[models.TapdWorkspace], + nextPage *srvhelper.NoPagintation, + err errors.Error, +) { + // construct the query and request + query := url.Values{} + query.Set("company_id", fmt.Sprintf("%v", connection.CompanyId)) + res, err := apiClient.Get("/workspaces/projects", query, nil) + if err != nil { + return + } + // parse the response + var resBody models.WorkspacesResponse + err = api.UnmarshalResponse(res, &resBody) + if err != nil { + return nil, nil, err + } + if resBody.Status != 1 { + return nil, nil, errors.BadInput.Wrap(err, "failed to get workspaces") + } + // tapd returns the whole freaking tree as a list, well...let's convert it to a tree + nodes := map[string]*Node{} + // convert the list to nodes + for i, workspace := range resBody.Data { + entry := &Entry{ + Type: api.RAS_ENTRY_TYPE_SCOPE, // default to scope + Id: fmt.Sprintf(`%d`, workspace.TapdWorkspace.Id), + Name: workspace.TapdWorkspace.Name, + ParentId: toStringPointer(workspace.TapdWorkspace.ParentId), + Data: &resBody.Data[i].TapdWorkspace, + } + nodes[entry.Id] = &Node{ + entry: entry, + } + } + // construct the tree + var root *Node + var current *Node + for _, node := range nodes { + // find parent and make sure it is a parent if any + parent := nodes[*node.entry.ParentId] + if parent != nil { + // make sure the parent is a group + parent.entry.Type = api.RAS_ENTRY_TYPE_GROUP + parent.entry.Data = nil + // add this node to the parent + parent.children = append(parent.children, node) + } else { + // or this is root and it must be a group + root = node + root.entry.Type = api.RAS_ENTRY_TYPE_GROUP + root.entry.Data = nil + } + if groupId == node.entry.Id { + current = node + } + } + generateFullNames(root, "") + // select the current group + if current == nil { + current = root + } + // sort children + sort.Sort(current.children) + // append to the final result + for _, node := range current.children { + children = append(children, *node.entry) + } + return +} + +type Entry = dsmodels.DsRemoteApiScopeListEntry[models.TapdWorkspace] +type Node struct { + entry *Entry + children Children +} +type Children []*Node + +func (a Children) Len() int { return len(a) } +func (a Children) Less(i, j int) bool { + if a[i].entry.Type != a[j].entry.Type { + return a[i].entry.Type < a[j].entry.Type + } + return a[i].entry.Name < a[j].entry.Name +} +func (a Children) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + +func generateFullNames(node *Node, prefix string) { + for _, child := range node.children { + child.entry.FullName = prefix + child.entry.Name + if child.entry.Type == api.RAS_ENTRY_TYPE_GROUP { + generateFullNames(child, child.entry.FullName+" / ") + } + } +} + +func toStringPointer(v any) *string { + s := fmt.Sprintf("%v", v) + return &s +} + +// RemoteScopes list all available scope for users +// @Summary list all available scope for users +// @Description list all available scope for users +// @Tags plugins/tapd +// @Accept application/json +// @Param connectionId path int false "connection ID" +// @Param groupId query string false "group ID" +// @Param pageToken query string false "page Token" +// @Success 200 {object} dsmodels.DsRemoteApiScopeList[models.TapdWorkspace] +// @Failure 400 {object} shared.ApiBody "Bad Request" +// @Failure 500 {object} shared.ApiBody "Internal Error" +// @Router /plugins/tapd/connections/{connectionId}/remote-scopes [GET] +func RemoteScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raScopeList.Get(input) +} + +// @Summary Remote server API proxy +// @Description Forward API requests to the specified remote server +// @Param connectionId path int true "connection ID" +// @Param path path string true "path to a API endpoint" +// @Tags plugins/github +// @Router /plugins/github/connections/{connectionId}/proxy/{path} [GET] +func Proxy(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return raProxy.Proxy(input) +} diff --git a/backend/plugins/tapd/api/scope.go b/backend/plugins/tapd/api/scope_api.go similarity index 88% rename from backend/plugins/tapd/api/scope.go rename to backend/plugins/tapd/api/scope_api.go index 673194113..8f020ccaa 100644 --- a/backend/plugins/tapd/api/scope.go +++ b/backend/plugins/tapd/api/scope_api.go @@ -26,12 +26,8 @@ import ( "github.com/apache/incubator-devlake/plugins/tapd/models" ) -type ScopeRes struct { - models.TapdWorkspace - api.ScopeResDoc[models.TapdScopeConfig] -} - -type ScopeReq api.ScopeReq[models.TapdWorkspace] +type PutScopesReqBody api.PutScopesReqBody[models.TapdWorkspace] +type ScopeDetail api.ScopeDetail[models.TapdWorkspace, models.TapdScopeConfig] // PutScope create or update tapd job // @Summary create or update tapd job @@ -39,13 +35,13 @@ type ScopeReq api.ScopeReq[models.TapdWorkspace] // @Tags plugins/tapd // @Accept application/json // @Param connectionId path int false "connection ID" -// @Param scope body ScopeReq true "json" +// @Param scope body PutScopesReqBody true "json" // @Success 200 {object} []models.TapdWorkspace // @Failure 400 {object} shared.ApiBody "Bad Request" // @Failure 500 {object} shared.ApiBody "Internal Error" // @Router /plugins/tapd/connections/{connectionId}/scopes [PUT] -func PutScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { - return scopeHelper.Put(input) +func PutScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeApi.PutMultiple(input) } // UpdateScope patch to tapd job @@ -61,7 +57,7 @@ func PutScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors // @Failure 500 {object} shared.ApiBody "Internal Error" // @Router /plugins/tapd/connections/{connectionId}/scopes/{scopeId} [PATCH] func UpdateScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { - return scopeHelper.Update(input) + return dsHelper.ScopeApi.Patch(input) } // GetScopeList get tapd jobs @@ -73,12 +69,12 @@ func UpdateScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, err // @Param pageSize query int false "page size, default 50" // @Param page query int false "page size, default 1" // @Param blueprints query bool false "also return blueprints using these scopes as part of the payload" -// @Success 200 {object} []ScopeRes +// @Success 200 {object} []ScopeDetail // @Failure 400 {object} shared.ApiBody "Bad Request" // @Failure 500 {object} shared.ApiBody "Internal Error" // @Router /plugins/tapd/connections/{connectionId}/scopes [GET] func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { - return scopeHelper.GetScopeList(input) + return dsHelper.ScopeApi.GetPage(input) } // GetScope get one tapd job @@ -87,13 +83,13 @@ func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, er // @Tags plugins/tapd // @Param connectionId path int false "connection ID" // @Param scopeId path string false "workspace ID" -// @Success 200 {object} ScopeRes +// @Success 200 {object} ScopeDetail // @Failure 400 {object} shared.ApiBody "Bad Request" // @Failure 500 {object} shared.ApiBody "Internal Error" // @Router /plugins/tapd/connections/{connectionId}/scopes/{scopeId} [GET] func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { input.Params["scopeId"] = strings.TrimLeft(input.Params["scopeId"], "/") - return scopeHelper.GetScope(input) + return dsHelper.ScopeApi.GetScopeDetail(input) } // DeleteScope delete plugin data associated with the scope and optionally the scope itself @@ -109,5 +105,5 @@ func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors // @Failure 500 {object} shared.ApiBody "Internal Error" // @Router /plugins/tapd/connections/{connectionId}/scopes/{scopeId} [DELETE] func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { - return scopeHelper.Delete(input) + return dsHelper.ScopeApi.Delete(input) } diff --git a/backend/plugins/tapd/api/scope_config.go b/backend/plugins/tapd/api/scope_config_api.go similarity index 89% rename from backend/plugins/tapd/api/scope_config.go rename to backend/plugins/tapd/api/scope_config_api.go index 7f64714d0..abca28949 100644 --- a/backend/plugins/tapd/api/scope_config.go +++ b/backend/plugins/tapd/api/scope_config_api.go @@ -33,8 +33,8 @@ import ( // @Failure 400 {object} shared.ApiBody "Bad Request" // @Failure 500 {object} shared.ApiBody "Internal Error" // @Router /plugins/tapd/connections/{connectionId}/scope-configs [POST] -func CreateScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { - return scHelper.Create(input) +func PostScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Post(input) } // UpdateScopeConfig update scope config for Tapd @@ -49,8 +49,8 @@ func CreateScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutpu // @Failure 400 {object} shared.ApiBody "Bad Request" // @Failure 500 {object} shared.ApiBody "Internal Error" // @Router /plugins/tapd/connections/{connectionId}/scope-configs/{id} [PATCH] -func UpdateScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { - return scHelper.Update(input) +func PatchScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { + return dsHelper.ScopeConfigApi.Patch(input) } // GetScopeConfig return one scope config @@ -64,7 +64,7 @@ func UpdateScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutpu // @Failure 500 {object} shared.ApiBody "Internal Error" // @Router /plugins/tapd/connections/{connectionId}/scope-configs/{id} [GET] func GetScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { - return scHelper.Get(input) + return dsHelper.ScopeConfigApi.GetDetail(input) } // GetScopeConfigList return all scope configs @@ -79,7 +79,7 @@ func GetScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, // @Failure 500 {object} shared.ApiBody "Internal Error" // @Router /plugins/tapd/connections/{connectionId}/scope-configs [GET] func GetScopeConfigList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { - return scHelper.List(input) + return dsHelper.ScopeConfigApi.GetAll(input) } // DeleteScopeConfig delete a scope config @@ -93,5 +93,5 @@ func GetScopeConfigList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutp // @Failure 500 {object} shared.ApiBody "Internal Error" // @Router /plugins/tapd/connections/{connectionId}/scope-configs/{id} [DELETE] func DeleteScopeConfig(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) { - return scHelper.Delete(input) + return dsHelper.ScopeConfigApi.Delete(input) } diff --git a/backend/plugins/tapd/impl/impl.go b/backend/plugins/tapd/impl/impl.go index 40835318f..326da6d16 100644 --- a/backend/plugins/tapd/impl/impl.go +++ b/backend/plugins/tapd/impl/impl.go @@ -211,25 +211,11 @@ func (p Tapd) PrepareTaskData(taskCtx plugin.TaskContext, options map[string]int if op.WorkspaceId != 0 { var scope *models.TapdWorkspace - // support v100 & advance mode - // If we still cannot find the record in db, we have to request from remote server and save it to db db := taskCtx.GetDal() err = db.First(&scope, dal.Where("connection_id = ? AND id = ?", op.ConnectionId, op.WorkspaceId)) - if err != nil && db.IsErrorNotFound(err) { - scope, err = api.GetApiWorkspace(op, tapdApiClient) - if err != nil { - return nil, err - } - logger.Debug(fmt.Sprintf("Current workspace: %d", scope.Id)) - err = db.CreateIfNotExist(&scope) - if err != nil { - return nil, err - } - } if err != nil { return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find workspace: %d", op.WorkspaceId)) } - op.WorkspaceId = scope.Id if op.ScopeConfigId == 0 { op.ScopeConfigId = scope.ScopeConfigId } @@ -301,22 +287,19 @@ func (p Tapd) ApiResources() map[string]map[string]plugin.ApiResourceHandler { "connections/:connectionId/scopes/:scopeId/latest-sync-state": { "GET": api.GetScopeLatestSyncState, }, - "connections/:connectionId/remote-scopes-prepare-token": { - "GET": api.PrepareFirstPageToken, - }, "connections/:connectionId/remote-scopes": { "GET": api.RemoteScopes, }, "connections/:connectionId/scopes": { "GET": api.GetScopeList, - "PUT": api.PutScope, + "PUT": api.PutScopes, }, "connections/:connectionId/scope-configs": { - "POST": api.CreateScopeConfig, + "POST": api.PostScopeConfig, "GET": api.GetScopeConfigList, }, "connections/:connectionId/scope-configs/:id": { - "PATCH": api.UpdateScopeConfig, + "PATCH": api.PatchScopeConfig, "GET": api.GetScopeConfig, "DELETE": api.DeleteScopeConfig, }, diff --git a/backend/plugins/tapd/models/connection.go b/backend/plugins/tapd/models/connection.go index 2238d712e..4d26aec67 100644 --- a/backend/plugins/tapd/models/connection.go +++ b/backend/plugins/tapd/models/connection.go @@ -25,6 +25,7 @@ import ( type TapdConn struct { helper.RestConnection `mapstructure:",squash"` helper.BasicAuth `mapstructure:",squash"` + CompanyId uint64 `gorm:"type:BIGINT" mapstructure:"companyId,string" json:"companyId,string" validate:"required"` } func (connection TapdConn) Sanitize() TapdConn { diff --git a/backend/plugins/tapd/models/migrationscripts/register.go b/backend/plugins/tapd/models/migrationscripts/20240415_add_company_id_to_connection.go similarity index 53% copy from backend/plugins/tapd/models/migrationscripts/register.go copy to backend/plugins/tapd/models/migrationscripts/20240415_add_company_id_to_connection.go index cd4f5d3a5..347e2de76 100644 --- a/backend/plugins/tapd/models/migrationscripts/register.go +++ b/backend/plugins/tapd/models/migrationscripts/20240415_add_company_id_to_connection.go @@ -18,20 +18,32 @@ limitations under the License. package migrationscripts import ( + "github.com/apache/incubator-devlake/core/context" + "github.com/apache/incubator-devlake/core/errors" "github.com/apache/incubator-devlake/core/plugin" + "github.com/apache/incubator-devlake/helpers/migrationhelper" ) -// All return all the migration scripts -func All() []plugin.MigrationScript { - return []plugin.MigrationScript{ - new(addInitTables), - new(encodeConnToken), - new(addTransformation), - new(deleteIssue), - new(modifyCustomFieldName), - new(addCustomFieldValue), - new(renameTr2ScopeConfig), - new(addRawParamTableForScope), - new(addConnIdToLabels), - } +var _ plugin.MigrationScript = (*addCompanyIdToConnection)(nil) + +type connection20240415 struct { + CompanyId uint64 +} + +func (connection20240415) TableName() string { + return "_tool_tapd_connections" +} + +type addCompanyIdToConnection struct{} + +func (script *addCompanyIdToConnection) Up(basicRes context.BasicRes) errors.Error { + return migrationhelper.AutoMigrateTables(basicRes, &connection20240415{}) +} + +func (*addCompanyIdToConnection) Version() uint64 { + return 20240415000000 +} + +func (script *addCompanyIdToConnection) Name() string { + return "add CompanyId to Connection" } diff --git a/backend/plugins/tapd/models/migrationscripts/register.go b/backend/plugins/tapd/models/migrationscripts/register.go index cd4f5d3a5..276ab8997 100644 --- a/backend/plugins/tapd/models/migrationscripts/register.go +++ b/backend/plugins/tapd/models/migrationscripts/register.go @@ -33,5 +33,6 @@ func All() []plugin.MigrationScript { new(renameTr2ScopeConfig), new(addRawParamTableForScope), new(addConnIdToLabels), + new(addCompanyIdToConnection), } } diff --git a/backend/plugins/tapd/tasks/task_data.go b/backend/plugins/tapd/tasks/task_data.go index 16b077bbb..467d055a8 100644 --- a/backend/plugins/tapd/tasks/task_data.go +++ b/backend/plugins/tapd/tasks/task_data.go @@ -26,12 +26,12 @@ import ( ) type TapdOptions struct { - ConnectionId uint64 `mapstruct:"connectionId"` - WorkspaceId uint64 `mapstruct:"workspaceId"` - PageSize uint64 `mapstruct:"pageSize"` - CstZone *time.Location - ScopeConfigId uint64 - ScopeConfig *models.TapdScopeConfig `json:"scopeConfig"` + ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId,omitempty"` + WorkspaceId uint64 `json:"workspaceId" mapstructure:"workspaceId,omitempty"` + PageSize uint64 `json:"pageSize,omitempty" mapstructure:"pageSize,omitempty"` + CstZone *time.Location `json:"cstZone,omitempty" mapstructure:"cstZone,omitempty"` + ScopeConfigId uint64 `json:"scopeConfigId,omitempty" mapstructure:"scopeConfigId,omitempty"` + ScopeConfig *models.TapdScopeConfig `json:"scopeConfig,omitempty" mapstructure:"pageSize,omitempty"` } type TapdTaskData struct {
