This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new eb9391413 refactor: gitlab adopts ds_helper (#6173)
eb9391413 is described below
commit eb9391413fb0d0aa2415860907666c85ca277484
Author: Klesh Wong <[email protected]>
AuthorDate: Sun Oct 8 10:43:14 2023 +0800
refactor: gitlab adopts ds_helper (#6173)
* refactor: gitlab adopts ds_helper
* fix: makeplan fails silently
* fix: not _raw_data fields for api output
* fix: connectionId is not picked by maptstructure
* fix: create a empty scopeConfig to avoid nil pointer checking
* refactor: put all data source 6 helpers into ds_helper
* fix: unable to bootstrap
* fix: test failed due to python remote plugin relies on _raw_data_origin
* fix: gitlab bp make plan
* fix: err should be checked immediately
---
backend/core/models/common/base.go | 2 +-
backend/core/plugin/hub.go | 15 +-
backend/core/runner/loader.go | 6 -
backend/helpers/pluginhelper/api/ds_helper.go | 33 ++-
backend/helpers/srvhelper/scope_service_helper.go | 16 ++
backend/plugins/github/api/blueprint_V200_test.go | 194 ----------------
backend/plugins/github/api/blueprint_v200.go | 57 ++---
backend/plugins/github/api/connection_api.go | 10 +-
backend/plugins/github/api/init.go | 10 +-
backend/plugins/github/api/scope_api.go | 10 +-
backend/plugins/github/api/scope_config_api.go | 10 +-
backend/plugins/gitlab/api/blueprint_V200_test.go | 248 ++++++++-------------
backend/plugins/gitlab/api/blueprint_v200.go | 63 +++---
.../api/{connection.go => connection_api.go} | 31 +--
backend/plugins/gitlab/api/init.go | 35 +--
.../plugins/gitlab/api/{scope.go => scope_api.go} | 30 ++-
.../api/{scope_config.go => scope_config_api.go} | 14 +-
backend/plugins/gitlab/impl/impl.go | 6 +-
backend/plugins/gitlab/models/project.go | 2 +-
backend/server/services/blueprint.go | 2 +-
backend/server/services/pipeline.go | 4 +
21 files changed, 259 insertions(+), 539 deletions(-)
diff --git a/backend/core/models/common/base.go
b/backend/core/models/common/base.go
index 0f4ec757c..b7dffd55b 100644
--- a/backend/core/models/common/base.go
+++ b/backend/core/models/common/base.go
@@ -80,7 +80,7 @@ func NewNoPKModel() NoPKModel {
}
type Scope struct {
- NoPKModel `json:"-" mapstructure:"-"`
+ NoPKModel
ConnectionId uint64 `json:"connectionId" gorm:"primaryKey"
validate:"required" mapstructure:"connectionId,omitempty"`
ScopeConfigId uint64 `json:"scopeConfigId,omitempty"
mapstructure:"scopeConfigId,omitempty"`
}
diff --git a/backend/core/plugin/hub.go b/backend/core/plugin/hub.go
index cef374222..911bfb6ba 100644
--- a/backend/core/plugin/hub.go
+++ b/backend/core/plugin/hub.go
@@ -19,8 +19,10 @@ package plugin
import (
"fmt"
- "github.com/apache/incubator-devlake/core/errors"
"strings"
+
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
)
// Allowing plugin to know each other
@@ -69,3 +71,14 @@ func FindPluginNameBySubPkgPath(subPkgPath string) (string,
errors.Error) {
}
return "", errors.Default.New(fmt.Sprintf("Unable to find plugin for
subPkgPath %s", subPkgPath))
}
+
+func InitPlugins(basicRes context.BasicRes) {
+ for pluginName, pluginMeta := range plugins {
+ if pluginEntry, ok := pluginMeta.(PluginInit); ok {
+ err := pluginEntry.Init(basicRes)
+ if err != nil {
+ panic(fmt.Errorf("failed to initialize plugin
%v due to %w", pluginName, err))
+ }
+ }
+ }
+}
diff --git a/backend/core/runner/loader.go b/backend/core/runner/loader.go
index 0690880c7..6e435a172 100644
--- a/backend/core/runner/loader.go
+++ b/backend/core/runner/loader.go
@@ -68,12 +68,6 @@ func LoadGoPlugins(basicRes context.BasicRes) errors.Error {
if !ok {
return errors.Default.New(fmt.Sprintf("%s
PluginEntry must implement PluginMeta interface", pluginName))
}
- if pluginEntry, ok :=
symPluginEntry.(plugin.PluginInit); ok {
- err = pluginEntry.Init(basicRes)
- if err != nil {
- return err
- }
- }
err = plugin.RegisterPlugin(pluginName, pluginMeta)
if err != nil {
return err
diff --git a/backend/helpers/pluginhelper/api/ds_helper.go
b/backend/helpers/pluginhelper/api/ds_helper.go
index bb0aa019f..a18916109 100644
--- a/backend/helpers/pluginhelper/api/ds_helper.go
+++ b/backend/helpers/pluginhelper/api/ds_helper.go
@@ -23,7 +23,20 @@ import (
"github.com/apache/incubator-devlake/helpers/srvhelper"
)
-func NewDataSourceHelpers[
+type DsHelper[
+ C plugin.ToolLayerConnection,
+ S plugin.ToolLayerScope,
+ SC plugin.ToolLayerScopeConfig,
+] struct {
+ ConnSrv *srvhelper.ConnectionSrvHelper[C, S, SC]
+ ConnApi *DsConnectionApiHelper[C, S, SC]
+ ScopeSrv *srvhelper.ScopeSrvHelper[C, S, SC]
+ ScopeApi *DsScopeApiHelper[C, S, SC]
+ ScopeConfigSrv *srvhelper.ScopeConfigSrvHelper[C, S, SC]
+ ScopeConfigApi *DsScopeConfigApiHelper[C, S, SC]
+}
+
+func NewDataSourceHelper[
C plugin.ToolLayerConnection,
S plugin.ToolLayerScope,
SC plugin.ToolLayerScopeConfig,
@@ -31,19 +44,19 @@ func NewDataSourceHelpers[
basicRes context.BasicRes,
pluginName string,
scopeSearchColumns []string,
-) (
- *srvhelper.ConnectionSrvHelper[C, S, SC],
- *DsConnectionApiHelper[C, S, SC],
- *srvhelper.ScopeSrvHelper[C, S, SC],
- *DsScopeApiHelper[C, S, SC],
- *srvhelper.ScopeConfigSrvHelper[C, S, SC],
- *DsScopeConfigApiHelper[C, S, SC],
-) {
+) *DsHelper[C, S, SC] {
connSrv := srvhelper.NewConnectionSrvHelper[C, S, SC](basicRes,
pluginName)
connApi := NewDsConnectionApiHelper[C, S, SC](basicRes, connSrv)
scopeSrv := srvhelper.NewScopeSrvHelper[C, S, SC](basicRes, pluginName,
scopeSearchColumns)
scopeApi := NewDsScopeApiHelper[C, S, SC](basicRes, scopeSrv)
scSrv := srvhelper.NewScopeConfigSrvHelper[C, S, SC](basicRes)
scApi := NewDsScopeConfigApiHelper[C, S, SC](basicRes, scSrv)
- return connSrv, connApi, scopeSrv, scopeApi, scSrv, scApi
+ return &DsHelper[C, S, SC]{
+ ConnSrv: connSrv,
+ ConnApi: connApi,
+ ScopeSrv: scopeSrv,
+ ScopeApi: scopeApi,
+ ScopeConfigSrv: scSrv,
+ ScopeConfigApi: scApi,
+ }
}
diff --git a/backend/helpers/srvhelper/scope_service_helper.go
b/backend/helpers/srvhelper/scope_service_helper.go
index f4fab4ce6..977b420fc 100644
--- a/backend/helpers/srvhelper/scope_service_helper.go
+++ b/backend/helpers/srvhelper/scope_service_helper.go
@@ -95,6 +95,22 @@ func (scopeSrv *ScopeSrvHelper[C, S, SC])
GetScopeDetail(includeBlueprints bool,
return scopeDetail, nil
}
+// MapScopeDetails returns scope details (scope and scopeConfig) for the given
blueprint scopes
+func (scopeSrv *ScopeSrvHelper[C, S, SC]) MapScopeDetails(connectionId uint64,
bpScopes []*models.BlueprintScope) ([]*ScopeDetail[S, SC], errors.Error) {
+ var err errors.Error
+ scopeDetails := make([]*ScopeDetail[S, SC], len(bpScopes))
+ for i, bpScope := range bpScopes {
+ scopeDetails[i], err = scopeSrv.GetScopeDetail(false,
connectionId, bpScope.ScopeId)
+ if err != nil {
+ return nil, err
+ }
+ if scopeDetails[i].ScopeConfig == nil {
+ scopeDetails[i].ScopeConfig = new(SC)
+ }
+ }
+ return scopeDetails, nil
+}
+
func (scopeSrv *ScopeSrvHelper[C, S, SC]) GetScopesPage(pagination
*ScopePagination) ([]*ScopeDetail[S, SC], int64, errors.Error) {
if pagination.ConnectionId < 1 {
return nil, 0, errors.BadInput.New("connectionId is required")
diff --git a/backend/plugins/github/api/blueprint_V200_test.go
b/backend/plugins/github/api/blueprint_V200_test.go
deleted file mode 100644
index 9d3ab4ee0..000000000
--- a/backend/plugins/github/api/blueprint_V200_test.go
+++ /dev/null
@@ -1,194 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package api
-
-import (
- "testing"
-
- "github.com/apache/incubator-devlake/core/dal"
- coreModels "github.com/apache/incubator-devlake/core/models"
- "github.com/apache/incubator-devlake/core/models/common"
- "github.com/apache/incubator-devlake/core/models/domainlayer"
- "github.com/apache/incubator-devlake/core/models/domainlayer/code"
- "github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
- "github.com/apache/incubator-devlake/core/plugin"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "github.com/apache/incubator-devlake/helpers/unithelper"
- mockdal "github.com/apache/incubator-devlake/mocks/core/dal"
- mocks "github.com/apache/incubator-devlake/mocks/core/dal"
- mockplugin "github.com/apache/incubator-devlake/mocks/core/plugin"
- "github.com/apache/incubator-devlake/plugins/github/models"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/mock"
-)
-
-func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
- connection := &models.GithubConnection{
- BaseConnection: helper.BaseConnection{
- Name: "github-test",
- Model: common.Model{
- ID: 1,
- },
- },
- GithubConn: models.GithubConn{
- RestConnection: helper.RestConnection{
- Endpoint: "https://api.github.com/";,
- Proxy: "",
- RateLimitPerHour: 0,
- },
- GithubAccessToken: models.GithubAccessToken{
- AccessToken: helper.AccessToken{
- Token: "123",
- },
- },
- },
- }
- mockMeta := mockplugin.NewPluginMeta(t)
-
mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/github")
- mockMeta.On("Name").Return("github").Maybe()
- err := plugin.RegisterPlugin("github", mockMeta)
- assert.Nil(t, err)
- // Refresh Global Variables and set the sql mock
- mockBasicRes(t)
-
- bs := &coreModels.BlueprintScope{
- ScopeId: "1",
- }
- bpScopes := make([]*coreModels.BlueprintScope, 0)
- bpScopes = append(bpScopes, bs)
-
- plan := make(coreModels.PipelinePlan, len(bpScopes))
- plan, err = makeDataSourcePipelinePlanV200(nil, plan, bpScopes,
connection)
- assert.Nil(t, err)
- scopes, err := makeScopesV200(bpScopes, connection)
- assert.Nil(t, err)
-
- expectPlan := coreModels.PipelinePlan{
- coreModels.PipelineStage{
- {
- Plugin: "github",
- Subtasks: []string{},
- Options: map[string]interface{}{
- "connectionId": uint64(1),
- "githubId": 12345,
- "name": "test/testRepo",
- },
- },
- {
- Plugin: "gitextractor",
- Options: map[string]interface{}{
- "proxy": "",
- "repoId": "github:GithubRepo:1:12345",
- "name": "test/testRepo",
- "url":
"https://git:123@this_is_cloneUrl";,
- },
- },
- },
- coreModels.PipelineStage{
- {
- Plugin: "refdiff",
- Options: map[string]interface{}{
- "repoId":
"github:GithubRepo:1:12345",
- "tagsLimit": 10,
- "tagsOrder": "reverse semver",
- "tagsPattern": "pattern",
- },
- },
- },
- }
- assert.Equal(t, expectPlan, plan)
- expectScopes := make([]plugin.Scope, 0)
- scopeRepo := &code.Repo{
- DomainEntity: domainlayer.DomainEntity{
- Id: "github:GithubRepo:1:12345",
- },
- Name: "test/testRepo",
- }
-
- scopeTicket := &ticket.Board{
- DomainEntity: domainlayer.DomainEntity{
- Id: "github:GithubRepo:1:12345",
- },
- Name: "test/testRepo",
- Description: "",
- Url: "",
- CreatedDate: nil,
- Type: "",
- }
-
- expectScopes = append(expectScopes, scopeRepo, scopeTicket)
- assert.Equal(t, expectScopes, scopes)
-}
-
-func mockBasicRes(t *testing.T) {
- testGithubRepo := &models.GithubRepo{
- Scope: common.Scope{
- ConnectionId: 1,
- ScopeConfigId: 1,
- },
- GithubId: 12345,
- Name: "testRepo",
- FullName: "test/testRepo",
- CloneUrl: "https://this_is_cloneUrl";,
- }
-
- testScopeConfig := &models.GithubScopeConfig{
- ScopeConfig: common.ScopeConfig{
- Model: common.Model{
- ID: 1,
- },
- Entities: []string{"CODE", "TICKET"},
- },
- Name: "github scope config",
- PrType: "hey,man,wasup",
- Refdiff: map[string]interface{}{
- "tagsPattern": "pattern",
- "tagsLimit": 10,
- "tagsOrder": "reverse semver",
- },
- }
- // Refresh Global Variables and set the sql mock
- mockRes := unithelper.DummyBasicRes(func(mockDal *mockdal.Dal) {
- mockDal.On("First", mock.AnythingOfType("*models.GithubRepo"),
mock.Anything).Run(func(args mock.Arguments) {
- dst := args.Get(0).(*models.GithubRepo)
- *dst = *testGithubRepo
- }).Return(nil)
-
- mockDal.On("First",
mock.AnythingOfType("*models.GithubScopeConfig"), mock.Anything).Run(func(args
mock.Arguments) {
- dst := args.Get(0).(*models.GithubScopeConfig)
- *dst = *testScopeConfig
- }).Return(nil)
-
- id := new(mocks.ColumnMeta)
- id.On("PrimayKey").Return(true)
- id.On("Name").Return("id")
- connectionId := new(mocks.ColumnMeta)
- connectionId.On("PrimayKey").Return(true)
- connectionId.On("Name").Return("connection_id")
- githubId := new(mocks.ColumnMeta)
- githubId.On("PrimayKey").Return(true)
- githubId.On("Name").Return("github_id")
-
- mockDal.On("GetColumns",
mock.AnythingOfType("GithubConnection"),
mock.Anything).Return([]dal.ColumnMeta{id}, nil)
- mockDal.On("GetColumns", mock.AnythingOfType("GithubRepo"),
mock.Anything).Return([]dal.ColumnMeta{connectionId, githubId}, nil)
- mockDal.On("GetColumns",
mock.AnythingOfType("GithubScopeConfig"),
mock.Anything).Return([]dal.ColumnMeta{id}, nil)
- })
- p := mockplugin.NewPluginMeta(t)
- p.On("Name").Return("dummy").Maybe()
- Init(mockRes, p)
-}
diff --git a/backend/plugins/github/api/blueprint_v200.go
b/backend/plugins/github/api/blueprint_v200.go
index 0ab128e4b..ee2a977b0 100644
--- a/backend/plugins/github/api/blueprint_v200.go
+++ b/backend/plugins/github/api/blueprint_v200.go
@@ -26,7 +26,6 @@ import (
"net/url"
"strings"
- "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
coreModels "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/models/domainlayer"
@@ -38,6 +37,7 @@ import (
"github.com/apache/incubator-devlake/core/utils"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
aha
"github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
+ "github.com/apache/incubator-devlake/helpers/srvhelper"
"github.com/apache/incubator-devlake/plugins/github/models"
"github.com/apache/incubator-devlake/plugins/github/tasks"
)
@@ -47,9 +47,12 @@ func MakeDataSourcePipelinePlanV200(
connectionId uint64,
bpScopes []*coreModels.BlueprintScope,
) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
- // get the connection info for url
- connection := &models.GithubConnection{}
- err := connectionHelper.FirstById(connection, connectionId)
+ // load connection, scope and scopeConfig from the db
+ connection, err := dsHelper.ConnSrv.FindByPk(connectionId)
+ if err != nil {
+ return nil, nil, err
+ }
+ scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId,
bpScopes)
if err != nil {
return nil, nil, err
}
@@ -61,12 +64,11 @@ func MakeDataSourcePipelinePlanV200(
return nil, nil, err
}
- plan := make(coreModels.PipelinePlan, len(bpScopes))
- plan, err = makeDataSourcePipelinePlanV200(subtaskMetas, plan,
bpScopes, connection)
+ plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails,
connection)
if err != nil {
return nil, nil, err
}
- scopes, err := makeScopesV200(bpScopes, connection)
+ scopes, err := makeScopesV200(scopeDetails, connection)
if err != nil {
return nil, nil, err
}
@@ -76,27 +78,16 @@ func MakeDataSourcePipelinePlanV200(
func makeDataSourcePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
- plan coreModels.PipelinePlan,
- bpScopes []*coreModels.BlueprintScope,
+ scopeDetails []*srvhelper.ScopeDetail[models.GithubRepo,
models.GithubScopeConfig],
connection *models.GithubConnection,
) (coreModels.PipelinePlan, errors.Error) {
- for i, bpScope := range bpScopes {
+ plan := make(coreModels.PipelinePlan, len(scopeDetails))
+ for i, scopeDetail := range scopeDetails {
+ githubRepo, scopeConfig := scopeDetail.Scope,
scopeDetail.ScopeConfig
stage := plan[i]
if stage == nil {
stage = coreModels.PipelineStage{}
}
- // get repo and scope config from db
- githubRepo, err := scopeSrv.FindByPk(connection.ID,
bpScope.ScopeId)
- if err != nil {
- return nil, err
- }
- var scopeConfig *models.GithubScopeConfig
- if githubRepo.ScopeConfigId != 0 {
- scopeConfig, err =
scSrv.FindByPk(githubRepo.ScopeConfigId)
- if err != nil {
- return nil, err
- }
- }
// refdiff
if scopeConfig != nil && scopeConfig.Refdiff != nil {
// add a new task to next stage
@@ -154,23 +145,13 @@ func makeDataSourcePipelinePlanV200(
return plan, nil
}
-func makeScopesV200(bpScopes []*coreModels.BlueprintScope, connection
*models.GithubConnection) ([]plugin.Scope, errors.Error) {
+func makeScopesV200(
+ scopeDetails []*srvhelper.ScopeDetail[models.GithubRepo,
models.GithubScopeConfig],
+ connection *models.GithubConnection,
+) ([]plugin.Scope, errors.Error) {
scopes := make([]plugin.Scope, 0)
- for _, bpScope := range bpScopes {
- githubRepo := &models.GithubRepo{}
- // get repo from db
- err := basicRes.GetDal().First(githubRepo,
dal.Where(`connection_id = ? AND github_id = ?`, connection.ID,
bpScope.ScopeId))
- if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("fail
to find repo%s", bpScope.ScopeId))
- }
-
- scopeConfig := &models.GithubScopeConfig{}
- // get scope configs from db
- db := basicRes.GetDal()
- err = db.First(scopeConfig, dal.Where(`id = ?`,
githubRepo.ScopeConfigId))
- if err != nil && !db.IsErrorNotFound(err) {
- return nil, err
- }
+ for _, scopeDetail := range scopeDetails {
+ githubRepo, scopeConfig := scopeDetail.Scope,
scopeDetail.ScopeConfig
if utils.StringsContains(scopeConfig.Entities,
plugin.DOMAIN_TYPE_CODE_REVIEW) ||
utils.StringsContains(scopeConfig.Entities,
plugin.DOMAIN_TYPE_CODE) ||
diff --git a/backend/plugins/github/api/connection_api.go
b/backend/plugins/github/api/connection_api.go
index 618d39fe0..b2531a17d 100644
--- a/backend/plugins/github/api/connection_api.go
+++ b/backend/plugins/github/api/connection_api.go
@@ -212,7 +212,7 @@ func TestConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/github/connections [POST]
func PostConnections(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- return connApi.Post(input)
+ return dsHelper.ConnApi.Post(input)
}
// @Summary patch github connection
@@ -224,7 +224,7 @@ func PostConnections(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/github/connections/{connectionId} [PATCH]
func PatchConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- return connApi.Patch(input)
+ return dsHelper.ConnApi.Patch(input)
}
// @Summary delete a github connection
@@ -236,7 +236,7 @@ func PatchConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/github/connections/{connectionId} [DELETE]
func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- return connApi.Delete(input)
+ return dsHelper.ConnApi.Delete(input)
}
// @Summary get all github connections
@@ -247,7 +247,7 @@ func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/github/connections [GET]
func ListConnections(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- return connApi.GetAll(input)
+ return dsHelper.ConnApi.GetAll(input)
}
// @Summary get github connection detail
@@ -258,5 +258,5 @@ func ListConnections(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/github/connections/{connectionId} [GET]
func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
- return connApi.GetDetail(input)
+ return dsHelper.ConnApi.GetDetail(input)
}
diff --git a/backend/plugins/github/api/init.go
b/backend/plugins/github/api/init.go
index a976677d9..6f6df7dfa 100644
--- a/backend/plugins/github/api/init.go
+++ b/backend/plugins/github/api/init.go
@@ -22,26 +22,20 @@ import (
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "github.com/apache/incubator-devlake/helpers/srvhelper"
"github.com/apache/incubator-devlake/plugins/github/models"
"github.com/go-playground/validator/v10"
)
var vld *validator.Validate
-// var connSrv *srvhelper.ConnectionSrvHelper[models.GithubConnection,
models.GithubRepo, models.GithubScopeConfig]
-var connApi *api.DsConnectionApiHelper[models.GithubConnection,
models.GithubRepo, models.GithubScopeConfig]
-var scopeSrv *srvhelper.ScopeSrvHelper[models.GithubConnection,
models.GithubRepo, models.GithubScopeConfig]
-var scopeApi *api.DsScopeApiHelper[models.GithubConnection, models.GithubRepo,
models.GithubScopeConfig]
-var scSrv *srvhelper.ScopeConfigSrvHelper[models.GithubConnection,
models.GithubRepo, models.GithubScopeConfig]
-var scApi *api.DsScopeConfigApiHelper[models.GithubConnection,
models.GithubRepo, models.GithubScopeConfig]
+var dsHelper *api.DsHelper[models.GithubConnection, models.GithubRepo,
models.GithubScopeConfig]
var connectionHelper *api.ConnectionApiHelper
var basicRes context.BasicRes
var remoteHelper *api.RemoteApiHelper[models.GithubConnection,
models.GithubRepo, repo, plugin.ApiGroup]
func Init(br context.BasicRes, p plugin.PluginMeta) {
basicRes = br
- _, connApi, scopeSrv, scopeApi, scSrv, scApi = api.NewDataSourceHelpers[
+ dsHelper = api.NewDataSourceHelper[
models.GithubConnection,
models.GithubRepo, models.GithubScopeConfig,
](
diff --git a/backend/plugins/github/api/scope_api.go
b/backend/plugins/github/api/scope_api.go
index 902069416..415adc4a2 100644
--- a/backend/plugins/github/api/scope_api.go
+++ b/backend/plugins/github/api/scope_api.go
@@ -39,7 +39,7 @@ type ScopeDetail api.ScopeDetail[models.GithubRepo,
models.GithubScopeConfig]
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/github/connections/{connectionId}/scopes [PUT]
func PutScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
- return scopeApi.PutMultiple(input)
+ return dsHelper.ScopeApi.PutMultiple(input)
}
// UpdateScope patch to github repo
@@ -55,7 +55,7 @@ func PutScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, error
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/github/connections/{connectionId}/scopes/{scopeId} [PATCH]
func PatchScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
- return scopeApi.Patch(input)
+ return dsHelper.ScopeApi.Patch(input)
}
// GetScopeList get Github repos
@@ -72,7 +72,7 @@ func PatchScope(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, erro
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/github/connections/{connectionId}/scopes [GET]
func GetScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
- return scopeApi.GetPage(input)
+ return dsHelper.ScopeApi.GetPage(input)
}
// GetScope get one Github repo
@@ -87,7 +87,7 @@ func GetScopes(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, error
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/github/connections/{connectionId}/scopes/{scopeId} [GET]
func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
- return scopeApi.GetScopeDetail(input)
+ return dsHelper.ScopeApi.GetScopeDetail(input)
}
// DeleteScope delete plugin data associated with the scope and optionally the
scope itself
@@ -103,5 +103,5 @@ func GetScope(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/github/connections/{connectionId}/scopes/{scopeId} [DELETE]
func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
- return scopeApi.Delete(input)
+ return dsHelper.ScopeApi.Delete(input)
}
diff --git a/backend/plugins/github/api/scope_config_api.go
b/backend/plugins/github/api/scope_config_api.go
index 351eeb24b..57d083066 100644
--- a/backend/plugins/github/api/scope_config_api.go
+++ b/backend/plugins/github/api/scope_config_api.go
@@ -34,7 +34,7 @@ import (
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/github/connections/{connectionId}/scope-configs [POST]
func PostScopeConfig(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- return scApi.Post(input)
+ return dsHelper.ScopeConfigApi.Post(input)
}
// PatchScopeConfig update scope config for Github
@@ -50,7 +50,7 @@ func PostScopeConfig(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/github/connections/{connectionId}/scope-configs/{id}
[PATCH]
func PatchScopeConfig(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- return scApi.Patch(input)
+ return dsHelper.ScopeConfigApi.Patch(input)
}
// GetScopeConfig return one scope config
@@ -64,7 +64,7 @@ func PatchScopeConfig(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/github/connections/{connectionId}/scope-configs/{id} [GET]
func GetScopeConfig(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- return scApi.GetDetail(input)
+ return dsHelper.ScopeConfigApi.GetDetail(input)
}
// GetScopeConfigList return all scope configs
@@ -79,7 +79,7 @@ func GetScopeConfig(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/github/connections/{connectionId}/scope-configs [GET]
func GetScopeConfigList(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- return scApi.GetAll(input)
+ return dsHelper.ScopeConfigApi.GetAll(input)
}
// DeleteScopeConfig delete a scope config
@@ -93,5 +93,5 @@ func GetScopeConfigList(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutp
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/github/connections/{connectionId}/scope-configs/{id}
[DELETE]
func DeleteScopeConfig(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- return scApi.Delete(input)
+ return dsHelper.ScopeConfigApi.Delete(input)
}
diff --git a/backend/plugins/gitlab/api/blueprint_V200_test.go
b/backend/plugins/gitlab/api/blueprint_V200_test.go
index 410d08572..94d667d90 100644
--- a/backend/plugins/gitlab/api/blueprint_V200_test.go
+++ b/backend/plugins/gitlab/api/blueprint_V200_test.go
@@ -18,106 +18,121 @@ limitations under the License.
package api
import (
- "strconv"
"testing"
- "time"
coreModels "github.com/apache/incubator-devlake/core/models"
- mockdal "github.com/apache/incubator-devlake/mocks/core/dal"
mockplugin "github.com/apache/incubator-devlake/mocks/core/plugin"
- "github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/common"
- "github.com/apache/incubator-devlake/core/models/domainlayer/code"
- "github.com/apache/incubator-devlake/core/models/domainlayer/devops"
- "github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/core/plugin"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "github.com/apache/incubator-devlake/helpers/unithelper"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/helpers/srvhelper"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
"github.com/apache/incubator-devlake/plugins/gitlab/tasks"
"github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/mock"
)
-func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
- const testConnectionID uint64 = 1
- const testScopeConfigId uint64 = 2
- const testID int = 37
- const testGitlabEndPoint string = "https://gitlab.com/api/v4/";
- const testHttpUrlToRepo string = "https://this_is_cloneUrl";
- const testToken string = "nddtf"
- const testName string = "gitlab-test"
- const pathWithNamespace string = "nddtf/gitlab-test"
- const testScopeConfigName string = "gitlab scope config"
- const testProxy string = ""
+func mockGitlabPlugin(t *testing.T) {
+ mockMeta := mockplugin.NewPluginMeta(t)
+
mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/gitlab")
+ mockMeta.On("Name").Return("dummy").Maybe()
+ err := plugin.RegisterPlugin("gitlab", mockMeta)
+ assert.Equal(t, err, nil)
+}
- bpScopes := []*coreModels.BlueprintScope{
- {
- ScopeId: strconv.Itoa(testID),
- },
- }
+func TestMakeScopes(t *testing.T) {
+ mockGitlabPlugin(t)
- var testGitlabProject = &models.GitlabProject{
- Scope: common.Scope{
- ConnectionId: testConnectionID,
- ScopeConfigId: testScopeConfigId,
- },
- GitlabId: testID,
- Name: testName,
- PathWithNamespace: pathWithNamespace,
- CreatedDate: &time.Time{},
- HttpUrlToRepo: testHttpUrlToRepo,
- }
+ const connectionId = 1
+ const gitlabProjectId = 37
+ const expectDomainScopeId = "gitlab:GitlabProject:1:37"
- var testScopeConfig = &models.GitlabScopeConfig{
- ScopeConfig: common.ScopeConfig{
- Model: common.Model{
- ID: testScopeConfigId,
+ actualScopes, err := makeScopeV200(
+ connectionId,
+ []*srvhelper.ScopeDetail[models.GitlabProject,
models.GitlabScopeConfig]{
+ {
+ Scope: models.GitlabProject{
+ Scope: common.Scope{
+ ConnectionId: connectionId,
+ },
+ GitlabId: gitlabProjectId,
+ },
+ ScopeConfig: &models.GitlabScopeConfig{
+ ScopeConfig: common.ScopeConfig{
+ Entities:
[]string{plugin.DOMAIN_TYPE_CODE, plugin.DOMAIN_TYPE_TICKET,
plugin.DOMAIN_TYPE_CICD},
+ },
+ },
},
- Entities: []string{plugin.DOMAIN_TYPE_CODE,
plugin.DOMAIN_TYPE_TICKET, plugin.DOMAIN_TYPE_CICD},
},
- Name: testScopeConfigName,
- PrType: "hey,man,wasup",
- Refdiff: map[string]interface{}{
- "tagsPattern": "pattern",
- "tagsLimit": 10,
- "tagsOrder": "reverse semver",
- },
- }
+ )
+ assert.Nil(t, err)
+ assert.Equal(t, 3, len(actualScopes))
+ assert.Equal(t, actualScopes[0].ScopeId(), expectDomainScopeId)
+ assert.Equal(t, actualScopes[1].ScopeId(), expectDomainScopeId)
+ assert.Equal(t, actualScopes[2].ScopeId(), expectDomainScopeId)
+}
- var testGitlabConnection = &models.GitlabConnection{
- BaseConnection: helper.BaseConnection{
- Name: testName,
- Model: common.Model{
- ID: testConnectionID,
- },
+func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
+ mockGitlabPlugin(t)
+
+ const connectionID uint64 = 1
+ const gitlabProjectId = 37
+ const scopeConfigId uint64 = 2
+ const scopeConfigName string = "gitlab scope config"
+ const gitlabEndPoint string = "https://gitlab.com/api/v4/";
+ const httpUrlToRepo string = "https://this_is_cloneUrl";
+ const gitlabToken string = "nddtf"
+ const gitlabProjectName string = "gitlab-test"
+ const pathWithNamespace string = "nddtf/gitlab-test"
+ const expectDomainScopeId = "gitlab:GitlabProject:1:37"
+
+ actualPlans, err := makePipelinePlanV200(
+ []plugin.SubTaskMeta{
+ tasks.ConvertProjectMeta,
+ tasks.CollectApiIssuesMeta,
+ tasks.ExtractApiIssuesMeta,
+ tasks.ConvertIssuesMeta,
+ tasks.ConvertIssueLabelsMeta,
+ tasks.CollectApiJobsMeta,
+ tasks.ExtractApiJobsMeta,
+ tasks.CollectApiPipelinesMeta,
+ tasks.ExtractApiPipelinesMeta,
},
- GitlabConn: models.GitlabConn{
- RestConnection: helper.RestConnection{
- Endpoint: testGitlabEndPoint,
- Proxy: testProxy,
- RateLimitPerHour: 0,
+ &models.GitlabConnection{
+ BaseConnection: api.BaseConnection{
+ Model: common.Model{
+ ID: connectionID,
+ },
},
- AccessToken: helper.AccessToken{
- Token: testToken,
+ GitlabConn: models.GitlabConn{
+ RestConnection: api.RestConnection{},
+ AccessToken: api.AccessToken{
+ Token: gitlabToken,
+ },
},
},
- }
-
- var expectRepoId = "gitlab:GitlabProject:1:37"
-
- var testSubTaskMeta = []plugin.SubTaskMeta{
- tasks.ConvertProjectMeta,
- tasks.CollectApiIssuesMeta,
- tasks.ExtractApiIssuesMeta,
- tasks.ConvertIssuesMeta,
- tasks.ConvertIssueLabelsMeta,
- tasks.CollectApiJobsMeta,
- tasks.ExtractApiJobsMeta,
- tasks.CollectApiPipelinesMeta,
- tasks.ExtractApiPipelinesMeta,
- }
+ []*srvhelper.ScopeDetail[models.GitlabProject,
models.GitlabScopeConfig]{
+ {
+ Scope: models.GitlabProject{
+ GitlabId: gitlabProjectId,
+ Name: gitlabProjectName,
+ HttpUrlToRepo: httpUrlToRepo,
+ },
+ ScopeConfig: &models.GitlabScopeConfig{
+ ScopeConfig: common.ScopeConfig{
+ Entities:
[]string{plugin.DOMAIN_TYPE_CODE, plugin.DOMAIN_TYPE_TICKET,
plugin.DOMAIN_TYPE_CICD},
+ },
+ PrType: "hey,man,wasup",
+ Refdiff: map[string]interface{}{
+ "tagsPattern": "pattern",
+ "tagsLimit": 10,
+ "tagsOrder": "reverse semver",
+ },
+ },
+ },
+ },
+ )
+ assert.Nil(t, err)
var expectPlans = coreModels.PipelinePlan{
{
@@ -135,17 +150,16 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
tasks.ExtractApiPipelinesMeta.Name,
},
Options: map[string]interface{}{
- "connectionId": uint64(1),
- "projectId": testID,
- "scopeConfigId": testScopeConfigId,
+ "connectionId": connectionID,
+ "projectId": gitlabProjectId,
},
},
{
Plugin: "gitextractor",
Options: map[string]interface{}{
"proxy": "",
- "repoId": expectRepoId,
- "name": testName,
+ "repoId": expectDomainScopeId,
+ "name": gitlabProjectName,
"url":
"https://git:nddtf@this_is_cloneUrl";,
},
},
@@ -162,71 +176,5 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
},
}
- expectRepo := code.NewRepo(expectRepoId, pathWithNamespace)
- expectRepo.ForkedFrom = testGitlabProject.ForkedFromProjectWebUrl
-
- expectCicdScope := devops.NewCicdScope(expectRepoId, pathWithNamespace)
- expectCicdScope.Description = ""
- expectCicdScope.Url = ""
-
- expectBoard := ticket.NewBoard(expectRepoId, pathWithNamespace)
- expectBoard.Description = ""
- expectBoard.Url = ""
- expectBoard.Type = ""
-
- var err errors.Error
-
- // register gitlab coreModels for NewDomainIdGenerator
- mockMeta := mockplugin.NewPluginMeta(t)
-
mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/gitlab")
- mockMeta.On("Name").Return("dummy").Maybe()
- err = plugin.RegisterPlugin("gitlab", mockMeta)
- assert.Equal(t, err, nil)
-
- // Refresh Global Variables and set the sql mock
- mockRes := unithelper.DummyBasicRes(func(mockDal *mockdal.Dal) {
- mockDal.On("First",
mock.AnythingOfType("*models.GitlabConnection"), mock.Anything).Run(func(args
mock.Arguments) {
- dst := args.Get(0).(*models.GitlabConnection)
- *dst = *testGitlabConnection
- }).Return(nil)
-
- mockDal.On("First",
mock.AnythingOfType("*models.GitlabProject"), mock.Anything).Run(func(args
mock.Arguments) {
- dst := args.Get(0).(*models.GitlabProject)
- *dst = *testGitlabProject
- }).Return(nil)
-
- mockDal.On("First",
mock.AnythingOfType("*models.GitlabScopeConfig"), mock.Anything).Run(func(args
mock.Arguments) {
- dst := args.Get(0).(*models.GitlabScopeConfig)
- *dst = *testScopeConfig
- }).Return(nil)
- })
- Init(mockRes, mockMeta)
-
- plans, scopes, err := MakePipelinePlanV200(testSubTaskMeta,
testConnectionID, bpScopes)
- assert.Equal(t, err, nil)
-
- assert.Equal(t, expectPlans, plans)
-
- // ignore CreatedDate UpdatedDate CreatedAt UpdatedAt checking
- expectRepo.CreatedDate = scopes[0].(*code.Repo).CreatedDate
- expectRepo.UpdatedDate = scopes[0].(*code.Repo).UpdatedDate
- expectRepo.CreatedAt = scopes[0].(*code.Repo).CreatedAt
- expectRepo.UpdatedAt = scopes[0].(*code.Repo).UpdatedAt
-
- expectCicdScope.CreatedDate = scopes[1].(*devops.CicdScope).CreatedDate
- expectCicdScope.UpdatedDate = scopes[1].(*devops.CicdScope).UpdatedDate
- expectCicdScope.CreatedAt = scopes[1].(*devops.CicdScope).CreatedAt
- expectCicdScope.UpdatedAt = scopes[1].(*devops.CicdScope).UpdatedAt
-
- expectBoard.CreatedDate = scopes[2].(*ticket.Board).CreatedDate
- expectBoard.CreatedAt = scopes[2].(*ticket.Board).CreatedAt
- expectBoard.UpdatedAt = scopes[2].(*ticket.Board).UpdatedAt
-
- var expectScopes = []plugin.Scope{
- expectRepo,
- expectCicdScope,
- expectBoard,
- }
-
- assert.Equal(t, expectScopes, scopes)
+ assert.Equal(t, expectPlans, actualPlans)
}
diff --git a/backend/plugins/gitlab/api/blueprint_v200.go
b/backend/plugins/gitlab/api/blueprint_v200.go
index 7b6fba3cb..31f283a40 100644
--- a/backend/plugins/gitlab/api/blueprint_v200.go
+++ b/backend/plugins/gitlab/api/blueprint_v200.go
@@ -23,7 +23,6 @@ import (
"io"
"net/http"
"net/url"
- "strconv"
"github.com/apache/incubator-devlake/plugins/gitlab/tasks"
@@ -38,27 +37,31 @@ import (
plugin "github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
aha
"github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
+ "github.com/apache/incubator-devlake/helpers/srvhelper"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
func MakePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
connectionId uint64,
- scope []*coreModels.BlueprintScope,
+ bpScopes []*coreModels.BlueprintScope,
) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
- var err errors.Error
- connection := new(models.GitlabConnection)
- err1 := connectionHelper.FirstById(connection, connectionId)
- if err1 != nil {
- return nil, nil, errors.Default.Wrap(err1, fmt.Sprintf("error
on get connection by id[%d]", connectionId))
+ // load connection, scope and scopeConfig from the db
+ connection, err := dsHelper.ConnSrv.FindByPk(connectionId)
+ if err != nil {
+ return nil, nil, err
+ }
+ scopeDetails, err := dsHelper.ScopeApi.MapScopeDetails(connectionId,
bpScopes)
+ if err != nil {
+ return nil, nil, err
}
- sc, err := makeScopeV200(connectionId, scope)
+ sc, err := makeScopeV200(connectionId, scopeDetails)
if err != nil {
return nil, nil, err
}
- pp, err := makePipelinePlanV200(subtaskMetas, scope, connection)
+ pp, err := makePipelinePlanV200(subtaskMetas, connection, scopeDetails)
if err != nil {
return nil, nil, err
}
@@ -66,21 +69,15 @@ func MakePipelinePlanV200(
return pp, sc, nil
}
-func makeScopeV200(connectionId uint64, scopes []*coreModels.BlueprintScope)
([]plugin.Scope, errors.Error) {
- sc := make([]plugin.Scope, 0, 3*len(scopes))
+func makeScopeV200(
+ connectionId uint64,
+ scopeDetails []*srvhelper.ScopeDetail[models.GitlabProject,
models.GitlabScopeConfig],
+) ([]plugin.Scope, errors.Error) {
+ sc := make([]plugin.Scope, 0, 3*len(scopeDetails))
- for _, scope := range scopes {
- intScopeId, err1 := strconv.Atoi(scope.ScopeId)
- if err1 != nil {
- return nil, errors.Default.Wrap(err1,
fmt.Sprintf("Failed to strconv.Atoi for scope.Id [%s]", scope.ScopeId))
- }
- id :=
didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connectionId,
intScopeId)
-
- // get repo from db
- gitlabProject, scopeConfig, err :=
scopeHelper.DbHelper().GetScopeAndConfig(connectionId, scope.ScopeId)
- if err != nil {
- return nil, err
- }
+ for _, scope := range scopeDetails {
+ gitlabProject, scopeConfig := scope.Scope, scope.ScopeConfig
+ id :=
didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connectionId,
gitlabProject.GitlabId)
if utils.StringsContains(scopeConfig.Entities,
plugin.DOMAIN_TYPE_CODE_REVIEW) ||
utils.StringsContains(scopeConfig.Entities,
plugin.DOMAIN_TYPE_CODE) {
@@ -111,30 +108,20 @@ func makeScopeV200(connectionId uint64, scopes
[]*coreModels.BlueprintScope) ([]
func makePipelinePlanV200(
subtaskMetas []plugin.SubTaskMeta,
- scopes []*coreModels.BlueprintScope,
connection *models.GitlabConnection,
+ scopeDetails []*srvhelper.ScopeDetail[models.GitlabProject,
models.GitlabScopeConfig],
) (coreModels.PipelinePlan, errors.Error) {
- plans := make(coreModels.PipelinePlan, 0, 3*len(scopes))
- for _, scope := range scopes {
+ plans := make(coreModels.PipelinePlan, 0, 3*len(scopeDetails))
+ for _, scope := range scopeDetails {
+ gitlabProject, scopeConfig := scope.Scope, scope.ScopeConfig
var stage coreModels.PipelineStage
var err errors.Error
// get repo
- gitlabProject, scopeConfig, err :=
scopeHelper.DbHelper().GetScopeAndConfig(connection.ID, scope.ScopeId)
- if err != nil {
- return nil, err
- }
-
- // get int scopeId
- intScopeId, err1 := strconv.Atoi(scope.ScopeId)
- if err != nil {
- return nil, errors.Default.Wrap(err1,
fmt.Sprintf("Failed to strconv.Atoi for scope.Id [%s]", scope.ScopeId))
- }
// gitlab main part
options := make(map[string]interface{})
options["connectionId"] = connection.ID
- options["projectId"] = intScopeId
- options["scopeConfigId"] = scopeConfig.ID
+ options["projectId"] = gitlabProject.GitlabId
// construct subtasks
subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas,
scopeConfig.Entities)
diff --git a/backend/plugins/gitlab/api/connection.go
b/backend/plugins/gitlab/api/connection_api.go
similarity index 84%
rename from backend/plugins/gitlab/api/connection.go
rename to backend/plugins/gitlab/api/connection_api.go
index 3de78ffa7..ef00ac349 100644
--- a/backend/plugins/gitlab/api/connection.go
+++ b/backend/plugins/gitlab/api/connection_api.go
@@ -90,13 +90,7 @@ func TestConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/gitlab/connections [POST]
func PostConnections(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- // update from request and save to database
- connection := &models.GitlabConnection{}
- err := connectionHelper.Create(connection, input)
- if err != nil {
- return nil, err
- }
- return &plugin.ApiResourceOutput{Body: connection, Status:
http.StatusOK}, nil
+ return dsHelper.ConnApi.Post(input)
}
// @Summary patch gitlab connection
@@ -108,12 +102,7 @@ func PostConnections(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/gitlab/connections/{connectionId} [PATCH]
func PatchConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- connection := &models.GitlabConnection{}
- err := connectionHelper.Patch(connection, input)
- if err != nil {
- return nil, err
- }
- return &plugin.ApiResourceOutput{Body: connection}, nil
+ return dsHelper.ConnApi.Patch(input)
}
// @Summary delete a gitlab connection
@@ -125,7 +114,7 @@ func PatchConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/gitlab/connections/{connectionId} [DELETE]
func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- return connectionHelper.Delete(&models.GitlabConnection{}, input)
+ return dsHelper.ConnApi.Delete(input)
}
// @Summary get all gitlab connections
@@ -136,12 +125,7 @@ func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/gitlab/connections [GET]
func ListConnections(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- var connections []models.GitlabConnection
- err := connectionHelper.List(&connections)
- if err != nil {
- return nil, err
- }
- return &plugin.ApiResourceOutput{Body: connections, Status:
http.StatusOK}, nil
+ return dsHelper.ConnApi.GetAll(input)
}
// @Summary get gitlab connection detail
@@ -152,10 +136,5 @@ func ListConnections(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/gitlab/connections/{connectionId} [GET]
func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
- connection := &models.GitlabConnection{}
- err := connectionHelper.First(connection, input.Params)
- if err != nil {
- return nil, err
- }
- return &plugin.ApiResourceOutput{Body: connection}, err
+ return dsHelper.ConnApi.GetDetail(input)
}
diff --git a/backend/plugins/gitlab/api/init.go
b/backend/plugins/gitlab/api/init.go
index 3637c03d1..4a88534c4 100644
--- a/backend/plugins/gitlab/api/init.go
+++ b/backend/plugins/gitlab/api/init.go
@@ -26,44 +26,33 @@ import (
)
var vld *validator.Validate
+
+var dsHelper *api.DsHelper[models.GitlabConnection, models.GitlabProject,
models.GitlabScopeConfig]
+
var connectionHelper *api.ConnectionApiHelper
-var scopeHelper *api.ScopeApiHelper[models.GitlabConnection,
models.GitlabProject, models.GitlabScopeConfig]
var remoteHelper *api.RemoteApiHelper[models.GitlabConnection,
models.GitlabProject, models.GitlabApiProject, models.GroupResponse]
var basicRes context.BasicRes
-var scHelper *api.ScopeConfigHelper[models.GitlabScopeConfig]
func Init(br context.BasicRes, p plugin.PluginMeta) {
-
basicRes = br
+ dsHelper = api.NewDataSourceHelper[
+ models.GitlabConnection,
+ models.GitlabProject, models.GitlabScopeConfig,
+ ](
+ br,
+ p.Name(),
+ []string{"name"},
+ )
+ // TODO: remove connectionHelper and refactor remoteHelper
vld = validator.New()
connectionHelper = api.NewConnectionHelper(
basicRes,
vld,
p.Name(),
)
- params := &api.ReflectionParameters{
- ScopeIdFieldName: "GitlabId",
- ScopeIdColumnName: "gitlab_id",
- RawScopeParamName: "ProjectId",
- SearchScopeParamName: "name",
- }
- scopeHelper = api.NewScopeHelper[models.GitlabConnection,
models.GitlabProject, models.GitlabScopeConfig](
- basicRes,
- vld,
- connectionHelper,
- api.NewScopeDatabaseHelperImpl[models.GitlabConnection,
models.GitlabProject, models.GitlabScopeConfig](
- basicRes, connectionHelper, params),
- params,
- nil,
- )
remoteHelper = api.NewRemoteHelper[models.GitlabConnection,
models.GitlabProject, models.GitlabApiProject, models.GroupResponse](
basicRes,
vld,
connectionHelper,
)
- scHelper = api.NewScopeConfigHelper[models.GitlabScopeConfig](
- basicRes,
- vld,
- p.Name(),
- )
}
diff --git a/backend/plugins/gitlab/api/scope.go
b/backend/plugins/gitlab/api/scope_api.go
similarity index 85%
rename from backend/plugins/gitlab/api/scope.go
rename to backend/plugins/gitlab/api/scope_api.go
index 42ceb7610..ebdb2e2eb 100644
--- a/backend/plugins/gitlab/api/scope.go
+++ b/backend/plugins/gitlab/api/scope_api.go
@@ -24,29 +24,25 @@ import (
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
-type ScopeRes struct {
- models.GitlabProject
- api.ScopeResDoc[models.GitlabScopeConfig]
-}
-
-type ScopeReq api.ScopeReq[models.GitlabProject]
+type PutScopesReqBody api.PutScopesReqBody[models.GitlabProject]
+type ScopeDetail api.ScopeDetail[models.GitlabProject,
models.GitlabScopeConfig]
-// PutScope create or update gitlab project
+// PutScopes create or update gitlab project
// @Summary create or update gitlab project
// @Description Create or update gitlab project
// @Tags plugins/gitlab
// @Accept application/json
// @Param connectionId path int false "connection ID"
-// @Param scope body ScopeReq true "json"
+// @Param scope body PutScopesReqBody true "json"
// @Success 200 {object} []models.GitlabProject
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/gitlab/connections/{connectionId}/scopes [PUT]
-func PutScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
- return scopeHelper.Put(input)
+func PutScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
+ return dsHelper.ScopeApi.PutMultiple(input)
}
-// UpdateScope patch to gitlab project
+// PatchScope patch to gitlab project
// @Summary patch to gitlab project
// @Description patch to gitlab project
// @Tags plugins/gitlab
@@ -58,8 +54,8 @@ func PutScope(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/gitlab/connections/{connectionId}/scopes/{scopeId} [PATCH]
-func UpdateScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
- return scopeHelper.Update(input)
+func PatchScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
+ return dsHelper.ScopeApi.Patch(input)
}
// GetScopeList get Gitlab projects
@@ -69,12 +65,12 @@ func UpdateScope(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, err
// @Param connectionId path int false "connection ID"
// @Param searchTerm query string false "search term for scope name"
// @Param blueprints query bool false "also return blueprints using these
scopes as part of the payload"
-// @Success 200 {object} []ScopeRes
+// @Success 200 {object} []ScopeDetail
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/gitlab/connections/{connectionId}/scopes/ [GET]
func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
- return scopeHelper.GetScopeList(input)
+ return dsHelper.ScopeApi.GetPage(input)
}
// GetScope get one Gitlab project
@@ -90,7 +86,7 @@ func GetScopeList(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, er
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/gitlab/connections/{connectionId}/scopes/{scopeId} [GET]
func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
- return scopeHelper.GetScope(input)
+ return dsHelper.ScopeApi.GetScopeDetail(input)
}
// DeleteScope delete plugin data associated with the scope and optionally the
scope itself
@@ -106,5 +102,5 @@ func GetScope(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/gitlab/connections/{connectionId}/scopes/{scopeId} [DELETE]
func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
- return scopeHelper.Delete(input)
+ return dsHelper.ScopeApi.Delete(input)
}
diff --git a/backend/plugins/gitlab/api/scope_config.go
b/backend/plugins/gitlab/api/scope_config_api.go
similarity index 91%
rename from backend/plugins/gitlab/api/scope_config.go
rename to backend/plugins/gitlab/api/scope_config_api.go
index 4ef4ed700..4ed8b36c0 100644
--- a/backend/plugins/gitlab/api/scope_config.go
+++ b/backend/plugins/gitlab/api/scope_config_api.go
@@ -34,10 +34,10 @@ import (
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/gitlab/connections/{connectionId}/scope-configs [POST]
func CreateScopeConfig(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- return scHelper.Create(input)
+ return dsHelper.ScopeConfigApi.Post(input)
}
-// UpdateScopeConfig update scope config for Gitlab
+// PatchScopeConfig update scope config for Gitlab
// @Summary update scope config for Gitlab
// @Description update scope config for Gitlab
// @Tags plugins/gitlab
@@ -49,8 +49,8 @@ func CreateScopeConfig(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutpu
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/gitlab/connections/{connectionId}/scope-configs/{id}
[PATCH]
-func UpdateScopeConfig(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- return scHelper.Update(input)
+func PatchScopeConfig(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ return dsHelper.ScopeConfigApi.Patch(input)
}
// GetScopeConfig return one scope config
@@ -64,7 +64,7 @@ func UpdateScopeConfig(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutpu
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/gitlab/connections/{connectionId}/scope-configs/{id} [GET]
func GetScopeConfig(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- return scHelper.Get(input)
+ return dsHelper.ScopeConfigApi.GetDetail(input)
}
// GetScopeConfigList return all scope configs
@@ -79,7 +79,7 @@ func GetScopeConfig(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/gitlab/connections/{connectionId}/scope-configs [GET]
func GetScopeConfigList(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- return scHelper.List(input)
+ return dsHelper.ScopeConfigApi.GetAll(input)
}
// DeleteScopeConfig delete a scope config
@@ -93,5 +93,5 @@ func GetScopeConfigList(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutp
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/gitlab/connections/{connectionId}/scope-configs/{id}
[DELETE]
func DeleteScopeConfig(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- return scHelper.Delete(input)
+ return dsHelper.ScopeConfigApi.Delete(input)
}
diff --git a/backend/plugins/gitlab/impl/impl.go
b/backend/plugins/gitlab/impl/impl.go
index 1233802cd..5b1f79dc5 100644
--- a/backend/plugins/gitlab/impl/impl.go
+++ b/backend/plugins/gitlab/impl/impl.go
@@ -229,7 +229,7 @@ func (p Gitlab) ApiResources()
map[string]map[string]plugin.ApiResourceHandler {
},
"connections/:connectionId/scopes/:scopeId": {
"GET": api.GetScope,
- "PATCH": api.UpdateScope,
+ "PATCH": api.PatchScope,
"DELETE": api.DeleteScope,
},
"connections/:connectionId/remote-scopes": {
@@ -240,14 +240,14 @@ func (p Gitlab) ApiResources()
map[string]map[string]plugin.ApiResourceHandler {
},
"connections/:connectionId/scopes": {
"GET": api.GetScopeList,
- "PUT": api.PutScope,
+ "PUT": api.PutScopes,
},
"connections/:connectionId/scope-configs": {
"POST": api.CreateScopeConfig,
"GET": api.GetScopeConfigList,
},
"connections/:connectionId/scope-configs/:id": {
- "PATCH": api.UpdateScopeConfig,
+ "PATCH": api.PatchScopeConfig,
"GET": api.GetScopeConfig,
"DELETE": api.DeleteScopeConfig,
},
diff --git a/backend/plugins/gitlab/models/project.go
b/backend/plugins/gitlab/models/project.go
index 5ea5d0f62..0ed8ccad5 100644
--- a/backend/plugins/gitlab/models/project.go
+++ b/backend/plugins/gitlab/models/project.go
@@ -29,7 +29,7 @@ import (
var _ plugin.ToolLayerScope = (*GitlabProject)(nil)
type GitlabProject struct {
- common.Scope
+ common.Scope `mapstructure:",squash" gorm:"embedded"`
GitlabId int `json:"gitlabId"
mapstructure:"gitlabId" validate:"required" gorm:"primaryKey"`
Name string `json:"name" mapstructure:"name"
gorm:"type:varchar(255)"`
Description string `json:"description"
mapstructure:"description"`
diff --git a/backend/server/services/blueprint.go
b/backend/server/services/blueprint.go
index ab4097a95..e295e694b 100644
--- a/backend/server/services/blueprint.go
+++ b/backend/server/services/blueprint.go
@@ -163,7 +163,7 @@ func validateBlueprintAndMakePlan(blueprint
*models.Blueprint) errors.Error {
} else if blueprint.Mode == models.BLUEPRINT_MODE_NORMAL {
var e errors.Error
blueprint.Plan, e = MakePlanForBlueprint(blueprint,
&blueprint.SyncPolicy)
- if err != nil {
+ if e != nil {
return e
}
}
diff --git a/backend/server/services/pipeline.go
b/backend/server/services/pipeline.go
index 97e6d06e0..0855341dd 100644
--- a/backend/server/services/pipeline.go
+++ b/backend/server/services/pipeline.go
@@ -29,6 +29,7 @@ import (
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models"
+ "github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/core/utils"
"github.com/apache/incubator-devlake/helpers/dbhelper"
"github.com/apache/incubator-devlake/impls/logruslog"
@@ -49,6 +50,9 @@ type PipelineQuery struct {
}
func pipelineServiceInit() {
+ // initilize plugin
+ plugin.InitPlugins(basicRes)
+
// notification
var notificationEndpoint = cfg.GetString("NOTIFICATION_ENDPOINT")
var notificationSecret = cfg.GetString("NOTIFICATION_SECRET")
