This is an automated email from the ASF dual-hosted git repository.
mappjzc pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 06170b3df refactor: clean up for scope config (#5361)
06170b3df is described below
commit 06170b3df7ab944299e1a205aae158a4bd92ec53
Author: Klesh Wong <[email protected]>
AuthorDate: Mon Jun 5 15:21:51 2023 +0800
refactor: clean up for scope config (#5361)
* refactor: clean up for scope config
1. delete bpv100 code
2. all plugins are using scopeHelper to load scope/scope_config
3. remove redundant variables. e.g., connectionHelper
* fix: rename api scope_config to scope-config
---
backend/core/models/dynamic_tabler.go | 7 +-
backend/core/plugin/plugin_blueprint.go | 26 ---
backend/core/plugin/plugin_meta.go | 12 +-
.../helpers/pluginhelper/api/graphql_collector.go | 9 +-
.../pluginhelper/api/scope_config_helper.go | 16 +-
.../helpers/pluginhelper/api/scope_db_helper.go | 39 +++-
.../pluginhelper/api/scope_generic_helper.go | 50 ++---
.../helpers/pluginhelper/api/scope_helper_test.go | 97 +++++-----
.../pluginhelper/api/transformation_rule_helper.go | 132 -------------
backend/helpers/unithelper/dummy_baesres.go | 1 +
backend/plugins/bamboo/api/blueprint_V200_test.go | 28 +--
backend/plugins/bamboo/api/blueprint_v200.go | 51 +----
backend/plugins/bamboo/impl/impl.go | 12 +-
.../plugins/bitbucket/api/blueprint_V200_test.go | 38 ++--
backend/plugins/bitbucket/api/blueprint_v200.go | 28 +--
backend/plugins/bitbucket/impl/impl.go | 8 +-
backend/plugins/customize/api/blueprint.go | 52 ------
backend/plugins/customize/impl/impl.go | 3 -
.../dora/e2e/connect_incident_to_deploy_test.go | 3 -
backend/plugins/dora/tasks/task_data.go | 13 +-
backend/plugins/gitee/api/blueprint.go | 208 ---------------------
backend/plugins/gitee/api/blueprint_test.go | 141 --------------
backend/plugins/gitee/api/swagger.go | 12 +-
backend/plugins/gitee/impl/impl.go | 36 ----
backend/plugins/gitee/models/connection.go | 2 +-
backend/plugins/gitee/tasks/issue_extractor.go | 5 +-
backend/plugins/gitee/tasks/pr_extractor.go | 5 +-
backend/plugins/gitee/tasks/pr_issue_enricher.go | 11 +-
backend/plugins/gitee/tasks/task_data.go | 41 +---
backend/plugins/github/api/blueprint_V200_test.go | 46 ++---
backend/plugins/github/api/blueprint_v200.go | 15 +-
backend/plugins/github/impl/impl.go | 7 +-
backend/plugins/github_graphql/impl/impl.go | 3 +-
backend/plugins/gitlab/api/blueprint_V200_test.go | 8 +-
backend/plugins/gitlab/api/blueprint_v200.go | 59 +-----
backend/plugins/gitlab/impl/impl.go | 6 +-
backend/plugins/jenkins/api/blueprint_v200.go | 39 +---
backend/plugins/jenkins/api/blueprint_v200_test.go | 40 ++--
backend/plugins/jenkins/impl/impl.go | 7 +-
backend/plugins/jira/api/blueprint_v200.go | 38 +---
backend/plugins/jira/api/blueprint_v200_test.go | 39 ++--
backend/plugins/jira/impl/impl.go | 6 +-
backend/plugins/pagerduty/api/blueprint_v200.go | 31 +--
backend/plugins/pagerduty/impl/impl.go | 4 +-
backend/plugins/sonarqube/api/blueprint.go | 66 -------
backend/plugins/sonarqube/api/blueprint_v200.go | 19 +-
.../plugins/sonarqube/api/blueprint_v200_test.go | 6 +-
backend/plugins/sonarqube/impl/impl.go | 2 +-
backend/plugins/tapd/api/blueprint_v200.go | 37 +---
backend/plugins/tapd/api/blueprint_v200_test.go | 40 ++--
backend/plugins/tapd/impl/impl.go | 4 +-
backend/plugins/teambition/api/blueprint200.go | 22 +--
backend/plugins/trello/api/blueprint_v200.go | 30 +--
backend/plugins/trello/impl/impl.go | 4 +-
backend/plugins/zentao/api/blueprint_V200_test.go | 73 ++++----
backend/plugins/zentao/api/blueprint_v200.go | 25 ++-
backend/plugins/zentao/api/init.go | 4 +
backend/plugins/zentao/impl/impl.go | 8 +
backend/server/services/blueprint.go | 3 +-
backend/server/services/blueprint_makeplan_v100.go | 77 --------
backend/server/services/remote/plugin/scope_api.go | 6 +-
.../services/remote/plugin/scope_db_helper.go | 10 +-
backend/test/e2e/remote/helper.go | 10 +-
backend/test/helper/api.go | 23 ---
64 files changed, 423 insertions(+), 1480 deletions(-)
diff --git a/backend/core/models/dynamic_tabler.go
b/backend/core/models/dynamic_tabler.go
index 2efd836ac..9c3c5cf8a 100644
--- a/backend/core/models/dynamic_tabler.go
+++ b/backend/core/models/dynamic_tabler.go
@@ -19,9 +19,10 @@ package models
import (
"encoding/json"
- "github.com/apache/incubator-devlake/core/errors"
"reflect"
+ "github.com/apache/incubator-devlake/core/errors"
+
"github.com/apache/incubator-devlake/core/dal"
)
@@ -86,6 +87,10 @@ func (d *DynamicTabler) Unwrap() any {
return d.wrapped
}
+func (d *DynamicTabler) UnwrapPtr() *any {
+ return &d.wrapped
+}
+
func (d *DynamicTabler) UnwrapSlice() []any {
var arr []any
slice := reflect.ValueOf(d.wrapped).Elem()
diff --git a/backend/core/plugin/plugin_blueprint.go
b/backend/core/plugin/plugin_blueprint.go
index 928dd5096..68aa1c51c 100644
--- a/backend/core/plugin/plugin_blueprint.go
+++ b/backend/core/plugin/plugin_blueprint.go
@@ -24,32 +24,6 @@ import (
"github.com/apache/incubator-devlake/core/errors"
)
-// PluginBlueprintV100 is used to support Blueprint Normal model, for Plugin
and Blueprint to
-// collaboarte and generate a sophisticated Pipeline Plan based on User
Settings.
-// V100 doesn't support Project, and being deprecated, please use
PluginBlueprintV200 instead
-type PluginBlueprintV100 interface {
- // MakePipelinePlan generates `pipeline.tasks` based on `version` and
`scope`
- //
- // `version` semver from `blueprint.settings.version`
- // `scope` arbitrary json.RawMessage, depends on `version`, for v1.0.0,
it is an Array of Objects
- MakePipelinePlan(connectionId uint64, scope []*BlueprintScopeV100)
(PipelinePlan, errors.Error)
-}
-
-// BlueprintConnectionV100 is the connection definition for protocol v1.0.0
-type BlueprintConnectionV100 struct {
- Plugin string `json:"plugin" validate:"required"`
- ConnectionId uint64 `json:"connectionId"
validate:"required"`
- SkipOnFail bool `json:"skipOnFail"`
- Scope []*BlueprintScopeV100 `json:"scope" validate:"required"`
-}
-
-// BlueprintScopeV100 is the scope definition for protocol v1.0.0
-type BlueprintScopeV100 struct {
- Entities []string `json:"entities"`
- Options json.RawMessage `json:"options"`
- Transformation json.RawMessage `json:"transformation"`
-}
-
/*
PluginBlueprintV200 for project support
diff --git a/backend/core/plugin/plugin_meta.go
b/backend/core/plugin/plugin_meta.go
index 37c135966..a261ac9b4 100644
--- a/backend/core/plugin/plugin_meta.go
+++ b/backend/core/plugin/plugin_meta.go
@@ -42,9 +42,9 @@ type PluginIcon interface {
}
// PluginSource abstracts data sources
-type PluginSource interface {
- Connection() interface{}
- Scope() interface{}
- // Deprecated: rename to ScopeConfig
- // TransformationRule() interface{}
-}
+// type PluginSource interface {
+// Connection() interface{}
+// Scope() interface{}
+// Deprecated: rename to ScopeConfig
+// ScopeConfig() interface{}
+// }
diff --git a/backend/helpers/pluginhelper/api/graphql_collector.go
b/backend/helpers/pluginhelper/api/graphql_collector.go
index 2007dbfc6..0347d5154 100644
--- a/backend/helpers/pluginhelper/api/graphql_collector.go
+++ b/backend/helpers/pluginhelper/api/graphql_collector.go
@@ -20,14 +20,15 @@ package api
import (
"context"
"encoding/json"
+ "net/http"
+ "reflect"
+ "time"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/common"
plugin "github.com/apache/incubator-devlake/core/plugin"
"github.com/merico-dev/graphql"
- "net/http"
- "reflect"
- "time"
)
// CursorPager contains pagination information for a graphql request
@@ -139,7 +140,7 @@ func (collector *GraphqlCollector) Execute() errors.Error {
// flush data if not incremental collection
if collector.args.Incremental {
- // re extract data for new transformation rules
+ // re extract data for new scope config
err = collector.ExtractExistRawData(divider)
if err != nil {
collector.checkError(err)
diff --git a/backend/helpers/pluginhelper/api/scope_config_helper.go
b/backend/helpers/pluginhelper/api/scope_config_helper.go
index 4425091ad..c6b5f48d1 100644
--- a/backend/helpers/pluginhelper/api/scope_config_helper.go
+++ b/backend/helpers/pluginhelper/api/scope_config_helper.go
@@ -30,14 +30,14 @@ import (
"github.com/go-playground/validator/v10"
)
-// ScopeConfigHelper is used to write the CURD of transformation rule
+// ScopeConfigHelper is used to write the CURD of scope config
type ScopeConfigHelper[Tr dal.Tabler] struct {
log log.Logger
db dal.Dal
validator *validator.Validate
}
-// NewScopeConfigHelper creates a ScopeConfigHelper for transformation rule
management
+// NewScopeConfigHelper creates a ScopeConfigHelper for scope config management
func NewScopeConfigHelper[Tr dal.Tabler](
basicRes context.BasicRes,
vld *validator.Validate,
@@ -59,11 +59,11 @@ func (t ScopeConfigHelper[Tr]) Create(input
*plugin.ApiResourceInput) (*plugin.A
}
var rule Tr
if err := DecodeMapStruct(input.Body, &rule, false); err != nil {
- return nil, errors.Default.Wrap(err, "error in decoding
transformation rule")
+ return nil, errors.Default.Wrap(err, "error in decoding scope
config")
}
if t.validator != nil {
if err := t.validator.Struct(rule); err != nil {
- return nil, errors.Default.Wrap(err, "error validating
transformation rule")
+ return nil, errors.Default.Wrap(err, "error validating
scope config")
}
}
valueConnectionId :=
reflect.ValueOf(&rule).Elem().FieldByName("ConnectionId")
@@ -73,7 +73,7 @@ func (t ScopeConfigHelper[Tr]) Create(input
*plugin.ApiResourceInput) (*plugin.A
if err := t.db.Create(&rule); err != nil {
if t.db.IsDuplicationError(err) {
- return nil, errors.BadInput.New("there was a
transformation rule with the same name, please choose another name")
+ return nil, errors.BadInput.New("there was a scope
config with the same name, please choose another name")
}
return nil, errors.BadInput.Wrap(err, "error on saving
ScopeConfig")
}
@@ -83,7 +83,7 @@ func (t ScopeConfigHelper[Tr]) Create(input
*plugin.ApiResourceInput) (*plugin.A
func (t ScopeConfigHelper[Tr]) Update(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
scopeConfigId, e := strconv.ParseUint(input.Params["id"], 10, 64)
if e != nil {
- return nil, errors.Default.Wrap(e, "the transformation rule ID
should be an integer")
+ return nil, errors.Default.Wrap(e, "the scope config ID should
be an integer")
}
var old Tr
err := t.db.First(&old, dal.Where("id = ?", scopeConfigId))
@@ -97,7 +97,7 @@ func (t ScopeConfigHelper[Tr]) Update(input
*plugin.ApiResourceInput) (*plugin.A
err = t.db.Update(&old, dal.Where("id = ?", scopeConfigId))
if err != nil {
if t.db.IsDuplicationError(err) {
- return nil, errors.BadInput.New("there was a
transformation rule with the same name, please choose another name")
+ return nil, errors.BadInput.New("there was a scope
config with the same name, please choose another name")
}
return nil, errors.BadInput.Wrap(err, "error on saving
ScopeConfig")
}
@@ -107,7 +107,7 @@ func (t ScopeConfigHelper[Tr]) Update(input
*plugin.ApiResourceInput) (*plugin.A
func (t ScopeConfigHelper[Tr]) Get(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
scopeConfigId, err := strconv.ParseUint(input.Params["id"], 10, 64)
if err != nil {
- return nil, errors.Default.Wrap(err, "the transformation rule
ID should be an integer")
+ return nil, errors.Default.Wrap(err, "the scope config ID
should be an integer")
}
var rule Tr
err = t.db.First(&rule, dal.Where("id = ?", scopeConfigId))
diff --git a/backend/helpers/pluginhelper/api/scope_db_helper.go
b/backend/helpers/pluginhelper/api/scope_db_helper.go
index 6e6bbe2ab..2a6074413 100644
--- a/backend/helpers/pluginhelper/api/scope_db_helper.go
+++ b/backend/helpers/pluginhelper/api/scope_db_helper.go
@@ -19,6 +19,8 @@ package api
import (
"fmt"
+ "reflect"
+
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
@@ -29,11 +31,12 @@ type ScopeDatabaseHelper[Conn any, Scope any, Tr any]
interface {
VerifyConnection(connectionId uint64) errors.Error
SaveScope(scopes []*Scope) errors.Error
UpdateScope(connectionId uint64, scopeId string, scope *Scope)
errors.Error
- GetScope(connectionId uint64, scopeId string) (Scope, errors.Error)
+ GetScope(connectionId uint64, scopeId string) (*Scope, errors.Error)
ListScopes(input *plugin.ApiResourceInput, connectionId uint64)
([]*Scope, errors.Error)
DeleteScope(connectionId uint64, scopeId string) errors.Error
- GetTransformationRule(ruleId uint64) (Tr, errors.Error)
- ListTransformationRules(ruleIds []uint64) ([]*Tr, errors.Error)
+ GetScopeConfig(ruleId uint64) (Tr, errors.Error)
+ ListScopeConfigs(ruleIds []uint64) ([]*Tr, errors.Error)
+ GetScopeAndConfig(connectionId uint64, scopeId string) (*Scope, *Tr,
errors.Error)
}
type ScopeDatabaseHelperImpl[Conn any, Scope any, Tr any] struct {
@@ -79,13 +82,33 @@ func (s *ScopeDatabaseHelperImpl[Conn, Scope, Tr])
UpdateScope(connectionId uint
return s.db.Update(&scope)
}
-func (s *ScopeDatabaseHelperImpl[Conn, Scope, Tr]) GetScope(connectionId
uint64, scopeId string) (Scope, errors.Error) {
+func (s *ScopeDatabaseHelperImpl[Conn, Scope, Tr]) GetScope(connectionId
uint64, scopeId string) (*Scope, errors.Error) {
query := dal.Where(fmt.Sprintf("connection_id = ? AND %s = ?",
s.params.ScopeIdColumnName), connectionId, scopeId)
- var scope Scope
- err := s.db.First(&scope, query)
+ scope := new(Scope)
+ err := s.db.First(scope, query)
return scope, err
}
+func (s *ScopeDatabaseHelperImpl[Conn, Scope, Tr])
GetScopeAndConfig(connectionId uint64, scopeId string) (*Scope, *Tr,
errors.Error) {
+ scope, err := s.GetScope(connectionId, scopeId)
+ if err != nil {
+ return nil, nil, err
+ }
+ scopeConfig := new(Tr)
+ scIdField := reflectField(scope, "ScopeConfigId")
+ if scIdField.IsValid() {
+ err = s.db.First(scopeConfig, dal.Where("id = ?",
scIdField.Uint()))
+ if err != nil {
+ return nil, nil, err
+ }
+ }
+ entitiesField := reflectField(scopeConfig, "Entities")
+ if entitiesField.IsValid() && entitiesField.IsNil() {
+ entitiesField.Set(reflect.ValueOf(plugin.DOMAIN_TYPES))
+ }
+ return scope, scopeConfig, nil
+}
+
func (s *ScopeDatabaseHelperImpl[Conn, Scope, Tr]) ListScopes(input
*plugin.ApiResourceInput, connectionId uint64) ([]*Scope, errors.Error) {
limit, offset := GetLimitOffset(input.Query, "pageSize", "page")
var scopes []*Scope
@@ -100,13 +123,13 @@ func (s *ScopeDatabaseHelperImpl[Conn, Scope, Tr])
DeleteScope(connectionId uint
return err
}
-func (s *ScopeDatabaseHelperImpl[Conn, Scope, Tr])
GetTransformationRule(ruleId uint64) (Tr, errors.Error) {
+func (s *ScopeDatabaseHelperImpl[Conn, Scope, Tr]) GetScopeConfig(ruleId
uint64) (Tr, errors.Error) {
var rule Tr
err := s.db.First(&rule, dal.Where("id = ?", ruleId))
return rule, err
}
-func (s *ScopeDatabaseHelperImpl[Conn, Scope, Tr])
ListTransformationRules(ruleIds []uint64) ([]*Tr, errors.Error) {
+func (s *ScopeDatabaseHelperImpl[Conn, Scope, Tr]) ListScopeConfigs(ruleIds
[]uint64) ([]*Tr, errors.Error) {
var rules []*Tr
err := s.db.All(&rules, dal.Where("id IN (?)", ruleIds))
return rules, err
diff --git a/backend/helpers/pluginhelper/api/scope_generic_helper.go
b/backend/helpers/pluginhelper/api/scope_generic_helper.go
index 78d161ec3..035593558 100644
--- a/backend/helpers/pluginhelper/api/scope_generic_helper.go
+++ b/backend/helpers/pluginhelper/api/scope_generic_helper.go
@@ -57,9 +57,9 @@ type (
opts *ScopeHelperOptions
}
ScopeRes[T any] struct {
- Scope T
`mapstructure:",squash"`
- TransformationRuleName string
`mapstructure:"transformationRuleName,omitempty"`
- Blueprints []*models.Blueprint
`mapstructure:"blueprints,omitempty"`
+ Scope T `mapstructure:",squash"`
+ ScopeConfigName string
`mapstructure:"scopeConfigName,omitempty"`
+ Blueprints []*models.Blueprint
`mapstructure:"blueprints,omitempty"`
}
ReflectionParameters struct {
ScopeIdFieldName string
@@ -124,6 +124,10 @@ func NewGenericScopeHelper[Conn any, Scope any, Tr any](
}
}
+func (c *GenericScopeApiHelper[Conn, Scope, Tr]) DbHelper()
ScopeDatabaseHelper[Conn, Scope, Tr] {
+ return c.dbHelper
+}
+
func (c *GenericScopeApiHelper[Conn, Scope, Tr]) PutScopes(input
*plugin.ApiResourceInput, scopes []*Scope) ([]*ScopeRes[Scope], errors.Error) {
params := c.extractFromReqParam(input)
if params.connectionId == 0 {
@@ -153,9 +157,9 @@ func (c *GenericScopeApiHelper[Conn, Scope, Tr])
PutScopes(input *plugin.ApiReso
return nil, errors.Default.Wrap(err, "error saving
scope")
}
}
- apiScopes, err := c.addTransformationName(scopes...)
+ apiScopes, err := c.addScopeConfigName(scopes...)
if err != nil {
- return nil, errors.Default.Wrap(err, "error associating
transformation to scope")
+ return nil, errors.Default.Wrap(err, "error associating scope
config to scope")
}
return apiScopes, nil
}
@@ -176,19 +180,19 @@ func (c *GenericScopeApiHelper[Conn, Scope, Tr])
UpdateScope(input *plugin.ApiRe
if err != nil {
return nil, err
}
- err = DecodeMapStruct(input.Body, &scope, false)
+ err = DecodeMapStruct(input.Body, scope, false)
if err != nil {
return nil, errors.Default.Wrap(err, "patch scope error")
}
- err = VerifyScope(&scope, c.validator)
+ err = VerifyScope(scope, c.validator)
if err != nil {
return nil, errors.Default.Wrap(err, "Invalid scope")
}
- err = c.dbHelper.UpdateScope(params.connectionId, params.scopeId,
&scope)
+ err = c.dbHelper.UpdateScope(params.connectionId, params.scopeId, scope)
if err != nil {
return nil, errors.Default.Wrap(err, "error on saving Scope")
}
- scopeRes, err := c.addTransformationName(&scope)
+ scopeRes, err := c.addScopeConfigName(scope)
if err != nil {
return nil, err
}
@@ -208,9 +212,9 @@ func (c *GenericScopeApiHelper[Conn, Scope, Tr])
GetScopes(input *plugin.ApiReso
if err != nil {
return nil, errors.Default.Wrap(err, fmt.Sprintf("error
verifying connection for connection ID %d", params.connectionId))
}
- apiScopes, err := c.addTransformationName(scopes...)
+ apiScopes, err := c.addScopeConfigName(scopes...)
if err != nil {
- return nil, errors.Default.Wrap(err, "error associating
transformations with scopes")
+ return nil, errors.Default.Wrap(err, "error associating scope
configs with scopes")
}
if params.loadBlueprints {
scopesById := c.mapByScopeId(apiScopes)
@@ -257,9 +261,9 @@ func (c *GenericScopeApiHelper[Conn, Scope, Tr])
GetScope(input *plugin.ApiResou
if err != nil {
return nil, errors.Default.Wrap(err, fmt.Sprintf("error
retrieving scope with scope ID %s", params.scopeId))
}
- apiScopes, err := c.addTransformationName(&scope)
+ apiScopes, err := c.addScopeConfigName(scope)
if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("error
associating transformation with scope %s", params.scopeId))
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("error
associating scope config with scope %s", params.scopeId))
}
scopeRes := apiScopes[0]
if params.loadBlueprints {
@@ -319,14 +323,14 @@ func (c *GenericScopeApiHelper[Conn, Scope, Tr])
DeleteScope(input *plugin.ApiRe
return nil
}
-func (c *GenericScopeApiHelper[Conn, Scope, Tr]) addTransformationName(scopes
...*Scope) ([]*ScopeRes[Scope], errors.Error) {
+func (c *GenericScopeApiHelper[Conn, Scope, Tr]) addScopeConfigName(scopes
...*Scope) ([]*ScopeRes[Scope], errors.Error) {
var ruleIds []uint64
for _, scope := range scopes {
- valueRepoRuleId := reflectField(scope, "TransformationRuleId")
+ valueRepoRuleId := reflectField(scope, "ScopeConfigId")
if !valueRepoRuleId.IsValid() {
break
}
- ruleId := reflectField(scope, "TransformationRuleId").Uint()
+ ruleId := reflectField(scope, "ScopeConfigId").Uint()
if ruleId > 0 {
ruleIds = append(ruleIds, ruleId)
}
@@ -334,7 +338,7 @@ func (c *GenericScopeApiHelper[Conn, Scope, Tr])
addTransformationName(scopes ..
var rules []*Tr
var err errors.Error
if len(ruleIds) > 0 {
- rules, err = c.dbHelper.ListTransformationRules(ruleIds)
+ rules, err = c.dbHelper.ListScopeConfigs(ruleIds)
if err != nil {
return nil, err
}
@@ -346,14 +350,14 @@ func (c *GenericScopeApiHelper[Conn, Scope, Tr])
addTransformationName(scopes ..
}
apiScopes := make([]*ScopeRes[Scope], 0)
for _, scope := range scopes {
- txRuleField := reflectField(scope, "TransformationRuleId")
- txRuleName := ""
- if txRuleField.IsValid() {
- txRuleName = names[txRuleField.Uint()]
+ scIdField := reflectField(scope, "ScopeConfigId")
+ scName := ""
+ if scIdField.IsValid() {
+ scName = names[scIdField.Uint()]
}
apiScopes = append(apiScopes, &ScopeRes[Scope]{
- Scope: *scope,
- TransformationRuleName: txRuleName,
+ Scope: *scope,
+ ScopeConfigName: scName,
})
}
return apiScopes, nil
diff --git a/backend/helpers/pluginhelper/api/scope_helper_test.go
b/backend/helpers/pluginhelper/api/scope_helper_test.go
index 86f934801..c173cc5d2 100644
--- a/backend/helpers/pluginhelper/api/scope_helper_test.go
+++ b/backend/helpers/pluginhelper/api/scope_helper_test.go
@@ -18,6 +18,10 @@ limitations under the License.
package api
import (
+ "reflect"
+ "testing"
+ "time"
+
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/unithelper"
@@ -28,9 +32,6 @@ import (
"github.com/stretchr/testify/mock"
"gorm.io/datatypes"
"gorm.io/gorm"
- "reflect"
- "testing"
- "time"
)
type TestModel struct {
@@ -39,20 +40,20 @@ type TestModel struct {
}
type TestRepo struct {
- ConnectionId uint64 `json:"connectionId" gorm:"primaryKey"
mapstructure:"connectionId,omitempty"`
- GithubId int `json:"githubId" gorm:"primaryKey"
mapstructure:"githubId"`
- Name string `json:"name" gorm:"type:varchar(255)"
mapstructure:"name,omitempty"`
- HTMLUrl string `json:"HTMLUrl"
gorm:"type:varchar(255)" mapstructure:"HTMLUrl,omitempty"`
- Description string `json:"description"
mapstructure:"description,omitempty"`
- TransformationRuleId uint64 `json:"transformationRuleId,omitempty"
mapstructure:"transformationRuleId,omitempty"`
- OwnerId int `json:"ownerId"
mapstructure:"ownerId,omitempty"`
- Language string `json:"language"
gorm:"type:varchar(255)" mapstructure:"language,omitempty"`
- ParentGithubId int `json:"parentId"
mapstructure:"parentGithubId,omitempty"`
- ParentHTMLUrl string `json:"parentHtmlUrl"
mapstructure:"parentHtmlUrl,omitempty"`
- CloneUrl string `json:"cloneUrl"
gorm:"type:varchar(255)" mapstructure:"cloneUrl,omitempty"`
- CreatedDate *time.Time `json:"createdDate" mapstructure:"-"`
- UpdatedDate *time.Time `json:"updatedDate" mapstructure:"-"`
- common.NoPKModel `json:"-" mapstructure:"-"`
+ ConnectionId uint64 `json:"connectionId" gorm:"primaryKey"
mapstructure:"connectionId,omitempty"`
+ GithubId int `json:"githubId" gorm:"primaryKey"
mapstructure:"githubId"`
+ Name string `json:"name" gorm:"type:varchar(255)"
mapstructure:"name,omitempty"`
+ HTMLUrl string `json:"HTMLUrl" gorm:"type:varchar(255)"
mapstructure:"HTMLUrl,omitempty"`
+ Description string `json:"description"
mapstructure:"description,omitempty"`
+ ScopeConfigId uint64 `json:"scopeConfigId,omitempty"
mapstructure:"scopeConfigId,omitempty"`
+ OwnerId int `json:"ownerId"
mapstructure:"ownerId,omitempty"`
+ Language string `json:"language" gorm:"type:varchar(255)"
mapstructure:"language,omitempty"`
+ ParentGithubId int `json:"parentId"
mapstructure:"parentGithubId,omitempty"`
+ ParentHTMLUrl string `json:"parentHtmlUrl"
mapstructure:"parentHtmlUrl,omitempty"`
+ CloneUrl string `json:"cloneUrl" gorm:"type:varchar(255)"
mapstructure:"cloneUrl,omitempty"`
+ CreatedDate *time.Time `json:"createdDate" mapstructure:"-"`
+ UpdatedDate *time.Time `json:"updatedDate" mapstructure:"-"`
+ common.NoPKModel `json:"-" mapstructure:"-"`
}
func (TestRepo) TableName() string {
@@ -113,7 +114,7 @@ func TestVerifyScope(t *testing.T) {
}
}
-type TestTransformationRule struct {
+type TestScopeConfig struct {
common.Model `mapstructure:"-"`
Name string `mapstructure:"name" json:"name"
gorm:"type:varchar(255);index:idx_name_github,unique" validate:"required"`
PrType string `mapstructure:"prType,omitempty"
json:"prType" gorm:"type:varchar(255)"`
@@ -130,8 +131,8 @@ type TestTransformationRule struct {
Refdiff datatypes.JSONMap
`mapstructure:"refdiff,omitempty" json:"refdiff" swaggertype:"object"
format:"json"`
}
-func (TestTransformationRule) TableName() string {
- return "_tool_github_transformation_rules"
+func (TestScopeConfig) TableName() string {
+ return "_tool_github_scope_configs"
}
func TestSetScopeFields(t *testing.T) {
@@ -265,40 +266,40 @@ func TestScopeApiHelper_Put(t *testing.T) {
input := &plugin.ApiResourceInput{Params:
map[string]string{"connectionId": "123"}, Body: map[string]interface{}{
"data": []map[string]interface{}{
{
- "HTMLUrl": "string",
- "githubId": 1,
- "cloneUrl": "string",
- "connectionId": 1,
- "createdAt": "string",
- "createdDate": "string",
- "description": "string",
- "language": "string",
- "name": "string",
- "owner": "string",
- "transformationRuleId": 0,
- "updatedAt": "string",
- "updatedDate": "string",
+ "HTMLUrl": "string",
+ "githubId": 1,
+ "cloneUrl": "string",
+ "connectionId": 1,
+ "createdAt": "string",
+ "createdDate": "string",
+ "description": "string",
+ "language": "string",
+ "name": "string",
+ "owner": "string",
+ "scopeConfigId": 0,
+ "updatedAt": "string",
+ "updatedDate": "string",
},
{
- "HTMLUrl": "11",
- "githubId": 2,
- "cloneUrl": "string",
- "connectionId": 1,
- "createdAt": "string",
- "createdDate": "string",
- "description": "string",
- "language": "string",
- "name": "string",
- "owner": "string",
- "transformationRuleId": 0,
- "updatedAt": "string",
- "updatedDate": "string",
+ "HTMLUrl": "11",
+ "githubId": 2,
+ "cloneUrl": "string",
+ "connectionId": 1,
+ "createdAt": "string",
+ "createdDate": "string",
+ "description": "string",
+ "language": "string",
+ "name": "string",
+ "owner": "string",
+ "scopeConfigId": 0,
+ "updatedAt": "string",
+ "updatedDate": "string",
}}}}
params := &ReflectionParameters{}
- dbHelper := NewScopeDatabaseHelperImpl[TestConnection, TestRepo,
TestTransformationRule](mockRes, connHelper, params)
+ dbHelper := NewScopeDatabaseHelperImpl[TestConnection, TestRepo,
TestScopeConfig](mockRes, connHelper, params)
// create a mock ScopeApiHelper with a mock database connection
- apiHelper := NewScopeHelper[TestConnection, TestRepo,
TestTransformationRule](mockRes, nil, connHelper, dbHelper, params, nil)
+ apiHelper := NewScopeHelper[TestConnection, TestRepo,
TestScopeConfig](mockRes, nil, connHelper, dbHelper, params, nil)
// test a successful call to Put
_, err := apiHelper.Put(input)
assert.NoError(t, err)
diff --git a/backend/helpers/pluginhelper/api/transformation_rule_helper.go
b/backend/helpers/pluginhelper/api/transformation_rule_helper.go
deleted file mode 100644
index d4aa52ba8..000000000
--- a/backend/helpers/pluginhelper/api/transformation_rule_helper.go
+++ /dev/null
@@ -1,132 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package api
-
-import (
- "net/http"
- "reflect"
- "strconv"
-
- "github.com/apache/incubator-devlake/core/context"
- "github.com/apache/incubator-devlake/core/dal"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/log"
- "github.com/apache/incubator-devlake/core/plugin"
- "github.com/go-playground/validator/v10"
-)
-
-// TransformationRuleHelper is used to write the CURD of transformation rule
-type TransformationRuleHelper[Tr dal.Tabler] struct {
- log log.Logger
- db dal.Dal
- validator *validator.Validate
-}
-
-// NewTransformationRuleHelper creates a TransformationRuleHelper for
transformation rule management
-func NewTransformationRuleHelper[Tr dal.Tabler](
- basicRes context.BasicRes,
- vld *validator.Validate,
-) *TransformationRuleHelper[Tr] {
- if vld == nil {
- vld = validator.New()
- }
- return &TransformationRuleHelper[Tr]{
- log: basicRes.GetLogger(),
- db: basicRes.GetDal(),
- validator: vld,
- }
-}
-
-func (t TransformationRuleHelper[Tr]) Create(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- connectionId, e := strconv.ParseUint(input.Params["connectionId"], 10,
64)
- if e != nil || connectionId == 0 {
- return nil, errors.Default.Wrap(e, "the connection ID should be
an non-zero integer")
- }
- var rule Tr
- if err := DecodeMapStruct(input.Body, &rule, false); err != nil {
- return nil, errors.Default.Wrap(err, "error in decoding
transformation rule")
- }
- if t.validator != nil {
- if err := t.validator.Struct(rule); err != nil {
- return nil, errors.Default.Wrap(err, "error validating
transformation rule")
- }
- }
- valueConnectionId :=
reflect.ValueOf(&rule).Elem().FieldByName("ConnectionId")
- if valueConnectionId.IsValid() {
- valueConnectionId.SetUint(connectionId)
- }
-
- if err := t.db.Create(&rule); err != nil {
- if t.db.IsDuplicationError(err) {
- return nil, errors.BadInput.New("there was a
transformation rule with the same name, please choose another name")
- }
- return nil, errors.BadInput.Wrap(err, "error on saving
TransformationRule")
- }
- return &plugin.ApiResourceOutput{Body: rule, Status: http.StatusOK}, nil
-}
-
-func (t TransformationRuleHelper[Tr]) Update(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- transformationRuleId, e := strconv.ParseUint(input.Params["id"], 10, 64)
- if e != nil {
- return nil, errors.Default.Wrap(e, "the transformation rule ID
should be an integer")
- }
- var old Tr
- err := t.db.First(&old, dal.Where("id = ?", transformationRuleId))
- if err != nil {
- return nil, errors.Default.Wrap(err, "error on saving
TransformationRule")
- }
- err = DecodeMapStruct(input.Body, &old, false)
- if err != nil {
- return nil, errors.Default.Wrap(err, "error decoding map into
transformationRule")
- }
- err = t.db.Update(&old, dal.Where("id = ?", transformationRuleId))
- if err != nil {
- if t.db.IsDuplicationError(err) {
- return nil, errors.BadInput.New("there was a
transformation rule with the same name, please choose another name")
- }
- return nil, errors.BadInput.Wrap(err, "error on saving
TransformationRule")
- }
- return &plugin.ApiResourceOutput{Body: old, Status: http.StatusOK}, nil
-}
-
-func (t TransformationRuleHelper[Tr]) Get(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- transformationRuleId, err := strconv.ParseUint(input.Params["id"], 10,
64)
- if err != nil {
- return nil, errors.Default.Wrap(err, "the transformation rule
ID should be an integer")
- }
- var rule Tr
- err = t.db.First(&rule, dal.Where("id = ?", transformationRuleId))
- if err != nil {
- return nil, errors.Default.Wrap(err, "error on get
TransformationRule")
- }
- return &plugin.ApiResourceOutput{Body: rule, Status: http.StatusOK}, nil
-}
-
-func (t TransformationRuleHelper[Tr]) List(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- connectionId, e := strconv.ParseUint(input.Params["connectionId"], 10,
64)
- if e != nil || connectionId == 0 {
- return nil, errors.Default.Wrap(e, "the connection ID should be
an non-zero integer")
- }
- var rules []Tr
- limit, offset := GetLimitOffset(input.Query, "pageSize", "page")
- err := t.db.All(&rules, dal.Where("connection_id = ?", connectionId),
dal.Limit(limit), dal.Offset(offset))
- if err != nil {
- return nil, errors.Default.Wrap(err, "error on get
TransformationRule list")
- }
- return &plugin.ApiResourceOutput{Body: rules, Status: http.StatusOK},
nil
-}
diff --git a/backend/helpers/unithelper/dummy_baesres.go
b/backend/helpers/unithelper/dummy_baesres.go
index 253d9b019..bd87969fb 100644
--- a/backend/helpers/unithelper/dummy_baesres.go
+++ b/backend/helpers/unithelper/dummy_baesres.go
@@ -34,5 +34,6 @@ func DummyBasicRes(callback func(mockDal *mockdal.Dal))
*mockcontext.BasicRes {
mockRes.On("GetDal").Return(mockDal)
mockRes.On("GetLogger").Return(mockLog)
mockRes.On("GetConfig", mock.Anything).Return("")
+ mockDal.On("AllTables").Return(nil, nil)
return mockRes
}
diff --git a/backend/plugins/bamboo/api/blueprint_V200_test.go
b/backend/plugins/bamboo/api/blueprint_V200_test.go
index 4e93e34de..2c171b5c2 100644
--- a/backend/plugins/bamboo/api/blueprint_V200_test.go
+++ b/backend/plugins/bamboo/api/blueprint_V200_test.go
@@ -31,7 +31,6 @@ import (
"github.com/apache/incubator-devlake/helpers/unithelper"
"github.com/apache/incubator-devlake/plugins/bamboo/models"
"github.com/apache/incubator-devlake/plugins/bamboo/tasks"
- "github.com/go-playground/validator/v10"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
@@ -131,21 +130,22 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
// Refresh Global Variables and set the sql mock
basicRes = unithelper.DummyBasicRes(func(mockDal *mockdal.Dal) {
- mockDal.On("First", mock.Anything, mock.Anything).Run(func(args
mock.Arguments) {
- switch dst := args.Get(0).(type) {
- case *models.BambooConnection:
- *dst = *testBambooConnection
- case *models.BambooProject:
- *dst = *testBambooProject
- case *models.BambooScopeConfig:
- *dst = *testScopeConfig
- }
+ mockDal.On("First",
mock.AnythingOfType("*models.BambooConnection"), mock.Anything).Run(func(args
mock.Arguments) {
+ dst := args.Get(0).(*models.BambooConnection)
+ *dst = *testBambooConnection
+ }).Return(nil)
+
+ mockDal.On("First",
mock.AnythingOfType("*models.BambooProject"), mock.Anything).Run(func(args
mock.Arguments) {
+ dst := args.Get(0).(*models.BambooProject)
+ *dst = *testBambooProject
+ }).Return(nil)
+
+ mockDal.On("First",
mock.AnythingOfType("*models.BambooScopeConfig"), mock.Anything).Run(func(args
mock.Arguments) {
+ dst := args.Get(0).(*models.BambooScopeConfig)
+ *dst = *testScopeConfig
}).Return(nil)
})
- connectionHelper = helper.NewConnectionHelper(
- basicRes,
- validator.New(),
- )
+ Init(basicRes)
plans, scopes, err := MakePipelinePlanV200(testSubTaskMeta,
testConnectionID, bpScopes, syncPolicy)
assert.Equal(t, err, nil)
diff --git a/backend/plugins/bamboo/api/blueprint_v200.go
b/backend/plugins/bamboo/api/blueprint_v200.go
index 7fdf12f43..88e2d7476 100644
--- a/backend/plugins/bamboo/api/blueprint_v200.go
+++ b/backend/plugins/bamboo/api/blueprint_v200.go
@@ -25,7 +25,6 @@ import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/utils"
- "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
plugin "github.com/apache/incubator-devlake/core/plugin"
@@ -65,12 +64,7 @@ func makeScopeV200(connectionId uint64, scopes
[]*plugin.BlueprintScopeV200) ([]
id :=
didgen.NewDomainIdGenerator(&models.BambooProject{}).Generate(connectionId,
scope.Id)
// get project from db
- project, err :=
GetProjectByConnectionIdAndscopeId(connectionId, scope.Id)
- if err != nil {
- return nil, err
- }
-
- scopeConfig, err := GetScopeConfigByproject(project)
+ project, scopeConfig, err :=
scopeHelper.DbHelper().GetScopeAndConfig(connectionId, scope.Id)
if err != nil {
return nil, err
}
@@ -96,12 +90,7 @@ func makePipelinePlanV200(
var stage plugin.PipelineStage
var err errors.Error
// get project
- project, err :=
GetProjectByConnectionIdAndscopeId(connection.ID, scope.Id)
- if err != nil {
- return nil, err
- }
-
- scopeConfig, err := GetScopeConfigByproject(project)
+ _, scopeConfig, err :=
scopeHelper.DbHelper().GetScopeAndConfig(connection.ID, scope.Id)
if err != nil {
return nil, err
}
@@ -128,39 +117,3 @@ func makePipelinePlanV200(
}
return plans, nil
}
-
-// GetProjectByConnectionIdAndscopeId get tbe project by the connectionId and
the scopeId
-func GetProjectByConnectionIdAndscopeId(connectionId uint64, scopeId string)
(*models.BambooProject, errors.Error) {
- key := scopeId
- project := &models.BambooProject{}
- db := basicRes.GetDal()
- err := db.First(project, dal.Where("connection_id = ? AND project_key =
?", connectionId, key))
- if err != nil {
- if db.IsErrorNotFound(err) {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("can
not find project by connection [%d] scope [%s]", connectionId, scopeId))
- }
- return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find
project by connection [%d] scope [%s]", connectionId, scopeId))
- }
-
- return project, nil
-}
-
-// GetScopeConfigByproject get the BambooScopeConfig by project
-func GetScopeConfigByproject(project *models.BambooProject)
(*models.BambooScopeConfig, errors.Error) {
- scopeConfig := &models.BambooScopeConfig{}
- scopeConfigId := project.ScopeConfigId
- if scopeConfigId != 0 {
- db := basicRes.GetDal()
- err := db.First(scopeConfig, dal.Where("id = ?", scopeConfigId))
- if err != nil {
- if db.IsErrorNotFound(err) {
- return nil, errors.Default.Wrap(err,
fmt.Sprintf("can not find ScopeConfig by ScopeConfig [%d]", scopeConfigId))
- }
- return nil, errors.Default.Wrap(err, fmt.Sprintf("fail
to find ScopeConfig by ScopeConfig [%d]", scopeConfigId))
- }
- } else {
- scopeConfig.ID = 0
- }
-
- return scopeConfig, nil
-}
diff --git a/backend/plugins/bamboo/impl/impl.go
b/backend/plugins/bamboo/impl/impl.go
index d8a88d488..714ad3bbf 100644
--- a/backend/plugins/bamboo/impl/impl.go
+++ b/backend/plugins/bamboo/impl/impl.go
@@ -38,10 +38,10 @@ var _ plugin.PluginInit = (*Bamboo)(nil)
var _ plugin.PluginTask = (*Bamboo)(nil)
var _ plugin.PluginModel = (*Bamboo)(nil)
var _ plugin.PluginMigration = (*Bamboo)(nil)
-var _ plugin.PluginBlueprintV100 = (*Bamboo)(nil)
var _ plugin.DataSourcePluginBlueprintV200 = (*Bamboo)(nil)
var _ plugin.CloseablePluginTask = (*Bamboo)(nil)
-var _ plugin.PluginSource = (*Bamboo)(nil)
+
+// var _ plugin.PluginSource = (*Bamboo)(nil)
type Bamboo struct{}
@@ -206,11 +206,11 @@ func (p Bamboo) ApiResources()
map[string]map[string]plugin.ApiResourceHandler {
"PATCH": api.PatchConnection,
"DELETE": api.DeleteConnection,
},
- "connections/:connectionId/scope_configs": {
+ "connections/:connectionId/scope-configs": {
"POST": api.CreateScopeConfig,
"GET": api.GetScopeConfigList,
},
- "connections/:connectionId/scope_configs/:id": {
+ "connections/:connectionId/scope-configs/:id": {
"PATCH": api.UpdateScopeConfig,
"GET": api.GetScopeConfig,
},
@@ -232,10 +232,6 @@ func (p Bamboo) ApiResources()
map[string]map[string]plugin.ApiResourceHandler {
}
}
-func (p Bamboo) MakePipelinePlan(connectionId uint64, scope
[]*plugin.BlueprintScopeV100) (plugin.PipelinePlan, errors.Error) {
- return nil, errors.Default.New("Bamboo does not support blueprint v100")
-}
-
func (p Bamboo) Close(taskCtx plugin.TaskContext) errors.Error {
data, ok := taskCtx.GetData().(*tasks.BambooTaskData)
if !ok {
diff --git a/backend/plugins/bitbucket/api/blueprint_V200_test.go
b/backend/plugins/bitbucket/api/blueprint_V200_test.go
index 0ebe39602..06f61c59d 100644
--- a/backend/plugins/bitbucket/api/blueprint_V200_test.go
+++ b/backend/plugins/bitbucket/api/blueprint_V200_test.go
@@ -26,7 +26,7 @@ import (
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- mockcontext "github.com/apache/incubator-devlake/mocks/core/context"
+ "github.com/apache/incubator-devlake/helpers/unithelper"
mockdal "github.com/apache/incubator-devlake/mocks/core/dal"
mockplugin "github.com/apache/incubator-devlake/mocks/core/plugin"
"github.com/apache/incubator-devlake/plugins/bitbucket/models"
@@ -59,7 +59,7 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
err := plugin.RegisterPlugin("bitbucket", mockMeta)
assert.Nil(t, err)
// Refresh Global Variables and set the sql mock
- basicRes = NewMockBasicRes()
+ mockBasicRes()
bs := &plugin.BlueprintScopeV200{
Id: "1",
}
@@ -70,7 +70,6 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
plan := make(plugin.PipelinePlan, len(bpScopes))
plan, err = makeDataSourcePipelinePlanV200(nil, plan, bpScopes,
connection, syncPolicy)
assert.Nil(t, err)
- basicRes = NewMockBasicRes()
scopes, err := makeScopesV200(bpScopes, connection)
assert.Nil(t, err)
@@ -129,8 +128,7 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
assert.Equal(t, expectScopes, scopes)
}
-// NewMockBasicRes FIXME ...
-func NewMockBasicRes() *mockcontext.BasicRes {
+func mockBasicRes() {
testBitbucketRepo := &models.BitbucketRepo{
ConnectionId: 1,
BitbucketId: "likyh/likyhphp",
@@ -154,21 +152,17 @@ func NewMockBasicRes() *mockcontext.BasicRes {
"tagsOrder": "reverse semver",
},
}
- mockRes := new(mockcontext.BasicRes)
- mockDal := new(mockdal.Dal)
-
- mockDal.On("First", mock.AnythingOfType("*models.BitbucketRepo"),
mock.Anything).Run(func(args mock.Arguments) {
- dst := args.Get(0).(*models.BitbucketRepo)
- *dst = *testBitbucketRepo
- }).Return(nil)
-
- mockDal.On("First",
mock.AnythingOfType("*models.BitbucketScopeConfig"),
mock.Anything).Run(func(args mock.Arguments) {
- dst := args.Get(0).(*models.BitbucketScopeConfig)
- *dst = *testScopeConfig
- }).Return(nil)
-
- mockRes.On("GetDal").Return(mockDal)
- mockRes.On("GetConfig", mock.Anything).Return("")
-
- return mockRes
+ // Refresh Global Variables and set the sql mock
+ mockRes := unithelper.DummyBasicRes(func(mockDal *mockdal.Dal) {
+ mockDal.On("First",
mock.AnythingOfType("*models.BitbucketRepo"), mock.Anything).Run(func(args
mock.Arguments) {
+ dst := args.Get(0).(*models.BitbucketRepo)
+ *dst = *testBitbucketRepo
+ }).Return(nil)
+
+ mockDal.On("First",
mock.AnythingOfType("*models.BitbucketScopeConfig"),
mock.Anything).Run(func(args mock.Arguments) {
+ dst := args.Get(0).(*models.BitbucketScopeConfig)
+ *dst = *testScopeConfig
+ }).Return(nil)
+ })
+ Init(mockRes)
}
diff --git a/backend/plugins/bitbucket/api/blueprint_v200.go
b/backend/plugins/bitbucket/api/blueprint_v200.go
index 421bac0fa..415c7725e 100644
--- a/backend/plugins/bitbucket/api/blueprint_v200.go
+++ b/backend/plugins/bitbucket/api/blueprint_v200.go
@@ -18,11 +18,9 @@ limitations under the License.
package api
import (
- "fmt"
"net/url"
"time"
- "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
@@ -34,11 +32,9 @@ import (
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/bitbucket/models"
"github.com/apache/incubator-devlake/plugins/bitbucket/tasks"
- "github.com/go-playground/validator/v10"
)
func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta,
connectionId uint64, bpScopes []*plugin.BlueprintScopeV200, syncPolicy
*plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope,
errors.Error) {
- connectionHelper := helper.NewConnectionHelper(basicRes,
validator.New())
// get the connection info for url
connection := &models.BitbucketConnection{}
err := connectionHelper.FirstById(connection, connectionId)
@@ -66,23 +62,14 @@ func makeDataSourcePipelinePlanV200(
connection *models.BitbucketConnection,
syncPolicy *plugin.BlueprintSyncPolicy,
) (plugin.PipelinePlan, errors.Error) {
- var err errors.Error
for i, bpScope := range bpScopes {
stage := plan[i]
if stage == nil {
stage = plugin.PipelineStage{}
}
- repo := &models.BitbucketRepo{}
- // get repo from db
- err = basicRes.GetDal().First(repo, dal.Where(`connection_id =
? AND bitbucket_id = ?`, connection.ID, bpScope.Id))
+ // get repo and scope config from db
+ repo, scopeConfig, err :=
scopeHelper.DbHelper().GetScopeAndConfig(connection.ID, bpScope.Id)
if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("fail
to find repo %s", bpScope.Id))
- }
- scopeConfig := &models.BitbucketScopeConfig{}
- // get scope configs from db
- db := basicRes.GetDal()
- err = db.First(scopeConfig, dal.Where(`id = ?`,
repo.ScopeConfigId))
- if err != nil && !db.IsErrorNotFound(err) {
return nil, err
}
// refdiff
@@ -153,18 +140,9 @@ func makeDataSourcePipelinePlanV200(
func makeScopesV200(bpScopes []*plugin.BlueprintScopeV200, connection
*models.BitbucketConnection) ([]plugin.Scope, errors.Error) {
scopes := make([]plugin.Scope, 0)
- db := basicRes.GetDal()
for _, bpScope := range bpScopes {
- repo := &models.BitbucketRepo{}
- // get repo from db
- err := db.First(repo, dal.Where(`connection_id = ? AND
bitbucket_id = ?`, connection.ID, bpScope.Id))
+ repo, scopeConfig, err :=
scopeHelper.DbHelper().GetScopeAndConfig(connection.ID, bpScope.Id)
if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("fail
to find repo%s", bpScope.Id))
- }
- scopeConfig := &models.BitbucketScopeConfig{}
- // get scope configs from db
- err = db.First(scopeConfig, dal.Where(`id = ?`,
repo.ScopeConfigId))
- if err != nil && !db.IsErrorNotFound(err) {
return nil, err
}
if utils.StringsContains(scopeConfig.Entities,
plugin.DOMAIN_TYPE_CODE_REVIEW) ||
diff --git a/backend/plugins/bitbucket/impl/impl.go
b/backend/plugins/bitbucket/impl/impl.go
index 72d5051d8..ab640b6f3 100644
--- a/backend/plugins/bitbucket/impl/impl.go
+++ b/backend/plugins/bitbucket/impl/impl.go
@@ -40,7 +40,9 @@ var _ plugin.PluginApi = (*Bitbucket)(nil)
var _ plugin.PluginModel = (*Bitbucket)(nil)
var _ plugin.PluginMigration = (*Bitbucket)(nil)
var _ plugin.CloseablePluginTask = (*Bitbucket)(nil)
-var _ plugin.PluginSource = (*Bitbucket)(nil)
+var _ plugin.DataSourcePluginBlueprintV200 = (*Bitbucket)(nil)
+
+// var _ plugin.PluginSource = (*Bitbucket)(nil)
type Bitbucket string
@@ -218,11 +220,11 @@ func (p Bitbucket) ApiResources()
map[string]map[string]plugin.ApiResourceHandle
"GET": api.GetScopeList,
"PUT": api.PutScope,
},
- "connections/:connectionId/scope_configs": {
+ "connections/:connectionId/scope-configs": {
"POST": api.CreateScopeConfig,
"GET": api.GetScopeConfigList,
},
- "connections/:connectionId/scope_configs/:id": {
+ "connections/:connectionId/scope-configs/:id": {
"PATCH": api.UpdateScopeConfig,
"GET": api.GetScopeConfig,
},
diff --git a/backend/plugins/customize/api/blueprint.go
b/backend/plugins/customize/api/blueprint.go
deleted file mode 100644
index 09a3cb796..000000000
--- a/backend/plugins/customize/api/blueprint.go
+++ /dev/null
@@ -1,52 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package api
-
-import (
- "encoding/json"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/plugin"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- _ "github.com/apache/incubator-devlake/server/api/shared"
-)
-
-func MakePipelinePlan(subtaskMetas []plugin.SubTaskMeta, connectionId uint64,
scope []*plugin.BlueprintScopeV100) (plugin.PipelinePlan, errors.Error) {
- var err error
- plan := make(plugin.PipelinePlan, len(scope))
- for i, scopeElem := range scope {
- taskOptions := make(map[string]interface{})
- err = json.Unmarshal(scopeElem.Options, &taskOptions)
- if err != nil {
- return nil, errors.Convert(err)
- }
-
- // subtasks
- subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas,
scopeElem.Entities)
- if err != nil {
- return nil, err
- }
- plan[i] = plugin.PipelineStage{
- {
- Plugin: "customize",
- Subtasks: subtasks,
- Options: taskOptions,
- },
- }
- }
- return plan, nil
-}
diff --git a/backend/plugins/customize/impl/impl.go
b/backend/plugins/customize/impl/impl.go
index 1c04999f5..2f8e77206 100644
--- a/backend/plugins/customize/impl/impl.go
+++ b/backend/plugins/customize/impl/impl.go
@@ -53,9 +53,6 @@ func (p Customize) SubTaskMetas() []plugin.SubTaskMeta {
}
}
-func (p Customize) MakePipelinePlan(connectionId uint64, scope
[]*plugin.BlueprintScopeV100) (plugin.PipelinePlan, errors.Error) {
- return api.MakePipelinePlan(p.SubTaskMetas(), connectionId, scope)
-}
func (p Customize) PrepareTaskData(taskCtx plugin.TaskContext, options
map[string]interface{}) (interface{}, errors.Error) {
var op tasks.Options
var err error
diff --git a/backend/plugins/dora/e2e/connect_incident_to_deploy_test.go
b/backend/plugins/dora/e2e/connect_incident_to_deploy_test.go
index b23684779..ee340d078 100644
--- a/backend/plugins/dora/e2e/connect_incident_to_deploy_test.go
+++ b/backend/plugins/dora/e2e/connect_incident_to_deploy_test.go
@@ -36,9 +36,6 @@ func TestConnectIncidentToDeploymentDataFlow(t *testing.T) {
taskData := &tasks.DoraTaskData{
Options: &tasks.DoraOptions{
ProjectName: "project1",
- TransformationRules: tasks.TransformationRules{
- ProductionPattern: "(?i)deploy",
- },
},
}
// import raw data table
diff --git a/backend/plugins/dora/tasks/task_data.go
b/backend/plugins/dora/tasks/task_data.go
index 96b7ca44b..150b2a0f1 100644
--- a/backend/plugins/dora/tasks/task_data.go
+++ b/backend/plugins/dora/tasks/task_data.go
@@ -26,17 +26,10 @@ type DoraApiParams struct {
ProjectName string
}
-type TransformationRules struct {
- ProductionPattern string `mapstructure:"productionPattern"
json:"productionPattern"`
- StagingPattern string `mapstructure:"stagingPattern"
json:"stagingPattern"`
- TestingPattern string `mapstructure:"testingPattern"
json:"testingPattern"`
-}
-
type DoraOptions struct {
- Tasks []string `json:"tasks,omitempty"`
- Since string
- ProjectName string `json:"projectName"`
- TransformationRules `mapstructure:"transformationRules"
json:"transformationRules"`
+ Tasks []string `json:"tasks,omitempty"`
+ Since string
+ ProjectName string `json:"projectName"`
}
type DoraTaskData struct {
diff --git a/backend/plugins/gitee/api/blueprint.go
b/backend/plugins/gitee/api/blueprint.go
deleted file mode 100644
index 9ed199f4e..000000000
--- a/backend/plugins/gitee/api/blueprint.go
+++ /dev/null
@@ -1,208 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package api
-
-import (
- "context"
- "encoding/json"
- "fmt"
- "io"
- "net/http"
- "net/url"
- "strings"
-
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
- "github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/core/utils"
- "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- aha
"github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
- "github.com/apache/incubator-devlake/plugins/gitee/models"
- "github.com/apache/incubator-devlake/plugins/gitee/tasks"
-)
-
-func MakePipelinePlan(subtaskMetas []plugin.SubTaskMeta, connectionId uint64,
scope []*plugin.BlueprintScopeV100) (plugin.PipelinePlan, errors.Error) {
- var err errors.Error
- connection := new(models.GiteeConnection)
- err = connectionHelper.FirstById(connection, connectionId)
- if err != nil {
- return nil, err
- }
- apiClient, err := api.NewApiClientFromConnection(context.TODO(),
basicRes, connection)
- if err != nil {
- return nil, err
- }
- plan, err := makePipelinePlan(subtaskMetas, scope, apiClient,
connection)
- if err != nil {
- return nil, err
- }
- return plan, nil
-}
-
-func makePipelinePlan(
- subtaskMetas []plugin.SubTaskMeta,
- scope []*plugin.BlueprintScopeV100,
- apiClient aha.ApiClientAbstract,
- connection *models.GiteeConnection,
-) (plugin.PipelinePlan, errors.Error) {
- var err errors.Error
- var repo *tasks.GiteeApiRepoResponse
- plan := make(plugin.PipelinePlan, len(scope))
- for i, scopeElem := range scope {
- // handle taskOptions and transformationRules, by dumping them
to taskOptions
- transformationRules := make(map[string]interface{})
- if len(scopeElem.Transformation) > 0 {
- err =
errors.Convert(json.Unmarshal(scopeElem.Transformation, &transformationRules))
- if err != nil {
- return nil, err
- }
- }
- // refdiff
- if refdiffRules, ok := transformationRules["refdiff"]; ok &&
refdiffRules != nil {
- // add a new task to next stage
- j := i + 1
- if j == len(plan) {
- plan = append(plan, nil)
- }
- plan[j] = plugin.PipelineStage{
- {
- Plugin: "refdiff",
- Options:
refdiffRules.(map[string]interface{}),
- },
- }
- // remove it from github transformationRules
- delete(transformationRules, "refdiff")
- }
- // construct task options for github
- options := make(map[string]interface{})
- err = errors.Convert(json.Unmarshal(scopeElem.Options,
&options))
- if err != nil {
- return nil, err
- }
- options["connectionId"] = connection.ID
- options["transformationRules"] = transformationRules
- // make sure task options is valid
- op, err := tasks.DecodeAndValidateTaskOptions(options)
- if err != nil {
- return nil, err
- }
- memorizedGetApiRepo := func() (*tasks.GiteeApiRepoResponse,
errors.Error) {
- if repo == nil {
- repo, err = getApiRepo(op, apiClient)
- }
- return repo, err
- }
-
- // construct subtasks
- subtasks, err := api.MakePipelinePlanSubtasks(subtaskMetas,
scopeElem.Entities)
- if err != nil {
- return nil, err
- }
- stage := plan[i]
- if stage == nil {
- stage = plugin.PipelineStage{}
- }
- stage = append(stage, &plugin.PipelineTask{
- Plugin: "gitee",
- Subtasks: subtasks,
- Options: options,
- })
- // collect git data by gitextractor if CODE was requested
- if utils.StringsContains(scopeElem.Entities,
plugin.DOMAIN_TYPE_CODE) {
- // here is the tricky part, we have to obtain the repo
id beforehand
- token := strings.Split(connection.Token, ",")[0]
- repo, err = memorizedGetApiRepo()
- if err != nil {
- return nil, err
- }
- cloneUrl, err :=
errors.Convert01(url.Parse(repo.HTMLUrl))
- if err != nil {
- return nil, err
- }
- cloneUrl.User = url.UserPassword("git", token)
- stage = append(stage, &plugin.PipelineTask{
- Plugin: "gitextractor",
- Options: map[string]interface{}{
- "url": cloneUrl.String(),
- "repoId":
didgen.NewDomainIdGenerator(&models.GiteeRepo{}).Generate(connection.ID,
repo.GiteeId),
- "proxy": connection.Proxy,
- },
- })
- }
- // dora
- if productionPattern, ok :=
transformationRules["productionPattern"]; ok && productionPattern != nil {
- j := i + 1
- if j == len(plan) {
- plan = append(plan, nil)
- }
- // add a new task to next stage
- if plan[j] != nil {
- j++
- }
- if j == len(plan) {
- plan = append(plan, nil)
- }
- repo, err = memorizedGetApiRepo()
- if err != nil {
- return nil, err
- }
-
- plan[j] = plugin.PipelineStage{
- {
- Plugin: "dora",
- Subtasks: []string{"EnrichTaskEnv"},
- Options: map[string]interface{}{
- "repoId":
didgen.NewDomainIdGenerator(&models.GiteeRepo{}).Generate(connection.ID,
repo.GiteeId),
- "transformationRules":
map[string]interface{}{
- "productionPattern":
productionPattern,
- },
- },
- },
- }
- // remove it from github transformationRules
- delete(transformationRules, "productionPattern")
- }
- plan[i] = stage
- repo = nil
- }
- return plan, nil
-}
-
-func getApiRepo(
- op *tasks.GiteeOptions,
- apiClient aha.ApiClientAbstract,
-) (*tasks.GiteeApiRepoResponse, errors.Error) {
- apiRepo := &tasks.GiteeApiRepoResponse{}
- res, err := apiClient.Get(fmt.Sprintf("repos/%s/%s", op.Owner,
op.Repo), nil, nil)
- if err != nil {
- return nil, err
- }
- defer res.Body.Close()
- if res.StatusCode != http.StatusOK {
- return nil,
errors.HttpStatus(res.StatusCode).New(fmt.Sprintf("unexpected status code when
requesting repo detail from %s", res.Request.URL.String()))
- }
- body, err := errors.Convert01(io.ReadAll(res.Body))
- if err != nil {
- return nil, err
- }
- err = errors.Convert(json.Unmarshal(body, apiRepo))
- if err != nil {
- return nil, err
- }
- return apiRepo, nil
-}
diff --git a/backend/plugins/gitee/api/blueprint_test.go
b/backend/plugins/gitee/api/blueprint_test.go
deleted file mode 100644
index b756e2a2f..000000000
--- a/backend/plugins/gitee/api/blueprint_test.go
+++ /dev/null
@@ -1,141 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package api
-
-import (
- "bytes"
- "encoding/json"
- "io"
- "net/http"
- "testing"
-
- "github.com/apache/incubator-devlake/core/models/common"
- "github.com/apache/incubator-devlake/core/plugin"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- mockplugin "github.com/apache/incubator-devlake/mocks/core/plugin"
- mockaha
"github.com/apache/incubator-devlake/mocks/helpers/pluginhelper/api/apihelperabstract"
- "github.com/apache/incubator-devlake/plugins/gitee/models"
- "github.com/apache/incubator-devlake/plugins/gitee/tasks"
- "github.com/stretchr/testify/assert"
- "github.com/stretchr/testify/mock"
-)
-
-func TestMakePipelinePlan(t *testing.T) {
- connection := &models.GiteeConnection{
- BaseConnection: helper.BaseConnection{
- Name: "gitee-test",
- Model: common.Model{
- ID: 1,
- },
- },
- GiteeConn: models.GiteeConn{
-
- RestConnection: helper.RestConnection{
- Endpoint: "https://api.github.com/";,
- Proxy: "",
- RateLimitPerHour: 0,
- },
- GiteeAccessToken: models.GiteeAccessToken{
- Token: "123",
- },
- },
- }
- mockApiCLient := mockaha.NewApiClientAbstract(t)
- repo := &tasks.GiteeApiRepoResponse{
- GiteeId: 12345,
- HTMLUrl: "https://this_is_cloneUrl";,
- }
- js, err := json.Marshal(repo)
- assert.Nil(t, err)
- res := &http.Response{}
- res.Body = io.NopCloser(bytes.NewBuffer(js))
- res.StatusCode = http.StatusOK
- mockApiCLient.On("Get", "repos/test/testRepo", mock.Anything,
mock.Anything).Return(res, nil)
- mockMeta := mockplugin.NewPluginMeta(t)
-
mockMeta.On("RootPkgPath").Return("github.com/apache/incubator-devlake/plugins/gitee")
- err = plugin.RegisterPlugin("gitee", mockMeta)
- assert.Nil(t, err)
- bs := &plugin.BlueprintScopeV100{
- Entities: []string{"CODE"},
- Options: json.RawMessage(`{
- "owner": "test",
- "repo": "testRepo"
- }`),
- Transformation: json.RawMessage(`{
- "prType": "hey,man,wasup",
- "refdiff": {
- "tagsPattern": "pattern",
- "tagsLimit": 10,
- "tagsOrder": "reverse semver"
- },
- "productionPattern": "xxxx"
- }`),
- }
- scopes := make([]*plugin.BlueprintScopeV100, 0)
- scopes = append(scopes, bs)
- plan, err := makePipelinePlan(nil, scopes, mockApiCLient, connection)
- assert.Nil(t, err)
-
- expectPlan := plugin.PipelinePlan{
- plugin.PipelineStage{
- {
- Plugin: "gitee",
- Subtasks: []string{},
- Options: map[string]interface{}{
- "connectionId": uint64(1),
- "owner": "test",
- "repo": "testRepo",
- "transformationRules":
map[string]interface{}{
- "prType": "hey,man,wasup",
- },
- },
- },
- {
- Plugin: "gitextractor",
- Options: map[string]interface{}{
- "proxy": "",
- "repoId": "gitee:GiteeRepo:1:12345",
- "url":
"https://git:123@this_is_cloneUrl";,
- },
- },
- },
- plugin.PipelineStage{
- {
- Plugin: "refdiff",
- Options: map[string]interface{}{
- "tagsLimit": float64(10),
- "tagsOrder": "reverse semver",
- "tagsPattern": "pattern",
- },
- },
- },
- plugin.PipelineStage{
- {
- Plugin: "dora",
- Subtasks: []string{"EnrichTaskEnv"},
- Options: map[string]interface{}{
- "repoId": "gitee:GiteeRepo:1:12345",
- "transformationRules":
map[string]interface{}{
- "productionPattern": "xxxx",
- },
- },
- },
- },
- }
- assert.Equal(t, expectPlan, plan)
-}
diff --git a/backend/plugins/gitee/api/swagger.go
b/backend/plugins/gitee/api/swagger.go
index d0d26fdeb..3a8b37785 100644
--- a/backend/plugins/gitee/api/swagger.go
+++ b/backend/plugins/gitee/api/swagger.go
@@ -25,7 +25,7 @@ package api
// @Router /pipelines/gitee/pipeline-plan [post]
func _() {}
-type CodeTransformationRules struct {
+type GiteeScopeConfig struct {
PrType string `mapstructure:"prType" json:"prType"`
PrComponent string `mapstructure:"prComponent"
json:"prComponent"`
PrBodyClosePattern string `mapstructure:"prBodyClosePattern"
json:"prBodyClosePattern"`
@@ -40,10 +40,10 @@ type GiteePipelinePlan [][]struct {
Plugin string `json:"plugin"`
Subtasks []string `json:"subtasks"`
Options struct {
- ConnectionID int `json:"connectionId"`
- Owner string `json:"owner"`
- Repo string `json:"repo"`
- Since string
- Transformation CodeTransformationRules `json:"transformation"`
+ ConnectionID int `json:"connectionId"`
+ Owner string `json:"owner"`
+ Repo string `json:"repo"`
+ Since string
+ ScopeConfig GiteeScopeConfig `json:"scopeConfig"`
} `json:"options"`
}
diff --git a/backend/plugins/gitee/impl/impl.go
b/backend/plugins/gitee/impl/impl.go
index 964c34c92..9ef766354 100644
--- a/backend/plugins/gitee/impl/impl.go
+++ b/backend/plugins/gitee/impl/impl.go
@@ -119,38 +119,6 @@ func (p Gitee) PrepareTaskData(taskCtx plugin.TaskContext,
options map[string]in
return nil, errors.BadInput.New("repo is required for Gitee
execution")
}
- if op.PrType == "" {
- op.PrType = "type/(.*)$"
- }
-
- if op.PrComponent == "" {
- op.PrComponent = "component/(.*)$"
- }
-
- if op.IssueSeverity == "" {
- op.IssueSeverity = "severity/(.*)$"
- }
-
- if op.IssuePriority == "" {
- op.IssuePriority = "^(highest|high|medium|low)$"
- }
-
- if op.IssueComponent == "" {
- op.IssueComponent = "component/(.*)$"
- }
-
- if op.IssueTypeBug == "" {
- op.IssueTypeBug = "^(bug|failure|error)$"
- }
-
- if op.IssueTypeIncident == "" {
- op.IssueTypeIncident = ""
- }
-
- if op.IssueTypeRequirement == "" {
- op.IssueTypeRequirement =
"^(feat|feature|proposal|requirement)$"
- }
-
if op.ConnectionId == 0 {
return nil, errors.BadInput.New("connectionId is invalid")
}
@@ -207,10 +175,6 @@ func (p Gitee) ApiResources()
map[string]map[string]plugin.ApiResourceHandler {
}
}
-func (p Gitee) MakePipelinePlan(connectionId uint64, scope
[]*plugin.BlueprintScopeV100) (plugin.PipelinePlan, errors.Error) {
- return api.MakePipelinePlan(p.SubTaskMetas(), connectionId, scope)
-}
-
func (p Gitee) Close(taskCtx plugin.TaskContext) errors.Error {
data, ok := taskCtx.GetData().(*tasks.GiteeTaskData)
if !ok {
diff --git a/backend/plugins/gitee/models/connection.go
b/backend/plugins/gitee/models/connection.go
index 4d280d4e5..fc94a7487 100644
--- a/backend/plugins/gitee/models/connection.go
+++ b/backend/plugins/gitee/models/connection.go
@@ -51,7 +51,7 @@ type ApiUserResponse struct {
Name string `json:"name"`
}
-type TransformationRules struct {
+type GiteeScopeConfig struct {
PrType string `mapstructure:"prType" env:"GITEE_PR_TYPE"
json:"prType"`
PrComponent string `mapstructure:"prComponent"
env:"GITEE_PR_COMPONENT" json:"prComponent"`
PrBodyClosePattern string `mapstructure:"prBodyClosePattern"
json:"prBodyClosePattern"`
diff --git a/backend/plugins/gitee/tasks/issue_extractor.go
b/backend/plugins/gitee/tasks/issue_extractor.go
index 50fcb57c4..abc2a7adf 100644
--- a/backend/plugins/gitee/tasks/issue_extractor.go
+++ b/backend/plugins/gitee/tasks/issue_extractor.go
@@ -19,12 +19,13 @@ package tasks
import (
"encoding/json"
+ "regexp"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitee/models"
- "regexp"
)
var ExtractApiIssuesMeta = plugin.SubTaskMeta{
@@ -97,7 +98,7 @@ type IssuesResponse struct {
func ExtractApiIssues(taskCtx plugin.SubTaskContext) errors.Error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_ISSUE_TABLE)
- config := data.Options.TransformationRules
+ config := data.Options.ScopeConfig
var issueSeverityRegex *regexp.Regexp
var issueComponentRegex *regexp.Regexp
var issuePriorityRegex *regexp.Regexp
diff --git a/backend/plugins/gitee/tasks/pr_extractor.go
b/backend/plugins/gitee/tasks/pr_extractor.go
index 9c670f9d2..112addd33 100644
--- a/backend/plugins/gitee/tasks/pr_extractor.go
+++ b/backend/plugins/gitee/tasks/pr_extractor.go
@@ -19,11 +19,12 @@ package tasks
import (
"encoding/json"
+ "regexp"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitee/models"
- "regexp"
)
var ExtractApiPullRequestsMeta = plugin.SubTaskMeta{
@@ -78,7 +79,7 @@ type GiteeApiPullResponse struct {
func ExtractApiPullRequests(taskCtx plugin.SubTaskContext) errors.Error {
rawDataSubTaskArgs, data := CreateRawDataSubTaskArgs(taskCtx,
RAW_PULL_REQUEST_TABLE)
- config := data.Options.TransformationRules
+ config := data.Options.ScopeConfig
var labelTypeRegex *regexp.Regexp
var labelComponentRegex *regexp.Regexp
var prType = config.PrType
diff --git a/backend/plugins/gitee/tasks/pr_issue_enricher.go
b/backend/plugins/gitee/tasks/pr_issue_enricher.go
index b4595f9bd..665ecb215 100644
--- a/backend/plugins/gitee/tasks/pr_issue_enricher.go
+++ b/backend/plugins/gitee/tasks/pr_issue_enricher.go
@@ -18,15 +18,16 @@ limitations under the License.
package tasks
import (
+ "reflect"
+ "regexp"
+ "strconv"
+ "strings"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitee/models"
- "reflect"
- "regexp"
- "strconv"
- "strings"
)
var EnrichPullRequestIssuesMeta = plugin.SubTaskMeta{
@@ -44,7 +45,7 @@ func EnrichPullRequestIssues(taskCtx plugin.SubTaskContext)
errors.Error {
var prBodyCloseRegex *regexp.Regexp
var err errors.Error
- prBodyClosePattern := data.Options.PrBodyClosePattern
+ prBodyClosePattern := data.Options.ScopeConfig.PrBodyClosePattern
//the pattern before the issue number, sometimes, the issue number is
#1098, sometimes it is https://xxx/#1098
prBodyClosePattern = strings.Replace(prBodyClosePattern, "%s",
data.Options.Owner, 1)
prBodyClosePattern = strings.Replace(prBodyClosePattern, "%s",
data.Options.Repo, 1)
diff --git a/backend/plugins/gitee/tasks/task_data.go
b/backend/plugins/gitee/tasks/task_data.go
index adda07c55..aa551fbc3 100644
--- a/backend/plugins/gitee/tasks/task_data.go
+++ b/backend/plugins/gitee/tasks/task_data.go
@@ -18,17 +18,18 @@ limitations under the License.
package tasks
import (
+ "time"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitee/models"
- "time"
)
type GiteeOptions struct {
- ConnectionId uint64 `json:"connectionId"`
- Owner string
- Repo string
- models.TransformationRules `mapstructure:"transformationRules"
json:"transformationRules"`
+ ConnectionId uint64 `json:"connectionId"`
+ Owner string
+ Repo string
+ ScopeConfig models.GiteeScopeConfig `mapstructure:"scopeConfig"
json:"scopeConfig"`
}
type GiteeTaskData struct {
@@ -50,36 +51,6 @@ func DecodeAndValidateTaskOptions(options
map[string]interface{}) (*GiteeOptions
if op.Repo == "" {
return nil, errors.BadInput.New("repo is required for GitHub
execution")
}
- if op.PrType == "" {
- op.PrType = "type/(.*)$"
- }
- if op.PrComponent == "" {
- op.PrComponent = "component/(.*)$"
- }
- if op.PrBodyClosePattern == "" {
- op.PrBodyClosePattern =
"(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and
)?(#|https:\\/\\/github.com\\/%s\\/%s\\/issues\\/)\\d+[ ]*)+)"
- }
- if op.IssueSeverity == "" {
- op.IssueSeverity = "severity/(.*)$"
- }
- if op.IssuePriority == "" {
- op.IssuePriority = "^(highest|high|medium|low)$"
- }
- if op.IssueComponent == "" {
- op.IssueComponent = "component/(.*)$"
- }
- if op.IssueTypeBug == "" {
- op.IssueTypeBug = "^(bug|failure|error)$"
- }
- if op.IssueTypeIncident == "" {
- op.IssueTypeIncident = ""
- }
- if op.IssueTypeRequirement == "" {
- op.IssueTypeRequirement =
"^(feat|feature|proposal|requirement)$"
- }
- if op.DeploymentPattern == "" {
- op.DeploymentPattern = "(?i)deploy"
- }
// find the needed GitHub now
if op.ConnectionId == 0 {
diff --git a/backend/plugins/github/api/blueprint_V200_test.go
b/backend/plugins/github/api/blueprint_V200_test.go
index 8ad53adf0..870a90128 100644
--- a/backend/plugins/github/api/blueprint_V200_test.go
+++ b/backend/plugins/github/api/blueprint_V200_test.go
@@ -26,7 +26,7 @@ import (
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- mockcontext "github.com/apache/incubator-devlake/mocks/core/context"
+ "github.com/apache/incubator-devlake/helpers/unithelper"
mockdal "github.com/apache/incubator-devlake/mocks/core/dal"
mockplugin "github.com/apache/incubator-devlake/mocks/core/plugin"
"github.com/apache/incubator-devlake/plugins/github/models"
@@ -60,7 +60,7 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
err := plugin.RegisterPlugin("github", mockMeta)
assert.Nil(t, err)
// Refresh Global Variables and set the sql mock
- basicRes = NewMockBasicRes()
+ mockBasicRes()
bs := &plugin.BlueprintScopeV200{
Id: "1",
}
@@ -71,7 +71,6 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
plan := make(plugin.PipelinePlan, len(bpScopes))
plan, err = makeDataSourcePipelinePlanV200(nil, plan, bpScopes,
connection, syncPolicy)
assert.Nil(t, err)
- basicRes = NewMockBasicRes()
scopes, err := makeScopesV200(bpScopes, connection)
assert.Nil(t, err)
@@ -131,8 +130,7 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
assert.Equal(t, expectScopes, scopes)
}
-// NewMockBasicRes FIXME ...
-func NewMockBasicRes() *mockcontext.BasicRes {
+func mockBasicRes() {
testGithubRepo := &models.GithubRepo{
ConnectionId: 1,
GithubId: 12345,
@@ -156,29 +154,17 @@ func NewMockBasicRes() *mockcontext.BasicRes {
"tagsOrder": "reverse semver",
},
}
- mockRes := new(mockcontext.BasicRes)
- mockDal := new(mockdal.Dal)
-
- mockDal.On("First", mock.AnythingOfType("*models.GithubRepo"),
mock.Anything).Run(func(args mock.Arguments) {
- dst := args.Get(0).(*models.GithubRepo)
- *dst = *testGithubRepo
- }).Return(nil)
-
- mockDal.On("First", mock.AnythingOfType("*models.GithubScopeConfig"),
mock.Anything).Run(func(args mock.Arguments) {
- dst := args.Get(0).(*models.GithubScopeConfig)
- *dst = *testScopeConfig
- }).Return(nil)
- // mockDal.On("First", mock.Anything, mock.Anything).Run(func(args
mock.Arguments) {
- // switch dst := args.Get(0).(type) {
- // case *models.GithubRepo:
- // *dst = *testGithubRepo
- // case *models.GithubScopeConfig:
- // *dst = *testScopeConfig
- // }
- // }).Return(nil)
-
- mockRes.On("GetDal").Return(mockDal)
- mockRes.On("GetConfig", mock.Anything).Return("")
-
- return mockRes
+ // Refresh Global Variables and set the sql mock
+ mockRes := unithelper.DummyBasicRes(func(mockDal *mockdal.Dal) {
+ mockDal.On("First", mock.AnythingOfType("*models.GithubRepo"),
mock.Anything).Run(func(args mock.Arguments) {
+ dst := args.Get(0).(*models.GithubRepo)
+ *dst = *testGithubRepo
+ }).Return(nil)
+
+ mockDal.On("First",
mock.AnythingOfType("*models.GithubScopeConfig"), mock.Anything).Run(func(args
mock.Arguments) {
+ dst := args.Get(0).(*models.GithubScopeConfig)
+ *dst = *testScopeConfig
+ }).Return(nil)
+ })
+ Init(mockRes)
}
diff --git a/backend/plugins/github/api/blueprint_v200.go
b/backend/plugins/github/api/blueprint_v200.go
index e0a0f4590..7fdc82e35 100644
--- a/backend/plugins/github/api/blueprint_v200.go
+++ b/backend/plugins/github/api/blueprint_v200.go
@@ -40,11 +40,9 @@ import (
aha
"github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
"github.com/apache/incubator-devlake/plugins/github/models"
"github.com/apache/incubator-devlake/plugins/github/tasks"
- "github.com/go-playground/validator/v10"
)
func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta,
connectionId uint64, bpScopes []*plugin.BlueprintScopeV200, syncPolicy
*plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope,
errors.Error) {
- connectionHelper := helper.NewConnectionHelper(basicRes,
validator.New())
// get the connection info for url
connection := &models.GithubConnection{}
err := connectionHelper.FirstById(connection, connectionId)
@@ -79,23 +77,14 @@ func makeDataSourcePipelinePlanV200(
connection *models.GithubConnection,
syncPolicy *plugin.BlueprintSyncPolicy,
) (plugin.PipelinePlan, errors.Error) {
- var err errors.Error
for i, bpScope := range bpScopes {
stage := plan[i]
if stage == nil {
stage = plugin.PipelineStage{}
}
- githubRepo := &models.GithubRepo{}
- // get repo from db
- err = basicRes.GetDal().First(githubRepo,
dal.Where(`connection_id = ? AND github_id = ?`, connection.ID, bpScope.Id))
+ // get repo and scope config from db
+ githubRepo, scopeConfig, err :=
scopeHelper.DbHelper().GetScopeAndConfig(connection.ID, bpScope.Id)
if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("fail
to find repo %s", bpScope.Id))
- }
- scopeConfig := &models.GithubScopeConfig{}
- // get scope configs from db
- db := basicRes.GetDal()
- err = db.First(scopeConfig, dal.Where(`id = ?`,
githubRepo.ScopeConfigId))
- if err != nil && !db.IsErrorNotFound(err) {
return nil, err
}
// refdiff
diff --git a/backend/plugins/github/impl/impl.go
b/backend/plugins/github/impl/impl.go
index 224c3a5e2..236cd9773 100644
--- a/backend/plugins/github/impl/impl.go
+++ b/backend/plugins/github/impl/impl.go
@@ -41,7 +41,8 @@ var _ plugin.PluginApi = (*Github)(nil)
var _ plugin.PluginModel = (*Github)(nil)
var _ plugin.DataSourcePluginBlueprintV200 = (*Github)(nil)
var _ plugin.CloseablePluginTask = (*Github)(nil)
-var _ plugin.PluginSource = (*Github)(nil)
+
+// var _ plugin.PluginSource = (*Github)(nil)
type Github struct{}
@@ -225,11 +226,11 @@ func (p Github) ApiResources()
map[string]map[string]plugin.ApiResourceHandler {
"GET": api.GetScopeList,
"PUT": api.PutScope,
},
- "connections/:connectionId/scope_configs": {
+ "connections/:connectionId/scope-configs": {
"POST": api.CreateScopeConfig,
"GET": api.GetScopeConfigList,
},
- "connections/:connectionId/scope_configs/:id": {
+ "connections/:connectionId/scope-configs/:id": {
"PATCH": api.UpdateScopeConfig,
"GET": api.GetScopeConfig,
},
diff --git a/backend/plugins/github_graphql/impl/impl.go
b/backend/plugins/github_graphql/impl/impl.go
index 9ed1251e5..38b3b90c3 100644
--- a/backend/plugins/github_graphql/impl/impl.go
+++ b/backend/plugins/github_graphql/impl/impl.go
@@ -46,7 +46,8 @@ var _ plugin.PluginTask = (*GithubGraphql)(nil)
var _ plugin.PluginApi = (*GithubGraphql)(nil)
var _ plugin.PluginModel = (*GithubGraphql)(nil)
var _ plugin.CloseablePluginTask = (*GithubGraphql)(nil)
-var _ plugin.PluginSource = (*GithubGraphql)(nil)
+
+// var _ plugin.PluginSource = (*GithubGraphql)(nil)
type GithubGraphql struct{}
diff --git a/backend/plugins/gitlab/api/blueprint_V200_test.go
b/backend/plugins/gitlab/api/blueprint_V200_test.go
index c0534a87b..0f4cc8bb3 100644
--- a/backend/plugins/gitlab/api/blueprint_V200_test.go
+++ b/backend/plugins/gitlab/api/blueprint_V200_test.go
@@ -35,7 +35,6 @@ import (
"github.com/apache/incubator-devlake/helpers/unithelper"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
"github.com/apache/incubator-devlake/plugins/gitlab/tasks"
- "github.com/go-playground/validator/v10"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
)
@@ -181,7 +180,7 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
assert.Equal(t, err, nil)
// Refresh Global Variables and set the sql mock
- basicRes = unithelper.DummyBasicRes(func(mockDal *mockdal.Dal) {
+ mockRes := unithelper.DummyBasicRes(func(mockDal *mockdal.Dal) {
mockDal.On("First",
mock.AnythingOfType("*models.GitlabConnection"), mock.Anything).Run(func(args
mock.Arguments) {
dst := args.Get(0).(*models.GitlabConnection)
*dst = *testGitlabConnection
@@ -197,10 +196,7 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
*dst = *testScopeConfig
}).Return(nil)
})
- connectionHelper = helper.NewConnectionHelper(
- basicRes,
- validator.New(),
- )
+ Init(mockRes)
plans, scopes, err := MakePipelinePlanV200(testSubTaskMeta,
testConnectionID, bpScopes, syncPolicy)
assert.Equal(t, err, nil)
diff --git a/backend/plugins/gitlab/api/blueprint_v200.go
b/backend/plugins/gitlab/api/blueprint_v200.go
index 4a493d78a..1a773030e 100644
--- a/backend/plugins/gitlab/api/blueprint_v200.go
+++ b/backend/plugins/gitlab/api/blueprint_v200.go
@@ -31,7 +31,6 @@ import (
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/utils"
- "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/models/domainlayer/code"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
@@ -79,12 +78,7 @@ func makeScopeV200(connectionId uint64, scopes
[]*plugin.BlueprintScopeV200) ([]
id :=
didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connectionId,
intScopeId)
// get repo from db
- gitlabProject, err :=
GetRepoByConnectionIdAndscopeId(connectionId, scope.Id)
- if err != nil {
- return nil, err
- }
-
- scopeConfig, err := GetScopeConfigByRepo(gitlabProject)
+ gitlabProject, scopeConfig, err :=
scopeHelper.DbHelper().GetScopeAndConfig(connectionId, scope.Id)
if err != nil {
return nil, err
}
@@ -128,13 +122,7 @@ func makePipelinePlanV200(
var stage plugin.PipelineStage
var err errors.Error
// get repo
- repo, err := GetRepoByConnectionIdAndscopeId(connection.ID,
scope.Id)
- if err != nil {
- return nil, err
- }
-
- // get scopeConfigId
- scopeConfig, err := GetScopeConfigByRepo(repo)
+ gitlabProject, scopeConfig, err :=
scopeHelper.DbHelper().GetScopeAndConfig(connection.ID, scope.Id)
if err != nil {
return nil, err
}
@@ -168,7 +156,7 @@ func makePipelinePlanV200(
// collect git data by gitextractor if CODE was requested
if utils.StringsContains(scopeConfig.Entities,
plugin.DOMAIN_TYPE_CODE) {
- cloneUrl, err :=
errors.Convert01(url.Parse(repo.HttpUrlToRepo))
+ cloneUrl, err :=
errors.Convert01(url.Parse(gitlabProject.HttpUrlToRepo))
if err != nil {
return nil, err
}
@@ -177,7 +165,7 @@ func makePipelinePlanV200(
Plugin: "gitextractor",
Options: map[string]interface{}{
"url": cloneUrl.String(),
- "repoId":
didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connection.ID,
repo.GitlabId),
+ "repoId":
didgen.NewDomainIdGenerator(&models.GitlabProject{}).Generate(connection.ID,
gitlabProject.GitlabId),
"proxy": connection.Proxy,
},
})
@@ -197,45 +185,6 @@ func makePipelinePlanV200(
return plans, nil
}
-// GetRepoByConnectionIdAndscopeId get tbe repo by the connectionId and the
scopeId
-func GetRepoByConnectionIdAndscopeId(connectionId uint64, scopeId string)
(*models.GitlabProject, errors.Error) {
- gitlabId, e := strconv.Atoi(scopeId)
- if e != nil {
- return nil, errors.Default.Wrap(e, fmt.Sprintf("scopeId %s is
not integer", scopeId))
- }
- repo := &models.GitlabProject{}
- db := basicRes.GetDal()
- err := db.First(repo, dal.Where("connection_id = ? AND gitlab_id = ?",
connectionId, gitlabId))
- if err != nil {
- if db.IsErrorNotFound(err) {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("can
not find repo by connection [%d] scope [%s]", connectionId, scopeId))
- }
- return nil, errors.Default.Wrap(err, fmt.Sprintf("fail to find
repo by connection [%d] scope [%s]", connectionId, scopeId))
- }
-
- return repo, nil
-}
-
-// GetScopeConfigByRepo get the GetScopeConfig by Repo
-func GetScopeConfigByRepo(repo *models.GitlabProject)
(*models.GitlabScopeConfig, errors.Error) {
- scopeConfigs := &models.GitlabScopeConfig{}
- scopeConfigId := repo.ScopeConfigId
- if scopeConfigId != 0 {
- db := basicRes.GetDal()
- err := db.First(scopeConfigs, dal.Where("id = ?",
scopeConfigId))
- if err != nil {
- if db.IsErrorNotFound(err) {
- return nil, errors.Default.Wrap(err,
fmt.Sprintf("can not find scopeConfig by id [%d]", scopeConfigId))
- }
- return nil, errors.Default.Wrap(err, fmt.Sprintf("fail
to find scopeConfig by id [%d]", scopeConfigId))
- }
- } else {
- scopeConfigs.ID = 0
- }
-
- return scopeConfigs, nil
-}
-
func GetApiProject(
op *tasks.GitlabOptions,
apiClient aha.ApiClientAbstract,
diff --git a/backend/plugins/gitlab/impl/impl.go
b/backend/plugins/gitlab/impl/impl.go
index 36fca3dec..67e2d0d17 100644
--- a/backend/plugins/gitlab/impl/impl.go
+++ b/backend/plugins/gitlab/impl/impl.go
@@ -41,7 +41,7 @@ var _ interface {
plugin.PluginMigration
plugin.DataSourcePluginBlueprintV200
plugin.CloseablePluginTask
- plugin.PluginSource
+ // plugin.PluginSource
} = (*Gitlab)(nil)
type Gitlab string
@@ -268,11 +268,11 @@ func (p Gitlab) ApiResources()
map[string]map[string]plugin.ApiResourceHandler {
"GET": api.GetScopeList,
"PUT": api.PutScope,
},
- "connections/:connectionId/scope_configs": {
+ "connections/:connectionId/scope-configs": {
"POST": api.CreateScopeConfig,
"GET": api.GetScopeConfigList,
},
- "connections/:connectionId/scope_configs/:id": {
+ "connections/:connectionId/scope-configs/:id": {
"PATCH": api.UpdateScopeConfig,
"GET": api.GetScopeConfig,
},
diff --git a/backend/plugins/jenkins/api/blueprint_v200.go
b/backend/plugins/jenkins/api/blueprint_v200.go
index c98391244..82ac7ad64 100644
--- a/backend/plugins/jenkins/api/blueprint_v200.go
+++ b/backend/plugins/jenkins/api/blueprint_v200.go
@@ -18,11 +18,8 @@ limitations under the License.
package api
import (
- "fmt"
"time"
- "github.com/apache/incubator-devlake/core/context"
- "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
@@ -35,11 +32,11 @@ import (
func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta,
connectionId uint64, bpScopes []*plugin.BlueprintScopeV200, syncPolicy
*plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope,
errors.Error) {
plan := make(plugin.PipelinePlan, len(bpScopes))
- plan, err := makeDataSourcePipelinePlanV200(basicRes, subtaskMetas,
plan, bpScopes, connectionId, syncPolicy)
+ plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, plan,
bpScopes, connectionId, syncPolicy)
if err != nil {
return nil, nil, err
}
- scopes, err := makeScopesV200(basicRes, bpScopes, connectionId)
+ scopes, err := makeScopesV200(bpScopes, connectionId)
if err != nil {
return nil, nil, err
}
@@ -47,23 +44,7 @@ func MakeDataSourcePipelinePlanV200(subtaskMetas
[]plugin.SubTaskMeta, connectio
return plan, scopes, nil
}
-func getScopeConfigByScopeId(basicRes context.BasicRes, connectionId uint64,
scopeId string) (*models.JenkinsScopeConfig, errors.Error) {
- db := basicRes.GetDal()
- scopeConfig := &models.JenkinsScopeConfig{}
- err := db.First(scopeConfig,
- dal.Select("c.*"),
- dal.From("_tool_jenkins_scope_configs c"),
- dal.Join("LEFT JOIN _tool_jenkins_jobs s ON (s.scope_config_id
= c.id)"),
- dal.Where("s.connection_id = ? AND s.full_name = ?",
connectionId, scopeId),
- )
- if err != nil {
- return nil, err
- }
- return scopeConfig, nil
-}
-
func makeDataSourcePipelinePlanV200(
- basicRes context.BasicRes,
subtaskMetas []plugin.SubTaskMeta,
plan plugin.PipelinePlan,
bpScopes []*plugin.BlueprintScopeV200,
@@ -85,7 +66,8 @@ func makeDataSourcePipelinePlanV200(
options["timeAfter"] =
syncPolicy.TimeAfter.Format(time.RFC3339)
}
- scopeConfig, err := getScopeConfigByScopeId(basicRes,
connectionId, bpScope.Id)
+ // get scope config from db
+ _, scopeConfig, err :=
scopeHelper.DbHelper().GetScopeAndConfig(connectionId, bpScope.Id)
if err != nil {
return nil, err
}
@@ -106,22 +88,13 @@ func makeDataSourcePipelinePlanV200(
}
func makeScopesV200(
- basicRes context.BasicRes,
bpScopes []*plugin.BlueprintScopeV200,
connectionId uint64,
) ([]plugin.Scope, errors.Error) {
scopes := make([]plugin.Scope, 0)
for _, bpScope := range bpScopes {
- jenkinsJob := &models.JenkinsJob{}
- // get repo from db
- err := basicRes.GetDal().First(jenkinsJob,
- dal.Where(`connection_id = ? and full_name = ?`,
- connectionId, bpScope.Id))
- if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("fail
to find jenkinsJob%s", bpScope.Id))
- }
-
- scopeConfig, err := getScopeConfigByScopeId(basicRes,
connectionId, bpScope.Id)
+ // get job and scope config from db
+ jenkinsJob, scopeConfig, err :=
scopeHelper.DbHelper().GetScopeAndConfig(connectionId, bpScope.Id)
if err != nil {
return nil, err
}
diff --git a/backend/plugins/jenkins/api/blueprint_v200_test.go
b/backend/plugins/jenkins/api/blueprint_v200_test.go
index 6b203e307..d409ac544 100644
--- a/backend/plugins/jenkins/api/blueprint_v200_test.go
+++ b/backend/plugins/jenkins/api/blueprint_v200_test.go
@@ -24,7 +24,7 @@ import (
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/devops"
"github.com/apache/incubator-devlake/core/plugin"
- mockcontext "github.com/apache/incubator-devlake/mocks/core/context"
+ "github.com/apache/incubator-devlake/helpers/unithelper"
mockdal "github.com/apache/incubator-devlake/mocks/core/dal"
mockplugin "github.com/apache/incubator-devlake/mocks/core/plugin"
"github.com/apache/incubator-devlake/plugins/jenkins/models"
@@ -45,12 +45,11 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
bpScopes := make([]*plugin.BlueprintScopeV200, 0)
bpScopes = append(bpScopes, bs)
- basicRes = NewMockBasicRes()
+ mockBasicRes()
plan := make(plugin.PipelinePlan, len(bpScopes))
- plan, err = makeDataSourcePipelinePlanV200(basicRes, nil, plan,
bpScopes, 1, syncPolicy)
+ plan, err = makeDataSourcePipelinePlanV200(nil, plan, bpScopes, 1,
syncPolicy)
assert.Nil(t, err)
- basicRes = NewMockBasicRes()
- scopes, err := makeScopesV200(basicRes, bpScopes, 1)
+ scopes, err := makeScopesV200(bpScopes, 1)
assert.Nil(t, err)
expectPlan := plugin.PipelinePlan{
@@ -79,8 +78,7 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
assert.Equal(t, expectScopes, scopes)
}
-// NewMockBasicRes FIXME ...
-func NewMockBasicRes() *mockcontext.BasicRes {
+func mockBasicRes() {
jenkinsJob := &models.JenkinsJob{
ConnectionId: 1,
FullName: "a/b/ccc",
@@ -92,20 +90,16 @@ func NewMockBasicRes() *mockcontext.BasicRes {
},
}
- mockRes := new(mockcontext.BasicRes)
- mockDal := new(mockdal.Dal)
-
- mockDal.On("First", mock.AnythingOfType("*models.JenkinsScopeConfig"),
mock.Anything).Run(func(args mock.Arguments) {
- dst := args.Get(0).(*models.JenkinsScopeConfig)
- *dst = *scopeConfig
- }).Return(nil)
- mockDal.On("First", mock.AnythingOfType("*models.JenkinsJob"),
mock.Anything).Run(func(args mock.Arguments) {
- dst := args.Get(0).(*models.JenkinsJob)
- *dst = *jenkinsJob
- }).Return(nil)
-
- mockRes.On("GetDal").Return(mockDal)
- mockRes.On("GetConfig", mock.Anything).Return("")
-
- return mockRes
+ // Refresh Global Variables and set the sql mock
+ mockRes := unithelper.DummyBasicRes(func(mockDal *mockdal.Dal) {
+ mockDal.On("First",
mock.AnythingOfType("*models.JenkinsScopeConfig"), mock.Anything).Run(func(args
mock.Arguments) {
+ dst := args.Get(0).(*models.JenkinsScopeConfig)
+ *dst = *scopeConfig
+ }).Return(nil)
+ mockDal.On("First", mock.AnythingOfType("*models.JenkinsJob"),
mock.Anything).Run(func(args mock.Arguments) {
+ dst := args.Get(0).(*models.JenkinsJob)
+ *dst = *jenkinsJob
+ }).Return(nil)
+ })
+ Init(mockRes)
}
diff --git a/backend/plugins/jenkins/impl/impl.go
b/backend/plugins/jenkins/impl/impl.go
index 01fb34425..62dbe55ea 100644
--- a/backend/plugins/jenkins/impl/impl.go
+++ b/backend/plugins/jenkins/impl/impl.go
@@ -42,7 +42,8 @@ var _ plugin.PluginApi = (*Jenkins)(nil)
var _ plugin.PluginModel = (*Jenkins)(nil)
var _ plugin.PluginMigration = (*Jenkins)(nil)
var _ plugin.CloseablePluginTask = (*Jenkins)(nil)
-var _ plugin.PluginSource = (*Jenkins)(nil)
+
+// var _ plugin.PluginSource = (*Jenkins)(nil)
var _ plugin.DataSourcePluginBlueprintV200 = (*Jenkins)(nil)
type Jenkins struct{}
@@ -188,11 +189,11 @@ func (p Jenkins) ApiResources()
map[string]map[string]plugin.ApiResourceHandler
"GET": api.GetScopeList,
"PUT": api.PutScope,
},
- "connections/:connectionId/scope_configs": {
+ "connections/:connectionId/scope-configs": {
"POST": api.CreateScopeConfig,
"GET": api.GetScopeConfigList,
},
- "connections/:connectionId/scope_configs/:id": {
+ "connections/:connectionId/scope-configs/:id": {
"PATCH": api.UpdateScopeConfig,
"GET": api.GetScopeConfig,
},
diff --git a/backend/plugins/jira/api/blueprint_v200.go
b/backend/plugins/jira/api/blueprint_v200.go
index 26b9a115d..2fcee1e09 100644
--- a/backend/plugins/jira/api/blueprint_v200.go
+++ b/backend/plugins/jira/api/blueprint_v200.go
@@ -18,11 +18,8 @@ limitations under the License.
package api
import (
- "fmt"
"time"
- "github.com/apache/incubator-devlake/core/context"
- "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
@@ -35,11 +32,11 @@ import (
func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta,
connectionId uint64, bpScopes []*plugin.BlueprintScopeV200, syncPolicy
*plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope,
errors.Error) {
plan := make(plugin.PipelinePlan, len(bpScopes))
- plan, err := makeDataSourcePipelinePlanV200(basicRes, subtaskMetas,
plan, bpScopes, connectionId, syncPolicy)
+ plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, plan,
bpScopes, connectionId, syncPolicy)
if err != nil {
return nil, nil, err
}
- scopes, err := makeScopesV200(basicRes, bpScopes, connectionId)
+ scopes, err := makeScopesV200(bpScopes, connectionId)
if err != nil {
return nil, nil, err
}
@@ -47,23 +44,7 @@ func MakeDataSourcePipelinePlanV200(subtaskMetas
[]plugin.SubTaskMeta, connectio
return plan, scopes, nil
}
-func getScopeConfigByScopeId(basicRes context.BasicRes, connectionId uint64,
scopeId string) (*models.JiraScopeConfig, errors.Error) {
- db := basicRes.GetDal()
- scopeConfig := &models.JiraScopeConfig{}
- err := db.First(scopeConfig,
- dal.Select("c.*"),
- dal.From("_tool_jira_scope_configs c"),
- dal.Join("LEFT JOIN _tool_jira_boards s ON (s.scope_config_id =
c.id)"),
- dal.Where("s.connection_id = ? AND s.board_id = ?",
connectionId, scopeId),
- )
- if err != nil {
- return nil, err
- }
- return scopeConfig, nil
-}
-
func makeDataSourcePipelinePlanV200(
- basicRes context.BasicRes,
subtaskMetas []plugin.SubTaskMeta,
plan plugin.PipelinePlan,
bpScopes []*plugin.BlueprintScopeV200,
@@ -83,7 +64,8 @@ func makeDataSourcePipelinePlanV200(
options["timeAfter"] =
syncPolicy.TimeAfter.Format(time.RFC3339)
}
- scopeConfig, err := getScopeConfigByScopeId(basicRes,
connectionId, bpScope.Id)
+ // get scope config from db
+ _, scopeConfig, err :=
scopeHelper.DbHelper().GetScopeAndConfig(connectionId, bpScope.Id)
if err != nil {
return nil, err
}
@@ -104,21 +86,13 @@ func makeDataSourcePipelinePlanV200(
}
func makeScopesV200(
- basicRes context.BasicRes,
bpScopes []*plugin.BlueprintScopeV200,
connectionId uint64,
) ([]plugin.Scope, errors.Error) {
scopes := make([]plugin.Scope, 0)
for _, bpScope := range bpScopes {
- jiraBoard := &models.JiraBoard{}
- // get repo from db
- err := basicRes.GetDal().First(jiraBoard,
- dal.Where(`connection_id = ? and board_id = ?`,
- connectionId, bpScope.Id))
- if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("fail
to find board %s", bpScope.Id))
- }
- scopeConfig, err := getScopeConfigByScopeId(basicRes,
connectionId, bpScope.Id)
+ // get board and scope config from db
+ jiraBoard, scopeConfig, err :=
scopeHelper.DbHelper().GetScopeAndConfig(connectionId, bpScope.Id)
if err != nil {
return nil, err
}
diff --git a/backend/plugins/jira/api/blueprint_v200_test.go
b/backend/plugins/jira/api/blueprint_v200_test.go
index da4fafc83..6f15c2568 100644
--- a/backend/plugins/jira/api/blueprint_v200_test.go
+++ b/backend/plugins/jira/api/blueprint_v200_test.go
@@ -24,7 +24,7 @@ import (
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/core/plugin"
- mockcontext "github.com/apache/incubator-devlake/mocks/core/context"
+ "github.com/apache/incubator-devlake/helpers/unithelper"
mockdal "github.com/apache/incubator-devlake/mocks/core/dal"
mockplugin "github.com/apache/incubator-devlake/mocks/core/plugin"
"github.com/apache/incubator-devlake/plugins/jira/models"
@@ -44,10 +44,10 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
bpScopes := make([]*plugin.BlueprintScopeV200, 0)
bpScopes = append(bpScopes, bs)
plan := make(plugin.PipelinePlan, len(bpScopes))
- basicRes := NewMockBasicRes()
- plan, err = makeDataSourcePipelinePlanV200(basicRes, nil, plan,
bpScopes, uint64(1), syncPolicy)
+ mockBasicRes()
+ plan, err = makeDataSourcePipelinePlanV200(nil, plan, bpScopes,
uint64(1), syncPolicy)
assert.Nil(t, err)
- scopes, err := makeScopesV200(basicRes, bpScopes, uint64(1))
+ scopes, err := makeScopesV200(bpScopes, uint64(1))
assert.Nil(t, err)
expectPlan := plugin.PipelinePlan{
@@ -76,8 +76,7 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
assert.Equal(t, expectScopes, scopes)
}
-// NewMockBasicRes FIXME ...
-func NewMockBasicRes() *mockcontext.BasicRes {
+func mockBasicRes() {
jiraBoard := &models.JiraBoard{
ConnectionId: 1,
BoardId: 10,
@@ -89,20 +88,16 @@ func NewMockBasicRes() *mockcontext.BasicRes {
},
}
- mockRes := new(mockcontext.BasicRes)
- mockDal := new(mockdal.Dal)
-
- mockDal.On("First", mock.AnythingOfType("*models.JiraScopeConfig"),
mock.Anything).Run(func(args mock.Arguments) {
- dst := args.Get(0).(*models.JiraScopeConfig)
- *dst = *scopeConfig
- }).Return(nil)
- mockDal.On("First", mock.AnythingOfType("*models.JiraBoard"),
mock.Anything).Run(func(args mock.Arguments) {
- dst := args.Get(0).(*models.JiraBoard)
- *dst = *jiraBoard
- }).Return(nil)
-
- mockRes.On("GetDal").Return(mockDal)
- mockRes.On("GetConfig", mock.Anything).Return("")
-
- return mockRes
+ // Refresh Global Variables and set the sql mock
+ mockRes := unithelper.DummyBasicRes(func(mockDal *mockdal.Dal) {
+ mockDal.On("First",
mock.AnythingOfType("*models.JiraScopeConfig"), mock.Anything).Run(func(args
mock.Arguments) {
+ dst := args.Get(0).(*models.JiraScopeConfig)
+ *dst = *scopeConfig
+ }).Return(nil)
+ mockDal.On("First", mock.AnythingOfType("*models.JiraBoard"),
mock.Anything).Run(func(args mock.Arguments) {
+ dst := args.Get(0).(*models.JiraBoard)
+ *dst = *jiraBoard
+ }).Return(nil)
+ })
+ Init(mockRes)
}
diff --git a/backend/plugins/jira/impl/impl.go
b/backend/plugins/jira/impl/impl.go
index b1949bd01..57cc0a03c 100644
--- a/backend/plugins/jira/impl/impl.go
+++ b/backend/plugins/jira/impl/impl.go
@@ -42,7 +42,7 @@ var _ interface {
plugin.PluginMigration
plugin.DataSourcePluginBlueprintV200
plugin.CloseablePluginTask
- plugin.PluginSource
+ // plugin.PluginSource
} = (*Jira)(nil)
type Jira struct {
@@ -294,11 +294,11 @@ func (p Jira) ApiResources()
map[string]map[string]plugin.ApiResourceHandler {
"GET": api.GetScopeList,
"PUT": api.PutScope,
},
- "connections/:connectionId/scope_configs": {
+ "connections/:connectionId/scope-configs": {
"POST": api.CreateScopeConfig,
"GET": api.GetScopeConfigList,
},
- "connections/:connectionId/scope_configs/:id": {
+ "connections/:connectionId/scope-configs/:id": {
"PATCH": api.UpdateScopeConfig,
"GET": api.GetScopeConfig,
},
diff --git a/backend/plugins/pagerduty/api/blueprint_v200.go
b/backend/plugins/pagerduty/api/blueprint_v200.go
index f45f55109..72ca47313 100644
--- a/backend/plugins/pagerduty/api/blueprint_v200.go
+++ b/backend/plugins/pagerduty/api/blueprint_v200.go
@@ -18,10 +18,8 @@ limitations under the License.
package api
import (
- "fmt"
"time"
- "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
@@ -31,15 +29,13 @@ import (
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/pagerduty/models"
"github.com/apache/incubator-devlake/plugins/pagerduty/tasks"
- "github.com/go-playground/validator/v10"
)
func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta,
connectionId uint64, bpScopes []*plugin.BlueprintScopeV200, syncPolicy
*plugin.BlueprintSyncPolicy,
) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
- connHelper := api.NewConnectionHelper(basicRes, validator.New())
// get the connection info for url
connection := &models.PagerDutyConnection{}
- err := connHelper.FirstById(connection, connectionId)
+ err := connectionHelper.FirstById(connection, connectionId)
if err != nil {
return nil, nil, err
}
@@ -64,19 +60,10 @@ func makeDataSourcePipelinePlanV200(
connection *models.PagerDutyConnection,
syncPolicy *plugin.BlueprintSyncPolicy,
) (plugin.PipelinePlan, errors.Error) {
- var err errors.Error
for i, bpScope := range bpScopes {
- service := &models.Service{}
- // get repo from db
- err = basicRes.GetDal().First(service, dal.Where(`connection_id
= ? AND id = ?`, connection.ID, bpScope.Id))
+ // get board and scope config from db
+ service, scopeConfig, err :=
scopeHelper.DbHelper().GetScopeAndConfig(connection.ID, bpScope.Id)
if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("fail
to find service %s", bpScope.Id))
- }
- scopeConfig := &models.PagerdutyScopeConfig{}
- // get scope configs from db
- db := basicRes.GetDal()
- err = db.First(scopeConfig, dal.Where(`id = ?`,
service.ScopeConfigId))
- if err != nil && !db.IsErrorNotFound(err) {
return nil, err
}
// construct task options for pagerduty
@@ -113,17 +100,9 @@ func makeDataSourcePipelinePlanV200(
func makeScopesV200(bpScopes []*plugin.BlueprintScopeV200, connection
*models.PagerDutyConnection) ([]plugin.Scope, errors.Error) {
scopes := make([]plugin.Scope, 0)
for _, bpScope := range bpScopes {
- db := basicRes.GetDal()
- // get service from db
- service := &models.Service{}
- err := db.First(service, dal.Where(`connection_id = ? AND id =
?`, connection.ID, bpScope.Id))
+ // get board and scope config from db
+ service, scopeConfig, err :=
scopeHelper.DbHelper().GetScopeAndConfig(connection.ID, bpScope.Id)
if err != nil {
- return nil, errors.Default.Wrap(err,
fmt.Sprintf("failed to find service: %s", bpScope.Id))
- }
- // get scope configs from db
- scopeConfig := &models.PagerdutyScopeConfig{}
- err = db.First(scopeConfig, dal.Where(`id = ?`,
service.ScopeConfigId))
- if err != nil && !db.IsErrorNotFound(err) {
return nil, err
}
// add board to scopes
diff --git a/backend/plugins/pagerduty/impl/impl.go
b/backend/plugins/pagerduty/impl/impl.go
index b3f1da041..caf2bde5e 100644
--- a/backend/plugins/pagerduty/impl/impl.go
+++ b/backend/plugins/pagerduty/impl/impl.go
@@ -148,11 +148,11 @@ func (p PagerDuty) ApiResources()
map[string]map[string]plugin.ApiResourceHandle
"PATCH": api.UpdateScope,
"DELETE": api.DeleteScope,
},
- "connections/:connectionId/scope_configs": {
+ "connections/:connectionId/scope-configs": {
"POST": api.CreateScopeConfig,
"GET": api.GetScopeConfigList,
},
- "connections/:connectionId/scope_configs/:id": {
+ "connections/:connectionId/scope-configs/:id": {
"PATCH": api.UpdateScopeConfig,
"GET": api.GetScopeConfig,
},
diff --git a/backend/plugins/sonarqube/api/blueprint.go
b/backend/plugins/sonarqube/api/blueprint.go
deleted file mode 100644
index 5d26b6b27..000000000
--- a/backend/plugins/sonarqube/api/blueprint.go
+++ /dev/null
@@ -1,66 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package api
-
-import (
- "encoding/json"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/plugin"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "github.com/apache/incubator-devlake/plugins/sonarqube/tasks"
-)
-
-func MakePipelinePlan(subtaskMetas []plugin.SubTaskMeta, connectionId uint64,
scope []*plugin.BlueprintScopeV100) (plugin.PipelinePlan, errors.Error) {
- var err error
- plan := make(plugin.PipelinePlan, len(scope))
- for i, scopeElem := range scope {
- taskOptions := make(map[string]interface{})
- err = json.Unmarshal(scopeElem.Options, &taskOptions)
- if err != nil {
- return nil, errors.Default.Wrap(err, "error
unmarshalling task options")
- }
- taskOptions["connectionId"] = connectionId
- /*
- var transformationRules tasks.TransformationRules
- if len(scopeElem.Transformation) > 0 {
- err = json.Unmarshal(scopeElem.Transformation,
&transformationRules)
- if err != nil {
- return nil, err
- }
- }
- */
- //taskOptions["transformationRules"] = transformationRules
- _, err := tasks.DecodeAndValidateTaskOptions(taskOptions)
- if err != nil {
- return nil, err
- }
- // subtasks
- subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas,
scopeElem.Entities)
- if err != nil {
- return nil, err
- }
- plan[i] = plugin.PipelineStage{
- {
- Plugin: "sonarqube",
- Subtasks: subtasks,
- Options: taskOptions,
- },
- }
- }
- return plan, nil
-}
diff --git a/backend/plugins/sonarqube/api/blueprint_v200.go
b/backend/plugins/sonarqube/api/blueprint_v200.go
index a60e50324..dd6656217 100644
--- a/backend/plugins/sonarqube/api/blueprint_v200.go
+++ b/backend/plugins/sonarqube/api/blueprint_v200.go
@@ -28,7 +28,6 @@ import (
"github.com/apache/incubator-devlake/core/models/domainlayer/codequality"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/core/utils"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
aha
"github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
"github.com/apache/incubator-devlake/plugins/sonarqube/models"
@@ -65,7 +64,7 @@ func makeDataSourcePipelinePlanV200(
options["connectionId"] = connectionId
options["projectKey"] = bpScope.Id
- subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas,
bpScope.Entities)
+ subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas,
plugin.DOMAIN_TYPES)
if err != nil {
return nil, err
}
@@ -92,15 +91,15 @@ func makeScopesV200(bpScopes []*plugin.BlueprintScopeV200,
connectionId uint64)
return nil, errors.Default.Wrap(err, fmt.Sprintf("fail
to find sonarqube project %s", bpScope.Id))
}
// add board to scopes
- if utils.StringsContains(bpScope.Entities,
plugin.DOMAIN_TYPE_CODE_QUALITY) {
- stProject := &codequality.CqProject{
- DomainEntity: domainlayer.DomainEntity{
- Id:
didgen.NewDomainIdGenerator(&models.SonarqubeProject{}).Generate(sonarqubeProject.ConnectionId,
sonarqubeProject.ProjectKey),
- },
- Name: sonarqubeProject.Name,
- }
- scopes = append(scopes, stProject)
+ // if utils.StringsContains(bpScope.Entities,
plugin.DOMAIN_TYPE_CODE_QUALITY) {
+ stProject := &codequality.CqProject{
+ DomainEntity: domainlayer.DomainEntity{
+ Id:
didgen.NewDomainIdGenerator(&models.SonarqubeProject{}).Generate(sonarqubeProject.ConnectionId,
sonarqubeProject.ProjectKey),
+ },
+ Name: sonarqubeProject.Name,
}
+ scopes = append(scopes, stProject)
+ // }
}
return scopes, nil
}
diff --git a/backend/plugins/sonarqube/api/blueprint_v200_test.go
b/backend/plugins/sonarqube/api/blueprint_v200_test.go
index 354dfc444..aaaf7d857 100644
--- a/backend/plugins/sonarqube/api/blueprint_v200_test.go
+++ b/backend/plugins/sonarqube/api/blueprint_v200_test.go
@@ -18,6 +18,8 @@ limitations under the License.
package api
import (
+ "testing"
+
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/codequality"
"github.com/apache/incubator-devlake/core/plugin"
@@ -27,7 +29,6 @@ import (
"github.com/apache/incubator-devlake/plugins/sonarqube/models"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
- "testing"
)
func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
@@ -36,8 +37,7 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
err := plugin.RegisterPlugin("sonarqube", mockMeta)
assert.Nil(t, err)
bs := &plugin.BlueprintScopeV200{
- Entities: []string{"CODEQUALITY"},
- Id: "f5a50c63-2e8f-4107-9014-853f6f467757",
+ Id: "f5a50c63-2e8f-4107-9014-853f6f467757",
}
syncPolicy := &plugin.BlueprintSyncPolicy{}
bpScopes := make([]*plugin.BlueprintScopeV200, 0)
diff --git a/backend/plugins/sonarqube/impl/impl.go
b/backend/plugins/sonarqube/impl/impl.go
index a63d0d273..9a447736b 100644
--- a/backend/plugins/sonarqube/impl/impl.go
+++ b/backend/plugins/sonarqube/impl/impl.go
@@ -41,7 +41,7 @@ var _ interface {
plugin.PluginMigration
plugin.DataSourcePluginBlueprintV200
plugin.CloseablePluginTask
- plugin.PluginSource
+ // plugin.PluginSource
} = (*Sonarqube)(nil)
type Sonarqube struct{}
diff --git a/backend/plugins/tapd/api/blueprint_v200.go
b/backend/plugins/tapd/api/blueprint_v200.go
index 648a269c5..77da4c5d7 100644
--- a/backend/plugins/tapd/api/blueprint_v200.go
+++ b/backend/plugins/tapd/api/blueprint_v200.go
@@ -18,12 +18,9 @@ limitations under the License.
package api
import (
- "fmt"
"strconv"
"time"
- "github.com/apache/incubator-devlake/core/context"
- "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
@@ -36,11 +33,11 @@ import (
func MakeDataSourcePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta,
connectionId uint64, bpScopes []*plugin.BlueprintScopeV200, syncPolicy
*plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, []plugin.Scope,
errors.Error) {
plan := make(plugin.PipelinePlan, len(bpScopes))
- plan, err := makeDataSourcePipelinePlanV200(basicRes, subtaskMetas,
plan, bpScopes, connectionId, syncPolicy)
+ plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, plan,
bpScopes, connectionId, syncPolicy)
if err != nil {
return nil, nil, err
}
- scopes, err := makeScopesV200(basicRes, bpScopes, connectionId)
+ scopes, err := makeScopesV200(bpScopes, connectionId)
if err != nil {
return nil, nil, err
}
@@ -48,23 +45,7 @@ func MakeDataSourcePipelinePlanV200(subtaskMetas
[]plugin.SubTaskMeta, connectio
return plan, scopes, nil
}
-func getScopeConfigByScopeId(basicRes context.BasicRes, connectionId uint64,
scopeId string) (*models.TapdScopeConfig, errors.Error) {
- db := basicRes.GetDal()
- scopeConfig := &models.TapdScopeConfig{}
- err := db.First(scopeConfig,
- dal.Select("c.*"),
- dal.From("_tool_tapd_scope_configs c"),
- dal.Join("LEFT JOIN _tool_tapd_workspaces s ON
(s.scope_config_id = c.id)"),
- dal.Where("s.connection_id = ? AND s.id = ?", connectionId,
scopeId),
- )
- if err != nil {
- return nil, err
- }
- return scopeConfig, nil
-}
-
func makeDataSourcePipelinePlanV200(
- basicRes context.BasicRes,
subtaskMetas []plugin.SubTaskMeta,
plan plugin.PipelinePlan,
bpScopes []*plugin.BlueprintScopeV200,
@@ -88,7 +69,7 @@ func makeDataSourcePipelinePlanV200(
options["timeAfter"] =
syncPolicy.TimeAfter.Format(time.RFC3339)
}
- scopeConfig, err := getScopeConfigByScopeId(basicRes,
connectionId, bpScope.Id)
+ _, scopeConfig, err :=
scopeHelper.DbHelper().GetScopeAndConfig(connectionId, bpScope.Id)
if err != nil {
return nil, err
}
@@ -109,22 +90,14 @@ func makeDataSourcePipelinePlanV200(
}
func makeScopesV200(
- basicRes context.BasicRes,
bpScopes []*plugin.BlueprintScopeV200,
connectionId uint64) ([]plugin.Scope, errors.Error,
) {
scopes := make([]plugin.Scope, 0)
for _, bpScope := range bpScopes {
- tapdWorkspace := &models.TapdWorkspace{}
- // get repo from db
- err := basicRes.GetDal().First(tapdWorkspace,
- dal.Where(`connection_id = ? and id = ?`,
- connectionId, bpScope.Id))
- if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("fail
to find wrokspace %s", bpScope.Id))
- }
+ // get workspace and scope config from db
- scopeConfig, err := getScopeConfigByScopeId(basicRes,
connectionId, bpScope.Id)
+ tapdWorkspace, scopeConfig, err :=
scopeHelper.DbHelper().GetScopeAndConfig(connectionId, bpScope.Id)
if err != nil {
return nil, err
}
diff --git a/backend/plugins/tapd/api/blueprint_v200_test.go
b/backend/plugins/tapd/api/blueprint_v200_test.go
index 043b973c6..0acdfff79 100644
--- a/backend/plugins/tapd/api/blueprint_v200_test.go
+++ b/backend/plugins/tapd/api/blueprint_v200_test.go
@@ -24,7 +24,7 @@ import (
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/core/plugin"
- mockcontext "github.com/apache/incubator-devlake/mocks/core/context"
+ "github.com/apache/incubator-devlake/helpers/unithelper"
mockdal "github.com/apache/incubator-devlake/mocks/core/dal"
mockplugin "github.com/apache/incubator-devlake/mocks/core/plugin"
"github.com/apache/incubator-devlake/plugins/tapd/models"
@@ -44,10 +44,10 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
bpScopes := make([]*plugin.BlueprintScopeV200, 0)
bpScopes = append(bpScopes, bs)
plan := make(plugin.PipelinePlan, len(bpScopes))
- basicRes := NewMockBasicRes()
- plan, err = makeDataSourcePipelinePlanV200(basicRes, nil, plan,
bpScopes, uint64(1), syncPolicy)
+ mockBasicRes()
+ plan, err = makeDataSourcePipelinePlanV200(nil, plan, bpScopes,
uint64(1), syncPolicy)
assert.Nil(t, err)
- scopes, err := makeScopesV200(basicRes, bpScopes, uint64(1))
+ scopes, err := makeScopesV200(bpScopes, uint64(1))
assert.Nil(t, err)
expectPlan := plugin.PipelinePlan{
@@ -76,8 +76,7 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
assert.Equal(t, expectScopes, scopes)
}
-// NewMockBasicRes FIXME ...
-func NewMockBasicRes() *mockcontext.BasicRes {
+func mockBasicRes() {
tapdWorkspace := &models.TapdWorkspace{
ConnectionId: 1,
Id: 10,
@@ -88,21 +87,16 @@ func NewMockBasicRes() *mockcontext.BasicRes {
Entities: []string{plugin.DOMAIN_TYPE_TICKET},
},
}
-
- mockRes := new(mockcontext.BasicRes)
- mockDal := new(mockdal.Dal)
-
- mockDal.On("First", mock.AnythingOfType("*models.TapdScopeConfig"),
mock.Anything).Run(func(args mock.Arguments) {
- dst := args.Get(0).(*models.TapdScopeConfig)
- *dst = *scopeConfig
- }).Return(nil)
- mockDal.On("First", mock.AnythingOfType("*models.TapdWorkspace"),
mock.Anything).Run(func(args mock.Arguments) {
- dst := args.Get(0).(*models.TapdWorkspace)
- *dst = *tapdWorkspace
- }).Return(nil)
-
- mockRes.On("GetDal").Return(mockDal)
- mockRes.On("GetConfig", mock.Anything).Return("")
-
- return mockRes
+ // Refresh Global Variables and set the sql mock
+ mockRes := unithelper.DummyBasicRes(func(mockDal *mockdal.Dal) {
+ mockDal.On("First",
mock.AnythingOfType("*models.TapdScopeConfig"), mock.Anything).Run(func(args
mock.Arguments) {
+ dst := args.Get(0).(*models.TapdScopeConfig)
+ *dst = *scopeConfig
+ }).Return(nil)
+ mockDal.On("First",
mock.AnythingOfType("*models.TapdWorkspace"), mock.Anything).Run(func(args
mock.Arguments) {
+ dst := args.Get(0).(*models.TapdWorkspace)
+ *dst = *tapdWorkspace
+ }).Return(nil)
+ })
+ Init(mockRes)
}
diff --git a/backend/plugins/tapd/impl/impl.go
b/backend/plugins/tapd/impl/impl.go
index f341b48b3..6339e1723 100644
--- a/backend/plugins/tapd/impl/impl.go
+++ b/backend/plugins/tapd/impl/impl.go
@@ -292,11 +292,11 @@ func (p Tapd) ApiResources()
map[string]map[string]plugin.ApiResourceHandler {
"GET": api.GetScopeList,
"PUT": api.PutScope,
},
- "connections/:connectionId/scope_configs": {
+ "connections/:connectionId/scope-configs": {
"POST": api.CreateScopeConfig,
"GET": api.GetScopeConfigList,
},
- "connections/:connectionId/scope_configs/:id": {
+ "connections/:connectionId/scope-configs/:id": {
"PATCH": api.UpdateScopeConfig,
"GET": api.GetScopeConfig,
},
diff --git a/backend/plugins/teambition/api/blueprint200.go
b/backend/plugins/teambition/api/blueprint200.go
index a4c6e044f..104a2c671 100644
--- a/backend/plugins/teambition/api/blueprint200.go
+++ b/backend/plugins/teambition/api/blueprint200.go
@@ -19,16 +19,16 @@ package api
import (
"fmt"
- "github.com/apache/incubator-devlake/plugins/teambition/models"
"time"
+ "github.com/apache/incubator-devlake/plugins/teambition/models"
+
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/core/utils"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
@@ -66,7 +66,7 @@ func makeDataSourcePipelinePlanV200(
options["timeAfter"] =
syncPolicy.TimeAfter.Format(time.RFC3339)
}
- subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas,
bpScope.Entities)
+ subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas,
plugin.DOMAIN_TYPES)
if err != nil {
return nil, err
}
@@ -93,15 +93,15 @@ func makeScopesV200(bpScopes []*plugin.BlueprintScopeV200,
connectionId uint64)
return nil, errors.Default.Wrap(err, fmt.Sprintf("fail
to find project %s", bpScope.Id))
}
// add board to scopes
- if utils.StringsContains(bpScope.Entities,
plugin.DOMAIN_TYPE_TICKET) {
- domainBoard := &ticket.Board{
- DomainEntity: domainlayer.DomainEntity{
- Id:
didgen.NewDomainIdGenerator(&models.TeambitionProject{}).Generate(project.ConnectionId,
project.Id),
- },
- Name: project.Name,
- }
- scopes = append(scopes, domainBoard)
+ // if utils.StringsContains(bpScope.Entities,
plugin.DOMAIN_TYPE_TICKET) {
+ domainBoard := &ticket.Board{
+ DomainEntity: domainlayer.DomainEntity{
+ Id:
didgen.NewDomainIdGenerator(&models.TeambitionProject{}).Generate(project.ConnectionId,
project.Id),
+ },
+ Name: project.Name,
}
+ scopes = append(scopes, domainBoard)
+ // }
}
return scopes, nil
}
diff --git a/backend/plugins/trello/api/blueprint_v200.go
b/backend/plugins/trello/api/blueprint_v200.go
index 74fc3a401..784a2e332 100644
--- a/backend/plugins/trello/api/blueprint_v200.go
+++ b/backend/plugins/trello/api/blueprint_v200.go
@@ -18,10 +18,8 @@ limitations under the License.
package api
import (
- "fmt"
"time"
- "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
@@ -53,20 +51,10 @@ func MakePipelinePlanV200(subtaskMetas
[]plugin.SubTaskMeta, connectionId uint64
func makeScopeV200(connectionId uint64, scopes []*plugin.BlueprintScopeV200)
([]plugin.Scope, errors.Error) {
sc := make([]plugin.Scope, 0, len(scopes))
- db := basicRes.GetDal()
for _, scope := range scopes {
- trelloBoard := &models.TrelloBoard{}
- // get board from db
- err := db.First(trelloBoard,
- dal.Where(`connection_id = ? and board_id = ?`,
connectionId, scope.Id))
+ trelloBoard, scopeConfig, err :=
scopeHelper.DbHelper().GetScopeAndConfig(connectionId, scope.Id)
if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("fail
to find board %s", scope.Id))
- }
- // get scope config from db
- scopeConfig := &models.TrelloScopeConfig{}
- err = db.First(scopeConfig, dal.Where(`id = ?`,
trelloBoard.ScopeConfigId))
- if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("fail
to find scope config %d", trelloBoard.ScopeConfigId))
+ return nil, err
}
// add board to scopes
if utils.StringsContains(scopeConfig.Entities,
plugin.DOMAIN_TYPE_TICKET) {
@@ -84,7 +72,6 @@ func makeScopeV200(connectionId uint64, scopes
[]*plugin.BlueprintScopeV200) ([]
}
func makePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta, plan
plugin.PipelinePlan, scopes []*plugin.BlueprintScopeV200, connectionId uint64,
syncPolicy *plugin.BlueprintSyncPolicy) (plugin.PipelinePlan, errors.Error) {
- db := basicRes.GetDal()
for i, scope := range scopes {
stage := plan[i]
if stage == nil {
@@ -99,18 +86,9 @@ func makePipelinePlanV200(subtaskMetas []plugin.SubTaskMeta,
plan plugin.Pipelin
options["timeAfter"] =
syncPolicy.TimeAfter.Format(time.RFC3339)
}
- // get board from db
- trelloBoard := &models.TrelloBoard{}
- err := db.First(trelloBoard,
- dal.Where(`connection_id = ? and board_id = ?`,
connectionId, scope.Id))
+ _, scopeConfig, err :=
scopeHelper.DbHelper().GetScopeAndConfig(connectionId, scope.Id)
if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("fail
to find board %s", scope.Id))
- }
- // get scope config from db
- scopeConfig := &models.TrelloScopeConfig{}
- err = db.First(scopeConfig, dal.Where(`id = ?`,
trelloBoard.ScopeConfigId))
- if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("fail
to find scope config %d", trelloBoard.ScopeConfigId))
+ return nil, err
}
// construct subtasks
subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas,
scopeConfig.Entities)
diff --git a/backend/plugins/trello/impl/impl.go
b/backend/plugins/trello/impl/impl.go
index 56dd6f0ac..0a3fb9b4f 100644
--- a/backend/plugins/trello/impl/impl.go
+++ b/backend/plugins/trello/impl/impl.go
@@ -159,11 +159,11 @@ func (p Trello) ApiResources()
map[string]map[string]plugin.ApiResourceHandler {
"connections/:connectionId/proxy/rest/*path": {
"GET": api.Proxy,
},
- "connections/:connectionId/scope_configs": {
+ "connections/:connectionId/scope-configs": {
"POST": api.CreateScopeConfig,
"GET": api.GetScopeConfigList,
},
- "connections/:connectionId/scope_configs/:id": {
+ "connections/:connectionId/scope-configs/:id": {
"PATCH": api.UpdateScopeConfig,
"GET": api.GetScopeConfig,
},
diff --git a/backend/plugins/zentao/api/blueprint_V200_test.go
b/backend/plugins/zentao/api/blueprint_V200_test.go
index cb96293e5..bb48c595c 100644
--- a/backend/plugins/zentao/api/blueprint_V200_test.go
+++ b/backend/plugins/zentao/api/blueprint_V200_test.go
@@ -25,7 +25,7 @@ import (
"github.com/apache/incubator-devlake/core/models/domainlayer/ticket"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- mockcontext "github.com/apache/incubator-devlake/mocks/core/context"
+ "github.com/apache/incubator-devlake/helpers/unithelper"
mockdal "github.com/apache/incubator-devlake/mocks/core/dal"
mockplugin "github.com/apache/incubator-devlake/mocks/core/plugin"
"github.com/apache/incubator-devlake/plugins/zentao/models"
@@ -58,14 +58,12 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
err := plugin.RegisterPlugin("zentao", mockMeta)
assert.Nil(t, err)
// Refresh Global Variables and set the sql mock
- basicRes = NewMockBasicRes()
+ mockBasicRes()
bs := &plugin.BlueprintScopeV200{
- Entities: []string{"TICKET"},
- Id: "project/1",
+ Id: "project/1",
}
bs2 := &plugin.BlueprintScopeV200{
- Entities: []string{"TICKET"},
- Id: "product/1",
+ Id: "product/1",
}
bpScopes := make([]*plugin.BlueprintScopeV200, 0)
bpScopes = append(bpScopes, bs, bs2)
@@ -74,7 +72,6 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
plan := make(plugin.PipelinePlan, len(bpScopes))
plan, scopes, err := makePipelinePlanV200(nil, plan, bpScopes,
connection, syncPolicy)
assert.Nil(t, err)
- basicRes = NewMockBasicRes()
expectPlan := plugin.PipelinePlan{
plugin.PipelineStage{
@@ -127,49 +124,47 @@ func TestMakeDataSourcePipelinePlanV200(t *testing.T) {
assert.Equal(t, expectScopes, scopes)
}
-// NewMockBasicRes FIXME ...
-func NewMockBasicRes() *mockcontext.BasicRes {
+// mockBasicRes FIXME ...
+func mockBasicRes() {
testZentaoProduct := &models.ZentaoProduct{
ConnectionId: 1,
Id: 1,
Name: "test/testRepo",
Type: `product/normal`,
- //TransformationRuleId: 1,
+ //ScopeConfigId: 1,
}
testZentaoProject := &models.ZentaoProject{
ConnectionId: 1,
Id: 1,
Name: "test/testRepo",
Type: `project`,
- //TransformationRuleId: 1,
+ //ScopeConfigId: 1,
}
- //testTransformationRule := &models.ZentaoTransformation{
- // Model: common.Model{
- // ID: 1,
- // },
- // Name: "Zentao transformation rule",
- //}
- mockRes := new(mockcontext.BasicRes)
- mockDal := new(mockdal.Dal)
-
- mockDal.On("First", mock.Anything, mock.Anything).Run(func(args
mock.Arguments) {
- dst := args.Get(0).(*models.ZentaoProject)
- *dst = *testZentaoProject
- }).Return(nil).Once()
-
- mockDal.On("First", mock.Anything, mock.Anything).Run(func(args
mock.Arguments) {
- dst := args.Get(0).(*models.ZentaoProduct)
- *dst = *testZentaoProduct
- }).Return(nil).Once()
-
- //mockDal.On("First", mock.Anything, mock.Anything).Run(func(args
mock.Arguments) {
- // dst := args.Get(0).(*models.ZentaoTransformation)
- // *dst = *testTransformationRule
- //}).Return(nil).Once()
-
- mockRes.On("GetDal").Return(mockDal)
- mockRes.On("GetConfig", mock.Anything).Return("")
-
- return mockRes
+ testScopeConfig := &models.ZentaoScopeConfig{
+ ScopeConfig: common.ScopeConfig{
+ Entities: []string{"TICKET"},
+ Model: common.Model{
+ ID: 1,
+ },
+ },
+ Name: "Zentao scope config",
+ }
+ mockRes := unithelper.DummyBasicRes(func(mockDal *mockdal.Dal) {
+ mockDal.On("First",
mock.AnythingOfType("*models.ZentaoProject"), mock.Anything).Run(func(args
mock.Arguments) {
+ dst := args.Get(0).(*models.ZentaoProject)
+ *dst = *testZentaoProject
+ }).Return(nil)
+
+ mockDal.On("First",
mock.AnythingOfType("*models.ZentaoProduct"), mock.Anything).Run(func(args
mock.Arguments) {
+ dst := args.Get(0).(*models.ZentaoProduct)
+ *dst = *testZentaoProduct
+ }).Return(nil)
+
+ mockDal.On("First",
mock.AnythingOfType("*models.ZentaoScopeConfig"), mock.Anything).Run(func(args
mock.Arguments) {
+ dst := args.Get(0).(*models.ZentaoScopeConfig)
+ *dst = *testScopeConfig
+ }).Return(nil)
+ })
+ Init(mockRes)
}
diff --git a/backend/plugins/zentao/api/blueprint_v200.go
b/backend/plugins/zentao/api/blueprint_v200.go
index 0352fb1f3..f64738992 100644
--- a/backend/plugins/zentao/api/blueprint_v200.go
+++ b/backend/plugins/zentao/api/blueprint_v200.go
@@ -18,11 +18,9 @@ limitations under the License.
package api
import (
- "fmt"
"strings"
"time"
- "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models/domainlayer"
"github.com/apache/incubator-devlake/core/models/domainlayer/didgen"
@@ -60,7 +58,6 @@ func makePipelinePlanV200(
connection *models.ZentaoConnection,
syncPolicy *plugin.BlueprintSyncPolicy,
) (plugin.PipelinePlan, []plugin.Scope, errors.Error) {
- var err errors.Error
domainScopes := make([]plugin.Scope, 0)
for i, bpScope := range bpScopes {
stage := plan[i]
@@ -74,16 +71,17 @@ func makePipelinePlanV200(
scopeType := strings.Split(bpScope.Id, `/`)[0]
scopeId := strings.Split(bpScope.Id, `/`)[1]
+
+ var entities []string
if scopeType == `project` {
- scope := &models.ZentaoProject{}
- // get repo from db
- err = basicRes.GetDal().First(scope,
dal.Where(`connection_id = ? AND id = ?`, connection.ID, scopeId))
+ scope, scopeConfig, err :=
projectScopeHelper.DbHelper().GetScopeAndConfig(connection.ID, scopeId)
if err != nil {
- return nil, nil, errors.Default.Wrap(err,
fmt.Sprintf("fail to find zentao project %s", bpScope.Id))
+ return nil, nil, err
}
op.ProjectId = scope.Id
+ entities = scopeConfig.Entities
- if utils.StringsContains(bpScope.Entities,
plugin.DOMAIN_TYPE_TICKET) {
+ if utils.StringsContains(scopeConfig.Entities,
plugin.DOMAIN_TYPE_TICKET) {
scopeTicket := &ticket.Board{
DomainEntity: domainlayer.DomainEntity{
Id:
didgen.NewDomainIdGenerator(&models.ZentaoProject{}).Generate(connection.ID,
scope.Id),
@@ -94,15 +92,14 @@ func makePipelinePlanV200(
domainScopes = append(domainScopes, scopeTicket)
}
} else {
- scope := &models.ZentaoProduct{}
- // get repo from db
- err = basicRes.GetDal().First(scope,
dal.Where(`connection_id = ? AND id = ?`, connection.ID, scopeId))
+ scope, scopeConfig, err :=
productScopeHelper.DbHelper().GetScopeAndConfig(connection.ID, scopeId)
if err != nil {
- return nil, nil, errors.Default.Wrap(err,
fmt.Sprintf("fail to find zentao product %s", bpScope.Id))
+ return nil, nil, err
}
op.ProductId = scope.Id
+ entities = scopeConfig.Entities
- if utils.StringsContains(bpScope.Entities,
plugin.DOMAIN_TYPE_TICKET) {
+ if utils.StringsContains(scopeConfig.Entities,
plugin.DOMAIN_TYPE_TICKET) {
scopeTicket := &ticket.Board{
DomainEntity: domainlayer.DomainEntity{
Id:
didgen.NewDomainIdGenerator(&models.ZentaoProduct{}).Generate(connection.ID,
scope.Id),
@@ -122,7 +119,7 @@ func makePipelinePlanV200(
return nil, nil, err
}
- subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas,
bpScope.Entities)
+ subtasks, err := helper.MakePipelinePlanSubtasks(subtaskMetas,
entities)
if err != nil {
return nil, nil, err
}
diff --git a/backend/plugins/zentao/api/init.go
b/backend/plugins/zentao/api/init.go
index 6eca63dab..ff47ba7fe 100644
--- a/backend/plugins/zentao/api/init.go
+++ b/backend/plugins/zentao/api/init.go
@@ -82,4 +82,8 @@ func Init(br context.BasicRes) {
vld,
connectionHelper,
)
+ scHelper = api.NewScopeConfigHelper[models.ZentaoScopeConfig](
+ basicRes,
+ vld,
+ )
}
diff --git a/backend/plugins/zentao/impl/impl.go
b/backend/plugins/zentao/impl/impl.go
index 922a0456b..cb90a71f6 100644
--- a/backend/plugins/zentao/impl/impl.go
+++ b/backend/plugins/zentao/impl/impl.go
@@ -212,6 +212,14 @@ func (p Zentao) ApiResources()
map[string]map[string]plugin.ApiResourceHandler {
"PATCH": api.UpdateProjectScope,
"DELETE": api.DeleteProjectScope,
},
+ "connections/:connectionId/scope-configs": {
+ "POST": api.CreateScopeConfig,
+ "GET": api.GetScopeConfigList,
+ },
+ "connections/:connectionId/scope-configs/:id": {
+ "PATCH": api.UpdateScopeConfig,
+ "GET": api.GetScopeConfig,
+ },
"connections/:connectionId/remote-scopes": {
"GET": api.RemoteScopes,
},
diff --git a/backend/server/services/blueprint.go
b/backend/server/services/blueprint.go
index 236dfc2bb..2bc547561 100644
--- a/backend/server/services/blueprint.go
+++ b/backend/server/services/blueprint.go
@@ -335,8 +335,7 @@ func MakePlanForBlueprint(blueprint *models.Blueprint,
skipCollectors bool) (plu
var plan plugin.PipelinePlan
switch bpSettings.Version {
case "1.0.0":
- // Notice: v1 not complete SkipOnFail & TimeAfter
- plan, err = GeneratePlanJsonV100(bpSettings)
+ return nil, errors.BadInput.New("Blueprint v1.0.0 had been
deprecated, please se v2.0.0 instead")
case "2.0.0":
// load project metric plugins and convert it to a map
metrics := make(map[string]json.RawMessage)
diff --git a/backend/server/services/blueprint_makeplan_v100.go
b/backend/server/services/blueprint_makeplan_v100.go
deleted file mode 100644
index bc3cfb411..000000000
--- a/backend/server/services/blueprint_makeplan_v100.go
+++ /dev/null
@@ -1,77 +0,0 @@
-/*
-Licensed to the Apache Software Foundation (ASF) under one or more
-contributor license agreements. See the NOTICE file distributed with
-this work for additional information regarding copyright ownership.
-The ASF licenses this file to You under the Apache License, Version 2.0
-(the "License"); you may not use this file except in compliance with
-the License. You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package services
-
-import (
- "encoding/json"
- "fmt"
- "github.com/apache/incubator-devlake/core/errors"
- "github.com/apache/incubator-devlake/core/models"
- "github.com/apache/incubator-devlake/core/plugin"
-)
-
-// GeneratePlanJsonV100 generates pipeline plan according v1.0.0 definition
-func GeneratePlanJsonV100(settings *models.BlueprintSettings)
(plugin.PipelinePlan, errors.Error) {
- connections := make([]*plugin.BlueprintConnectionV100, 0)
- err := errors.Convert(json.Unmarshal(settings.Connections,
&connections))
- if err != nil {
- return nil, err
- }
- hasDoraEnrich := false
- doraRules := make(map[string]interface{})
- plans := make([]plugin.PipelinePlan, len(connections))
- for i, connection := range connections {
- if len(connection.Scope) == 0 {
- return nil,
errors.Default.New(fmt.Sprintf("connections[%d].scope is empty", i))
- }
- p, err := plugin.GetPlugin(connection.Plugin)
- if err != nil {
- return nil, err
- }
- if pluginBp, ok := p.(plugin.PluginBlueprintV100); ok {
- plans[i], err =
pluginBp.MakePipelinePlan(connection.ConnectionId, connection.Scope)
- if err != nil {
- return nil, err
- }
- } else {
- return nil, errors.Default.New(fmt.Sprintf("plugin %s
does not support blueprint protocol version 1.0.0", connection.Plugin))
- }
- for _, stage := range plans[i] {
- for _, task := range stage {
- if task.Plugin == "dora" {
- hasDoraEnrich = true
- for k, v := range task.Options {
- doraRules[k] = v
- }
- }
- }
- }
- }
- mergedPipelinePlan := ParallelizePipelinePlans(plans...)
- if hasDoraEnrich {
- plan := plugin.PipelineStage{
- &plugin.PipelineTask{
- Plugin: "dora",
- Subtasks:
[]string{"calculateChangeLeadTimeOld", "connectIncidentToDeploymentOld"},
- Options: doraRules,
- },
- }
- mergedPipelinePlan = append(mergedPipelinePlan, plan)
- }
- return mergedPipelinePlan, nil
-}
diff --git a/backend/server/services/remote/plugin/scope_api.go
b/backend/server/services/remote/plugin/scope_api.go
index bc768fa68..6d89c2422 100644
--- a/backend/server/services/remote/plugin/scope_api.go
+++ b/backend/server/services/remote/plugin/scope_api.go
@@ -108,9 +108,9 @@ func convertScopeResponse(scopes
...*api.ScopeRes[models.RemoteScope]) ([]map[st
for i, scope := range scopes {
resMap := map[string]any{}
err := models.MapTo(api.ScopeRes[map[string]any]{
- Scope: nil, //ignore intentionally
- TransformationRuleName: scope.TransformationRuleName,
- Blueprints: scope.Blueprints,
+ Scope: nil, //ignore intentionally
+ ScopeConfigName: scope.ScopeConfigName,
+ Blueprints: scope.Blueprints,
}, &resMap)
if err != nil {
return nil, err
diff --git a/backend/server/services/remote/plugin/scope_db_helper.go
b/backend/server/services/remote/plugin/scope_db_helper.go
index e9fdac371..e44d425f3 100644
--- a/backend/server/services/remote/plugin/scope_db_helper.go
+++ b/backend/server/services/remote/plugin/scope_db_helper.go
@@ -19,14 +19,15 @@ package plugin
import (
"fmt"
+ "reflect"
+ "time"
+
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/server/services/remote/models"
- "reflect"
- "time"
)
type ScopeDatabaseHelperImpl struct {
@@ -74,14 +75,15 @@ func (s *ScopeDatabaseHelperImpl) UpdateScope(connectionId
uint64, scopeId strin
return s.save([]*models.RemoteScope{scope}, nil, &now)
}
-func (s *ScopeDatabaseHelperImpl) GetScope(connectionId uint64, scopeId
string) (models.RemoteScope, errors.Error) {
+func (s *ScopeDatabaseHelperImpl) GetScope(connectionId uint64, scopeId
string) (*models.RemoteScope, errors.Error) {
query := dal.Where(fmt.Sprintf("connection_id = ? AND %s = ?",
s.params.ScopeIdColumnName), connectionId, scopeId)
scope := s.pa.scopeType.New()
err := api.CallDB(s.db.First, scope, query)
if err != nil {
return nil, errors.Default.Wrap(err, "could not get scope")
}
- return scope.Unwrap(), nil
+ // @keon @camille: not sure if this is correct
+ return (*models.RemoteScope)(scope.UnwrapPtr()), nil
}
func (s *ScopeDatabaseHelperImpl) ListScopes(input *plugin.ApiResourceInput,
connectionId uint64) ([]*models.RemoteScope, errors.Error) {
diff --git a/backend/test/e2e/remote/helper.go
b/backend/test/e2e/remote/helper.go
index d706a19bc..d2fc4ebae 100644
--- a/backend/test/e2e/remote/helper.go
+++ b/backend/test/e2e/remote/helper.go
@@ -19,14 +19,15 @@ package remote
import (
"fmt"
- "github.com/apache/incubator-devlake/core/models"
- "github.com/apache/incubator-devlake/core/plugin"
- "github.com/stretchr/testify/require"
"os"
"path/filepath"
"testing"
"time"
+ "github.com/apache/incubator-devlake/core/models"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/stretchr/testify/require"
+
"github.com/apache/incubator-devlake/test/helper"
)
@@ -129,9 +130,6 @@ func CreateTestBlueprints(t *testing.T, client
*helper.DevlakeClient, count int)
{
Id: scope.Id,
Name: "Test scope",
- Entities: []string{
-
plugin.DOMAIN_TYPE_CICD,
- },
},
},
},
diff --git a/backend/test/helper/api.go b/backend/test/helper/api.go
index 7765c95a1..807a16c30 100644
--- a/backend/test/helper/api.go
+++ b/backend/test/helper/api.go
@@ -273,29 +273,6 @@ func (d *DevlakeClient) SearchRemoteScopes(query
SearchRemoteScopesQuery) Search
nil, nil)
}
-// CreateBasicBlueprint FIXME
-func (d *DevlakeClient) CreateBasicBlueprint(name string, connection
*plugin.BlueprintConnectionV100) models.Blueprint {
- settings := &models.BlueprintSettings{
- Version: "1.0.0",
- Connections:
ToJson([]*plugin.BlueprintConnectionV100{connection}),
- }
- blueprint := models.Blueprint{
- Name: name,
- Mode: models.BLUEPRINT_MODE_NORMAL,
- Plan: nil,
- Enable: true,
- CronConfig: "manual",
- IsManual: true,
- Settings: ToJson(settings),
- }
- d.testCtx.Helper()
- blueprint = sendHttpRequest[models.Blueprint](d.testCtx, d.timeout,
debugInfo{
- print: true,
- inlineJson: false,
- }, http.MethodPost, fmt.Sprintf("%s/blueprints", d.Endpoint), nil,
&blueprint)
- return blueprint
-}
-
// TriggerBlueprint FIXME
func (d *DevlakeClient) TriggerBlueprint(blueprintId uint64) models.Pipeline {
d.testCtx.Helper()
