This is an automated email from the ASF dual-hosted git repository.
hez pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 42b730a6d [feat-5503]: Deletable scopes and connections (#5506)
42b730a6d is described below
commit 42b730a6da302e18ae854282bfeef537a6fe186d
Author: Keon Amini <[email protected]>
AuthorDate: Fri Jun 16 18:20:46 2023 -0500
[feat-5503]: Deletable scopes and connections (#5506)
* feat: Scopes and Connection are only deleted if they are deletable
* feat: auto-delete a project's blueprint when that project is deleted
* test: Added integration test cases
* fix: Allow clean scope data in spite of existing references to that scope
* test: Added scope data integrity validations to tests
fix: fixed python setting the wrong scope_id on CICDScope
fix: scope_helper now excludes the scope model from data deletion
* fix: remove double-delete of blueprint in delete-project
* fix: minor fixes to get the tests passing
---
backend/Makefile | 8 +-
backend/core/errors/types.go | 12 +-
backend/core/plugin/plugin_api.go | 5 +
.../helpers/pluginhelper/api/connection_helper.go | 15 +-
.../pluginhelper/api/scope_generic_helper.go | 162 +++++++++++-------
backend/helpers/pluginhelper/api/scope_helper.go | 4 +-
.../pluginhelper/services/blueprint_helper.go | 36 ++++
backend/plugins/ae/api/connection.go | 8 +-
backend/plugins/bamboo/api/connection.go | 8 +-
backend/plugins/bamboo/api/scope.go | 1 +
backend/plugins/bitbucket/api/connection.go | 8 +-
backend/plugins/bitbucket/api/scope.go | 1 +
backend/plugins/feishu/api/connection.go | 8 +-
backend/plugins/gitee/api/connection.go | 8 +-
backend/plugins/github/api/connection.go | 8 +-
backend/plugins/github/api/scope.go | 1 +
backend/plugins/gitlab/api/connection.go | 8 +-
backend/plugins/gitlab/api/scope.go | 1 +
backend/plugins/jenkins/api/connection.go | 8 +-
backend/plugins/jenkins/api/scope.go | 1 +
backend/plugins/jira/api/connection.go | 8 +-
backend/plugins/jira/api/scope.go | 1 +
backend/plugins/pagerduty/api/connection.go | 8 +-
backend/plugins/pagerduty/api/scope.go | 1 +
backend/plugins/slack/api/connection.go | 8 +-
backend/plugins/sonarqube/api/connection.go | 8 +-
backend/plugins/sonarqube/api/scope.go | 1 +
backend/plugins/tapd/api/connection.go | 8 +-
backend/plugins/tapd/api/scope.go | 1 +
backend/plugins/teambition/api/connection.go | 8 +-
backend/plugins/trello/api/connection.go | 8 +-
backend/plugins/trello/api/scope.go | 1 +
backend/plugins/webhook/api/connection.go | 8 +-
backend/plugins/zentao/api/connection.go | 8 +-
backend/plugins/zentao/api/scope.go | 2 +
backend/python/pydevlake/pydevlake/plugin.py | 2 +-
backend/server/api/router.go | 15 +-
backend/server/services/project.go | 38 +++--
.../services/remote/plugin/connection_api.go | 5 +-
.../server/services/remote/plugin/plugin_impl.go | 10 +-
backend/server/services/remote/plugin/scope_api.go | 4 +-
backend/test/e2e/remote/helper.go | 63 ++++++-
backend/test/e2e/remote/python_plugin_test.go | 89 +++++++++-
backend/test/helper/api.go | 187 ++++++++++++---------
backend/test/helper/client.go | 101 ++++++++---
backend/test/helper/client_factory.go | 10 ++
46 files changed, 685 insertions(+), 229 deletions(-)
diff --git a/backend/Makefile b/backend/Makefile
index 01d1f343d..b1bee77a7 100644
--- a/backend/Makefile
+++ b/backend/Makefile
@@ -110,16 +110,20 @@ e2e-plugins-test:
done; \
exit $$exit_code
-e2e-test:
+e2e-test-init:
export ENV_PATH=$(shell readlink -f .env);\
set -e;\
go run ./test/init.go || exit $$?;\
+
+e2e-test-run:
exit_code=0;\
- for m in $$(go list ./test/e2e/... | grep -v manual/); do \
+ for m in $$(go list ./test/e2e/... | grep -v manual); do \
echo $$m; go test -p 1 -timeout 300s -v $$m || exit_code=$$?; \
done; \
exit $$exit_code
+e2e-test: e2e-test-init e2e-test-run
+
integration-test:
export ENV_PATH=$(shell readlink -f .env);\
set -e;\
diff --git a/backend/core/errors/types.go b/backend/core/errors/types.go
index 3b12e0f68..a4af60ef2 100644
--- a/backend/core/errors/types.go
+++ b/backend/core/errors/types.go
@@ -27,13 +27,17 @@ var (
// Default special error type. If it's wrapping another error, then it
will take the type of that error if it's an Error. Otherwise, it equates to
Internal.
Default = register(nil)
- SubtaskErr = register(&Type{meta: "subtask"})
- NotFound = register(&Type{httpCode: http.StatusNotFound, meta:
"not-found"})
+ SubtaskErr = register(&Type{meta: "subtask"})
+ //400+
BadInput = register(&Type{httpCode: http.StatusBadRequest, meta:
"bad-input"})
Unauthorized = register(&Type{httpCode: http.StatusUnauthorized, meta:
"unauthorized"})
Forbidden = register(&Type{httpCode: http.StatusForbidden, meta:
"forbidden"})
- Internal = register(&Type{httpCode: http.StatusInternalServerError,
meta: "internal"})
- Timeout = register(&Type{httpCode: http.StatusGatewayTimeout,
meta: "timeout"})
+ NotFound = register(&Type{httpCode: http.StatusNotFound, meta:
"not-found"})
+ Conflict = register(&Type{httpCode: http.StatusConflict, meta:
"internal"})
+
+ //500+
+ Internal = register(&Type{httpCode: http.StatusInternalServerError,
meta: "internal"})
+ Timeout = register(&Type{httpCode: http.StatusGatewayTimeout, meta:
"timeout"})
//cached values
typesByHttpCode = newSyncMap[int, *Type]()
diff --git a/backend/core/plugin/plugin_api.go
b/backend/core/plugin/plugin_api.go
index 50b19072b..9ee36c2ff 100644
--- a/backend/core/plugin/plugin_api.go
+++ b/backend/core/plugin/plugin_api.go
@@ -31,6 +31,11 @@ type ApiResourceInput struct {
Request *http.Request
}
+// GetPlugin get the plugin in context
+func (input *ApiResourceInput) GetPlugin() string {
+ return input.Params["plugin"]
+}
+
// OutputFile is the file returned
type OutputFile struct {
ContentType string
diff --git a/backend/helpers/pluginhelper/api/connection_helper.go
b/backend/helpers/pluginhelper/api/connection_helper.go
index b49f59719..fd2d4c205 100644
--- a/backend/helpers/pluginhelper/api/connection_helper.go
+++ b/backend/helpers/pluginhelper/api/connection_helper.go
@@ -18,6 +18,7 @@ limitations under the License.
package api
import (
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/services"
"strconv"
"github.com/apache/incubator-devlake/core/context"
@@ -35,6 +36,7 @@ type ConnectionApiHelper struct {
log log.Logger
db dal.Dal
validator *validator.Validate
+ bpManager *services.BlueprintManager
}
// NewConnectionHelper creates a ConnectionHelper for connection management
@@ -50,6 +52,7 @@ func NewConnectionHelper(
log: basicRes.GetLogger(),
db: basicRes.GetDal(),
validator: vld,
+ bpManager:
services.NewBlueprintManager(basicRes.GetDal()),
}
}
@@ -101,8 +104,16 @@ func (c *ConnectionApiHelper) List(connections
interface{}) errors.Error {
}
// Delete connection
-func (c *ConnectionApiHelper) Delete(connection interface{}) errors.Error {
- return CallDB(c.db.Delete, connection)
+func (c *ConnectionApiHelper) Delete(plugin string, connection interface{})
(*services.BlueprintProjectPairs, errors.Error) {
+ connectionId := reflectField(connection, "ID").Uint()
+ referencingBps, err := c.bpManager.GetBlueprintsByConnection(plugin,
connectionId)
+ if err != nil {
+ return nil, err
+ }
+ if len(referencingBps) > 0 {
+ return services.NewBlueprintProjectPairs(referencingBps),
errors.Conflict.New("Found one or more references to this connection")
+ }
+ return nil, CallDB(c.db.Delete, connection)
}
func (c *ConnectionApiHelper) merge(connection interface{}, body
map[string]interface{}) errors.Error {
diff --git a/backend/helpers/pluginhelper/api/scope_generic_helper.go
b/backend/helpers/pluginhelper/api/scope_generic_helper.go
index 904a72d55..fb0d2dc18 100644
--- a/backend/helpers/pluginhelper/api/scope_generic_helper.go
+++ b/backend/helpers/pluginhelper/api/scope_generic_helper.go
@@ -46,12 +46,12 @@ var (
type NoScopeConfig struct{}
type (
- GenericScopeApiHelper[Conn any, Scope any, Tr any] struct {
+ GenericScopeApiHelper[Conn any, Scope any, ScopeConfig any] struct {
log log.Logger
db dal.Dal
validator *validator.Validate
reflectionParams *ReflectionParameters
- dbHelper ScopeDatabaseHelper[Conn, Scope, Tr]
+ dbHelper ScopeDatabaseHelper[Conn, Scope, ScopeConfig]
bpManager *serviceHelper.BlueprintManager
connHelper *ConnectionApiHelper
opts *ScopeHelperOptions
@@ -63,6 +63,8 @@ type (
ScopeConfig *ScopeConfig
`mapstructure:"scopeConfig,omitempty" json:"scopeConfig"`
Blueprints []*models.Blueprint
`mapstructure:"blueprints,omitempty" json:"blueprints"`
}
+ // Alias, for swagger purposes
+ ScopeRefDoc =
serviceHelper.BlueprintProjectPairs
ScopeRes[Scope any, ScopeConfig any] struct {
Scope *Scope
`mapstructure:",squash"` // ideally we need this field to be embedded in the
struct
ScopeResDoc[ScopeConfig] `mapstructure:",squash"` // however,
only this type of embeding is supported as of golang 1.20
@@ -99,14 +101,14 @@ type (
}
)
-func NewGenericScopeHelper[Conn any, Scope any, Tr any](
+func NewGenericScopeHelper[Conn any, Scope any, ScopeConfig any](
basicRes context.BasicRes,
vld *validator.Validate,
connHelper *ConnectionApiHelper,
- dbHelper ScopeDatabaseHelper[Conn, Scope, Tr],
+ dbHelper ScopeDatabaseHelper[Conn, Scope, ScopeConfig],
params *ReflectionParameters,
opts *ScopeHelperOptions,
-) *GenericScopeApiHelper[Conn, Scope, Tr] {
+) *GenericScopeApiHelper[Conn, Scope, ScopeConfig] {
if connHelper == nil {
panic("nil connHelper")
}
@@ -120,14 +122,7 @@ func NewGenericScopeHelper[Conn any, Scope any, Tr any](
if opts == nil {
opts = &ScopeHelperOptions{}
}
- tablesCacheLoader.Do(func() {
- var err errors.Error
- tablesCache, err = basicRes.GetDal().AllTables()
- if err != nil {
- panic(err)
- }
- })
- return &GenericScopeApiHelper[Conn, Scope, Tr]{
+ return &GenericScopeApiHelper[Conn, Scope, ScopeConfig]{
log: basicRes.GetLogger(),
db: basicRes.GetDal(),
validator: vld,
@@ -139,11 +134,11 @@ func NewGenericScopeHelper[Conn any, Scope any, Tr any](
}
}
-func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) DbHelper()
ScopeDatabaseHelper[Conn, Scope, Tr] {
+func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) DbHelper()
ScopeDatabaseHelper[Conn, Scope, ScopeConfig] {
return gs.dbHelper
}
-func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) PutScopes(input
*plugin.ApiResourceInput, scopes []*Scope) ([]*ScopeRes[Scope, Tr],
errors.Error) {
+func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) PutScopes(input
*plugin.ApiResourceInput, scopes []*Scope) ([]*ScopeRes[Scope, ScopeConfig],
errors.Error) {
params, err := gs.extractFromReqParam(input, false)
if err != nil {
return nil, err
@@ -179,7 +174,7 @@ func (gs *GenericScopeApiHelper[Conn, Scope, Tr])
PutScopes(input *plugin.ApiRes
return apiScopes, nil
}
-func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) UpdateScope(input
*plugin.ApiResourceInput) (*ScopeRes[Scope, Tr], errors.Error) {
+func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) UpdateScope(input
*plugin.ApiResourceInput) (*ScopeRes[Scope, ScopeConfig], errors.Error) {
params, err := gs.extractFromReqParam(input, true)
if err != nil {
return nil, err
@@ -211,7 +206,7 @@ func (gs *GenericScopeApiHelper[Conn, Scope, Tr])
UpdateScope(input *plugin.ApiR
return scopeRes[0], nil
}
-func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) GetScopes(input
*plugin.ApiResourceInput) ([]*ScopeRes[Scope, Tr], errors.Error) {
+func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) GetScopes(input
*plugin.ApiResourceInput) ([]*ScopeRes[Scope, ScopeConfig], errors.Error) {
params, err := gs.extractFromGetReqParam(input, false)
if err != nil {
return nil, errors.BadInput.New("invalid path params:
\"connectionId\" not set")
@@ -258,7 +253,7 @@ func (gs *GenericScopeApiHelper[Conn, Scope, Tr])
GetScopes(input *plugin.ApiRes
return apiScopes, nil
}
-func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) GetScope(input
*plugin.ApiResourceInput) (*ScopeRes[Scope, Tr], errors.Error) {
+func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) GetScope(input
*plugin.ApiResourceInput) (*ScopeRes[Scope, ScopeConfig], errors.Error) {
params, err := gs.extractFromGetReqParam(input, true)
if err != nil {
return nil, err
@@ -288,49 +283,45 @@ func (gs *GenericScopeApiHelper[Conn, Scope, Tr])
GetScope(input *plugin.ApiReso
return scopeRes, nil
}
-func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) DeleteScope(input
*plugin.ApiResourceInput) errors.Error {
+func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) DeleteScope(input
*plugin.ApiResourceInput) (*serviceHelper.BlueprintProjectPairs, errors.Error) {
params, err := gs.extractFromDeleteReqParam(input)
if err != nil {
- return err
+ return nil, err
}
err = gs.dbHelper.VerifyConnection(params.connectionId)
if err != nil {
- return errors.Default.Wrap(err, fmt.Sprintf("error verifying
connection for connection ID %d", params.connectionId))
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("error
verifying connection for connection ID %d", params.connectionId))
}
- scopeParamValue := params.scopeId
- if gs.opts.GetScopeParamValue != nil {
- scopeParamValue, err = gs.opts.GetScopeParamValue(gs.db,
params.scopeId)
+ if refs, err := gs.getScopeReferences(input.GetPlugin(),
params.connectionId, params.scopeId); err != nil || refs != nil {
if err != nil {
- return errors.Default.Wrap(err, fmt.Sprintf("error
extracting scope parameter name for scope %s", params.scopeId))
+ return nil, err
}
+ if err = gs.deleteScopeData(params.plugin, params.scopeId); err
!= nil {
+ return nil, err
+ }
+ return refs, errors.Conflict.New("Found one or more references
to this scope")
}
- // find all tables for this plugin
- tables, err := gs.getAffectedTables(params.plugin)
- if err != nil {
- return errors.Default.Wrap(err, fmt.Sprintf("error getting
database tables managed by plugin %s", params.plugin))
- }
- err = gs.transactionalDelete(tables, scopeParamValue)
- if err != nil {
- return errors.Default.Wrap(err, fmt.Sprintf("error deleting
data bound to scope %s for plugin %s", params.scopeId, params.plugin))
+ if err = gs.deleteScopeData(params.plugin, params.scopeId); err != nil {
+ return nil, err
}
if !params.deleteDataOnly {
// Delete the scope itself
err = gs.dbHelper.DeleteScope(params.connectionId,
params.scopeId)
if err != nil {
- return errors.Default.Wrap(err, fmt.Sprintf("error
deleting scope %s", params.scopeId))
+ return nil, errors.Default.Wrap(err, fmt.Sprintf("error
deleting scope %s", params.scopeId))
}
err = gs.updateBlueprints(params.connectionId, params.plugin,
params.scopeId)
if err != nil {
- return err
+ return nil, err
}
}
- return nil
+ return nil, nil
}
-func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) addScopeConfig(scopes
...*Scope) ([]*ScopeRes[Scope, Tr], errors.Error) {
- apiScopes := make([]*ScopeRes[Scope, Tr], len(scopes))
+func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig])
addScopeConfig(scopes ...*Scope) ([]*ScopeRes[Scope, ScopeConfig],
errors.Error) {
+ apiScopes := make([]*ScopeRes[Scope, ScopeConfig], len(scopes))
for i, scope := range scopes {
- apiScopes[i] = &ScopeRes[Scope, Tr]{
+ apiScopes[i] = &ScopeRes[Scope, ScopeConfig]{
Scope: scope,
}
scIdField := reflectField(scope, "ScopeConfigId")
@@ -345,8 +336,20 @@ func (gs *GenericScopeApiHelper[Conn, Scope, Tr])
addScopeConfig(scopes ...*Scop
return apiScopes, nil
}
-func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) mapByScopeId(scopes
[]*ScopeRes[Scope, Tr]) map[string]*ScopeRes[Scope, Tr] {
- scopeMap := map[string]*ScopeRes[Scope, Tr]{}
+func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig])
getScopeReferences(pluginName string, connectionId uint64, scopeId string)
(*serviceHelper.BlueprintProjectPairs, errors.Error) {
+ blueprintMap, err := gs.bpManager.GetBlueprintsByScopes(connectionId,
pluginName, scopeId)
+ if err != nil {
+ return nil, err
+ }
+ blueprints := blueprintMap[scopeId]
+ if len(blueprints) == 0 {
+ return nil, nil
+ }
+ return serviceHelper.NewBlueprintProjectPairs(blueprints), nil
+}
+
+func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) mapByScopeId(scopes
[]*ScopeRes[Scope, ScopeConfig]) map[string]*ScopeRes[Scope, ScopeConfig] {
+ scopeMap := map[string]*ScopeRes[Scope, ScopeConfig]{}
for _, scope := range scopes {
scopeId := fmt.Sprintf("%v", reflectField(scope.Scope,
gs.reflectionParams.ScopeIdFieldName).Interface())
scopeMap[scopeId] = scope
@@ -354,7 +357,7 @@ func (gs *GenericScopeApiHelper[Conn, Scope, Tr])
mapByScopeId(scopes []*ScopeRe
return scopeMap
}
-func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) extractFromReqParam(input
*plugin.ApiResourceInput, withScopeId bool) (*requestParams, errors.Error) {
+func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig])
extractFromReqParam(input *plugin.ApiResourceInput, withScopeId bool)
(*requestParams, errors.Error) {
connectionId, err := strconv.ParseUint(input.Params["connectionId"],
10, 64)
if err != nil {
return nil, errors.BadInput.Wrap(err, "Invalid
\"connectionId\"")
@@ -370,7 +373,7 @@ func (gs *GenericScopeApiHelper[Conn, Scope, Tr])
extractFromReqParam(input *plu
scopeId = scopeId[1:]
}
}
- pluginName := input.Params["plugin"]
+ pluginName := input.GetPlugin()
return &requestParams{
connectionId: connectionId,
plugin: pluginName,
@@ -378,7 +381,7 @@ func (gs *GenericScopeApiHelper[Conn, Scope, Tr])
extractFromReqParam(input *plu
}, nil
}
-func (gs *GenericScopeApiHelper[Conn, Scope, Tr])
extractFromDeleteReqParam(input *plugin.ApiResourceInput)
(*deleteRequestParams, errors.Error) {
+func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig])
extractFromDeleteReqParam(input *plugin.ApiResourceInput)
(*deleteRequestParams, errors.Error) {
params, err := gs.extractFromReqParam(input, true)
if err != nil {
return nil, err
@@ -399,7 +402,7 @@ func (gs *GenericScopeApiHelper[Conn, Scope, Tr])
extractFromDeleteReqParam(inpu
}, nil
}
-func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) extractFromGetReqParam(input
*plugin.ApiResourceInput, withScopeId bool) (*getRequestParams, errors.Error) {
+func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig])
extractFromGetReqParam(input *plugin.ApiResourceInput, withScopeId bool)
(*getRequestParams, errors.Error) {
params, err := gs.extractFromReqParam(input, withScopeId)
if err != nil {
return nil, err
@@ -420,7 +423,7 @@ func (gs *GenericScopeApiHelper[Conn, Scope, Tr])
extractFromGetReqParam(input *
}, nil
}
-func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) getRawParams(connectionId
uint64, scopeId any) string {
+func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig])
getRawParams(connectionId uint64, scopeId any) string {
paramsMap := map[string]any{
"ConnectionId": connectionId,
gs.reflectionParams.RawScopeParamName: scopeId,
@@ -432,7 +435,7 @@ func (gs *GenericScopeApiHelper[Conn, Scope, Tr])
getRawParams(connectionId uint
return string(b)
}
-func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) setScopeFields(p
interface{}, connectionId uint64, createdDate *time.Time, updatedDate
*time.Time) {
+func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) setScopeFields(p
interface{}, connectionId uint64, createdDate *time.Time, updatedDate
*time.Time) {
pType := reflect.TypeOf(p)
if pType.Kind() != reflect.Ptr {
panic("expected a pointer to a struct")
@@ -500,7 +503,7 @@ func returnPrimaryKeyValue(p interface{}) string {
return result
}
-func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) verifyScope(scope
interface{}, vld *validator.Validate) errors.Error {
+func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) verifyScope(scope
interface{}, vld *validator.Validate) errors.Error {
if gs.opts.IsRemote {
return nil
}
@@ -514,7 +517,7 @@ func (gs *GenericScopeApiHelper[Conn, Scope, Tr])
verifyScope(scope interface{},
return nil
}
-func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) validatePrimaryKeys(scopes
[]*Scope) errors.Error {
+func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig])
validatePrimaryKeys(scopes []*Scope) errors.Error {
if gs.opts.IsRemote {
return nil
}
@@ -531,7 +534,7 @@ func (gs *GenericScopeApiHelper[Conn, Scope, Tr])
validatePrimaryKeys(scopes []*
return nil
}
-func (gs *GenericScopeApiHelper[Conn, Scope, Tr])
updateBlueprints(connectionId uint64, pluginName string, scopeId string)
errors.Error {
+func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig])
updateBlueprints(connectionId uint64, pluginName string, scopeId string)
errors.Error {
blueprintsMap, err := gs.bpManager.GetBlueprintsByScopes(connectionId,
pluginName, scopeId)
if err != nil {
return errors.Default.Wrap(err, fmt.Sprintf("error retrieving
scope with scope ID %s", scopeId))
@@ -570,7 +573,28 @@ func (gs *GenericScopeApiHelper[Conn, Scope, Tr])
updateBlueprints(connectionId
return nil
}
-func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) transactionalDelete(tables
[]string, scopeId string) errors.Error {
+func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig])
deleteScopeData(plugin string, scopeId string) errors.Error {
+ var err errors.Error
+ scopeParamValue := scopeId
+ if gs.opts.GetScopeParamValue != nil {
+ scopeParamValue, err = gs.opts.GetScopeParamValue(gs.db,
scopeId)
+ if err != nil {
+ return errors.Default.Wrap(err, fmt.Sprintf("error
extracting scope parameter name for scope %s", scopeId))
+ }
+ }
+ // find all tables for this plugin
+ tables, err := gs.getAffectedTables(plugin)
+ if err != nil {
+ return errors.Default.Wrap(err, fmt.Sprintf("error getting
database tables managed by plugin %s", plugin))
+ }
+ err = gs.transactionalDelete(tables, scopeParamValue)
+ if err != nil {
+ return errors.Default.Wrap(err, fmt.Sprintf("error deleting
data bound to scope %s for plugin %s", scopeId, plugin))
+ }
+ return nil
+}
+
+func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig])
transactionalDelete(tables []string, scopeId string) errors.Error {
tx := gs.db.Begin()
for _, table := range tables {
query := createDeleteQuery(table,
gs.reflectionParams.RawScopeParamName, scopeId)
@@ -617,7 +641,15 @@ func createDeleteQuery(tableName string, scopeIdKey
string, scopeId string) stri
return query
}
-func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) getAffectedTables(pluginName
string) ([]string, errors.Error) {
+func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) lazyCacheTables()
errors.Error {
+ var err errors.Error
+ tablesCacheLoader.Do(func() {
+ tablesCache, err = gs.db.AllTables()
+ })
+ return err
+}
+
+func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig])
getAffectedTables(pluginName string) ([]string, errors.Error) {
var tables []string
meta, err := plugin.GetPlugin(pluginName)
if err != nil {
@@ -626,6 +658,9 @@ func (gs *GenericScopeApiHelper[Conn, Scope, Tr])
getAffectedTables(pluginName s
if pluginModel, ok := meta.(plugin.PluginModel); !ok {
return nil, errors.Default.New(fmt.Sprintf("plugin \"%s\" does
not implement listing its tables", pluginName))
} else {
+ if err = gs.lazyCacheTables(); err != nil {
+ return nil, err
+ }
// collect raw tables
for _, table := range tablesCache {
if strings.HasPrefix(table, "_raw_"+pluginName) {
@@ -633,20 +668,18 @@ func (gs *GenericScopeApiHelper[Conn, Scope, Tr])
getAffectedTables(pluginName s
}
}
// collect tool tables
- tablesInfo := pluginModel.GetTablesInfo()
- for _, table := range tablesInfo {
- // we only care about tables with RawOrigin
- ok = hasField(table, "RawDataParams")
- if ok {
- tables = append(tables, table.TableName())
+ toolModels := pluginModel.GetTablesInfo()
+ for _, toolModel := range toolModels {
+ if !isScopeModel(toolModel) && hasField(toolModel,
"RawDataParams") {
+ tables = append(tables, toolModel.TableName())
}
}
// collect domain tables
- for _, domainTable := range domaininfo.GetDomainTablesInfo() {
+ for _, domainModel := range domaininfo.GetDomainTablesInfo() {
// we only care about tables with RawOrigin
- ok = hasField(domainTable, "RawDataParams")
+ ok = hasField(domainModel, "RawDataParams")
if ok {
- tables = append(tables, domainTable.TableName())
+ tables = append(tables, domainModel.TableName())
}
}
// additional tables
@@ -655,3 +688,10 @@ func (gs *GenericScopeApiHelper[Conn, Scope, Tr])
getAffectedTables(pluginName s
gs.log.Debug("Discovered %d tables used by plugin \"%s\": %v",
len(tables), pluginName, tables)
return tables, nil
}
+
+func isScopeModel(obj dal.Tabler) bool {
+ if _, ok := obj.(plugin.ToolLayerScope); ok {
+ return true
+ }
+ return reflectField(obj, "ScopeConfigId").IsValid()
+}
diff --git a/backend/helpers/pluginhelper/api/scope_helper.go
b/backend/helpers/pluginhelper/api/scope_helper.go
index c26c661df..91d8fdd0d 100644
--- a/backend/helpers/pluginhelper/api/scope_helper.go
+++ b/backend/helpers/pluginhelper/api/scope_helper.go
@@ -95,9 +95,9 @@ func (c *ScopeApiHelper[Conn, Scope, Tr]) GetScope(input
*plugin.ApiResourceInpu
}
func (c *ScopeApiHelper[Conn, Scope, Tr]) Delete(input
*plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
- err := c.DeleteScope(input)
+ refs, err := c.DeleteScope(input)
if err != nil {
- return nil, err
+ return &plugin.ApiResourceOutput{Body: refs, Status:
err.GetType().GetHttpCode()}, nil
}
return &plugin.ApiResourceOutput{Body: nil, Status: http.StatusOK}, nil
}
diff --git a/backend/helpers/pluginhelper/services/blueprint_helper.go
b/backend/helpers/pluginhelper/services/blueprint_helper.go
index d88a98ce8..6282c21fe 100644
--- a/backend/helpers/pluginhelper/services/blueprint_helper.go
+++ b/backend/helpers/pluginhelper/services/blueprint_helper.go
@@ -38,12 +38,26 @@ type GetBlueprintQuery struct {
PageSize int
}
+type BlueprintProjectPairs struct {
+ Projects []string `json:"projects"`
+ Blueprints []string `json:"blueprints"`
+}
+
func NewBlueprintManager(db dal.Dal) *BlueprintManager {
return &BlueprintManager{
db: db,
}
}
+func NewBlueprintProjectPairs(bps []*models.Blueprint) *BlueprintProjectPairs {
+ pairs := &BlueprintProjectPairs{}
+ for _, bp := range bps {
+ pairs.Blueprints = append(pairs.Blueprints, bp.Name)
+ pairs.Projects = append(pairs.Projects, bp.ProjectName)
+ }
+ return pairs
+}
+
// SaveDbBlueprint accepts a Blueprint instance and upsert it to database
func (b *BlueprintManager) SaveDbBlueprint(blueprint *models.Blueprint)
errors.Error {
var err error
@@ -160,6 +174,28 @@ func (b *BlueprintManager)
GetBlueprintsByScopes(connectionId uint64, pluginName
return scopeMap, nil
}
+// GetBlueprintsByScopes returns all blueprints that have these scopeIds and
this connection Id
+func (b *BlueprintManager) GetBlueprintsByConnection(plugin string,
connectionId uint64) ([]*models.Blueprint, errors.Error) {
+ bps, _, err := b.GetDbBlueprints(&GetBlueprintQuery{})
+ if err != nil {
+ return nil, err
+ }
+ var filteredBps []*models.Blueprint
+ for _, bp := range bps {
+ connections, err := bp.GetConnections()
+ if err != nil {
+ return nil, err
+ }
+ for _, connection := range connections {
+ if connection.ConnectionId == connectionId &&
connection.Plugin == plugin {
+ filteredBps = append(filteredBps, bp)
+ break
+ }
+ }
+ }
+ return filteredBps, nil
+}
+
// GetDbBlueprintByProjectName returns the detail of a given projectName
func (b *BlueprintManager) GetDbBlueprintByProjectName(projectName string)
(*models.Blueprint, errors.Error) {
dbBlueprint := &models.Blueprint{}
diff --git a/backend/plugins/ae/api/connection.go
b/backend/plugins/ae/api/connection.go
index 7cc80df0f..00a9d3c99 100644
--- a/backend/plugins/ae/api/connection.go
+++ b/backend/plugins/ae/api/connection.go
@@ -19,6 +19,7 @@ package api
import (
"context"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/services"
"net/http"
"github.com/apache/incubator-devlake/core/errors"
@@ -134,6 +135,7 @@ func PatchConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Tags plugins/ae
// @Success 200 {object} models.AeConnection "Success"
// @Failure 400 {string} errcode.Error "Bad Request"
+// @Failure 409 {object} services.BlueprintProjectPairs "References exist to
this connection"
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/ae/connections/{connectionId} [DELETE]
func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
@@ -142,6 +144,10 @@ func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput
if err != nil {
return nil, err
}
- err = connectionHelper.Delete(connection)
+ var refs *services.BlueprintProjectPairs
+ refs, err = connectionHelper.Delete(input.GetPlugin(), connection)
+ if err != nil {
+ return &plugin.ApiResourceOutput{Body: refs, Status:
err.GetType().GetHttpCode()}, err
+ }
return &plugin.ApiResourceOutput{Body: connection}, err
}
diff --git a/backend/plugins/bamboo/api/connection.go
b/backend/plugins/bamboo/api/connection.go
index ca627af97..7c1995ac8 100644
--- a/backend/plugins/bamboo/api/connection.go
+++ b/backend/plugins/bamboo/api/connection.go
@@ -19,6 +19,7 @@ package api
import (
"context"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/services"
"net/http"
"github.com/apache/incubator-devlake/core/errors"
@@ -107,6 +108,7 @@ func PatchConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Tags plugins/bamboo
// @Success 200 {object} models.BambooConnection
// @Failure 400 {string} errcode.Error "Bad Request"
+// @Failure 409 {object} services.BlueprintProjectPairs "References exist to
this connection"
// @Failure 500 {string} errcode.Error "Internel Error"
// @Router /plugins/bamboo/connections/{connectionId} [DELETE]
func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
@@ -115,7 +117,11 @@ func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput
if err != nil {
return nil, err
}
- err = connectionHelper.Delete(connection)
+ var refs *services.BlueprintProjectPairs
+ refs, err = connectionHelper.Delete(input.GetPlugin(), connection)
+ if err != nil {
+ return &plugin.ApiResourceOutput{Body: refs, Status:
err.GetType().GetHttpCode()}, err
+ }
return &plugin.ApiResourceOutput{Body: connection}, err
}
diff --git a/backend/plugins/bamboo/api/scope.go
b/backend/plugins/bamboo/api/scope.go
index 0989000b7..62172865c 100644
--- a/backend/plugins/bamboo/api/scope.go
+++ b/backend/plugins/bamboo/api/scope.go
@@ -101,6 +101,7 @@ func GetScope(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors
// @Param delete_data_only query bool false "Only delete the scope data, not
the scope itself"
// @Success 200
// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 409 {object} api.ScopeRefDoc "References exist to this scope"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/bamboo/connections/{connectionId}/scopes/{scopeId} [DELETE]
func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
diff --git a/backend/plugins/bitbucket/api/connection.go
b/backend/plugins/bitbucket/api/connection.go
index 64e5d7c31..0f19f90f5 100644
--- a/backend/plugins/bitbucket/api/connection.go
+++ b/backend/plugins/bitbucket/api/connection.go
@@ -19,6 +19,7 @@ package api
import (
"context"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/services"
"net/http"
"github.com/apache/incubator-devlake/server/api/shared"
@@ -114,6 +115,7 @@ func PatchConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Tags plugins/bitbucket
// @Success 200 {object} models.BitbucketConnection
// @Failure 400 {string} errcode.Error "Bad Request"
+// @Failure 409 {object} services.BlueprintProjectPairs "References exist to
this connection"
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/bitbucket/connections/{connectionId} [DELETE]
func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
@@ -122,7 +124,11 @@ func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput
if err != nil {
return nil, err
}
- err = connectionHelper.Delete(connection)
+ var refs *services.BlueprintProjectPairs
+ refs, err = connectionHelper.Delete(input.GetPlugin(), connection)
+ if err != nil {
+ return &plugin.ApiResourceOutput{Body: refs, Status:
err.GetType().GetHttpCode()}, err
+ }
return &plugin.ApiResourceOutput{Body: connection}, err
}
diff --git a/backend/plugins/bitbucket/api/scope.go
b/backend/plugins/bitbucket/api/scope.go
index a22adbfe0..1639fa96a 100644
--- a/backend/plugins/bitbucket/api/scope.go
+++ b/backend/plugins/bitbucket/api/scope.go
@@ -105,6 +105,7 @@ func GetScope(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors
// @Param delete_data_only query bool false "Only delete the scope data, not
the scope itself"
// @Success 200
// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 409 {object} api.ScopeRefDoc "References exist to this scope"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/bitbucket/connections/{connectionId}/scopes/{scopeId}
[DELETE]
func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
diff --git a/backend/plugins/feishu/api/connection.go
b/backend/plugins/feishu/api/connection.go
index bcf48f3c3..a570f6208 100644
--- a/backend/plugins/feishu/api/connection.go
+++ b/backend/plugins/feishu/api/connection.go
@@ -19,6 +19,7 @@ package api
import (
"context"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/services"
"github.com/apache/incubator-devlake/server/api/shared"
"net/http"
@@ -100,6 +101,7 @@ func PatchConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Tags plugins/feishu
// @Success 200 {object} models.FeishuConnection
// @Failure 400 {string} errcode.Error "Bad Request"
+// @Failure 409 {object} services.BlueprintProjectPairs "References exist to
this connection"
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/feishu/connections/{connectionId} [DELETE]
func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
@@ -108,7 +110,11 @@ func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput
if err != nil {
return nil, err
}
- err = connectionHelper.Delete(connection)
+ var refs *services.BlueprintProjectPairs
+ refs, err = connectionHelper.Delete(input.GetPlugin(), connection)
+ if err != nil {
+ return &plugin.ApiResourceOutput{Body: refs, Status:
err.GetType().GetHttpCode()}, err
+ }
return &plugin.ApiResourceOutput{Body: connection}, err
}
diff --git a/backend/plugins/gitee/api/connection.go
b/backend/plugins/gitee/api/connection.go
index 8fac8a660..0bb87fa59 100644
--- a/backend/plugins/gitee/api/connection.go
+++ b/backend/plugins/gitee/api/connection.go
@@ -19,6 +19,7 @@ package api
import (
"context"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/services"
"net/http"
"github.com/apache/incubator-devlake/server/api/shared"
@@ -117,6 +118,7 @@ func PatchConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Tags plugins/gitee
// @Success 200 {object} models.GiteeConnection
// @Failure 400 {string} errcode.Error "Bad Request"
+// @Failure 409 {object} services.BlueprintProjectPairs "References exist to
this connection"
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/gitee/connections/{connectionId} [DELETE]
func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
@@ -125,7 +127,11 @@ func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput
if err != nil {
return nil, err
}
- err = connectionHelper.Delete(connection)
+ var refs *services.BlueprintProjectPairs
+ refs, err = connectionHelper.Delete(input.GetPlugin(), connection)
+ if err != nil {
+ return &plugin.ApiResourceOutput{Body: refs, Status:
err.GetType().GetHttpCode()}, err
+ }
return &plugin.ApiResourceOutput{Body: connection}, err
}
diff --git a/backend/plugins/github/api/connection.go
b/backend/plugins/github/api/connection.go
index f2e240267..72894c500 100644
--- a/backend/plugins/github/api/connection.go
+++ b/backend/plugins/github/api/connection.go
@@ -20,6 +20,7 @@ package api
import (
"context"
"fmt"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/services"
"net/http"
"strings"
@@ -242,6 +243,7 @@ func PatchConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Tags plugins/github
// @Success 200 {object} models.GithubConnection
// @Failure 400 {string} errcode.Error "Bad Request"
+// @Failure 409 {object} services.BlueprintProjectPairs "References exist to
this connection"
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/github/connections/{connectionId} [DELETE]
func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
@@ -250,7 +252,11 @@ func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput
if err != nil {
return nil, err
}
- err = connectionHelper.Delete(connection)
+ var refs *services.BlueprintProjectPairs
+ refs, err = connectionHelper.Delete(input.GetPlugin(), connection)
+ if err != nil {
+ return &plugin.ApiResourceOutput{Body: refs, Status:
err.GetType().GetHttpCode()}, err
+ }
return &plugin.ApiResourceOutput{Body: connection}, err
}
diff --git a/backend/plugins/github/api/scope.go
b/backend/plugins/github/api/scope.go
index 1f0102c37..8ca575a11 100644
--- a/backend/plugins/github/api/scope.go
+++ b/backend/plugins/github/api/scope.go
@@ -101,6 +101,7 @@ func GetScope(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors
// @Param delete_data_only query bool false "Only delete the scope data, not
the scope itself"
// @Success 200
// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 409 {object} api.ScopeRefDoc "References exist to this scope"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/github/connections/{connectionId}/scopes/{scopeId} [DELETE]
func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
diff --git a/backend/plugins/gitlab/api/connection.go
b/backend/plugins/gitlab/api/connection.go
index d23d9dc3e..df6bece61 100644
--- a/backend/plugins/gitlab/api/connection.go
+++ b/backend/plugins/gitlab/api/connection.go
@@ -20,6 +20,7 @@ package api
import (
"context"
"fmt"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/services"
"net/http"
"net/url"
@@ -121,6 +122,7 @@ func PatchConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Tags plugins/gitlab
// @Success 200 {object} models.GitlabConnection
// @Failure 400 {string} errcode.Error "Bad Request"
+// @Failure 409 {object} services.BlueprintProjectPairs "References exist to
this connection"
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/gitlab/connections/{connectionId} [DELETE]
func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
@@ -129,7 +131,11 @@ func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput
if err != nil {
return nil, err
}
- err = connectionHelper.Delete(connection)
+ var refs *services.BlueprintProjectPairs
+ refs, err = connectionHelper.Delete(input.GetPlugin(), connection)
+ if err != nil {
+ return &plugin.ApiResourceOutput{Body: refs, Status:
err.GetType().GetHttpCode()}, err
+ }
return &plugin.ApiResourceOutput{Body: connection}, err
}
diff --git a/backend/plugins/gitlab/api/scope.go
b/backend/plugins/gitlab/api/scope.go
index 73dc5da45..705ca6c2e 100644
--- a/backend/plugins/gitlab/api/scope.go
+++ b/backend/plugins/gitlab/api/scope.go
@@ -101,6 +101,7 @@ func GetScope(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors
// @Param delete_data_only query bool false "Only delete the scope data, not
the scope itself"
// @Success 200
// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 409 {object} api.ScopeRefDoc "References exist to this scope"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/gitlab/connections/{connectionId}/scopes/{scopeId} [DELETE]
func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
diff --git a/backend/plugins/jenkins/api/connection.go
b/backend/plugins/jenkins/api/connection.go
index 9ade3fa6b..634063497 100644
--- a/backend/plugins/jenkins/api/connection.go
+++ b/backend/plugins/jenkins/api/connection.go
@@ -19,6 +19,7 @@ package api
import (
"context"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/services"
"net/http"
"strings"
@@ -123,6 +124,7 @@ func PatchConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Tags plugins/jenkins
// @Success 200 {object} models.JenkinsConnection
// @Failure 400 {string} errcode.Error "Bad Request"
+// @Failure 409 {object} services.BlueprintProjectPairs "References exist to
this connection"
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/jenkins/connections/{connectionId} [DELETE]
func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
@@ -131,7 +133,11 @@ func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput
if err != nil {
return nil, err
}
- err = connectionHelper.Delete(connection)
+ var refs *services.BlueprintProjectPairs
+ refs, err = connectionHelper.Delete(input.GetPlugin(), connection)
+ if err != nil {
+ return &plugin.ApiResourceOutput{Body: refs, Status:
err.GetType().GetHttpCode()}, err
+ }
return &plugin.ApiResourceOutput{Body: connection}, err
}
diff --git a/backend/plugins/jenkins/api/scope.go
b/backend/plugins/jenkins/api/scope.go
index 3bef31f9a..f1165bb1e 100644
--- a/backend/plugins/jenkins/api/scope.go
+++ b/backend/plugins/jenkins/api/scope.go
@@ -105,6 +105,7 @@ func GetScope(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors
// @Param delete_data_only query bool false "Only delete the scope data, not
the scope itself"
// @Success 200
// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 409 {object} api.ScopeRefDoc "References exist to this scope"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/jenkins/connections/{connectionId}/scopes/{scopeId}
[DELETE]
func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
diff --git a/backend/plugins/jira/api/connection.go
b/backend/plugins/jira/api/connection.go
index c0513685f..419af61c5 100644
--- a/backend/plugins/jira/api/connection.go
+++ b/backend/plugins/jira/api/connection.go
@@ -20,6 +20,7 @@ package api
import (
"context"
"fmt"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/services"
"net/http"
"net/url"
"strings"
@@ -168,6 +169,7 @@ func PatchConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Tags plugins/jira
// @Success 200 {object} models.JiraConnection
// @Failure 400 {string} errcode.Error "Bad Request"
+// @Failure 409 {object} services.BlueprintProjectPairs "References exist to
this connection"
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/jira/connections/{connectionId} [DELETE]
func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
@@ -176,7 +178,11 @@ func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput
if err != nil {
return nil, err
}
- err = connectionHelper.Delete(connection)
+ var refs *services.BlueprintProjectPairs
+ refs, err = connectionHelper.Delete(input.GetPlugin(), connection)
+ if err != nil {
+ return &plugin.ApiResourceOutput{Body: refs, Status:
err.GetType().GetHttpCode()}, err
+ }
return &plugin.ApiResourceOutput{Body: connection}, err
}
diff --git a/backend/plugins/jira/api/scope.go
b/backend/plugins/jira/api/scope.go
index 93feb8034..b76893bfa 100644
--- a/backend/plugins/jira/api/scope.go
+++ b/backend/plugins/jira/api/scope.go
@@ -109,6 +109,7 @@ func GetScope(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors
// @Param delete_data_only query bool false "Only delete the scope data, not
the scope itself"
// @Success 200
// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 409 {object} api.ScopeRefDoc "References exist to this scope"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/jira/connections/{connectionId}/scopes/{scopeId} [DELETE]
func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
diff --git a/backend/plugins/pagerduty/api/connection.go
b/backend/plugins/pagerduty/api/connection.go
index 932e585e9..040a0ea1f 100644
--- a/backend/plugins/pagerduty/api/connection.go
+++ b/backend/plugins/pagerduty/api/connection.go
@@ -19,6 +19,7 @@ package api
import (
"context"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/services"
"net/http"
"github.com/apache/incubator-devlake/core/errors"
@@ -99,6 +100,7 @@ func PatchConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Tags plugins/pagerduty
// @Success 200 {object} models.PagerDutyConnection
// @Failure 400 {string} errcode.Error "Bad Request"
+// @Failure 409 {object} services.BlueprintProjectPairs "References exist to
this connection"
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/pagerduty/connections/{connectionId} [DELETE]
func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
@@ -107,7 +109,11 @@ func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput
if err != nil {
return nil, err
}
- err = connectionHelper.Delete(connection)
+ var refs *services.BlueprintProjectPairs
+ refs, err = connectionHelper.Delete(input.GetPlugin(), connection)
+ if err != nil {
+ return &plugin.ApiResourceOutput{Body: refs, Status:
err.GetType().GetHttpCode()}, err
+ }
return &plugin.ApiResourceOutput{Body: connection}, err
}
diff --git a/backend/plugins/pagerduty/api/scope.go
b/backend/plugins/pagerduty/api/scope.go
index b86e55b72..871de3d65 100644
--- a/backend/plugins/pagerduty/api/scope.go
+++ b/backend/plugins/pagerduty/api/scope.go
@@ -102,6 +102,7 @@ func GetScope(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors
// @Param delete_data_only query bool false "Only delete the scope data, not
the scope itself"
// @Success 200
// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 409 {object} api.ScopeRefDoc "References exist to this scope"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/pagerduty/connections/{connectionId}/scopes/{serviceId}
[DELETE]
func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
diff --git a/backend/plugins/slack/api/connection.go
b/backend/plugins/slack/api/connection.go
index b1bf7ccd3..303c74fb4 100644
--- a/backend/plugins/slack/api/connection.go
+++ b/backend/plugins/slack/api/connection.go
@@ -19,6 +19,7 @@ package api
import (
"context"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/services"
"github.com/apache/incubator-devlake/server/api/shared"
"net/http"
@@ -100,6 +101,7 @@ func PatchConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Tags plugins/slack
// @Success 200 {object} models.SlackConnection
// @Failure 400 {string} errcode.Error "Bad Request"
+// @Failure 409 {object} services.BlueprintProjectPairs "References exist to
this connection"
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/slack/connections/{connectionId} [DELETE]
func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
@@ -108,7 +110,11 @@ func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput
if err != nil {
return nil, err
}
- err = connectionHelper.Delete(connection)
+ var refs *services.BlueprintProjectPairs
+ refs, err = connectionHelper.Delete(input.GetPlugin(), connection)
+ if err != nil {
+ return &plugin.ApiResourceOutput{Body: refs, Status:
err.GetType().GetHttpCode()}, err
+ }
return &plugin.ApiResourceOutput{Body: connection}, err
}
diff --git a/backend/plugins/sonarqube/api/connection.go
b/backend/plugins/sonarqube/api/connection.go
index 9dc000e61..3a1bb781e 100644
--- a/backend/plugins/sonarqube/api/connection.go
+++ b/backend/plugins/sonarqube/api/connection.go
@@ -19,6 +19,7 @@ package api
import (
"context"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/services"
"net/http"
"github.com/apache/incubator-devlake/core/errors"
@@ -128,6 +129,7 @@ func PatchConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Param connectionId path int false "connection ID"
// @Success 200 {object} models.SonarqubeConnection
// @Failure 400 {string} errcode.Error "Bad Request"
+// @Failure 409 {object} services.BlueprintProjectPairs "References exist to
this connection"
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/sonarqube/connections/{connectionId} [DELETE]
func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
@@ -136,7 +138,11 @@ func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput
if err != nil {
return nil, err
}
- err = connectionHelper.Delete(connection)
+ var refs *services.BlueprintProjectPairs
+ refs, err = connectionHelper.Delete(input.GetPlugin(), connection)
+ if err != nil {
+ return &plugin.ApiResourceOutput{Body: refs, Status:
err.GetType().GetHttpCode()}, err
+ }
return &plugin.ApiResourceOutput{Body: connection}, err
}
diff --git a/backend/plugins/sonarqube/api/scope.go
b/backend/plugins/sonarqube/api/scope.go
index 880f3bf46..e31f36ddd 100644
--- a/backend/plugins/sonarqube/api/scope.go
+++ b/backend/plugins/sonarqube/api/scope.go
@@ -100,6 +100,7 @@ func GetScope(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors
// @Param delete_data_only query bool false "Only delete the scope data, not
the scope itself"
// @Success 200
// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 409 {object} api.ScopeRefDoc "References exist to this scope"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/sonarqube/connections/{connectionId}/scopes/{scopeId}
[DELETE]
func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
diff --git a/backend/plugins/tapd/api/connection.go
b/backend/plugins/tapd/api/connection.go
index 3bca819bc..7545709b3 100644
--- a/backend/plugins/tapd/api/connection.go
+++ b/backend/plugins/tapd/api/connection.go
@@ -20,6 +20,7 @@ package api
import (
"context"
"fmt"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/services"
"net/http"
"github.com/apache/incubator-devlake/server/api/shared"
@@ -119,6 +120,7 @@ func PatchConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Tags plugins/tapd
// @Success 200 {object} models.TapdConnection "Success"
// @Failure 400 {string} errcode.Error "Bad Request"
+// @Failure 409 {object} services.BlueprintProjectPairs "References exist to
this connection"
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/tapd/connections/{connectionId} [DELETE]
func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
@@ -127,7 +129,11 @@ func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput
if err != nil {
return nil, err
}
- err = connectionHelper.Delete(connection)
+ var refs *services.BlueprintProjectPairs
+ refs, err = connectionHelper.Delete(input.GetPlugin(), connection)
+ if err != nil {
+ return &plugin.ApiResourceOutput{Body: refs, Status:
err.GetType().GetHttpCode()}, err
+ }
return &plugin.ApiResourceOutput{Body: connection}, err
}
diff --git a/backend/plugins/tapd/api/scope.go
b/backend/plugins/tapd/api/scope.go
index 04d8b5f60..54da0d3f8 100644
--- a/backend/plugins/tapd/api/scope.go
+++ b/backend/plugins/tapd/api/scope.go
@@ -104,6 +104,7 @@ func GetScope(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors
// @Param delete_data_only query bool false "Only delete the scope data, not
the scope itself"
// @Success 200
// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 409 {object} api.ScopeRefDoc "References exist to this scope"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/tapd/connections/{connectionId}/scopes/{scopeId} [DELETE]
func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
diff --git a/backend/plugins/teambition/api/connection.go
b/backend/plugins/teambition/api/connection.go
index a163848d2..009124591 100644
--- a/backend/plugins/teambition/api/connection.go
+++ b/backend/plugins/teambition/api/connection.go
@@ -20,6 +20,7 @@ package api
import (
"context"
"fmt"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/services"
"net/http"
"github.com/apache/incubator-devlake/core/errors"
@@ -128,6 +129,7 @@ func PatchConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Tags plugins/teambition
// @Success 200 {object} models.TeambitionConnection
// @Failure 400 {string} errcode.Error "Bad Request"
+// @Failure 409 {object} services.BlueprintProjectPairs "References exist to
this connection"
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/teambition/connections/{connectionId} [DELETE]
func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
@@ -136,7 +138,11 @@ func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput
if err != nil {
return nil, err
}
- err = connectionHelper.Delete(connection)
+ var refs *services.BlueprintProjectPairs
+ refs, err = connectionHelper.Delete(input.GetPlugin(), connection)
+ if err != nil {
+ return &plugin.ApiResourceOutput{Body: refs, Status:
err.GetType().GetHttpCode()}, err
+ }
return &plugin.ApiResourceOutput{Body: connection}, err
}
diff --git a/backend/plugins/trello/api/connection.go
b/backend/plugins/trello/api/connection.go
index be7378f7d..d10a900b5 100644
--- a/backend/plugins/trello/api/connection.go
+++ b/backend/plugins/trello/api/connection.go
@@ -19,6 +19,7 @@ package api
import (
"context"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/services"
"net/http"
"github.com/apache/incubator-devlake/server/api/shared"
@@ -114,6 +115,7 @@ func PatchConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Tags plugins/trello
// @Success 200 {object} models.TrelloConnection
// @Failure 400 {string} errcode.Error "Bad Request"
+// @Failure 409 {object} services.BlueprintProjectPairs "References exist to
this connection"
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/trello/connections/{connectionId} [DELETE]
func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
@@ -122,7 +124,11 @@ func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput
if err != nil {
return nil, err
}
- err = connectionHelper.Delete(connection)
+ var refs *services.BlueprintProjectPairs
+ refs, err = connectionHelper.Delete(input.GetPlugin(), connection)
+ if err != nil {
+ return &plugin.ApiResourceOutput{Body: refs, Status:
err.GetType().GetHttpCode()}, err
+ }
return &plugin.ApiResourceOutput{Body: connection}, err
}
diff --git a/backend/plugins/trello/api/scope.go
b/backend/plugins/trello/api/scope.go
index 5b2100724..94aa605de 100644
--- a/backend/plugins/trello/api/scope.go
+++ b/backend/plugins/trello/api/scope.go
@@ -99,6 +99,7 @@ func GetScope(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors
// @Param delete_data_only query bool false "Only delete the scope data, not
the scope itself"
// @Success 200
// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 409 {object} api.ScopeRefDoc "References exist to this scope"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/trello/connections/{connectionId}/scopes/{scopeId} [DELETE]
func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
diff --git a/backend/plugins/webhook/api/connection.go
b/backend/plugins/webhook/api/connection.go
index 6698c184b..fea459af8 100644
--- a/backend/plugins/webhook/api/connection.go
+++ b/backend/plugins/webhook/api/connection.go
@@ -19,6 +19,7 @@ package api
import (
"fmt"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/services"
"net/http"
"github.com/apache/incubator-devlake/core/errors"
@@ -69,6 +70,7 @@ func PatchConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Tags plugins/webhook
// @Success 200 {object} models.WebhookConnection
// @Failure 400 {string} errcode.Error "Bad Request"
+// @Failure 409 {object} services.BlueprintProjectPairs "References exist to
this connection"
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/webhook/connections/{connectionId} [DELETE]
func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
@@ -77,7 +79,11 @@ func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput
if err != nil {
return nil, err
}
- err = connectionHelper.Delete(connection)
+ var refs *services.BlueprintProjectPairs
+ refs, err = connectionHelper.Delete(input.GetPlugin(), connection)
+ if err != nil {
+ return &plugin.ApiResourceOutput{Body: refs, Status:
err.GetType().GetHttpCode()}, err
+ }
return &plugin.ApiResourceOutput{Body: connection}, err
}
diff --git a/backend/plugins/zentao/api/connection.go
b/backend/plugins/zentao/api/connection.go
index ccf5cf2de..429cd4587 100644
--- a/backend/plugins/zentao/api/connection.go
+++ b/backend/plugins/zentao/api/connection.go
@@ -19,6 +19,7 @@ package api
import (
"context"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/services"
"net/http"
"github.com/apache/incubator-devlake/server/api/shared"
@@ -103,6 +104,7 @@ func PatchConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Tags plugins/zentao
// @Success 200 {object} models.ZentaoConnection
// @Failure 400 {string} errcode.Error "Bad Request"
+// @Failure 409 {object} services.BlueprintProjectPairs "References exist to
this connection"
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/zentao/connections/{connectionId} [DELETE]
func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
@@ -111,7 +113,11 @@ func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput
if err != nil {
return nil, err
}
- err = connectionHelper.Delete(connection)
+ var refs *services.BlueprintProjectPairs
+ refs, err = connectionHelper.Delete(input.GetPlugin(), connection)
+ if err != nil {
+ return &plugin.ApiResourceOutput{Body: refs, Status:
err.GetType().GetHttpCode()}, err
+ }
return &plugin.ApiResourceOutput{Body: connection}, err
}
diff --git a/backend/plugins/zentao/api/scope.go
b/backend/plugins/zentao/api/scope.go
index 8772c52c9..2a9731f89 100644
--- a/backend/plugins/zentao/api/scope.go
+++ b/backend/plugins/zentao/api/scope.go
@@ -139,6 +139,7 @@ func GetProjectScope(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Param delete_data_only query bool false "Only delete the scope data, not
the scope itself"
// @Success 200
// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 409 {object} api.ScopeRefDoc "References exist to this scope"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/zentao/connections/{connectionId}/scopes/product/{scopeId}
[DELETE]
func DeleteProductScope(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
@@ -154,6 +155,7 @@ func DeleteProductScope(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutp
// @Param delete_data_only query bool false "Only delete the scope data, not
the scope itself"
// @Success 200
// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 409 {object} api.ScopeRefDoc "References exist to this scope"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/zentao/connections/{connectionId}/scopes/project/{scopeId}
[DELETE]
func DeleteProjectScope(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
diff --git a/backend/python/pydevlake/pydevlake/plugin.py
b/backend/python/pydevlake/pydevlake/plugin.py
index e6cbe8e96..2a9461cf0 100644
--- a/backend/python/pydevlake/pydevlake/plugin.py
+++ b/backend/python/pydevlake/pydevlake/plugin.py
@@ -139,7 +139,7 @@ class Plugin(ABC):
for tool_scope, _ in scope_config_pairs:
for scope in self.domain_scopes(tool_scope):
scope.id = tool_scope.domain_id()
- scope.raw_data_params = raw_data_params(connection.id,
scope.id)
+ scope.raw_data_params = raw_data_params(connection.id,
tool_scope.id)
domain_scopes.append(
msg.DynamicDomainScope(
type_name=type(scope).__name__,
diff --git a/backend/server/api/router.go b/backend/server/api/router.go
index 55ccba612..5acdd8d57 100644
--- a/backend/server/api/router.go
+++ b/backend/server/api/router.go
@@ -19,6 +19,8 @@ package api
import (
"fmt"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/impls/logruslog"
"net/http"
"strings"
@@ -96,7 +98,7 @@ func registerPluginEndpoints(r *gin.Engine, pluginName
string, apiResources map[
func handlePluginCall(pluginName string, handler plugin.ApiResourceHandler)
func(c *gin.Context) {
return func(c *gin.Context) {
- var err error
+ var err errors.Error
input := &plugin.ApiResourceInput{}
input.Params = make(map[string]string)
if len(c.Params) > 0 {
@@ -110,8 +112,8 @@ func handlePluginCall(pluginName string, handler
plugin.ApiResourceHandler) func
if
strings.HasPrefix(c.Request.Header.Get("Content-Type"), "multipart/form-data;")
{
input.Request = c.Request
} else {
- err = c.ShouldBindJSON(&input.Body)
- if err != nil && err.Error() != "EOF" {
+ err2 := c.ShouldBindJSON(&input.Body)
+ if err2 != nil && err2.Error() != "EOF" {
shared.ApiOutputError(c, err)
return
}
@@ -119,7 +121,12 @@ func handlePluginCall(pluginName string, handler
plugin.ApiResourceHandler) func
}
output, err := handler(input)
if err != nil {
- shared.ApiOutputError(c, err)
+ if output != nil && output.Body != nil {
+ logruslog.Global.Error(err, "")
+ shared.ApiOutputSuccess(c, output.Body,
err.GetType().GetHttpCode())
+ } else {
+ shared.ApiOutputError(c, err)
+ }
} else if output != nil {
status := output.Status
if status < http.StatusContinue {
diff --git a/backend/server/services/project.go
b/backend/server/services/project.go
index 4413a8225..48594bbeb 100644
--- a/backend/server/services/project.go
+++ b/backend/server/services/project.go
@@ -253,7 +253,15 @@ func DeleteProject(name string) errors.Error {
if name == "" {
return errors.BadInput.New("project name is missing")
}
- var err errors.Error
+ // verify exists
+ _, err := getProjectByName(db, name)
+ if err != nil {
+ return err
+ }
+ err = deleteProjectBlueprint(name)
+ if err != nil {
+ return err
+ }
tx := db.Begin()
defer func() {
if r := recover(); r != nil || err != nil {
@@ -263,10 +271,6 @@ func DeleteProject(name string) errors.Error {
}
}
}()
- _, err = getProjectByName(tx, name)
- if err != nil {
- return err
- }
err = tx.Delete(&models.Project{}, dal.Where("name = ?", name))
if err != nil {
return errors.Default.Wrap(err, "error deleting project")
@@ -287,20 +291,20 @@ func DeleteProject(name string) errors.Error {
if err != nil {
return errors.Default.Wrap(err, "error deleting project Issue
metric")
}
- err = tx.Commit()
- if err != nil {
- return err
- }
- bp, err := bpManager.GetDbBlueprintByProjectName(name)
+ return tx.Commit()
+}
+
+func deleteProjectBlueprint(projectName string) errors.Error {
+ bp, err := bpManager.GetDbBlueprintByProjectName(projectName)
if err != nil {
- if tx.IsErrorNotFound(err) {
- return nil
+ if !db.IsErrorNotFound(err) {
+ return errors.Default.Wrap(err, fmt.Sprintf("error
finding blueprint associated with project %s", projectName))
+ }
+ } else {
+ err = bpManager.DeleteBlueprint(bp.ID)
+ if err != nil {
+ return errors.Default.Wrap(err, fmt.Sprintf("error
deleting blueprint associated with project %s", projectName))
}
- return err
- }
- err = bpManager.DeleteBlueprint(bp.ID)
- if err != nil {
- return err
}
return nil
}
diff --git a/backend/server/services/remote/plugin/connection_api.go
b/backend/server/services/remote/plugin/connection_api.go
index 93cdc8751..4a4929aae 100644
--- a/backend/server/services/remote/plugin/connection_api.go
+++ b/backend/server/services/remote/plugin/connection_api.go
@@ -97,7 +97,10 @@ func (pa *pluginAPI) DeleteConnection(input
*plugin.ApiResourceInput) (*plugin.A
if err != nil {
return nil, err
}
- err = pa.helper.Delete(connection)
+ refs, err := pa.helper.Delete(input.GetPlugin(), connection)
+ if err != nil {
+ return &plugin.ApiResourceOutput{Body: refs, Status:
err.GetType().GetHttpCode()}, err
+ }
conn := connection.Unwrap()
return &plugin.ApiResourceOutput{Body: conn}, err
}
diff --git a/backend/server/services/remote/plugin/plugin_impl.go
b/backend/server/services/remote/plugin/plugin_impl.go
index 0ac2daa06..f07f86388 100644
--- a/backend/server/services/remote/plugin/plugin_impl.go
+++ b/backend/server/services/remote/plugin/plugin_impl.go
@@ -68,13 +68,17 @@ func newPlugin(info *models.PluginInfo, invoker
bridge.Invoker) (*remotePluginIm
if err != nil {
return nil, errors.Default.Wrap(err, fmt.Sprintf("Couldn't load
ScopeConfig type for plugin %s", info.Name))
}
- toolModelTablers := make([]*coreModels.DynamicTabler,
len(info.ToolModelInfos))
- for i, toolModelInfo := range info.ToolModelInfos {
+ // put the scope and connection models in the tool list to be
consistent with Go plugins
+ toolModelTablers := []*coreModels.DynamicTabler{
+ connectionTabler.New(),
+ scopeTabler.New(),
+ }
+ for _, toolModelInfo := range info.ToolModelInfos {
toolModelTabler, err :=
toolModelInfo.LoadDynamicTabler(common.NoPKModel{})
if err != nil {
return nil, errors.Default.Wrap(err,
fmt.Sprintf("Couldn't load ToolModel type for plugin %s", info.Name))
}
- toolModelTablers[i] = toolModelTabler.New()
+ toolModelTablers = append(toolModelTablers,
toolModelTabler.New())
}
openApiSpec, err := doc.GenerateOpenApiSpec(info)
if err != nil {
diff --git a/backend/server/services/remote/plugin/scope_api.go
b/backend/server/services/remote/plugin/scope_api.go
index 9ff150a5a..1f65bd84c 100644
--- a/backend/server/services/remote/plugin/scope_api.go
+++ b/backend/server/services/remote/plugin/scope_api.go
@@ -95,9 +95,9 @@ func (pa *pluginAPI) GetScope(input *plugin.ApiResourceInput)
(*plugin.ApiResour
}
func (pa *pluginAPI) DeleteScope(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- err := scopeHelper.DeleteScope(input)
+ refs, err := scopeHelper.DeleteScope(input)
if err != nil {
- return nil, err
+ return &plugin.ApiResourceOutput{Body: refs, Status:
err.GetType().GetHttpCode()}, nil
}
return &plugin.ApiResourceOutput{Body: nil, Status: http.StatusOK}, nil
}
diff --git a/backend/test/e2e/remote/helper.go
b/backend/test/e2e/remote/helper.go
index 11999e8ad..6038bc6cd 100644
--- a/backend/test/e2e/remote/helper.go
+++ b/backend/test/e2e/remote/helper.go
@@ -19,6 +19,9 @@ package remote
import (
"fmt"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/services"
+ "net/http"
"os"
"path/filepath"
"testing"
@@ -37,6 +40,20 @@ const (
FAKE_PLUGIN_DIR = "python/test/fakeplugin"
)
+var (
+ PluginDataTables = []string{
+ "_raw_fake_fakepipelinestream",
+ "_tool_fakeplugin_fakepipelines",
+ "cicd_pipelines",
+ }
+ PluginConfigTables = []string{
+ "_tool_fakeplugin_fakescopeconfigs",
+ "_tool_fakeplugin_fakeconnections",
+ "cicd_scopes",
+ }
+ PluginScopeTable = "_tool_fakeplugin_fakeprojects"
+)
+
type (
FakePluginConnection struct {
Id uint64 `json:"id"`
@@ -68,7 +85,8 @@ type (
func ConnectLocalServer(t *testing.T) *helper.DevlakeClient {
fmt.Println("Connect to server")
client := helper.StartDevLakeServer(t, nil)
- client.SetTimeout(30 * time.Second)
+ client.SetTimeout(60 * time.Second)
+ client.SetPipelineTimeout(60 * time.Second)
return client
}
@@ -105,7 +123,11 @@ func CreateTestScope(client *helper.DevlakeClient, config
*FakeScopeConfig, conn
}
func CreateTestScopeConfig(client *helper.DevlakeClient, connectionId uint64)
*FakeScopeConfig {
- config :=
helper.Cast[FakeScopeConfig](client.CreateScopeConfig(PLUGIN_NAME,
connectionId, FakeScopeConfig{Name: "Scope config", Env: "test env", Entities:
[]string{"CICD"}}))
+ config :=
helper.Cast[FakeScopeConfig](client.CreateScopeConfig(PLUGIN_NAME,
connectionId, FakeScopeConfig{
+ Name: "Scope config",
+ Env: "test env",
+ Entities: []string{plugin.DOMAIN_TYPE_CICD},
+ }))
return &config
}
@@ -154,3 +176,40 @@ func CreateTestBlueprints(t *testing.T, client
*helper.DevlakeClient, count int)
scope: scope,
}
}
+
+func DeleteScopeWithDataIntegrityValidation(t *testing.T, client
*helper.DevlakeClient, connectionId uint64, scopeId string, deleteDataOnly
bool) services.BlueprintProjectPairs {
+ db := client.GetDal()
+ for _, table := range PluginDataTables {
+ count, err := db.Count(dal.From(table))
+ require.NoError(t, err)
+ require.Greaterf(t, int(count), 0, fmt.Sprintf("no data was
found in table: %s", table))
+ }
+ configData := map[string]int{}
+ for _, table := range PluginConfigTables {
+ count, err := db.Count(dal.From(table))
+ require.NoError(t, err)
+ require.Greaterf(t, int(count), 0, fmt.Sprintf("no data was
found in table: %s", table))
+ configData[table] = int(count)
+ }
+ refs := client.DeleteScope(PLUGIN_NAME, connectionId, scopeId,
deleteDataOnly)
+ for _, table := range PluginDataTables {
+ count, err := db.Count(dal.From(table))
+ require.NoError(t, err)
+ require.Equalf(t, 0, int(count), fmt.Sprintf("data was found in
table: %s", table))
+ }
+ if !deleteDataOnly && client.LastReturnedStatusCode() == http.StatusOK {
+ count, err := db.Count(dal.From(PluginScopeTable))
+ require.NoError(t, err)
+ require.Equalf(t, 0, int(count), fmt.Sprintf("data was found in
table: %s", PluginScopeTable))
+ } else {
+ count, err := db.Count(dal.From(PluginScopeTable))
+ require.NoError(t, err)
+ require.Greaterf(t, int(count), 0, fmt.Sprintf("no data was
found in table: %s", PluginScopeTable))
+ }
+ for _, table := range PluginConfigTables {
+ count, err := db.Count(dal.From(table))
+ require.NoError(t, err)
+ require.Equalf(t, configData[table], int(count),
fmt.Sprintf("data was unexpectedly changed in table: %s", table))
+ }
+ return refs
+}
diff --git a/backend/test/e2e/remote/python_plugin_test.go
b/backend/test/e2e/remote/python_plugin_test.go
index 2e6e8e83d..6e64923ac 100644
--- a/backend/test/e2e/remote/python_plugin_test.go
+++ b/backend/test/e2e/remote/python_plugin_test.go
@@ -18,6 +18,7 @@ limitations under the License.
package remote
import (
+ "net/http"
"testing"
"github.com/apache/incubator-devlake/core/models"
@@ -36,6 +37,30 @@ func TestCreateConnection(t *testing.T) {
require.Equal(t, TOKEN, conns[0].Token)
}
+func TestDeleteConnection(t *testing.T) {
+ client := CreateClient(t)
+
+ CreateTestConnection(client)
+
+ conns := client.ListConnections(PLUGIN_NAME)
+ require.Equal(t, 1, len(conns))
+ require.Equal(t, TOKEN, conns[0].Token)
+ refs := client.DeleteConnection(PLUGIN_NAME, conns[0].ID)
+ require.Equal(t, 0, len(refs.Projects))
+ require.Equal(t, 0, len(refs.Blueprints))
+}
+
+func TestDeleteConnection_Conflict(t *testing.T) {
+ client := CreateClient(t)
+ _ = CreateTestBlueprints(t, client, 1)
+ conns := client.ListConnections(PLUGIN_NAME)
+ require.Equal(t, 1, len(conns))
+ require.Equal(t, TOKEN, conns[0].Token)
+ refs :=
client.SetExpectedStatusCode(http.StatusConflict).DeleteConnection(PLUGIN_NAME,
conns[0].ID)
+ require.Equal(t, 1, len(refs.Projects))
+ require.Equal(t, 1, len(refs.Blueprints))
+}
+
func TestRemoteScopeGroups(t *testing.T) {
client := CreateClient(t)
connection := CreateTestConnection(client)
@@ -122,16 +147,18 @@ func TestRunPipeline(t *testing.T) {
require.Equal(t, "", pipeline.ErrorName)
}
-func TestBlueprintV200_withScopeDeletion(t *testing.T) {
+func TestBlueprintV200_withScopeDeletion_Conflict(t *testing.T) {
client := CreateClient(t)
params := CreateTestBlueprints(t, client, 1)
client.TriggerBlueprint(params.blueprints[0].ID)
scopesResponse := client.ListScopes(PLUGIN_NAME, params.connection.ID,
true)
require.Equal(t, 1, len(scopesResponse))
require.Equal(t, 1, len(scopesResponse[0].Blueprints))
- client.DeleteScope(PLUGIN_NAME, params.connection.ID, params.scope.Id,
false)
+ refs := DeleteScopeWithDataIntegrityValidation(t,
client.SetExpectedStatusCode(http.StatusConflict), params.connection.ID,
params.scope.Id, false)
+ require.Equal(t, 1, len(refs.Blueprints))
+ require.Equal(t, 1, len(refs.Projects))
scopesResponse = client.ListScopes(PLUGIN_NAME, params.connection.ID,
true)
- require.Equal(t, 0, len(scopesResponse))
+ require.Equal(t, 1, len(scopesResponse))
bpsResult := client.ListBlueprints()
require.Equal(t, 1, len(bpsResult.Blueprints))
}
@@ -145,16 +172,60 @@ func TestBlueprintV200_withBlueprintDeletion(t
*testing.T) {
require.Equal(t, 2, len(scopesResponse[0].Blueprints))
client.DeleteBlueprint(params.blueprints[0].ID)
scopesResponse = client.ListScopes(PLUGIN_NAME, params.connection.ID,
true)
- require.Equal(t, 1, len(scopesResponse)) //scopes are NOT
cascade-deleted when bp is deleted
+ require.Equal(t, 1, len(scopesResponse))
bpsList := client.ListBlueprints()
require.Equal(t, 1, len(bpsList.Blueprints))
require.Equal(t, params.blueprints[1].ID, bpsList.Blueprints[0].ID)
+ projectsResponse := client.ListProjects()
+ require.Equal(t, 2, len(projectsResponse.Projects))
+}
+
+func TestBlueprintV200_withBlueprintDeletion_thenScopeDeletion(t *testing.T) {
+ client := CreateClient(t)
+ params := CreateTestBlueprints(t, client, 1)
+ client.TriggerBlueprint(params.blueprints[0].ID)
+ scopesResponse := client.ListScopes(PLUGIN_NAME, params.connection.ID,
true)
+ require.Equal(t, 1, len(scopesResponse))
+ require.Equal(t, 1, len(scopesResponse[0].Blueprints))
+ client.DeleteBlueprint(params.blueprints[0].ID)
+ scopesResponse = client.ListScopes(PLUGIN_NAME, params.connection.ID,
true)
+ require.Equal(t, 1, len(scopesResponse))
+ bpsList := client.ListBlueprints()
+ require.Equal(t, 0, len(bpsList.Blueprints))
+ refs := DeleteScopeWithDataIntegrityValidation(t, client,
params.connection.ID, params.scope.Id, false)
+ require.Equal(t, 0, len(refs.Blueprints))
+ require.Equal(t, 0, len(refs.Projects))
+ scopesResponse = client.ListScopes(PLUGIN_NAME, params.connection.ID,
true)
+ require.Equal(t, 0, len(scopesResponse))
+ projectsResponse := client.ListProjects()
+ require.Equal(t, 1, len(projectsResponse.Projects))
+}
+
+func TestBlueprintV200_withProjectDeletion_thenScopeDeletion(t *testing.T) {
+ client := CreateClient(t)
+ params := CreateTestBlueprints(t, client, 1)
+ client.TriggerBlueprint(params.blueprints[0].ID)
+ scopesResponse := client.ListScopes(PLUGIN_NAME, params.connection.ID,
true)
+ require.Equal(t, 1, len(scopesResponse))
+ require.Equal(t, 1, len(scopesResponse[0].Blueprints))
+ client.DeleteProject(params.projects[0].Name)
+ scopesResponse = client.ListScopes(PLUGIN_NAME, params.connection.ID,
true)
+ require.Equal(t, 1, len(scopesResponse))
+ bpsList := client.ListBlueprints()
+ require.Equal(t, 0, len(bpsList.Blueprints))
+ refs := DeleteScopeWithDataIntegrityValidation(t, client,
params.connection.ID, params.scope.Id, false)
+ require.Equal(t, 0, len(refs.Blueprints))
+ require.Equal(t, 0, len(refs.Projects))
+ scopesResponse = client.ListScopes(PLUGIN_NAME, params.connection.ID,
true)
+ require.Equal(t, 0, len(scopesResponse))
+ projectsResponse := client.ListProjects()
+ require.Equal(t, 0, len(projectsResponse.Projects))
}
func TestCreateScopeConfig(t *testing.T) {
client := CreateClient(t)
connection := CreateTestConnection(client)
- scopeConfig := FakeScopeConfig{Name: "Scope config", Env: "test env",
Entities: []string{"CICD"}}
+ scopeConfig := FakeScopeConfig{Name: "Scope config", Env: "test env",
Entities: []string{plugin.DOMAIN_TYPE_CICD}}
res := client.CreateScopeConfig(PLUGIN_NAME, connection.ID, scopeConfig)
scopeConfig = helper.Cast[FakeScopeConfig](res)
@@ -163,18 +234,18 @@ func TestCreateScopeConfig(t *testing.T) {
scopeConfig = helper.Cast[FakeScopeConfig](res)
require.Equal(t, "Scope config", scopeConfig.Name)
require.Equal(t, "test env", scopeConfig.Env)
- require.Equal(t, []string{"CICD"}, scopeConfig.Entities)
+ require.Equal(t, []string{plugin.DOMAIN_TYPE_CICD},
scopeConfig.Entities)
}
func TestUpdateScopeConfig(t *testing.T) {
client := CreateClient(t)
connection := CreateTestConnection(client)
res := client.CreateScopeConfig(PLUGIN_NAME, connection.ID,
FakeScopeConfig{Name: "old name", Env: "old env", Entities: []string{}})
- oldscopeConfig := helper.Cast[FakeScopeConfig](res)
+ oldScopeConfig := helper.Cast[FakeScopeConfig](res)
- client.PatchScopeConfig(PLUGIN_NAME, connection.ID, oldscopeConfig.Id,
FakeScopeConfig{Name: "new name", Env: "new env", Entities: []string{"CICD"}})
+ client.PatchScopeConfig(PLUGIN_NAME, connection.ID, oldScopeConfig.Id,
FakeScopeConfig{Name: "new name", Env: "new env", Entities:
[]string{plugin.DOMAIN_TYPE_CICD}})
- res = client.GetScopeConfig(PLUGIN_NAME, connection.ID,
oldscopeConfig.Id)
+ res = client.GetScopeConfig(PLUGIN_NAME, connection.ID,
oldScopeConfig.Id)
scopeConfig := helper.Cast[FakeScopeConfig](res)
require.Equal(t, "new name", scopeConfig.Name)
require.Equal(t, "new env", scopeConfig.Env)
diff --git a/backend/test/helper/api.go b/backend/test/helper/api.go
index e02111092..022458e2f 100644
--- a/backend/test/helper/api.go
+++ b/backend/test/helper/api.go
@@ -19,6 +19,7 @@ package helper
import (
"fmt"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/services"
"net/http"
"reflect"
"strings"
@@ -35,18 +36,20 @@ import (
// CreateConnection FIXME
func (d *DevlakeClient) TestConnection(pluginName string, connection any) {
d.testCtx.Helper()
- _ = sendHttpRequest[Connection](d.testCtx, d.timeout, debugInfo{
- print: true,
- inlineJson: false,
+ _ = sendHttpRequest[Connection](d.testCtx, d.timeout, &testContext{
+ client: d,
+ printPayload: true,
+ inlineJson: false,
}, http.MethodPost, fmt.Sprintf("%s/plugins/%s/test", d.Endpoint,
pluginName), nil, connection)
}
// CreateConnection FIXME
func (d *DevlakeClient) CreateConnection(pluginName string, connection any)
*Connection {
d.testCtx.Helper()
- created := sendHttpRequest[Connection](d.testCtx, d.timeout, debugInfo{
- print: true,
- inlineJson: false,
+ created := sendHttpRequest[Connection](d.testCtx, d.timeout,
&testContext{
+ client: d,
+ printPayload: true,
+ inlineJson: false,
}, http.MethodPost, fmt.Sprintf("%s/plugins/%s/connections",
d.Endpoint, pluginName), nil, connection)
return &created
}
@@ -54,13 +57,25 @@ func (d *DevlakeClient) CreateConnection(pluginName string,
connection any) *Con
// ListConnections FIXME
func (d *DevlakeClient) ListConnections(pluginName string) []*Connection {
d.testCtx.Helper()
- all := sendHttpRequest[[]*Connection](d.testCtx, d.timeout, debugInfo{
- print: true,
- inlineJson: false,
+ all := sendHttpRequest[[]*Connection](d.testCtx, d.timeout,
&testContext{
+ client: d,
+ printPayload: true,
+ inlineJson: false,
}, http.MethodGet, fmt.Sprintf("%s/plugins/%s/connections", d.Endpoint,
pluginName), nil, nil)
return all
}
+// DeleteConnection FIXME
+func (d *DevlakeClient) DeleteConnection(pluginName string, connectionId
uint64) services.BlueprintProjectPairs {
+ d.testCtx.Helper()
+ refs := sendHttpRequest[services.BlueprintProjectPairs](d.testCtx,
d.timeout, &testContext{
+ client: d,
+ printPayload: true,
+ inlineJson: false,
+ }, http.MethodDelete, fmt.Sprintf("%s/plugins/%s/connections/%d",
d.Endpoint, pluginName, connectionId), nil, nil)
+ return refs
+}
+
// CreateBasicBlueprintV2 FIXME
func (d *DevlakeClient) CreateBasicBlueprintV2(name string, config
*BlueprintV2Config) models.Blueprint {
settings := &models.BlueprintSettings{
@@ -83,31 +98,35 @@ func (d *DevlakeClient) CreateBasicBlueprintV2(name string,
config *BlueprintV2C
Settings: ToJson(settings),
}
d.testCtx.Helper()
- blueprint = sendHttpRequest[models.Blueprint](d.testCtx, d.timeout,
debugInfo{
- print: true,
- inlineJson: false,
+ blueprint = sendHttpRequest[models.Blueprint](d.testCtx, d.timeout,
&testContext{
+ client: d,
+ printPayload: true,
+ inlineJson: false,
}, http.MethodPost, fmt.Sprintf("%s/blueprints", d.Endpoint), nil,
&blueprint)
return blueprint
}
func (d *DevlakeClient) ListBlueprints() blueprints.PaginatedBlueprint {
- return sendHttpRequest[blueprints.PaginatedBlueprint](d.testCtx,
d.timeout, debugInfo{
- print: true,
- inlineJson: false,
+ return sendHttpRequest[blueprints.PaginatedBlueprint](d.testCtx,
d.timeout, &testContext{
+ client: d,
+ printPayload: true,
+ inlineJson: false,
}, http.MethodGet, fmt.Sprintf("%s/blueprints", d.Endpoint), nil, nil)
}
func (d *DevlakeClient) GetBlueprint(blueprintId uint64) models.Blueprint {
- return sendHttpRequest[models.Blueprint](d.testCtx, d.timeout,
debugInfo{
- print: true,
- inlineJson: false,
+ return sendHttpRequest[models.Blueprint](d.testCtx, d.timeout,
&testContext{
+ client: d,
+ printPayload: true,
+ inlineJson: false,
}, http.MethodGet, fmt.Sprintf("%s/blueprints/%d", d.Endpoint,
blueprintId), nil, nil)
}
func (d *DevlakeClient) DeleteBlueprint(blueprintId uint64) {
- sendHttpRequest[any](d.testCtx, d.timeout, debugInfo{
- print: true,
- inlineJson: false,
+ sendHttpRequest[any](d.testCtx, d.timeout, &testContext{
+ client: d,
+ printPayload: true,
+ inlineJson: false,
}, http.MethodDelete, fmt.Sprintf("%s/blueprints/%d", d.Endpoint,
blueprintId), nil, nil)
}
@@ -131,9 +150,10 @@ func (d *DevlakeClient) CreateProject(project
*ProjectConfig) models.ApiOutputPr
Enable: true,
})
}
- return sendHttpRequest[models.ApiOutputProject](d.testCtx, d.timeout,
debugInfo{
- print: true,
- inlineJson: false,
+ return sendHttpRequest[models.ApiOutputProject](d.testCtx, d.timeout,
&testContext{
+ client: d,
+ printPayload: true,
+ inlineJson: false,
}, http.MethodPost, fmt.Sprintf("%s/projects", d.Endpoint), nil,
&models.ApiInputProject{
BaseProject: models.BaseProject{
Name: project.ProjectName,
@@ -145,23 +165,26 @@ func (d *DevlakeClient) CreateProject(project
*ProjectConfig) models.ApiOutputPr
}
func (d *DevlakeClient) GetProject(projectName string) models.ApiOutputProject
{
- return sendHttpRequest[models.ApiOutputProject](d.testCtx, d.timeout,
debugInfo{
- print: true,
- inlineJson: false,
+ return sendHttpRequest[models.ApiOutputProject](d.testCtx, d.timeout,
&testContext{
+ client: d,
+ printPayload: true,
+ inlineJson: false,
}, http.MethodGet, fmt.Sprintf("%s/projects/%s", d.Endpoint,
projectName), nil, nil)
}
func (d *DevlakeClient) ListProjects() apiProject.PaginatedProjects {
- return sendHttpRequest[apiProject.PaginatedProjects](d.testCtx,
d.timeout, debugInfo{
- print: true,
- inlineJson: false,
+ return sendHttpRequest[apiProject.PaginatedProjects](d.testCtx,
d.timeout, &testContext{
+ client: d,
+ printPayload: true,
+ inlineJson: false,
}, http.MethodGet, fmt.Sprintf("%s/projects", d.Endpoint), nil, nil)
}
func (d *DevlakeClient) DeleteProject(projectName string) {
- sendHttpRequest[any](d.testCtx, d.timeout, debugInfo{
- print: true,
- inlineJson: false,
+ sendHttpRequest[any](d.testCtx, d.timeout, &testContext{
+ client: d,
+ printPayload: true,
+ inlineJson: false,
}, http.MethodDelete, fmt.Sprintf("%s/projects/%s", d.Endpoint,
projectName), nil, nil)
}
@@ -169,23 +192,26 @@ func (d *DevlakeClient) CreateScopes(pluginName string,
connectionId uint64, sco
request := map[string]any{
"data": scopes,
}
- return sendHttpRequest[any](d.testCtx, d.timeout, debugInfo{
- print: true,
- inlineJson: false,
+ return sendHttpRequest[any](d.testCtx, d.timeout, &testContext{
+ client: d,
+ printPayload: true,
+ inlineJson: false,
}, http.MethodPut, fmt.Sprintf("%s/plugins/%s/connections/%d/scopes",
d.Endpoint, pluginName, connectionId), nil, request)
}
func (d *DevlakeClient) UpdateScope(pluginName string, connectionId uint64,
scopeId string, scope any) any {
- return sendHttpRequest[any](d.testCtx, d.timeout, debugInfo{
- print: true,
- inlineJson: false,
+ return sendHttpRequest[any](d.testCtx, d.timeout, &testContext{
+ client: d,
+ printPayload: true,
+ inlineJson: false,
}, http.MethodPatch,
fmt.Sprintf("%s/plugins/%s/connections/%d/scopes/%s", d.Endpoint, pluginName,
connectionId, scopeId), nil, scope)
}
func (d *DevlakeClient) ListScopes(pluginName string, connectionId uint64,
listBlueprints bool) []ScopeResponse {
- scopesRaw := sendHttpRequest[[]map[string]any](d.testCtx, d.timeout,
debugInfo{
- print: true,
- inlineJson: false,
+ scopesRaw := sendHttpRequest[[]map[string]any](d.testCtx, d.timeout,
&testContext{
+ client: d,
+ printPayload: true,
+ inlineJson: false,
}, http.MethodGet,
fmt.Sprintf("%s/plugins/%s/connections/%d/scopes?blueprints=%v", d.Endpoint,
pluginName, connectionId, listBlueprints), nil, nil)
var responses []ScopeResponse
for _, scopeRaw := range scopesRaw {
@@ -195,47 +221,53 @@ func (d *DevlakeClient) ListScopes(pluginName string,
connectionId uint64, listB
}
func (d *DevlakeClient) GetScope(pluginName string, connectionId uint64,
scopeId string, listBlueprints bool) any {
- return sendHttpRequest[api.ScopeRes[any, any]](d.testCtx, d.timeout,
debugInfo{
- print: true,
- inlineJson: false,
+ return sendHttpRequest[api.ScopeRes[any, any]](d.testCtx, d.timeout,
&testContext{
+ client: d,
+ printPayload: true,
+ inlineJson: false,
}, http.MethodGet,
fmt.Sprintf("%s/plugins/%s/connections/%d/scopes/%s?blueprints=%v", d.Endpoint,
pluginName, connectionId, scopeId, listBlueprints), nil, nil)
}
-func (d *DevlakeClient) DeleteScope(pluginName string, connectionId uint64,
scopeId string, deleteDataOnly bool) {
- sendHttpRequest[any](d.testCtx, d.timeout, debugInfo{
- print: true,
- inlineJson: false,
+func (d *DevlakeClient) DeleteScope(pluginName string, connectionId uint64,
scopeId string, deleteDataOnly bool) services.BlueprintProjectPairs {
+ return sendHttpRequest[services.BlueprintProjectPairs](d.testCtx,
d.timeout, &testContext{
+ client: d,
+ printPayload: true,
+ inlineJson: false,
}, http.MethodDelete,
fmt.Sprintf("%s/plugins/%s/connections/%d/scopes/%s?delete_data_only=%v",
d.Endpoint, pluginName, connectionId, scopeId, deleteDataOnly), nil, nil)
}
func (d *DevlakeClient) CreateScopeConfig(pluginName string, connectionId
uint64, scopeConfig any) any {
- return sendHttpRequest[any](d.testCtx, d.timeout, debugInfo{
- print: true,
- inlineJson: false,
+ return sendHttpRequest[any](d.testCtx, d.timeout, &testContext{
+ client: d,
+ printPayload: true,
+ inlineJson: false,
}, http.MethodPost,
fmt.Sprintf("%s/plugins/%s/connections/%d/scope-configs",
d.Endpoint, pluginName, connectionId), nil, scopeConfig)
}
func (d *DevlakeClient) PatchScopeConfig(pluginName string, connectionId
uint64, scopeConfigId uint64, scopeConfig any) any {
- return sendHttpRequest[any](d.testCtx, d.timeout, debugInfo{
- print: true,
- inlineJson: false,
+ return sendHttpRequest[any](d.testCtx, d.timeout, &testContext{
+ client: d,
+ printPayload: true,
+ inlineJson: false,
}, http.MethodPatch,
fmt.Sprintf("%s/plugins/%s/connections/%d/scope-configs/%d",
d.Endpoint, pluginName, connectionId, scopeConfigId), nil,
scopeConfig)
}
func (d *DevlakeClient) ListScopeConfigs(pluginName string, connectionId
uint64) []any {
- return sendHttpRequest[[]any](d.testCtx, d.timeout, debugInfo{
- print: true,
- inlineJson: false,
+ return sendHttpRequest[[]any](d.testCtx, d.timeout, &testContext{
+ client: d,
+ printPayload: true,
+ inlineJson: false,
}, http.MethodGet,
fmt.Sprintf("%s/plugins/%s/connections/%d/scope-configs?pageSize=20&page=1",
d.Endpoint, pluginName, connectionId), nil, nil)
}
func (d *DevlakeClient) GetScopeConfig(pluginName string, connectionId uint64,
scopeConfigId uint64) any {
- return sendHttpRequest[any](d.testCtx, d.timeout, debugInfo{
- print: true,
- inlineJson: false,
+ return sendHttpRequest[any](d.testCtx, d.timeout, &testContext{
+ client: d,
+ printPayload: true,
+ inlineJson: false,
}, http.MethodGet,
fmt.Sprintf("%s/plugins/%s/connections/%d/scope-configs/%d",
d.Endpoint, pluginName, connectionId, scopeConfigId), nil, nil)
}
@@ -258,17 +290,19 @@ func (d *DevlakeClient) RemoteScopes(query
RemoteScopesQuery) RemoteScopesOutput
if len(query.Params) > 0 {
url = url + "?" + mapToQueryString(query.Params)
}
- return sendHttpRequest[RemoteScopesOutput](d.testCtx, d.timeout,
debugInfo{
- print: true,
- inlineJson: false,
+ return sendHttpRequest[RemoteScopesOutput](d.testCtx, d.timeout,
&testContext{
+ client: d,
+ printPayload: true,
+ inlineJson: false,
}, http.MethodGet, url, nil, nil)
}
// SearchRemoteScopes makes calls to the "scope API" indirectly. "Search" is
the remote endpoint to hit.
func (d *DevlakeClient) SearchRemoteScopes(query SearchRemoteScopesQuery)
SearchRemoteScopesOutput {
- return sendHttpRequest[SearchRemoteScopesOutput](d.testCtx, d.timeout,
debugInfo{
- print: true,
- inlineJson: false,
+ return sendHttpRequest[SearchRemoteScopesOutput](d.testCtx, d.timeout,
&testContext{
+ client: d,
+ printPayload: true,
+ inlineJson: false,
}, http.MethodGet,
fmt.Sprintf("%s/plugins/%s/connections/%d/search-remote-scopes?search=%s&page=%d&pageSize=%d&%s",
d.Endpoint,
query.PluginName,
@@ -283,9 +317,10 @@ func (d *DevlakeClient) SearchRemoteScopes(query
SearchRemoteScopesQuery) Search
// TriggerBlueprint FIXME
func (d *DevlakeClient) TriggerBlueprint(blueprintId uint64) models.Pipeline {
d.testCtx.Helper()
- pipeline := sendHttpRequest[models.Pipeline](d.testCtx, d.timeout,
debugInfo{
- print: true,
- inlineJson: false,
+ pipeline := sendHttpRequest[models.Pipeline](d.testCtx, d.timeout,
&testContext{
+ client: d,
+ printPayload: true,
+ inlineJson: false,
}, http.MethodPost, fmt.Sprintf("%s/blueprints/%d/trigger", d.Endpoint,
blueprintId), nil, nil)
return d.monitorPipeline(pipeline.ID)
}
@@ -293,9 +328,10 @@ func (d *DevlakeClient) TriggerBlueprint(blueprintId
uint64) models.Pipeline {
// RunPipeline FIXME
func (d *DevlakeClient) RunPipeline(pipeline models.NewPipeline)
models.Pipeline {
d.testCtx.Helper()
- pipelineResult := sendHttpRequest[models.Pipeline](d.testCtx,
d.timeout, debugInfo{
- print: true,
- inlineJson: false,
+ pipelineResult := sendHttpRequest[models.Pipeline](d.testCtx,
d.timeout, &testContext{
+ client: d,
+ printPayload: true,
+ inlineJson: false,
}, http.MethodPost, fmt.Sprintf("%s/pipelines", d.Endpoint), nil,
&pipeline)
return d.monitorPipeline(pipelineResult.ID)
}
@@ -316,8 +352,9 @@ func (d *DevlakeClient) monitorPipeline(id uint64)
models.Pipeline {
coloredPrintf("calling:\n\t%s %s\nwith:\n%s\n", http.MethodGet,
endpoint, string(ToCleanJson(false, nil)))
var pipelineResult models.Pipeline
require.NoError(d.testCtx, runWithTimeout(d.pipelineTimeout, func()
(bool, errors.Error) {
- pipelineResult = sendHttpRequest[models.Pipeline](d.testCtx,
d.pipelineTimeout, debugInfo{
- print: false,
+ pipelineResult = sendHttpRequest[models.Pipeline](d.testCtx,
d.pipelineTimeout, &testContext{
+ client: d,
+ printPayload: false,
}, http.MethodGet, fmt.Sprintf("%s/pipelines/%d", d.Endpoint,
id), nil, nil)
if pipelineResult.Status == models.TASK_COMPLETED {
coloredPrintf("result: %s\n", ToCleanJson(true,
&pipelineResult))
diff --git a/backend/test/helper/client.go b/backend/test/helper/client.go
index 320dee298..ccf9902e4 100644
--- a/backend/test/helper/client.go
+++ b/backend/test/helper/client.go
@@ -76,14 +76,17 @@ func init() {
// DevlakeClient FIXME
type (
DevlakeClient struct {
- Endpoint string
- db *gorm.DB
- log log.Logger
- cfg *viper.Viper
- testCtx *testing.T
- basicRes corectx.BasicRes
- timeout time.Duration
- pipelineTimeout time.Duration
+ Endpoint string
+ db *gorm.DB
+ log log.Logger
+ cfg *viper.Viper
+ testCtx *testing.T
+ basicRes corectx.BasicRes
+ timeout time.Duration
+ pipelineTimeout time.Duration
+ expectedStatusCode int
+ lastReturnedStatusCode int
+ isRemote bool
}
LocalClientConfig struct {
ServerPort uint
@@ -96,18 +99,37 @@ type (
PipelineTimeout time.Duration
}
RemoteClientConfig struct {
- Endpoint string
+ Endpoint string
+ DbURL string
+ TruncateDb bool
}
)
// ConnectRemoteServer returns a client to an existing server based on the
config
-func ConnectRemoteServer(t *testing.T, cfg *RemoteClientConfig) *DevlakeClient
{
- return &DevlakeClient{
- Endpoint: cfg.Endpoint,
- db: nil,
- log: nil,
+func ConnectRemoteServer(t *testing.T, clientConfig *RemoteClientConfig)
*DevlakeClient {
+ var db *gorm.DB
+ var err errors.Error
+ logger := logruslog.Global.Nested("test")
+ cfg := config.GetConfig()
+ if clientConfig.DbURL != "" {
+ cfg.Set("DB_URL", clientConfig.DbURL)
+ db, err = runner.NewGormDb(cfg, logger)
+ require.NoError(t, err)
+ }
+ logger.Info("Connecting to remote server: %s", clientConfig.Endpoint)
+ client := &DevlakeClient{
+ isRemote: true,
+ Endpoint: clientConfig.Endpoint,
+ db: db,
+ cfg: cfg,
+ log: logger,
testCtx: t,
+ basicRes: contextimpl.NewDefaultBasicRes(cfg, logger,
dalgorm.NewDalgorm(db)),
}
+ client.prepareDB(&LocalClientConfig{
+ TruncateDb: clientConfig.TruncateDb,
+ })
+ return client
}
// ConnectLocalServer spins up a local server from the config and returns a
client connected to it
@@ -174,6 +196,17 @@ func (d *DevlakeClient) SetPipelineTimeout(timeout
time.Duration) {
d.pipelineTimeout = timeout
}
+// SetExpectedStatusCode override the expected status code of the next API
call. If it's anything but this, the test will fail.
+func (d *DevlakeClient) SetExpectedStatusCode(code int) *DevlakeClient {
+ d.expectedStatusCode = code
+ return d
+}
+
+// SetExpectedStatusCode return the last http status code
+func (d *DevlakeClient) LastReturnedStatusCode() int {
+ return d.lastReturnedStatusCode
+}
+
// GetDal get a reference to the dal.Dal used by the server
func (d *DevlakeClient) GetDal() dal.Dal {
return dalgorm.NewDalgorm(d.db)
@@ -181,6 +214,9 @@ func (d *DevlakeClient) GetDal() dal.Dal {
// AwaitPluginAvailability wait for this plugin to become available on the
server given a timeout. Returns false if this condition does not get met.
func (d *DevlakeClient) AwaitPluginAvailability(pluginName string, timeout
time.Duration) {
+ if d.isRemote {
+ return
+ }
err := runWithTimeout(timeout, func() (bool, errors.Error) {
_, err := plugin.GetPlugin(pluginName)
return err == nil, nil
@@ -339,11 +375,14 @@ func runWithTimeout(timeout time.Duration, f func()
(bool, errors.Error)) errors
}
}
-func sendHttpRequest[Res any](t *testing.T, timeout time.Duration, debug
debugInfo, httpMethod string, endpoint string, headers map[string]string, body
any) Res {
+func sendHttpRequest[Res any](t *testing.T, timeout time.Duration, ctx
*testContext, httpMethod string, endpoint string, headers map[string]string,
body any) Res {
t.Helper()
+ defer func() {
+ ctx.client.expectedStatusCode = 0
+ }()
b := ToJson(body)
- if debug.print {
- coloredPrintf("calling:\n\t%s %s\nwith:\n%s\n", httpMethod,
endpoint, string(ToCleanJson(debug.inlineJson, body)))
+ if ctx.printPayload {
+ coloredPrintf("calling:\n\t%s %s\nwith:\n%s\n", httpMethod,
endpoint, string(ToCleanJson(ctx.inlineJson, body)))
}
var result Res
err := runWithTimeout(timeout, func() (bool, errors.Error) {
@@ -360,19 +399,26 @@ func sendHttpRequest[Res any](t *testing.T, timeout
time.Duration, debug debugIn
if err != nil {
return false, errors.Convert(err)
}
- if response.StatusCode >= 300 {
- if err = response.Body.Close(); err != nil {
- return false, errors.Convert(err)
+ defer func() {
+ ctx.client.lastReturnedStatusCode = response.StatusCode
+ }()
+ if ctx.client.expectedStatusCode > 0 || response.StatusCode >=
300 {
+ if ctx.client.expectedStatusCode == 0 ||
ctx.client.expectedStatusCode != response.StatusCode {
+ if response.StatusCode >= 300 {
+ if err = response.Body.Close(); err !=
nil {
+ return false,
errors.Convert(err)
+ }
+ response.Close = true
+ return false,
errors.HttpStatus(response.StatusCode).New(fmt.Sprintf("unexpected http status
code calling [%s] %s: %d", httpMethod, endpoint, response.StatusCode))
+ }
}
- response.Close = true
- return false,
errors.HttpStatus(response.StatusCode).New(fmt.Sprintf("unexpected http status
code calling [%s] %s: %d", httpMethod, endpoint, response.StatusCode))
}
b, _ = io.ReadAll(response.Body)
if err = json.Unmarshal(b, &result); err != nil {
return false, errors.Convert(err)
}
- if debug.print {
- coloredPrintf("result: %s\n",
ToCleanJson(debug.inlineJson, b))
+ if ctx.printPayload {
+ coloredPrintf("result: %s\n",
ToCleanJson(ctx.inlineJson, b))
}
if err = response.Body.Close(); err != nil {
return false, errors.Convert(err)
@@ -390,7 +436,8 @@ func coloredPrintf(msg string, args ...any) {
fmt.Printf(colorifier, msg)
}
-type debugInfo struct {
- print bool
- inlineJson bool
+type testContext struct {
+ printPayload bool
+ inlineJson bool
+ client *DevlakeClient
}
diff --git a/backend/test/helper/client_factory.go
b/backend/test/helper/client_factory.go
index 68559f0cd..a4aa1c4c0 100644
--- a/backend/test/helper/client_factory.go
+++ b/backend/test/helper/client_factory.go
@@ -35,3 +35,13 @@ func StartDevLakeServer(t *testing.T, loadedGoPlugins
map[string]plugin.PluginMe
})
return client
}
+
+// Connect to an existing DevLake server with default config. Tables are
truncated. Useful for troubleshooting outside the IDE.
+func ConnectDevLakeServer(t *testing.T) *DevlakeClient {
+ client := ConnectRemoteServer(t, &RemoteClientConfig{
+ Endpoint: "http://localhost:8089";,
+ DbURL: config.GetConfig().GetString("E2E_DB_URL"),
+ TruncateDb: true,
+ })
+ return client
+}
