This is an automated email from the ASF dual-hosted git repository.
mappjzc pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 8246916a8 fix: Add missing RawScopeParamName field to plugins (#5449)
8246916a8 is described below
commit 8246916a8cf2efc5946c3ea9680daec23541a33e
Author: Keon Amini <[email protected]>
AuthorDate: Tue Jun 13 02:51:58 2023 -0500
fix: Add missing RawScopeParamName field to plugins (#5449)
* fix: Fill in missing RawScopeParamName for all plugins that were missing
it.
* Added validation for ReflectionParams to prevent this issue in the future.
* Return [] instead of nill for empty BP scopes from backend to fix UI
issue.
* Handle the case for non-string scope IDs - delete was not working for
them.
* fix: allow unconfigured ScopeConfig for python plugins - python code
needs adjustment
---
backend/core/models/blueprint.go | 3 +
.../pluginhelper/api/scope_generic_helper.go | 195 ++++++++++++---------
.../helpers/pluginhelper/api/scope_helper_test.go | 78 +++++----
backend/plugins/bamboo/api/init.go | 1 +
backend/plugins/bitbucket/api/init.go | 1 +
backend/plugins/gitlab/api/init.go | 1 +
backend/plugins/jenkins/api/init.go | 1 +
backend/plugins/jira/api/init.go | 1 +
backend/plugins/sonarqube/api/init.go | 1 +
backend/plugins/tapd/api/init.go | 1 +
backend/plugins/trello/api/init.go | 1 +
backend/plugins/zentao/api/init.go | 2 +
backend/server/services/project.go | 13 +-
.../server/services/remote/plugin/default_api.go | 6 +-
.../server/services/remote/plugin/plugin_impl.go | 12 +-
backend/test/e2e/manual/azuredevops/azure_test.go | 9 +
backend/test/helper/api.go | 7 +
17 files changed, 202 insertions(+), 131 deletions(-)
diff --git a/backend/core/models/blueprint.go b/backend/core/models/blueprint.go
index e8b63c44e..8f0b25d46 100644
--- a/backend/core/models/blueprint.go
+++ b/backend/core/models/blueprint.go
@@ -76,6 +76,9 @@ func (bps *BlueprintSettings) UpdateConnections(updater
func(c *plugin.Blueprint
if err != nil {
return err
}
+ if conn.Scopes == nil {
+ conn.Scopes = []*plugin.BlueprintScopeV200{} //UI
expects this to be []
+ }
conns[i] = conn
}
bps.Connections, err = errors.Convert01(json.Marshal(&conns))
diff --git a/backend/helpers/pluginhelper/api/scope_generic_helper.go
b/backend/helpers/pluginhelper/api/scope_generic_helper.go
index d6b045564..acc612872 100644
--- a/backend/helpers/pluginhelper/api/scope_generic_helper.go
+++ b/backend/helpers/pluginhelper/api/scope_generic_helper.go
@@ -68,12 +68,17 @@ type (
ScopeResDoc[ScopeConfig] `mapstructure:",squash"` // however,
only this type of embeding is supported as of golang 1.20
}
ReflectionParameters struct {
- ScopeIdFieldName string
- ScopeIdColumnName string
- RawScopeParamName string
+ // This corresponds to the struct field of the scope struct's
ID field
+ ScopeIdFieldName string `validate:"required"`
+ // This corresponds to the database column name of the scope
struct's ID (typically primary key) field
+ ScopeIdColumnName string `validate:"required"`
+ // This corresponds to the scope field on the ApiParams struct
of a plugin.
+ RawScopeParamName string `validate:"required"`
}
ScopeHelperOptions struct {
+ // Define this if the raw params doesn't store the ScopeId but
a different attribute of the Scope (e.g. Name)
GetScopeParamValue func(db dal.Dal, scopeId string) (string,
errors.Error)
+ IsRemote bool
}
)
@@ -108,6 +113,10 @@ func NewGenericScopeHelper[Conn any, Scope any, Tr any](
if params == nil {
panic("reflection params not provided")
}
+ err := vld.Struct(params)
+ if err != nil {
+ panic(err)
+ }
if opts == nil {
opts = &ScopeHelperOptions{}
}
@@ -130,56 +139,56 @@ func NewGenericScopeHelper[Conn any, Scope any, Tr any](
}
}
-func (c *GenericScopeApiHelper[Conn, Scope, Tr]) DbHelper()
ScopeDatabaseHelper[Conn, Scope, Tr] {
- return c.dbHelper
+func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) DbHelper()
ScopeDatabaseHelper[Conn, Scope, Tr] {
+ return gs.dbHelper
}
-func (c *GenericScopeApiHelper[Conn, Scope, Tr]) PutScopes(input
*plugin.ApiResourceInput, scopes []*Scope) ([]*ScopeRes[Scope, Tr],
errors.Error) {
- params, err := c.extractFromReqParam(input, false)
+func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) PutScopes(input
*plugin.ApiResourceInput, scopes []*Scope) ([]*ScopeRes[Scope, Tr],
errors.Error) {
+ params, err := gs.extractFromReqParam(input, false)
if err != nil {
return nil, err
}
- err = c.dbHelper.VerifyConnection(params.connectionId)
+ err = gs.dbHelper.VerifyConnection(params.connectionId)
if err != nil {
return nil, errors.Default.Wrap(err, fmt.Sprintf("error
verifying connection for connection ID %d", params.connectionId))
}
- err = c.validatePrimaryKeys(scopes)
+ err = gs.validatePrimaryKeys(scopes)
if err != nil {
return nil, err
}
now := time.Now()
for _, scope := range scopes {
// Set the connection ID, CreatedDate, and UpdatedDate fields
- setScopeFields(scope, params.connectionId, &now, &now)
- err = VerifyScope(scope, c.validator)
+ gs.setScopeFields(scope, params.connectionId, &now, &now)
+ err = gs.verifyScope(scope, gs.validator)
if err != nil {
return nil, errors.Default.Wrap(err, "error verifying
scope")
}
}
// Save the scopes to the database
if len(scopes) > 0 {
- err = c.dbHelper.SaveScope(scopes)
+ err = gs.dbHelper.SaveScope(scopes)
if err != nil {
return nil, errors.Default.Wrap(err, "error saving
scope")
}
}
- apiScopes, err := c.addScopeConfig(scopes...)
+ apiScopes, err := gs.addScopeConfig(scopes...)
if err != nil {
return nil, errors.Default.Wrap(err, "error associating scope
config to scope")
}
return apiScopes, nil
}
-func (c *GenericScopeApiHelper[Conn, Scope, Tr]) UpdateScope(input
*plugin.ApiResourceInput) (*ScopeRes[Scope, Tr], errors.Error) {
- params, err := c.extractFromReqParam(input, true)
+func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) UpdateScope(input
*plugin.ApiResourceInput) (*ScopeRes[Scope, Tr], errors.Error) {
+ params, err := gs.extractFromReqParam(input, true)
if err != nil {
return nil, err
}
- err = c.dbHelper.VerifyConnection(params.connectionId)
+ err = gs.dbHelper.VerifyConnection(params.connectionId)
if err != nil {
return nil, err
}
- scope, err := c.dbHelper.GetScope(params.connectionId, params.scopeId)
+ scope, err := gs.dbHelper.GetScope(params.connectionId, params.scopeId)
if err != nil {
return nil, err
}
@@ -187,46 +196,46 @@ func (c *GenericScopeApiHelper[Conn, Scope, Tr])
UpdateScope(input *plugin.ApiRe
if err != nil {
return nil, errors.Default.Wrap(err, "patch scope error")
}
- err = VerifyScope(scope, c.validator)
+ err = gs.verifyScope(scope, gs.validator)
if err != nil {
return nil, errors.Default.Wrap(err, "Invalid scope")
}
- err = c.dbHelper.UpdateScope(params.connectionId, params.scopeId, scope)
+ err = gs.dbHelper.UpdateScope(params.connectionId, params.scopeId,
scope)
if err != nil {
return nil, errors.Default.Wrap(err, "error on saving Scope")
}
- scopeRes, err := c.addScopeConfig(scope)
+ scopeRes, err := gs.addScopeConfig(scope)
if err != nil {
return nil, err
}
return scopeRes[0], nil
}
-func (c *GenericScopeApiHelper[Conn, Scope, Tr]) GetScopes(input
*plugin.ApiResourceInput) ([]*ScopeRes[Scope, Tr], errors.Error) {
- params, err := c.extractFromGetReqParam(input, false)
+func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) GetScopes(input
*plugin.ApiResourceInput) ([]*ScopeRes[Scope, Tr], errors.Error) {
+ params, err := gs.extractFromGetReqParam(input, false)
if err != nil {
return nil, errors.BadInput.New("invalid path params:
\"connectionId\" not set")
}
- err = c.dbHelper.VerifyConnection(params.connectionId)
+ err = gs.dbHelper.VerifyConnection(params.connectionId)
if err != nil {
return nil, errors.Default.Wrap(err, fmt.Sprintf("error
verifying connection for connection ID %d", params.connectionId))
}
- scopes, err := c.dbHelper.ListScopes(input, params.connectionId)
+ scopes, err := gs.dbHelper.ListScopes(input, params.connectionId)
if err != nil {
return nil, errors.Default.Wrap(err, fmt.Sprintf("error
verifying connection for connection ID %d", params.connectionId))
}
- apiScopes, err := c.addScopeConfig(scopes...)
+ apiScopes, err := gs.addScopeConfig(scopes...)
if err != nil {
return nil, errors.Default.Wrap(err, "error associating scope
configs with scopes")
}
// return empty array rather than nil in case of no scopes
if len(apiScopes) > 0 && params.loadBlueprints {
- scopesById := c.mapByScopeId(apiScopes)
+ scopesById := gs.mapByScopeId(apiScopes)
var scopeIds []string
for id := range scopesById {
scopeIds = append(scopeIds, id)
}
- blueprintMap, err :=
c.bpManager.GetBlueprintsByScopes(params.connectionId, scopeIds...)
+ blueprintMap, err :=
gs.bpManager.GetBlueprintsByScopes(params.connectionId, scopeIds...)
if err != nil {
return nil, errors.Default.Wrap(err, fmt.Sprintf("error
getting blueprints for scopes from connection %d", params.connectionId))
}
@@ -243,32 +252,32 @@ func (c *GenericScopeApiHelper[Conn, Scope, Tr])
GetScopes(input *plugin.ApiReso
for bpId := range blueprintMap {
danglingIds = append(danglingIds, bpId)
}
- c.log.Warn(nil, "The following dangling scopes were
found: %v", danglingIds)
+ gs.log.Warn(nil, "The following dangling scopes were
found: %v", danglingIds)
}
}
return apiScopes, nil
}
-func (c *GenericScopeApiHelper[Conn, Scope, Tr]) GetScope(input
*plugin.ApiResourceInput) (*ScopeRes[Scope, Tr], errors.Error) {
- params, err := c.extractFromGetReqParam(input, true)
+func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) GetScope(input
*plugin.ApiResourceInput) (*ScopeRes[Scope, Tr], errors.Error) {
+ params, err := gs.extractFromGetReqParam(input, true)
if err != nil {
return nil, err
}
- err = c.dbHelper.VerifyConnection(params.connectionId)
+ err = gs.dbHelper.VerifyConnection(params.connectionId)
if err != nil {
return nil, errors.Default.Wrap(err, fmt.Sprintf("error
verifying connection for connection ID %d", params.connectionId))
}
- scope, err := c.dbHelper.GetScope(params.connectionId, params.scopeId)
+ scope, err := gs.dbHelper.GetScope(params.connectionId, params.scopeId)
if err != nil {
return nil, errors.Default.Wrap(err, fmt.Sprintf("error
retrieving scope with scope ID %s", params.scopeId))
}
- apiScopes, err := c.addScopeConfig(scope)
+ apiScopes, err := gs.addScopeConfig(scope)
if err != nil {
return nil, errors.Default.Wrap(err, fmt.Sprintf("error
associating scope config with scope %s", params.scopeId))
}
scopeRes := apiScopes[0]
if params.loadBlueprints {
- blueprintMap, err :=
c.bpManager.GetBlueprintsByScopes(params.connectionId, params.scopeId)
+ blueprintMap, err :=
gs.bpManager.GetBlueprintsByScopes(params.connectionId, params.scopeId)
if err != nil {
return nil, errors.Default.Wrap(err, fmt.Sprintf("error
getting blueprints for scope with scope ID %s", params.scopeId))
}
@@ -279,41 +288,38 @@ func (c *GenericScopeApiHelper[Conn, Scope, Tr])
GetScope(input *plugin.ApiResou
return scopeRes, nil
}
-func (c *GenericScopeApiHelper[Conn, Scope, Tr]) DeleteScope(input
*plugin.ApiResourceInput) errors.Error {
- params, err := c.extractFromDeleteReqParam(input)
+func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) DeleteScope(input
*plugin.ApiResourceInput) errors.Error {
+ params, err := gs.extractFromDeleteReqParam(input)
if err != nil {
return err
}
- err = c.dbHelper.VerifyConnection(params.connectionId)
+ err = gs.dbHelper.VerifyConnection(params.connectionId)
if err != nil {
return errors.Default.Wrap(err, fmt.Sprintf("error verifying
connection for connection ID %d", params.connectionId))
}
- // delete all the plugin records referencing this scope
- if c.reflectionParams.RawScopeParamName != "" {
- scopeParamValue := params.scopeId
- if c.opts.GetScopeParamValue != nil {
- scopeParamValue, err = c.opts.GetScopeParamValue(c.db,
params.scopeId)
- if err != nil {
- return errors.Default.Wrap(err,
fmt.Sprintf("error extracting scope parameter name for scope %s",
params.scopeId))
- }
- }
- // find all tables for this plugin
- tables, err := getAffectedTables(params.plugin)
- if err != nil {
- return errors.Default.Wrap(err, fmt.Sprintf("error
getting database tables managed by plugin %s", params.plugin))
- }
- err = c.transactionalDelete(tables, scopeParamValue)
+ scopeParamValue := params.scopeId
+ if gs.opts.GetScopeParamValue != nil {
+ scopeParamValue, err = gs.opts.GetScopeParamValue(gs.db,
params.scopeId)
if err != nil {
- return errors.Default.Wrap(err, fmt.Sprintf("error
deleting data bound to scope %s for plugin %s", params.scopeId, params.plugin))
+ return errors.Default.Wrap(err, fmt.Sprintf("error
extracting scope parameter name for scope %s", params.scopeId))
}
}
+ // find all tables for this plugin
+ tables, err := getAffectedTables(params.plugin)
+ if err != nil {
+ return errors.Default.Wrap(err, fmt.Sprintf("error getting
database tables managed by plugin %s", params.plugin))
+ }
+ err = gs.transactionalDelete(tables, scopeParamValue)
+ if err != nil {
+ return errors.Default.Wrap(err, fmt.Sprintf("error deleting
data bound to scope %s for plugin %s", params.scopeId, params.plugin))
+ }
if !params.deleteDataOnly {
// Delete the scope itself
- err = c.dbHelper.DeleteScope(params.connectionId,
params.scopeId)
+ err = gs.dbHelper.DeleteScope(params.connectionId,
params.scopeId)
if err != nil {
return errors.Default.Wrap(err, fmt.Sprintf("error
deleting scope %s", params.scopeId))
}
- err = c.updateBlueprints(params.connectionId, params.scopeId)
+ err = gs.updateBlueprints(params.connectionId, params.scopeId)
if err != nil {
return err
}
@@ -321,7 +327,7 @@ func (c *GenericScopeApiHelper[Conn, Scope, Tr])
DeleteScope(input *plugin.ApiRe
return nil
}
-func (c *GenericScopeApiHelper[Conn, Scope, Tr]) addScopeConfig(scopes
...*Scope) ([]*ScopeRes[Scope, Tr], errors.Error) {
+func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) addScopeConfig(scopes
...*Scope) ([]*ScopeRes[Scope, Tr], errors.Error) {
apiScopes := make([]*ScopeRes[Scope, Tr], len(scopes))
for i, scope := range scopes {
apiScopes[i] = &ScopeRes[Scope, Tr]{
@@ -329,7 +335,7 @@ func (c *GenericScopeApiHelper[Conn, Scope, Tr])
addScopeConfig(scopes ...*Scope
}
scIdField := reflectField(scope, "ScopeConfigId")
if scIdField.IsValid() && scIdField.Uint() > 0 {
- scopeConfig, err :=
c.dbHelper.GetScopeConfig(scIdField.Uint())
+ scopeConfig, err :=
gs.dbHelper.GetScopeConfig(scIdField.Uint())
if err != nil {
return nil, err
}
@@ -339,16 +345,16 @@ func (c *GenericScopeApiHelper[Conn, Scope, Tr])
addScopeConfig(scopes ...*Scope
return apiScopes, nil
}
-func (c *GenericScopeApiHelper[Conn, Scope, Tr]) mapByScopeId(scopes
[]*ScopeRes[Scope, Tr]) map[string]*ScopeRes[Scope, Tr] {
+func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) mapByScopeId(scopes
[]*ScopeRes[Scope, Tr]) map[string]*ScopeRes[Scope, Tr] {
scopeMap := map[string]*ScopeRes[Scope, Tr]{}
for _, scope := range scopes {
- scopeId := fmt.Sprintf("%v", reflectField(scope.Scope,
c.reflectionParams.ScopeIdFieldName).Interface())
+ scopeId := fmt.Sprintf("%v", reflectField(scope.Scope,
gs.reflectionParams.ScopeIdFieldName).Interface())
scopeMap[scopeId] = scope
}
return scopeMap
}
-func (c *GenericScopeApiHelper[Conn, Scope, Tr]) extractFromReqParam(input
*plugin.ApiResourceInput, withScopeId bool) (*requestParams, errors.Error) {
+func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) extractFromReqParam(input
*plugin.ApiResourceInput, withScopeId bool) (*requestParams, errors.Error) {
connectionId, err := strconv.ParseUint(input.Params["connectionId"],
10, 64)
if err != nil {
return nil, errors.BadInput.Wrap(err, "Invalid
\"connectionId\"")
@@ -372,8 +378,8 @@ func (c *GenericScopeApiHelper[Conn, Scope, Tr])
extractFromReqParam(input *plug
}, nil
}
-func (c *GenericScopeApiHelper[Conn, Scope, Tr])
extractFromDeleteReqParam(input *plugin.ApiResourceInput)
(*deleteRequestParams, errors.Error) {
- params, err := c.extractFromReqParam(input, true)
+func (gs *GenericScopeApiHelper[Conn, Scope, Tr])
extractFromDeleteReqParam(input *plugin.ApiResourceInput)
(*deleteRequestParams, errors.Error) {
+ params, err := gs.extractFromReqParam(input, true)
if err != nil {
return nil, err
}
@@ -393,8 +399,8 @@ func (c *GenericScopeApiHelper[Conn, Scope, Tr])
extractFromDeleteReqParam(input
}, nil
}
-func (c *GenericScopeApiHelper[Conn, Scope, Tr]) extractFromGetReqParam(input
*plugin.ApiResourceInput, withScopeId bool) (*getRequestParams, errors.Error) {
- params, err := c.extractFromReqParam(input, withScopeId)
+func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) extractFromGetReqParam(input
*plugin.ApiResourceInput, withScopeId bool) (*getRequestParams, errors.Error) {
+ params, err := gs.extractFromReqParam(input, withScopeId)
if err != nil {
return nil, err
}
@@ -414,7 +420,19 @@ func (c *GenericScopeApiHelper[Conn, Scope, Tr])
extractFromGetReqParam(input *p
}, nil
}
-func setScopeFields(p interface{}, connectionId uint64, createdDate
*time.Time, updatedDate *time.Time) {
+func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) getRawParams(connectionId
uint64, scopeId any) string {
+ paramsMap := map[string]any{
+ "ConnectionId": connectionId,
+ gs.reflectionParams.RawScopeParamName: scopeId,
+ }
+ b, err := json.Marshal(paramsMap)
+ if err != nil {
+ panic(err)
+ }
+ return string(b)
+}
+
+func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) setScopeFields(p
interface{}, connectionId uint64, createdDate *time.Time, updatedDate
*time.Time) {
pType := reflect.TypeOf(p)
if pType.Kind() != reflect.Ptr {
panic("expected a pointer to a struct")
@@ -424,6 +442,14 @@ func setScopeFields(p interface{}, connectionId uint64,
createdDate *time.Time,
connIdField := pValue.FieldByName("ConnectionId")
connIdField.SetUint(connectionId)
+ // set raw params
+ rawParams := pValue.FieldByName("RawDataParams")
+ if !rawParams.IsValid() {
+ panic("scope is missing the field \"RawDataParams\"")
+ }
+ scopeIdField := pValue.FieldByName(gs.reflectionParams.ScopeIdFieldName)
+ rawParams.Set(reflect.ValueOf(gs.getRawParams(connectionId,
scopeIdField.Interface())))
+
// set CreatedDate
createdDateField := pValue.FieldByName("CreatedDate")
if createdDateField.IsValid() &&
createdDateField.Type().AssignableTo(reflect.TypeOf(createdDate)) {
@@ -474,21 +500,22 @@ func returnPrimaryKeyValue(p interface{}) string {
return result
}
-func VerifyScope(scope interface{}, vld *validator.Validate) errors.Error {
- if vld != nil {
- pType := reflect.TypeOf(scope)
- if pType.Kind() != reflect.Ptr {
- panic("expected a pointer to a struct")
- }
- if err := vld.Struct(scope); err != nil {
- return errors.Default.Wrap(err, "error validating
target")
- }
+func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) verifyScope(scope
interface{}, vld *validator.Validate) errors.Error {
+ if gs.opts.IsRemote {
+ return nil
+ }
+ pType := reflect.TypeOf(scope)
+ if pType.Kind() != reflect.Ptr {
+ panic("expected a pointer to a struct")
+ }
+ if err := vld.Struct(scope); err != nil {
+ return errors.Default.Wrap(err, "error validating target")
}
return nil
}
-func (c *GenericScopeApiHelper[Conn, Scope, Tr]) validatePrimaryKeys(scopes
[]*Scope) errors.Error {
- if c.validator == nil {
+func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) validatePrimaryKeys(scopes
[]*Scope) errors.Error {
+ if gs.opts.IsRemote {
return nil
}
keeper := make(map[string]struct{})
@@ -504,8 +531,8 @@ func (c *GenericScopeApiHelper[Conn, Scope, Tr])
validatePrimaryKeys(scopes []*S
return nil
}
-func (c *GenericScopeApiHelper[Conn, Scope, Tr]) updateBlueprints(connectionId
uint64, scopeId string) errors.Error {
- blueprintsMap, err := c.bpManager.GetBlueprintsByScopes(connectionId,
scopeId)
+func (gs *GenericScopeApiHelper[Conn, Scope, Tr])
updateBlueprints(connectionId uint64, scopeId string) errors.Error {
+ blueprintsMap, err := gs.bpManager.GetBlueprintsByScopes(connectionId,
scopeId)
if err != nil {
return errors.Default.Wrap(err, fmt.Sprintf("error retrieving
scope with scope ID %s", scopeId))
}
@@ -534,7 +561,7 @@ func (c *GenericScopeApiHelper[Conn, Scope, Tr])
updateBlueprints(connectionId u
if err != nil {
return errors.Default.Wrap(err,
fmt.Sprintf("error writing new settings into blueprint %s", blueprint.Name))
}
- err = c.bpManager.SaveDbBlueprint(blueprint)
+ err = gs.bpManager.SaveDbBlueprint(blueprint)
if err != nil {
return errors.Default.Wrap(err,
fmt.Sprintf("error saving the updated blueprint %s", blueprint.Name))
}
@@ -543,15 +570,15 @@ func (c *GenericScopeApiHelper[Conn, Scope, Tr])
updateBlueprints(connectionId u
return nil
}
-func (c *GenericScopeApiHelper[Conn, Scope, Tr]) transactionalDelete(tables
[]string, scopeId string) errors.Error {
- tx := c.db.Begin()
+func (gs *GenericScopeApiHelper[Conn, Scope, Tr]) transactionalDelete(tables
[]string, scopeId string) errors.Error {
+ tx := gs.db.Begin()
for _, table := range tables {
- query := createDeleteQuery(table,
c.reflectionParams.RawScopeParamName, scopeId)
+ query := createDeleteQuery(table,
gs.reflectionParams.RawScopeParamName, scopeId)
err := tx.Exec(query)
if err != nil {
err2 := tx.Rollback()
if err2 != nil {
- c.log.Warn(err2, fmt.Sprintf("error rolling
back table data deletion transaction. query was %s", query))
+ gs.log.Warn(err2, fmt.Sprintf("error rolling
back table data deletion transaction. query was %s", query))
}
return err
}
@@ -586,7 +613,7 @@ func createDeleteQuery(tableName string, scopeIdKey string,
scopeId string) stri
} else if strings.HasPrefix(tableName, "_raw_") {
column = "params"
}
- query := `DELETE FROM ` + tableName + ` WHERE ` + column + ` LIKE '%"`
+ scopeIdKey + `":"` + scopeId + `"%'`
+ query := `DELETE FROM ` + tableName + ` WHERE ` + column + ` LIKE '%"`
+ scopeIdKey + `":%` + scopeId + `%'`
return query
}
diff --git a/backend/helpers/pluginhelper/api/scope_helper_test.go
b/backend/helpers/pluginhelper/api/scope_helper_test.go
index c173cc5d2..b05701693 100644
--- a/backend/helpers/pluginhelper/api/scope_helper_test.go
+++ b/backend/helpers/pluginhelper/api/scope_helper_test.go
@@ -74,6 +74,7 @@ func (TestConnection) TableName() string {
}
func TestVerifyScope(t *testing.T) {
+ apiHelper := createMockScopeHelper[TestRepo]("GithubId")
testCases := []struct {
name string
model TestModel
@@ -106,7 +107,7 @@ func TestVerifyScope(t *testing.T) {
}
for _, tc := range testCases {
- err := VerifyScope(&tc.model, validator.New())
+ err := apiHelper.verifyScope(&tc.model, validator.New())
if (err != nil) != tc.wantErr {
t.Errorf("unexpected error value - got: %v, want: %v",
err, tc.wantErr)
}
@@ -137,7 +138,7 @@ func (TestScopeConfig) TableName() string {
func TestSetScopeFields(t *testing.T) {
// create a struct
- var p struct {
+ type P struct {
ConnectionId uint64 `json:"connectionId"
mapstructure:"connectionId" gorm:"primaryKey"`
GitlabId int `json:"gitlabId" mapstructure:"gitlabId"
gorm:"primaryKey"`
@@ -145,12 +146,14 @@ func TestSetScopeFields(t *testing.T) {
UpdatedDate *time.Time `json:"updatedDate"
mapstructure:"-"`
common.NoPKModel `json:"-" mapstructure:"-"`
}
+ p := P{}
+ apiHelper := createMockScopeHelper[P]("GitlabId")
// call setScopeFields to assign value
connectionId := uint64(123)
createdDate := time.Now()
updatedDate := &createdDate
- setScopeFields(&p, connectionId, &createdDate, updatedDate)
+ apiHelper.setScopeFields(&p, connectionId, &createdDate, updatedDate)
// verify fields
if p.ConnectionId != connectionId {
@@ -167,7 +170,7 @@ func TestSetScopeFields(t *testing.T) {
t.Errorf("UpdatedDate not set correctly, expected: %v, got:
%v", updatedDate, p.UpdatedDate)
}
- setScopeFields(&p, connectionId, &createdDate, nil)
+ apiHelper.setScopeFields(&p, connectionId, &createdDate, nil)
// verify fields
if p.ConnectionId != connectionId {
@@ -182,13 +185,15 @@ func TestSetScopeFields(t *testing.T) {
t.Errorf("UpdatedDate not set correctly, expected: %v, got:
%v", nil, p.UpdatedDate)
}
- var p1 struct {
+ type P1 struct {
ConnectionId uint64 `json:"connectionId"
mapstructure:"connectionId" gorm:"primaryKey"`
GitlabId int `json:"gitlabId" mapstructure:"gitlabId"
gorm:"primaryKey"`
common.NoPKModel `json:"-" mapstructure:"-"`
}
- setScopeFields(&p1, connectionId, &createdDate, &createdDate)
+ apiHelper2 := createMockScopeHelper[P1]("GitlabId")
+ p1 := P1{}
+ apiHelper2.setScopeFields(&p1, connectionId, &createdDate, &createdDate)
}
@@ -240,28 +245,7 @@ func TestReturnPrimaryKeyValue(t *testing.T) {
}
func TestScopeApiHelper_Put(t *testing.T) {
- mockDal := new(mockdal.Dal)
- mockLogger := unithelper.DummyLogger()
- mockRes := new(mockcontext.BasicRes)
-
- mockRes.On("GetDal").Return(mockDal)
- mockRes.On("GetConfig", mock.Anything).Return("")
- mockRes.On("GetLogger").Return(mockLogger)
-
- // we expect total 2 deletion calls after all code got carried out
- mockDal.On("Delete", mock.Anything, mock.Anything).Return(nil).Twice()
- mockDal.On("GetPrimaryKeyFields", mock.Anything).Return(
- []reflect.StructField{
- {Name: "ID", Type: reflect.TypeOf("")},
- },
- )
- mockDal.On("CreateOrUpdate", mock.Anything, mock.Anything).Return(nil)
- mockDal.On("First", mock.Anything, mock.Anything).Return(nil)
- mockDal.On("All", mock.Anything, mock.Anything).Return(nil)
- mockDal.On("AllTables").Return(nil, nil)
-
- connHelper := NewConnectionHelper(mockRes, nil)
-
+ apiHelper := createMockScopeHelper[TestRepo]("GithubId")
// create a mock input, scopes, and connection
input := &plugin.ApiResourceInput{Params:
map[string]string{"connectionId": "123"}, Body: map[string]interface{}{
"data": []map[string]interface{}{
@@ -295,12 +279,40 @@ func TestScopeApiHelper_Put(t *testing.T) {
"updatedAt": "string",
"updatedDate": "string",
}}}}
-
- params := &ReflectionParameters{}
- dbHelper := NewScopeDatabaseHelperImpl[TestConnection, TestRepo,
TestScopeConfig](mockRes, connHelper, params)
- // create a mock ScopeApiHelper with a mock database connection
- apiHelper := NewScopeHelper[TestConnection, TestRepo,
TestScopeConfig](mockRes, nil, connHelper, dbHelper, params, nil)
// test a successful call to Put
_, err := apiHelper.Put(input)
assert.NoError(t, err)
}
+
+func createMockScopeHelper[Repo any](scopeIdFieldName string)
*ScopeApiHelper[TestConnection, Repo, TestScopeConfig] {
+ mockDal := new(mockdal.Dal)
+ mockLogger := unithelper.DummyLogger()
+ mockRes := new(mockcontext.BasicRes)
+
+ mockRes.On("GetDal").Return(mockDal)
+ mockRes.On("GetConfig", mock.Anything).Return("")
+ mockRes.On("GetLogger").Return(mockLogger)
+
+ // we expect total 2 deletion calls after all code got carried out
+ mockDal.On("Delete", mock.Anything, mock.Anything).Return(nil).Twice()
+ mockDal.On("GetPrimaryKeyFields", mock.Anything).Return(
+ []reflect.StructField{
+ {Name: "ID", Type: reflect.TypeOf("")},
+ },
+ )
+ mockDal.On("CreateOrUpdate", mock.Anything, mock.Anything).Return(nil)
+ mockDal.On("First", mock.Anything, mock.Anything).Return(nil)
+ mockDal.On("All", mock.Anything, mock.Anything).Return(nil)
+ mockDal.On("AllTables").Return(nil, nil)
+
+ connHelper := NewConnectionHelper(mockRes, nil)
+
+ params := &ReflectionParameters{
+ ScopeIdFieldName: scopeIdFieldName,
+ ScopeIdColumnName: "scope_id",
+ RawScopeParamName: "ScopeId",
+ }
+ dbHelper := NewScopeDatabaseHelperImpl[TestConnection, Repo,
TestScopeConfig](mockRes, connHelper, params)
+ // create a mock ScopeApiHelper with a mock database connection
+ return NewScopeHelper[TestConnection, Repo, TestScopeConfig](mockRes,
validator.New(), connHelper, dbHelper, params, nil)
+}
diff --git a/backend/plugins/bamboo/api/init.go
b/backend/plugins/bamboo/api/init.go
index a99863e72..251bc19e6 100644
--- a/backend/plugins/bamboo/api/init.go
+++ b/backend/plugins/bamboo/api/init.go
@@ -42,6 +42,7 @@ func Init(br context.BasicRes) {
params := &api.ReflectionParameters{
ScopeIdFieldName: "ProjectKey",
ScopeIdColumnName: "project_key",
+ RawScopeParamName: "ProjectKey",
}
scopeHelper = api.NewScopeHelper[models.BambooConnection,
models.BambooProject, models.BambooScopeConfig](
basicRes,
diff --git a/backend/plugins/bitbucket/api/init.go
b/backend/plugins/bitbucket/api/init.go
index 1aef2f092..9b6ae8f92 100644
--- a/backend/plugins/bitbucket/api/init.go
+++ b/backend/plugins/bitbucket/api/init.go
@@ -41,6 +41,7 @@ func Init(br context.BasicRes) {
params := &api.ReflectionParameters{
ScopeIdFieldName: "BitbucketId",
ScopeIdColumnName: "bitbucket_id",
+ RawScopeParamName: "FullName",
}
scopeHelper = api.NewScopeHelper[models.BitbucketConnection,
models.BitbucketRepo, models.BitbucketScopeConfig](
basicRes,
diff --git a/backend/plugins/gitlab/api/init.go
b/backend/plugins/gitlab/api/init.go
index 302cc5198..347d103ad 100644
--- a/backend/plugins/gitlab/api/init.go
+++ b/backend/plugins/gitlab/api/init.go
@@ -41,6 +41,7 @@ func Init(br context.BasicRes) {
params := &api.ReflectionParameters{
ScopeIdFieldName: "GitlabId",
ScopeIdColumnName: "gitlab_id",
+ RawScopeParamName: "ProjectId",
}
scopeHelper = api.NewScopeHelper[models.GitlabConnection,
models.GitlabProject, models.GitlabScopeConfig](
basicRes,
diff --git a/backend/plugins/jenkins/api/init.go
b/backend/plugins/jenkins/api/init.go
index c6914494a..aadcae11b 100644
--- a/backend/plugins/jenkins/api/init.go
+++ b/backend/plugins/jenkins/api/init.go
@@ -42,6 +42,7 @@ func Init(br context.BasicRes) {
params := &api.ReflectionParameters{
ScopeIdFieldName: "FullName",
ScopeIdColumnName: "full_name",
+ RawScopeParamName: "FullName",
}
scopeHelper = api.NewScopeHelper[models.JenkinsConnection,
models.JenkinsJob, models.JenkinsScopeConfig](
basicRes,
diff --git a/backend/plugins/jira/api/init.go b/backend/plugins/jira/api/init.go
index c8ff1b570..9564d0678 100644
--- a/backend/plugins/jira/api/init.go
+++ b/backend/plugins/jira/api/init.go
@@ -42,6 +42,7 @@ func Init(br context.BasicRes) {
params := &api.ReflectionParameters{
ScopeIdFieldName: "BoardId",
ScopeIdColumnName: "board_id",
+ RawScopeParamName: "BoardId",
}
scopeHelper = api.NewScopeHelper[models.JiraConnection,
models.JiraBoard, models.JiraScopeConfig](
basicRes,
diff --git a/backend/plugins/sonarqube/api/init.go
b/backend/plugins/sonarqube/api/init.go
index 2a5273b04..52c2bd0a2 100644
--- a/backend/plugins/sonarqube/api/init.go
+++ b/backend/plugins/sonarqube/api/init.go
@@ -40,6 +40,7 @@ func Init(br context.BasicRes) {
params := &api.ReflectionParameters{
ScopeIdFieldName: "ProjectKey",
ScopeIdColumnName: "project_key",
+ RawScopeParamName: "ProjectKey",
}
scopeHelper = api.NewScopeHelper[models.SonarqubeConnection,
models.SonarqubeProject, any](
basicRes,
diff --git a/backend/plugins/tapd/api/init.go b/backend/plugins/tapd/api/init.go
index c034961a8..2726f40e0 100644
--- a/backend/plugins/tapd/api/init.go
+++ b/backend/plugins/tapd/api/init.go
@@ -41,6 +41,7 @@ func Init(br context.BasicRes) {
params := &api.ReflectionParameters{
ScopeIdFieldName: "Id",
ScopeIdColumnName: "id",
+ RawScopeParamName: "WorkSpaceId",
}
scopeHelper = api.NewScopeHelper[models.TapdConnection,
models.TapdWorkspace, models.TapdScopeConfig](
basicRes,
diff --git a/backend/plugins/trello/api/init.go
b/backend/plugins/trello/api/init.go
index d0faa3424..6b8096d34 100644
--- a/backend/plugins/trello/api/init.go
+++ b/backend/plugins/trello/api/init.go
@@ -40,6 +40,7 @@ func Init(br context.BasicRes) {
params := &api.ReflectionParameters{
ScopeIdFieldName: "BoardId",
ScopeIdColumnName: "board_id",
+ RawScopeParamName: "BoardId",
}
scopeHelper = api.NewScopeHelper[models.TrelloConnection,
models.TrelloBoard, models.TrelloScopeConfig](
basicRes,
diff --git a/backend/plugins/zentao/api/init.go
b/backend/plugins/zentao/api/init.go
index ff47ba7fe..7695bf784 100644
--- a/backend/plugins/zentao/api/init.go
+++ b/backend/plugins/zentao/api/init.go
@@ -49,6 +49,7 @@ func Init(br context.BasicRes) {
productParams := &api.ReflectionParameters{
ScopeIdFieldName: "Id",
ScopeIdColumnName: "id",
+ RawScopeParamName: "ProductId",
}
productScopeHelper = api.NewScopeHelper[models.ZentaoConnection,
models.ZentaoProduct, models.ZentaoScopeConfig](
basicRes,
@@ -62,6 +63,7 @@ func Init(br context.BasicRes) {
projectParams := &api.ReflectionParameters{
ScopeIdFieldName: "Project",
ScopeIdColumnName: "project",
+ RawScopeParamName: "ProjectId",
}
projectScopeHelper = api.NewScopeHelper[models.ZentaoConnection,
models.ZentaoProject, models.ZentaoScopeConfig](
basicRes,
diff --git a/backend/server/services/project.go
b/backend/server/services/project.go
index 5635464e2..4413a8225 100644
--- a/backend/server/services/project.go
+++ b/backend/server/services/project.go
@@ -269,23 +269,23 @@ func DeleteProject(name string) errors.Error {
}
err = tx.Delete(&models.Project{}, dal.Where("name = ?", name))
if err != nil {
- return err
+ return errors.Default.Wrap(err, "error deleting project")
}
err = tx.Delete(&crossdomain.ProjectMapping{}, dal.Where("project_name
= ?", name))
if err != nil {
- return err
+ return errors.Default.Wrap(err, "error deleting project")
}
err = tx.Delete(&models.ProjectMetricSetting{}, dal.Where("project_name
= ?", name))
if err != nil {
- return err
+ return errors.Default.Wrap(err, "error deleting project metric
setting")
}
err = tx.Delete(&crossdomain.ProjectPrMetric{}, dal.Where("project_name
= ?", name))
if err != nil {
- return err
+ return errors.Default.Wrap(err, "error deleting project PR
metric")
}
err = tx.Delete(&crossdomain.ProjectIssueMetric{},
dal.Where("project_name = ?", name))
if err != nil {
- return err
+ return errors.Default.Wrap(err, "error deleting project Issue
metric")
}
err = tx.Commit()
if err != nil {
@@ -293,6 +293,9 @@ func DeleteProject(name string) errors.Error {
}
bp, err := bpManager.GetDbBlueprintByProjectName(name)
if err != nil {
+ if tx.IsErrorNotFound(err) {
+ return nil
+ }
return err
}
err = bpManager.DeleteBlueprint(bp.ID)
diff --git a/backend/server/services/remote/plugin/default_api.go
b/backend/server/services/remote/plugin/default_api.go
index ac100c67f..eac7cdea8 100644
--- a/backend/server/services/remote/plugin/default_api.go
+++ b/backend/server/services/remote/plugin/default_api.go
@@ -99,10 +99,12 @@ func createScopeHelper(pa *pluginAPI)
*api.GenericScopeApiHelper[remoteModel.Rem
}
return api.NewGenericScopeHelper[remoteModel.RemoteConnection,
remoteModel.RemoteScope, remoteModel.RemoteScopeConfig](
basicRes,
- nil,
+ vld,
connectionHelper,
NewScopeDatabaseHelperImpl(pa, basicRes, params),
params,
- &api.ScopeHelperOptions{},
+ &api.ScopeHelperOptions{
+ IsRemote: true,
+ },
)
}
diff --git a/backend/server/services/remote/plugin/plugin_impl.go
b/backend/server/services/remote/plugin/plugin_impl.go
index b86659914..ac63a8ffe 100644
--- a/backend/server/services/remote/plugin/plugin_impl.go
+++ b/backend/server/services/remote/plugin/plugin_impl.go
@@ -175,13 +175,11 @@ func (p *remotePluginImpl) getScopeAndConfig(db dal.Dal,
connectionId uint64, sc
return nil, nil, errors.BadInput.Wrap(err, "Invalid scope")
}
wrappedScopeConfig := p.scopeConfigTabler.New()
- err = api.CallDB(db.First, wrappedScopeConfig,
dal.From(p.scopeConfigTabler.TableName()), dal.Where("id = ?",
scope.ScopeConfigId))
- if err != nil {
- return nil, nil, err
- }
-
- if err != nil {
- return nil, nil, err
+ if scope.ScopeConfigId != 0 {
+ err = api.CallDB(db.First, wrappedScopeConfig,
dal.From(p.scopeConfigTabler.TableName()), dal.Where("id = ?",
scope.ScopeConfigId))
+ if err != nil {
+ return nil, nil, err
+ }
}
return wrappedScope.Unwrap(), wrappedScopeConfig.Unwrap(), nil
}
diff --git a/backend/test/e2e/manual/azuredevops/azure_test.go
b/backend/test/e2e/manual/azuredevops/azure_test.go
index 613a97e04..012068700 100644
--- a/backend/test/e2e/manual/azuredevops/azure_test.go
+++ b/backend/test/e2e/manual/azuredevops/azure_test.go
@@ -24,6 +24,7 @@ import (
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/plugin"
gitextractor
"github.com/apache/incubator-devlake/plugins/gitextractor/impl"
+ pluginmodels
"github.com/apache/incubator-devlake/plugins/pagerduty/models"
"github.com/apache/incubator-devlake/test/helper"
"github.com/stretchr/testify/require"
"testing"
@@ -130,6 +131,14 @@ func TestAzure(t *testing.T) {
// run the bp
pipeline := client.TriggerBlueprint(bp.ID)
require.Equal(t, models.TASK_COMPLETED, pipeline.Status)
+ createdScopesList := client.ListScopes(azurePlugin,
connection.ID, true)
+ require.True(t, len(createdScopesList) > 0)
+ for _, scope := range createdScopesList {
+ scopeCast :=
helper.Cast[pluginmodels.Service](scope.Scope)
+ fmt.Printf("Deleting scope %s\n", scopeCast.Id)
+ client.DeleteScope(azurePlugin, connection.ID,
scopeCast.Id, false)
+ fmt.Printf("Deleted scope %s\n", scopeCast.Id)
+ }
})
fmt.Println("========DONE=======")
}
diff --git a/backend/test/helper/api.go b/backend/test/helper/api.go
index 78d5837d2..e02111092 100644
--- a/backend/test/helper/api.go
+++ b/backend/test/helper/api.go
@@ -158,6 +158,13 @@ func (d *DevlakeClient) ListProjects()
apiProject.PaginatedProjects {
}, http.MethodGet, fmt.Sprintf("%s/projects", d.Endpoint), nil, nil)
}
+func (d *DevlakeClient) DeleteProject(projectName string) {
+ sendHttpRequest[any](d.testCtx, d.timeout, debugInfo{
+ print: true,
+ inlineJson: false,
+ }, http.MethodDelete, fmt.Sprintf("%s/projects/%s", d.Endpoint,
projectName), nil, nil)
+}
+
func (d *DevlakeClient) CreateScopes(pluginName string, connectionId uint64,
scopes ...any) any {
request := map[string]any{
"data": scopes,
