This is an automated email from the ASF dual-hosted git repository.
abeizn pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 2464f2c50 refactor: unify connection/scope/scopeConfig helpers (#6146)
2464f2c50 is described below
commit 2464f2c5069988bc5769daa63b853a7e15a446a9
Author: Klesh Wong <[email protected]>
AuthorDate: Thu Sep 28 14:22:10 2023 +0800
refactor: unify connection/scope/scopeConfig helpers (#6146)
* refactor: unify connection/scope/scopeConfig helpers
* fix: linting
* feat: python pick up VIRTUAL_ENV automatically
* fix: e2e test
* fix: createWithMap failed with nil clause ptr
* fix: select may or may not targeting the table
* fix: integration test
* fix: conflict error checking should be nested in the non-nil condition
* fix: remove debugging code
* fix: client side should not be initlializing plugins
* fix: typo and incorect comments
* fix: scope detail api should output scopeConfig and blueprints as well
* fix: make ParsePagination private
* fix: toDsRefs should not imply deletion
* feat: validate pagination
---
backend/core/runner/run_task.go | 7 +-
.../helpers/pluginhelper/api/connection_auths.go | 5 +
.../pluginhelper/api/ds_connection_api_helper.go | 75 ++++++
backend/helpers/pluginhelper/api/ds_helper.go | 49 ++++
.../pluginhelper/api/ds_scope_api_helper.go | 119 +++++++++
.../pluginhelper/api/ds_scope_config_api_helper.go | 92 +++++++
.../helpers/pluginhelper/api/model_api_helper.go | 184 ++++++++++++++
.../helpers/srvhelper/connection_service_helper.go | 82 ++++++
backend/helpers/srvhelper/dsrefs.go | 45 ++++
backend/helpers/srvhelper/model_service_helper.go | 215 ++++++++++++++++
backend/helpers/srvhelper/pagination.go | 42 ++++
.../srvhelper/scope_config_service_helper.go | 66 +++++
backend/helpers/srvhelper/scope_service_helper.go | 276 +++++++++++++++++++++
backend/helpers/utils/mapstructure.go | 20 +-
backend/helpers/utils/mapstructure_test.go | 17 ++
backend/impls/dalgorm/dalgorm.go | 32 ++-
backend/plugins/github/api/blueprint_v200.go | 9 +-
.../api/{connection.go => connection_api.go} | 31 +--
backend/plugins/github/api/init.go | 59 ++---
.../plugins/github/api/{scope.go => scope_api.go} | 37 ++-
.../api/{scope_config.go => scope_config_api.go} | 18 +-
backend/plugins/github/impl/impl.go | 12 +-
backend/python/test/fakeplugin/run.sh | 3 +
backend/test/helper/client.go | 8 -
24 files changed, 1381 insertions(+), 122 deletions(-)
diff --git a/backend/core/runner/run_task.go b/backend/core/runner/run_task.go
index 4cc5eb251..a04e65fe6 100644
--- a/backend/core/runner/run_task.go
+++ b/backend/core/runner/run_task.go
@@ -107,14 +107,11 @@ func RunTask(
}
}
// update finishedTasks
- dbe := db.UpdateColumn(
+ errors.Must(db.UpdateColumn(
&models.Pipeline{},
"finished_tasks", dal.Expr("finished_tasks + 1"),
dal.Where("id=?", task.PipelineId),
- )
- if dbe != nil {
- logger.Error(dbe, "update pipeline state failed")
- }
+ ))
// not return err if the `SkipOnFail` is true and the error is
not canceled
if dbPipeline.SkipOnFail && !errors.Is(err, gocontext.Canceled)
{
err = nil
diff --git a/backend/helpers/pluginhelper/api/connection_auths.go
b/backend/helpers/pluginhelper/api/connection_auths.go
index 6219b9a5d..09116415b 100644
--- a/backend/helpers/pluginhelper/api/connection_auths.go
+++ b/backend/helpers/pluginhelper/api/connection_auths.go
@@ -144,6 +144,11 @@ func (ma *MultiAuth)
SetupAuthenticationForConnection(connection plugin.ApiConne
return apiAuthenticator.SetupAuthentication(req)
}
+func (ma *MultiAuth) CustomValidate(connection interface{}, v
*validator.Validate) errors.Error {
+ return ma.ValidateConnection(connection, v)
+}
+
+// TODO: deprecated, rename to CustomValidate instead
func (ma *MultiAuth) ValidateConnection(connection interface{}, v
*validator.Validate) errors.Error {
// the idea is to filtered out errors from unselected Authentication
struct
validationErrors := v.Struct(connection).(validator.ValidationErrors)
diff --git a/backend/helpers/pluginhelper/api/ds_connection_api_helper.go
b/backend/helpers/pluginhelper/api/ds_connection_api_helper.go
new file mode 100644
index 000000000..8d7bf5a55
--- /dev/null
+++ b/backend/helpers/pluginhelper/api/ds_connection_api_helper.go
@@ -0,0 +1,75 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "strconv"
+
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/srvhelper"
+ "github.com/apache/incubator-devlake/server/api/shared"
+)
+
+// DsConnectionApiHelper
+type DsConnectionApiHelper[C plugin.ToolLayerConnection, S
plugin.ToolLayerScope, SC plugin.ToolLayerScopeConfig] struct {
+ *ModelApiHelper[C]
+ *srvhelper.ConnectionSrvHelper[C, S, SC]
+}
+
+func NewDsConnectionApiHelper[C plugin.ToolLayerConnection, S
plugin.ToolLayerScope, SC plugin.ToolLayerScopeConfig](
+ basicRes context.BasicRes,
+ connSrvHelper *srvhelper.ConnectionSrvHelper[C, S, SC],
+) *DsConnectionApiHelper[C, S, SC] {
+ return &DsConnectionApiHelper[C, S, SC]{
+ ModelApiHelper: NewModelApiHelper[C](basicRes,
connSrvHelper.ModelSrvHelper, []string{"connectionId"}),
+ ConnectionSrvHelper: connSrvHelper,
+ }
+}
+
+func (connApi *DsConnectionApiHelper[C, S, SC]) Delete(input
*plugin.ApiResourceInput) (out *plugin.ApiResourceOutput, err errors.Error) {
+ var conn *C
+ conn, err = connApi.FindByPk(input)
+ if err != nil {
+ return nil, err
+ }
+ refs, err := connApi.ConnectionSrvHelper.DeleteConnection(conn)
+ if err != nil {
+ return &plugin.ApiResourceOutput{Body: &shared.ApiBody{
+ Success: false,
+ Message: err.Error(),
+ Data: refs,
+ }, Status: err.GetType().GetHttpCode()}, nil
+ }
+ return &plugin.ApiResourceOutput{
+ Body: conn,
+ }, nil
+}
+
+func extractConnectionId(input *plugin.ApiResourceInput) (uint64,
errors.Error) {
+ connectionId, ok := input.Params["connectionId"]
+ if !ok {
+ return 0, errors.BadInput.New("connectionId is required")
+ }
+ id, err := strconv.ParseUint(connectionId, 10, 64)
+ if err != nil {
+ return 0, errors.BadInput.Wrap(err, "connectionId must be a
number")
+ }
+ return id, nil
+}
diff --git a/backend/helpers/pluginhelper/api/ds_helper.go
b/backend/helpers/pluginhelper/api/ds_helper.go
new file mode 100644
index 000000000..bb0aa019f
--- /dev/null
+++ b/backend/helpers/pluginhelper/api/ds_helper.go
@@ -0,0 +1,49 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/srvhelper"
+)
+
+func NewDataSourceHelpers[
+ C plugin.ToolLayerConnection,
+ S plugin.ToolLayerScope,
+ SC plugin.ToolLayerScopeConfig,
+](
+ basicRes context.BasicRes,
+ pluginName string,
+ scopeSearchColumns []string,
+) (
+ *srvhelper.ConnectionSrvHelper[C, S, SC],
+ *DsConnectionApiHelper[C, S, SC],
+ *srvhelper.ScopeSrvHelper[C, S, SC],
+ *DsScopeApiHelper[C, S, SC],
+ *srvhelper.ScopeConfigSrvHelper[C, S, SC],
+ *DsScopeConfigApiHelper[C, S, SC],
+) {
+ connSrv := srvhelper.NewConnectionSrvHelper[C, S, SC](basicRes,
pluginName)
+ connApi := NewDsConnectionApiHelper[C, S, SC](basicRes, connSrv)
+ scopeSrv := srvhelper.NewScopeSrvHelper[C, S, SC](basicRes, pluginName,
scopeSearchColumns)
+ scopeApi := NewDsScopeApiHelper[C, S, SC](basicRes, scopeSrv)
+ scSrv := srvhelper.NewScopeConfigSrvHelper[C, S, SC](basicRes)
+ scApi := NewDsScopeConfigApiHelper[C, S, SC](basicRes, scSrv)
+ return connSrv, connApi, scopeSrv, scopeApi, scSrv, scApi
+}
diff --git a/backend/helpers/pluginhelper/api/ds_scope_api_helper.go
b/backend/helpers/pluginhelper/api/ds_scope_api_helper.go
new file mode 100644
index 000000000..279d9b1ae
--- /dev/null
+++ b/backend/helpers/pluginhelper/api/ds_scope_api_helper.go
@@ -0,0 +1,119 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/srvhelper"
+ "github.com/apache/incubator-devlake/server/api/shared"
+)
+
+type PutScopesReqBody[T any] struct {
+ Data []*T `json:"data"`
+}
+
+type ScopeDetail[S plugin.ToolLayerScope, SC plugin.ToolLayerScopeConfig]
srvhelper.ScopeDetail[S, SC]
+
+type DsScopeApiHelper[C plugin.ToolLayerConnection, S plugin.ToolLayerScope,
SC plugin.ToolLayerScopeConfig] struct {
+ *ModelApiHelper[S]
+ *srvhelper.ScopeSrvHelper[C, S, SC]
+}
+
+func NewDsScopeApiHelper[C plugin.ToolLayerConnection, S
plugin.ToolLayerScope, SC plugin.ToolLayerScopeConfig](
+ basicRes context.BasicRes,
+ srvHelper *srvhelper.ScopeSrvHelper[C, S, SC],
+) *DsScopeApiHelper[C, S, SC] {
+ return &DsScopeApiHelper[C, S, SC]{
+ ModelApiHelper: NewModelApiHelper[S](basicRes,
srvHelper.ModelSrvHelper, []string{"connectionId", "scopeId"}),
+ ScopeSrvHelper: srvHelper,
+ }
+}
+
+func (scopeApi *DsScopeApiHelper[C, S, SC]) GetPage(input
*plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ pagination, err := parsePagination[srvhelper.ScopePagination](input)
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "failed to decode
pathvars into pagination")
+ }
+ scopes, count, err := scopeApi.ScopeSrvHelper.GetScopesPage(pagination)
+ if err != nil {
+ return nil, err
+ }
+ return &plugin.ApiResourceOutput{
+ Body: map[string]interface{}{
+ "count": count,
+ "scopes": scopes,
+ },
+ }, nil
+}
+
+func (scopeApi *DsScopeApiHelper[C, S, SC]) GetScopeDetail(input
*plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ pkv, err := scopeApi.ExtractPkValues(input)
+ if err != nil {
+ return nil, err
+ }
+ scopeDetail, err :=
scopeApi.ScopeSrvHelper.GetScopeDetail(input.Query.Get("blueprints") == "true",
pkv...)
+ if err != nil {
+ return nil, err
+ }
+ return &plugin.ApiResourceOutput{
+ Body: scopeDetail,
+ }, nil
+}
+
+func (scopeApi *DsScopeApiHelper[C, S, SC]) PutMultiple(input
*plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ // fix data[].connectionId
+ connectionId, err := extractConnectionId(input)
+ if err != nil {
+ return nil, err
+ }
+ data, ok := input.Body["data"].([]interface{})
+ if !ok {
+ return nil, errors.BadInput.New("invalid data")
+ }
+ for _, row := range data {
+ dict, ok := row.(map[string]interface{})
+ if !ok {
+ return nil, errors.BadInput.New("invalid data row")
+ }
+ dict["connectionId"] = connectionId
+ }
+ return scopeApi.ModelApiHelper.PutMultiple(input)
+}
+
+func (scopeApi *DsScopeApiHelper[C, S, SC]) Delete(input
*plugin.ApiResourceInput) (*plugin.ApiResourceOutput, errors.Error) {
+ var scope *S
+ scope, err := scopeApi.FindByPk(input)
+ if err != nil {
+ return nil, err
+ }
+ // time.Sleep(1 * time.Minute) # uncomment this line if you were to
verify pipelines get blocked while deleting data
+ // check referencing blueprints
+ refs, err := scopeApi.ScopeSrvHelper.DeleteScope(scope,
input.Query.Get("delete_data_only") == "true")
+ if err != nil {
+ return &plugin.ApiResourceOutput{Body: &shared.ApiBody{
+ Success: false,
+ Message: err.Error(),
+ Data: refs,
+ }, Status: err.GetType().GetHttpCode()}, nil
+ }
+ return &plugin.ApiResourceOutput{
+ Body: scope,
+ }, nil
+}
diff --git a/backend/helpers/pluginhelper/api/ds_scope_config_api_helper.go
b/backend/helpers/pluginhelper/api/ds_scope_config_api_helper.go
new file mode 100644
index 000000000..6a3e7cd19
--- /dev/null
+++ b/backend/helpers/pluginhelper/api/ds_scope_config_api_helper.go
@@ -0,0 +1,92 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/srvhelper"
+ "github.com/apache/incubator-devlake/server/api/shared"
+)
+
+// DsScopeConfigApiHelper
+type DsScopeConfigApiHelper[C plugin.ToolLayerConnection, S
plugin.ToolLayerScope, SC plugin.ToolLayerScopeConfig] struct {
+ *ModelApiHelper[SC]
+ *srvhelper.ScopeConfigSrvHelper[C, S, SC]
+}
+
+func NewDsScopeConfigApiHelper[C plugin.ToolLayerConnection, S
plugin.ToolLayerScope, SC plugin.ToolLayerScopeConfig](
+ basicRes context.BasicRes,
+ dalHelper *srvhelper.ScopeConfigSrvHelper[C, S, SC],
+) *DsScopeConfigApiHelper[C, S, SC] {
+ return &DsScopeConfigApiHelper[C, S, SC]{
+ ModelApiHelper: NewModelApiHelper[SC](basicRes,
dalHelper.ModelSrvHelper, []string{"scopeConfigId"}),
+ ScopeConfigSrvHelper: dalHelper,
+ }
+}
+
+func (connApi *DsScopeConfigApiHelper[C, S, SC]) GetAll(input
*plugin.ApiResourceInput) (out *plugin.ApiResourceOutput, err errors.Error) {
+ connectionId, err := extractConnectionId(input)
+ if err != nil {
+ return nil, err
+ }
+ scopeConfigs :=
errors.Must1(connApi.ScopeConfigSrvHelper.GetAllByConnectionId(connectionId))
+ return &plugin.ApiResourceOutput{
+ Body: scopeConfigs,
+ }, nil
+}
+
+func (connApi *DsScopeConfigApiHelper[C, S, SC]) Post(input
*plugin.ApiResourceInput) (out *plugin.ApiResourceOutput, err errors.Error) {
+ // fix connectionId
+ connectionId, err := extractConnectionId(input)
+ if err != nil {
+ return nil, err
+ }
+ input.Body["connectionId"] = connectionId
+ return connApi.ModelApiHelper.Post(input)
+}
+
+func (connApi *DsScopeConfigApiHelper[C, S, SC]) Patch(input
*plugin.ApiResourceInput) (out *plugin.ApiResourceOutput, err errors.Error) {
+ // fix connectionId
+ connectionId, err := extractConnectionId(input)
+ if err != nil {
+ return nil, err
+ }
+ input.Body["connectionId"] = connectionId
+ return connApi.ModelApiHelper.Patch(input)
+}
+
+func (connApi *DsScopeConfigApiHelper[C, S, SC]) Delete(input
*plugin.ApiResourceInput) (out *plugin.ApiResourceOutput, err errors.Error) {
+ var scopeConfig *SC
+ scopeConfig, err = connApi.FindByPk(input)
+ if err != nil {
+ return nil, err
+ }
+ refs, err := connApi.ScopeConfigSrvHelper.DeleteScopeConfig(scopeConfig)
+ if err != nil {
+ return &plugin.ApiResourceOutput{Body: &shared.ApiBody{
+ Success: false,
+ Message: err.Error(),
+ Data: refs,
+ }, Status: err.GetType().GetHttpCode()}, nil
+ }
+ return &plugin.ApiResourceOutput{
+ Body: scopeConfig,
+ }, nil
+}
diff --git a/backend/helpers/pluginhelper/api/model_api_helper.go
b/backend/helpers/pluginhelper/api/model_api_helper.go
new file mode 100644
index 000000000..ce4d18be5
--- /dev/null
+++ b/backend/helpers/pluginhelper/api/model_api_helper.go
@@ -0,0 +1,184 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "fmt"
+ "net/http"
+
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/log"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/srvhelper"
+ "github.com/apache/incubator-devlake/helpers/utils"
+ "github.com/go-playground/validator/v10"
+)
+
+var vld = validator.New()
+
+type ModelApiHelper[M dal.Tabler] struct {
+ basicRes context.BasicRes
+ dalHelper *srvhelper.ModelSrvHelper[M]
+ log log.Logger
+ modelName string
+ pkPathVarNames []string
+}
+
+func NewModelApiHelper[M dal.Tabler](
+ basicRes context.BasicRes,
+ dalHelper *srvhelper.ModelSrvHelper[M],
+ pkPathVarNames []string, // path variable names of primary key
+) *ModelApiHelper[M] {
+ m := new(M)
+ modelName := fmt.Sprintf("%T", m)
+ return &ModelApiHelper[M]{
+ basicRes: basicRes,
+ dalHelper: dalHelper,
+ log:
basicRes.GetLogger().Nested(fmt.Sprintf("%s_dal", modelName)),
+ modelName: modelName,
+ pkPathVarNames: pkPathVarNames,
+ }
+}
+
+func (self *ModelApiHelper[M]) Post(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ model := new(M)
+ err := utils.DecodeMapStruct(input.Body, model, false)
+ if err != nil {
+ return nil, err
+ }
+ err = self.dalHelper.Create(model)
+ if err != nil {
+ return nil, err
+ }
+ return &plugin.ApiResourceOutput{
+ Status: http.StatusCreated,
+ Body: model,
+ }, nil
+}
+
+func (self *ModelApiHelper[M]) ExtractPkValues(input *plugin.ApiResourceInput)
([]interface{}, errors.Error) {
+ pkv := make([]interface{}, len(self.pkPathVarNames))
+ for i, pkn := range self.pkPathVarNames {
+ var ok bool
+ pkv[i], ok = input.Params[pkn]
+ if !ok {
+ return nil, errors.BadInput.New(fmt.Sprintf("missing
path variable %s", pkn))
+ }
+ }
+ return pkv, nil
+}
+
+func (self *ModelApiHelper[M]) FindByPk(input *plugin.ApiResourceInput) (*M,
errors.Error) {
+ pkv, err := self.ExtractPkValues(input)
+ if err != nil {
+ return nil, err
+ }
+ return self.dalHelper.FindByPk(pkv...)
+}
+
+func (self *ModelApiHelper[M]) GetDetail(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ model, err := self.FindByPk(input)
+ if err != nil {
+ return nil, err
+ }
+ return &plugin.ApiResourceOutput{
+ Body: model,
+ }, nil
+}
+
+func (self *ModelApiHelper[M]) Patch(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ model, err := self.FindByPk(input)
+ if err != nil {
+ return nil, err
+ }
+ err = utils.DecodeMapStruct(input.Body, model, true)
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, fmt.Sprintf("faled to
patch %s", self.modelName))
+ }
+ err = self.dalHelper.Update(model)
+ if err != nil {
+ return nil, err
+ }
+ return &plugin.ApiResourceOutput{
+ Body: model,
+ }, nil
+}
+
+func (self *ModelApiHelper[M]) Delete(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ model, err := self.FindByPk(input)
+ if err != nil {
+ return nil, err
+ }
+ err = self.dalHelper.DeleteModel(model)
+ if err != nil {
+ return nil, err
+ }
+ return &plugin.ApiResourceOutput{
+ Body: model,
+ }, nil
+}
+
+func (self *ModelApiHelper[M]) GetAll(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ all, err := self.dalHelper.GetAll()
+ return &plugin.ApiResourceOutput{
+ Body: all,
+ }, err
+}
+
+func (self *ModelApiHelper[M]) PutMultiple(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ var req struct {
+ Data []M `json:"data"`
+ }
+ err := utils.DecodeMapStruct(input.Body, &req, false)
+ if err != nil {
+ return nil, err
+ }
+ for i, item := range req.Data {
+ err := self.dalHelper.CreateOrUpdate(&item)
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err,
fmt.Sprintf("failed to save item %d", i))
+ }
+ }
+ return &plugin.ApiResourceOutput{
+ Body: req.Data,
+ }, nil
+}
+
+func parsePagination[P any](input *plugin.ApiResourceInput, query
...dal.Clause) (*P, errors.Error) {
+ if !input.Query.Has("page") {
+ input.Query.Set("page", "1")
+ }
+ if !input.Query.Has("pageSize") {
+ input.Query.Set("pageSize", "100")
+ }
+ pagination := new(P)
+ err := utils.DecodeMapStruct(input.Query, pagination, false)
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "faild to decode
pagination from query string")
+ }
+ err = utils.DecodeMapStruct(input.Params, pagination, false)
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "faild to decode
pagination from path variables")
+ }
+ if e := vld.Struct(pagination); e != nil {
+ return nil, errors.BadInput.Wrap(e, "invalid pagination
parameters")
+ }
+ return pagination, nil
+}
diff --git a/backend/helpers/srvhelper/connection_service_helper.go
b/backend/helpers/srvhelper/connection_service_helper.go
new file mode 100644
index 000000000..a62650314
--- /dev/null
+++ b/backend/helpers/srvhelper/connection_service_helper.go
@@ -0,0 +1,82 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package srvhelper
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models"
+ "github.com/apache/incubator-devlake/core/plugin"
+)
+
+// ConnectionSrvHelper
+type ConnectionSrvHelper[C plugin.ToolLayerConnection, S
plugin.ToolLayerScope, SC plugin.ToolLayerScopeConfig] struct {
+ *ModelSrvHelper[C]
+ pluginName string
+}
+
+// NewConnectionSrvHelper creates a ConnectionDalHelper for connection
management
+func NewConnectionSrvHelper[
+ C plugin.ToolLayerConnection,
+ S plugin.ToolLayerScope,
+ SC plugin.ToolLayerScopeConfig,
+](
+ basicRes context.BasicRes,
+ pluginName string,
+) *ConnectionSrvHelper[C, S, SC] {
+ return &ConnectionSrvHelper[C, S, SC]{
+ ModelSrvHelper: NewModelSrvHelper[C](basicRes),
+ pluginName: pluginName,
+ }
+}
+
+func (connSrv *ConnectionSrvHelper[C, S, SC]) DeleteConnection(connection *C)
(refs *DsRefs, err errors.Error) {
+ err = connSrv.ModelSrvHelper.NoRunningPipeline(func(tx dal.Transaction)
errors.Error {
+ // make sure no blueprint is using the connection
+ connectionId := (*connection).ConnectionId()
+ refs =
toDsRefs(connSrv.getAllBlueprinsByConnection(connectionId))
+ if refs != nil {
+ return errors.Conflict.New("Cannot delete the scope
because it is referenced by blueprints")
+ }
+ scopeCount := errors.Must1(connSrv.db.Count(dal.From(new(S)),
dal.Where("connection_id = ?", connectionId)))
+ if scopeCount > 0 {
+ return errors.Conflict.New("Please delete all data
scope(s) before you delete this Data Connection.")
+ }
+ errors.Must(tx.Delete(connection))
+ errors.Must(connSrv.db.Delete(new(SC), dal.Where("connection_id
= ?", connectionId)))
+ return nil
+ })
+ return
+}
+
+func (connSrv *ConnectionSrvHelper[C, S, SC])
getAllBlueprinsByConnection(connectionId uint64) []*models.Blueprint {
+ blueprints := make([]*models.Blueprint, 0)
+ errors.Must(connSrv.db.All(
+ &blueprints,
+ dal.From("_devlake_blueprints bp"),
+ dal.Join("JOIN _devlake_blueprint_connections cn ON
cn.blueprint_id = bp.id"),
+ dal.Where(
+ "mode = ? AND cn.connection_id = ? AND cn.plugin_name =
?",
+ "NORMAL",
+ connectionId,
+ connSrv.pluginName,
+ ),
+ ))
+ return blueprints
+}
diff --git a/backend/helpers/srvhelper/dsrefs.go
b/backend/helpers/srvhelper/dsrefs.go
new file mode 100644
index 000000000..afb7bd450
--- /dev/null
+++ b/backend/helpers/srvhelper/dsrefs.go
@@ -0,0 +1,45 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package srvhelper
+
+import (
+ "github.com/apache/incubator-devlake/core/models"
+)
+
+type DsRefs struct {
+ Blueprints []string `json:"blueprints"`
+ Projects []string `json:"projects"`
+}
+
+func toDsRefs(blueprints []*models.Blueprint) *DsRefs {
+ if len(blueprints) > 0 {
+ blueprintNames := make([]string, 0, len(blueprints))
+ projectNames := make([]string, 0, len(blueprints))
+ for _, bp := range blueprints {
+ blueprintNames = append(blueprintNames, bp.Name)
+ if bp.ProjectName != "" {
+ projectNames = append(projectNames,
bp.ProjectName)
+ }
+ }
+ return &DsRefs{
+ Blueprints: blueprintNames,
+ Projects: projectNames,
+ }
+ }
+ return nil
+}
diff --git a/backend/helpers/srvhelper/model_service_helper.go
b/backend/helpers/srvhelper/model_service_helper.go
new file mode 100644
index 000000000..7acb87c96
--- /dev/null
+++ b/backend/helpers/srvhelper/model_service_helper.go
@@ -0,0 +1,215 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package srvhelper
+
+import (
+ "fmt"
+ "time"
+
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/log"
+ "github.com/apache/incubator-devlake/core/models"
+ "github.com/apache/incubator-devlake/helpers/dbhelper"
+ "github.com/go-playground/validator/v10"
+)
+
+type CustomValidator interface {
+ CustomValidate(entity interface{}, validate *validator.Validate)
errors.Error
+}
+
+type ModelSrvHelper[M dal.Tabler] struct {
+ basicRes context.BasicRes
+ log log.Logger
+ db dal.Dal
+ validator *validator.Validate
+ modelName string
+ pk []dal.ColumnMeta
+ pkWhere string
+ pkCount int
+ searchColumns []string
+}
+
+func NewModelSrvHelper[M dal.Tabler](basicRes context.BasicRes)
*ModelSrvHelper[M] {
+ m := new(M)
+ modelName := fmt.Sprintf("%T", m)
+ db := basicRes.GetDal()
+ if db == nil {
+ db = basicRes.GetDal()
+ }
+ pk := errors.Must1(dal.GetPrimarykeyColumns(db, *m))
+ pkWhere := ""
+ for _, col := range pk {
+ if pkWhere != "" {
+ pkWhere += " AND "
+ }
+ pkWhere += fmt.Sprintf("%s = ? ", col.Name())
+ }
+ return &ModelSrvHelper[M]{
+ basicRes: basicRes,
+ log: basicRes.GetLogger().Nested(fmt.Sprintf("%s_dal",
modelName)),
+ db: db,
+ validator: validator.New(),
+ modelName: modelName,
+ pk: pk,
+ pkWhere: pkWhere,
+ pkCount: len(pk),
+ }
+}
+
+func (srv *ModelSrvHelper[M]) NewTx(tx dal.Transaction) *ModelSrvHelper[M] {
+ helper := new(ModelSrvHelper[M])
+ *helper = *srv
+ helper.db = tx
+ return helper
+}
+
+func (srv *ModelSrvHelper[M]) ValidateModel(model *M) errors.Error {
+ // the model can validate itself
+ if customValidator, ok := (interface{}(model)).(CustomValidator); ok {
+ return customValidator.CustomValidate(model, srv.validator)
+ }
+ // basic validator
+ if e := srv.validator.Struct(model); e != nil {
+ return errors.BadInput.Wrap(e, "validation faild")
+ }
+ return nil
+}
+
+// Create validates given model and insert it into database if validation
passed
+func (srv *ModelSrvHelper[M]) Create(model *M) errors.Error {
+ println("create model")
+ err := srv.ValidateModel(model)
+ if err != nil {
+ return err
+ }
+ err = srv.db.Create(model)
+ if err != nil {
+ if srv.db.IsDuplicationError(err) {
+ return errors.Conflict.Wrap(err, fmt.Sprintf("%s
already exists", srv.modelName))
+ }
+ return err
+ }
+ return err
+}
+
+// Update validates given model and update it into database if validation
passed
+func (srv *ModelSrvHelper[M]) Update(model *M) errors.Error {
+ err := srv.ValidateModel(model)
+ if err != nil {
+ if srv.db.IsDuplicationError(err) {
+ return errors.Conflict.Wrap(err, fmt.Sprintf("%s
already exists", srv.modelName))
+ }
+ return err
+ }
+ return srv.db.Update(model)
+}
+
+// CreateOrUpdate validates given model and insert or update it into database
if validation passed
+func (srv *ModelSrvHelper[M]) CreateOrUpdate(model *M) errors.Error {
+ err := srv.ValidateModel(model)
+ if err != nil {
+ return err
+ }
+ return srv.db.CreateOrUpdate(model)
+}
+
+// DeleteModel deletes given model from database
+func (srv *ModelSrvHelper[M]) DeleteModel(model *M) errors.Error {
+ return srv.db.Delete(model)
+}
+
+// FindByPk returns model with given primary key from database
+func (srv *ModelSrvHelper[M]) FindByPk(pk ...interface{}) (*M, errors.Error) {
+ if len(pk) != srv.pkCount {
+ return nil, errors.BadInput.New("invalid primary key")
+ }
+ model := new(M)
+ err := srv.db.First(model, dal.Where(srv.pkWhere, pk...))
+ if err != nil {
+ if srv.db.IsErrorNotFound(err) {
+ return nil, errors.NotFound.Wrap(err, fmt.Sprintf("%s
not found", srv.modelName))
+ }
+ return nil, err
+ }
+ return model, nil
+}
+
+// GetAll returns all models from database
+func (srv *ModelSrvHelper[M]) GetAll() ([]*M, errors.Error) {
+ array := make([]*M, 0)
+ return array, srv.db.All(&array)
+}
+
+func (srv *ModelSrvHelper[M]) GetPage(pagination *Pagination, query
...dal.Clause) ([]*M, int64, errors.Error) {
+ query = append(query, dal.From(new(M)))
+ // process keyword
+ searchTerm := pagination.SearchTerm
+ if searchTerm != "" && len(srv.searchColumns) > 0 {
+ sql := ""
+ value := "%" + searchTerm + "%"
+ values := make([]interface{}, len(srv.searchColumns))
+ for i, field := range srv.searchColumns {
+ if sql != "" {
+ sql += " OR "
+ }
+ sql += fmt.Sprintf("%s LIKE ?", field)
+ values[i] = value
+ }
+ sql = fmt.Sprintf("(%s)", sql)
+ query = append(query,
+ dal.Where(sql, values...),
+ )
+ }
+ count, err := srv.db.Count(query...)
+ if err != nil {
+ return nil, 0, err
+ }
+ query = append(query, dal.Limit(pagination.GetLimit()),
dal.Offset(pagination.GetOffset()))
+ var scopes []*M
+ return scopes, count, srv.db.All(&scopes, query...)
+}
+
+func (srv *ModelSrvHelper[M]) NoRunningPipeline(fn func(tx dal.Transaction)
errors.Error, tablesToLock ...*dal.LockTable) (err errors.Error) {
+ // make sure no pipeline is running
+ tablesToLock = append(tablesToLock, &dal.LockTable{Table:
"_devlake_pipelines", Exclusive: true})
+ txHelper := dbhelper.NewTxHelper(srv.basicRes, &err)
+ defer txHelper.End()
+ tx := txHelper.Begin()
+ err = txHelper.LockTablesTimeout(2*time.Second, tablesToLock)
+ if err != nil {
+ err = errors.Conflict.Wrap(err, "lock pipelines table timedout")
+ return
+ }
+ count := errors.Must1(tx.Count(
+ dal.From("_devlake_pipelines"),
+ dal.Where("status = ?", models.TASK_RUNNING),
+ ))
+ if count > 0 {
+ err = errors.Conflict.New("at least one pipeline is running")
+ return
+ }
+ // time.Sleep(1 * time.Minute) # uncomment this line if you were to
verify pipelines get blocked while deleting data
+ // creating a nested transaction to avoid mysql complaining about
table(s) NOT being locked
+ nextedTxHelper := dbhelper.NewTxHelper(srv.basicRes, &err)
+ defer nextedTxHelper.End()
+ nestedTX := nextedTxHelper.Begin()
+ err = fn(nestedTX)
+ return
+}
diff --git a/backend/helpers/srvhelper/pagination.go
b/backend/helpers/srvhelper/pagination.go
new file mode 100644
index 000000000..8b6b96300
--- /dev/null
+++ b/backend/helpers/srvhelper/pagination.go
@@ -0,0 +1,42 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package srvhelper
+
+type Pagination struct {
+ Page int `json:"page" mapstructure:"page" validate:"min=1"`
+ PageSize int `json:"pageSize" mapstructure:"pageSize"
validate:"min=1,max=1000"`
+ SearchTerm string `json:"searchTerm" mapstructure:"searchTerm"`
+}
+
+func (pagination *Pagination) GetPage() int {
+ if pagination.Page > 0 {
+ return pagination.Page
+ }
+ return 1
+}
+
+func (pagination *Pagination) GetLimit() int {
+ if pagination.PageSize > 0 {
+ return pagination.PageSize
+ }
+ return 100
+}
+
+func (pagination *Pagination) GetOffset() int {
+ return (pagination.GetPage() - 1) * pagination.GetLimit()
+}
diff --git a/backend/helpers/srvhelper/scope_config_service_helper.go
b/backend/helpers/srvhelper/scope_config_service_helper.go
new file mode 100644
index 000000000..3ce1cad9a
--- /dev/null
+++ b/backend/helpers/srvhelper/scope_config_service_helper.go
@@ -0,0 +1,66 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package srvhelper
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+)
+
+// ScopeConfigSrvHelper
+type ScopeConfigSrvHelper[C plugin.ToolLayerConnection, S
plugin.ToolLayerScope, SC plugin.ToolLayerScopeConfig] struct {
+ *ModelSrvHelper[SC]
+}
+
+func NewScopeConfigSrvHelper[
+ C plugin.ToolLayerConnection,
+ S plugin.ToolLayerScope,
+ SC plugin.ToolLayerScopeConfig,
+](basicRes context.BasicRes) *ScopeConfigSrvHelper[C, S, SC] {
+ return &ScopeConfigSrvHelper[C, S, SC]{
+ ModelSrvHelper: NewModelSrvHelper[SC](basicRes),
+ }
+}
+
+func (scopeConfigSrv *ScopeConfigSrvHelper[C, S, SC])
GetAllByConnectionId(connectionId uint64) ([]*SC, errors.Error) {
+ var scopeConfigs []*SC
+ err := scopeConfigSrv.db.All(&scopeConfigs,
+ dal.Where("connection_id = ?", connectionId),
+ dal.Orderby("id DESC"),
+ )
+ return scopeConfigs, err
+}
+
+func (scopeConfigSrv *ScopeConfigSrvHelper[C, S, SC])
DeleteScopeConfig(scopeConfig *SC) (refs []*S, err errors.Error) {
+ err = scopeConfigSrv.ModelSrvHelper.NoRunningPipeline(func(tx
dal.Transaction) errors.Error {
+ // make sure no scope is using the scopeConfig
+ sc := *scopeConfig
+ errors.Must(tx.All(
+ &refs,
+ dal.Where("connection_id = ? AND scope_config_id = ?",
sc.ScopeConfigConnectionId(), sc.ScopeConfigId()),
+ ))
+ if len(refs) > 0 {
+ return errors.Conflict.New("Please delete all data
scope(s) before you delete this ScopeConfig.")
+ }
+ errors.Must(tx.Delete(scopeConfig))
+ return nil
+ })
+ return
+}
diff --git a/backend/helpers/srvhelper/scope_service_helper.go
b/backend/helpers/srvhelper/scope_service_helper.go
new file mode 100644
index 000000000..f4fab4ce6
--- /dev/null
+++ b/backend/helpers/srvhelper/scope_service_helper.go
@@ -0,0 +1,276 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package srvhelper
+
+import (
+ "fmt"
+ "reflect"
+ "strings"
+
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models"
+ "github.com/apache/incubator-devlake/core/models/domainlayer/domaininfo"
+ "github.com/apache/incubator-devlake/core/plugin"
+)
+
+type ScopePagination struct {
+ Pagination `mapstructure:",squash"`
+ ConnectionId uint64 `json:"connectionId" mapstructure:"connectionId"
validate:"required"`
+ Blueprints bool `json:"blueprints" mapstructure:"blueprints"`
+}
+
+type ScopeDetail[S plugin.ToolLayerScope, SC plugin.ToolLayerScopeConfig]
struct {
+ Scope S `json:"scope"`
+ ScopeConfig *SC `json:"scopeConfig,omitempty"`
+ Blueprints []*models.Blueprint `json:"blueprints,omitempty"`
+}
+
+type ScopeSrvHelper[C plugin.ToolLayerConnection, S plugin.ToolLayerScope, SC
plugin.ToolLayerScopeConfig] struct {
+ *ModelSrvHelper[S]
+ pluginName string
+ searchColumns []string
+}
+
+// NewScopeSrvHelper creates a ScopeDalHelper for scope management
+func NewScopeSrvHelper[
+ C plugin.ToolLayerConnection,
+ S plugin.ToolLayerScope,
+ SC plugin.ToolLayerScopeConfig,
+](
+ basicRes context.BasicRes,
+ pluginName string,
+ searchColumns []string,
+) *ScopeSrvHelper[C, S, SC] {
+ return &ScopeSrvHelper[C, S, SC]{
+ ModelSrvHelper: NewModelSrvHelper[S](basicRes),
+ pluginName: pluginName,
+ searchColumns: searchColumns,
+ }
+}
+
+func (scopeSrv *ScopeSrvHelper[C, S, SC]) Validate(scope *S) errors.Error {
+ connectionId := (*scope).ScopeConnectionId()
+ connectionCount := errors.Must1(scopeSrv.db.Count(dal.From(new(SC)),
dal.Where("id = ?", connectionId)))
+ if connectionCount == 0 {
+ return errors.BadInput.New("connectionId is invalid")
+ }
+ scopeConfigId := (*scope).ScopeScopeConfigId()
+ scopeConfigCount := errors.Must1(scopeSrv.db.Count(dal.From(new(SC)),
dal.Where("id = ?", scopeConfigId)))
+ if scopeConfigCount == 0 {
+ return errors.BadInput.New("scopeConfigId is invalid")
+ }
+ return nil
+}
+
+func (scopeSrv *ScopeSrvHelper[C, S, SC]) GetScopeDetail(includeBlueprints
bool, pkv ...interface{}) (*ScopeDetail[S, SC], errors.Error) {
+ scope, err := scopeSrv.ModelSrvHelper.FindByPk(pkv...)
+ if err != nil {
+ return nil, err
+ }
+ s := *scope
+ scopeDetail := &ScopeDetail[S, SC]{
+ Scope: s,
+ ScopeConfig: scopeSrv.getScopeConfig(s.ScopeScopeConfigId()),
+ }
+ if includeBlueprints {
+ scopeDetail.Blueprints =
scopeSrv.getAllBlueprinsByScope(s.ScopeConnectionId(), s.ScopeId())
+ }
+ return scopeDetail, nil
+}
+
+func (scopeSrv *ScopeSrvHelper[C, S, SC]) GetScopesPage(pagination
*ScopePagination) ([]*ScopeDetail[S, SC], int64, errors.Error) {
+ if pagination.ConnectionId < 1 {
+ return nil, 0, errors.BadInput.New("connectionId is required")
+ }
+ scopes, count, err := scopeSrv.ModelSrvHelper.GetPage(
+ &pagination.Pagination,
+ dal.Where("connection_id = ?", pagination.ConnectionId),
+ )
+ if err != nil {
+ return nil, 0, err
+ }
+
+ data := make([]*ScopeDetail[S, SC], len(scopes))
+ for i, s := range scopes {
+ // load blueprints
+ scope := (*s)
+ scopeDetail := &ScopeDetail[S, SC]{
+ Scope: scope,
+ ScopeConfig:
scopeSrv.getScopeConfig(scope.ScopeScopeConfigId()),
+ }
+ if pagination.Blueprints {
+ scopeDetail.Blueprints =
scopeSrv.getAllBlueprinsByScope(scope.ScopeConnectionId(), scope.ScopeId())
+ }
+ data[i] = scopeDetail
+ }
+ return data, count, nil
+}
+
+func (scopeSrv *ScopeSrvHelper[C, S, SC]) DeleteScope(scope *S, dataOnly bool)
(refs *DsRefs, err errors.Error) {
+ err = scopeSrv.ModelSrvHelper.NoRunningPipeline(func(tx
dal.Transaction) errors.Error {
+ s := (*scope)
+ // check referencing blueprints
+ if !dataOnly {
+ refs =
toDsRefs(scopeSrv.getAllBlueprinsByScope(s.ScopeConnectionId(), s.ScopeId()))
+ if refs != nil {
+ return errors.Conflict.New("Cannot delete the
scope because it is referenced by blueprints")
+ }
+ errors.Must(tx.Delete(scope))
+ }
+ // delete data
+ scopeSrv.deleteScopeData(s, tx)
+ return nil
+ })
+ return
+}
+
+func (scopeSrv *ScopeSrvHelper[C, S, SC]) getScopeConfig(scopeConfigId uint64)
*SC {
+ if scopeConfigId < 1 {
+ return nil
+ }
+ scopeConfig := new(SC)
+ errors.Must(scopeSrv.db.First(
+ scopeConfig,
+ dal.Where(
+ "id = ?",
+ scopeConfigId,
+ ),
+ ))
+ return scopeConfig
+}
+
+func (scopeSrv *ScopeSrvHelper[C, S, SC]) getAllBlueprinsByScope(connectionId
uint64, scopeId string) []*models.Blueprint {
+ blueprints := make([]*models.Blueprint, 0)
+ errors.Must(scopeSrv.db.All(
+ &blueprints,
+ dal.From("_devlake_blueprints bp"),
+ dal.Join("JOIN _devlake_blueprint_scopes sc ON sc.blueprint_id
= bp.id"),
+ dal.Where(
+ "mode = ? AND sc.connection_id = ? AND sc.plugin_name =
? AND sc.scope_id = ?",
+ "NORMAL",
+ connectionId,
+ scopeSrv.pluginName,
+ scopeId,
+ ),
+ ))
+ return blueprints
+}
+
+func (scopeSrv *ScopeSrvHelper[C, S, SC]) deleteScopeData(scope
plugin.ToolLayerScope, tx dal.Transaction) {
+ rawDataParams := plugin.MarshalScopeParams(scope.ScopeParams())
+ generateWhereClause := func(table string) (string, []any) {
+ var where string
+ var params []interface{}
+ if strings.HasPrefix(table, "_raw_") {
+ // raw table: should check connection and scope
+ where = "params = ?"
+ params = []interface{}{rawDataParams}
+ } else if strings.HasPrefix(table, "_tool_") {
+ // tool layer table: should check connection and scope
+ where = "_raw_data_params = ?"
+ params = []interface{}{rawDataParams}
+ } else {
+ // framework tables: should check plugin, connection
and scope
+ if table == (models.CollectorLatestState{}.TableName())
{
+ // diff sync state
+ where = "raw_data_table LIKE ? AND
raw_data_params = ?"
+ } else {
+ // domain layer table
+ where = "_raw_data_table LIKE ? AND
_raw_data_params = ?"
+ }
+ rawDataTablePrefix := fmt.Sprintf("_raw_%s%%",
scopeSrv.pluginName)
+ params = []interface{}{rawDataTablePrefix,
rawDataParams}
+ }
+ return where, params
+ }
+ tables := errors.Must1(scopeSrv.getAffectedTables())
+ for _, table := range tables {
+ where, params := generateWhereClause(table)
+ scopeSrv.log.Info("deleting data from table %s with WHERE
\"%s\" and params: \"%v\"", table, where, params)
+ sql := fmt.Sprintf("DELETE FROM %s WHERE %s", table, where)
+ errors.Must(tx.Exec(sql, params...))
+ }
+}
+
+func (scopeSrv *ScopeSrvHelper[C, S, SC]) getAffectedTables() ([]string,
errors.Error) {
+ var tables []string
+ meta, err := plugin.GetPlugin(scopeSrv.pluginName)
+ if err != nil {
+ return nil, err
+ }
+ if pluginModel, ok := meta.(plugin.PluginModel); !ok {
+ panic(errors.Default.New(fmt.Sprintf("plugin \"%s\" does not
implement listing its tables", scopeSrv.pluginName)))
+ } else {
+ // Unfortunately, can't cache the tables because Python creates
some tables on a per-demand basis, so such a cache would possibly get outdated.
+ // It's a rare scenario in practice, but might as well play it
safe and sacrifice some performance here
+ var allTables []string
+ if allTables, err = scopeSrv.db.AllTables(); err != nil {
+ return nil, err
+ }
+ // collect raw tables
+ for _, table := range allTables {
+ if strings.HasPrefix(table,
"_raw_"+scopeSrv.pluginName) {
+ tables = append(tables, table)
+ }
+ }
+ // collect tool tables
+ toolModels := pluginModel.GetTablesInfo()
+ for _, toolModel := range toolModels {
+ if !isScopeModel(toolModel) && hasField(toolModel,
"RawDataParams") {
+ tables = append(tables, toolModel.TableName())
+ }
+ }
+ // collect domain tables
+ for _, domainModel := range domaininfo.GetDomainTablesInfo() {
+ // we only care about tables with RawOrigin
+ ok = hasField(domainModel, "RawDataParams")
+ if ok {
+ tables = append(tables, domainModel.TableName())
+ }
+ }
+ // additional tables
+ tables = append(tables,
models.CollectorLatestState{}.TableName())
+ }
+ scopeSrv.log.Debug("Discovered %d tables used by plugin \"%s\": %v",
len(tables), scopeSrv.pluginName, tables)
+ return tables, nil
+}
+
+// TODO: sort out the follow functions
+func isScopeModel(obj dal.Tabler) bool {
+ _, ok := obj.(plugin.ToolLayerScope)
+ return ok
+}
+
+func hasField(obj any, fieldName string) bool {
+ obj = models.UnwrapObject(obj)
+ _, ok := reflectType(obj).FieldByName(fieldName)
+ return ok
+}
+
+func reflectType(obj any) reflect.Type {
+ obj = models.UnwrapObject(obj)
+ typ := reflect.TypeOf(obj)
+ kind := typ.Kind()
+ for kind == reflect.Ptr {
+ typ = typ.Elem()
+ kind = typ.Kind()
+ }
+ return typ
+}
diff --git a/backend/helpers/utils/mapstructure.go
b/backend/helpers/utils/mapstructure.go
index d7a109ebb..22ea85949 100644
--- a/backend/helpers/utils/mapstructure.go
+++ b/backend/helpers/utils/mapstructure.go
@@ -38,6 +38,17 @@ func DecodeHook(f reflect.Type, t reflect.Type, data
interface{}) (interface{},
if t == reflect.TypeOf(json.RawMessage{}) {
return json.Marshal(data)
}
+ // to support decoding url.Values (query string) to non-array variables
+ if t.Kind() != reflect.Slice && t.Kind() != reflect.Array &&
+ (f.Kind() == reflect.Slice || f.Kind() == reflect.Array) {
+ v := reflect.ValueOf(data)
+ if v.Len() == 1 {
+ data = v.Index(0).Interface()
+ var result interface{}
+ err := DecodeMapStruct(data, &result, true)
+ return result, err
+ }
+ }
if t != reflect.TypeOf(common.Iso8601Time{}) && t !=
reflect.TypeOf(time.Time{}) {
return data, nil
@@ -65,12 +76,13 @@ func DecodeHook(f reflect.Type, t reflect.Type, data
interface{}) (interface{},
}
// DecodeMapStruct with time.Time and Iso8601Time support
-func DecodeMapStruct(input map[string]interface{}, result interface{},
zeroFields bool) errors.Error {
+func DecodeMapStruct(input interface{}, result interface{}, zeroFields bool)
errors.Error {
result = models.UnwrapObject(result)
decoder, err := mapstructure.NewDecoder(&mapstructure.DecoderConfig{
- ZeroFields: zeroFields,
- DecodeHook: mapstructure.ComposeDecodeHookFunc(DecodeHook),
- Result: result,
+ ZeroFields: zeroFields,
+ DecodeHook:
mapstructure.ComposeDecodeHookFunc(DecodeHook),
+ Result: result,
+ WeaklyTypedInput: true,
})
if err != nil {
return errors.Convert(err)
diff --git a/backend/helpers/utils/mapstructure_test.go
b/backend/helpers/utils/mapstructure_test.go
index 3758632d1..755dea4d2 100644
--- a/backend/helpers/utils/mapstructure_test.go
+++ b/backend/helpers/utils/mapstructure_test.go
@@ -20,6 +20,7 @@ package utils
import (
"encoding/json"
"fmt"
+ "net/url"
"testing"
"time"
@@ -138,3 +139,19 @@ func TestIso8601Time(t *testing.T) {
assert.Equal(t, expected, record4.Created.UTC())
}
}
+
+func TestDecodeMapStructUrlVales(t *testing.T) {
+ query := &url.Values{}
+ query.Set("page", "1")
+ query.Set("pageSize", "100")
+
+ var pagination struct {
+ Page int `mapstructure:"page"`
+ PageSize int `mapstructure:"pageSize"`
+ }
+
+ err := DecodeMapStruct(query, &pagination, true)
+ assert.Nil(t, err)
+ assert.Equal(t, 1, pagination.Page)
+ assert.Equal(t, 100, pagination.PageSize)
+}
diff --git a/backend/impls/dalgorm/dalgorm.go b/backend/impls/dalgorm/dalgorm.go
index 21c0f5bdc..c5518a889 100644
--- a/backend/impls/dalgorm/dalgorm.go
+++ b/backend/impls/dalgorm/dalgorm.go
@@ -25,6 +25,7 @@ import (
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/models"
"github.com/apache/incubator-devlake/core/utils"
"gorm.io/gorm"
@@ -105,6 +106,8 @@ func buildTx(tx *gorm.DB, clauses []dal.Clause) *gorm.DB {
Alias: dd.Alias,
Raw: dd.Raw,
})
+ case models.DynamicTabler:
+ tx = tx.Table(dd.TableName())
default:
tx = tx.Model(d)
}
@@ -138,8 +141,26 @@ func (d *Dalgorm) Exec(query string, params
...interface{}) errors.Error {
return d.convertGormError(d.db.Exec(query,
transformParams(params)...).Error)
}
+func (d *Dalgorm) unwrapDynamic(entityPtr *interface{}, clausesPtr
*[]dal.Clause) {
+ if dynamic, ok := (*entityPtr).(models.DynamicTabler); ok {
+ if clausesPtr != nil {
+ *clausesPtr = append(*clausesPtr,
dal.From(dynamic.TableName()))
+ }
+ *entityPtr = dynamic.Unwrap()
+ } else if clausesPtr != nil {
+ // try to add dal.From if it does not exist
+ for _, c := range *clausesPtr {
+ if c.Type == dal.FromClause {
+ return
+ }
+ }
+ *clausesPtr = append(*clausesPtr, dal.From(*entityPtr))
+ }
+}
+
// AutoMigrate runs auto migration for given models
func (d *Dalgorm) AutoMigrate(entity interface{}, clauses ...dal.Clause)
errors.Error {
+ d.unwrapDynamic(&entity, &clauses)
err := buildTx(d.db, clauses).AutoMigrate(entity)
if err == nil {
// fix pg cache plan error
@@ -170,11 +191,13 @@ func (d *Dalgorm) Fetch(cursor dal.Rows, dst interface{})
errors.Error {
// All loads matched rows from database to `dst`, USE IT WITH COUTIOUS!!
func (d *Dalgorm) All(dst interface{}, clauses ...dal.Clause) errors.Error {
+ d.unwrapDynamic(&dst, &clauses)
return d.convertGormError(buildTx(d.db, clauses).Find(dst).Error)
}
// First loads first matched row from database to `dst`, error will be
returned if no records were found
func (d *Dalgorm) First(dst interface{}, clauses ...dal.Clause) errors.Error {
+ d.unwrapDynamic(&dst, &clauses)
return d.convertGormError(buildTx(d.db, clauses).First(dst).Error)
}
@@ -192,45 +215,52 @@ func (d *Dalgorm) Pluck(column string, dest interface{},
clauses ...dal.Clause)
// Create insert record to database
func (d *Dalgorm) Create(entity interface{}, clauses ...dal.Clause)
errors.Error {
+ d.unwrapDynamic(&entity, &clauses)
return d.convertGormError(buildTx(d.db, clauses).Create(entity).Error)
}
// CreateWithMap insert record to database
func (d *Dalgorm) CreateWithMap(entity interface{}, record
map[string]interface{}) errors.Error {
+ d.unwrapDynamic(&entity, nil)
return d.convertGormError(buildTx(d.db,
nil).Model(entity).Clauses(clause.OnConflict{UpdateAll:
true}).Create(record).Error)
}
// Update updates record
func (d *Dalgorm) Update(entity interface{}, clauses ...dal.Clause)
errors.Error {
+ d.unwrapDynamic(&entity, &clauses)
return d.convertGormError(buildTx(d.db, clauses).Save(entity).Error)
}
// CreateOrUpdate tries to create the record, or fallback to update all if
failed
func (d *Dalgorm) CreateOrUpdate(entity interface{}, clauses ...dal.Clause)
errors.Error {
+ d.unwrapDynamic(&entity, &clauses)
return d.convertGormError(buildTx(d.db,
clauses).Clauses(clause.OnConflict{UpdateAll: true}).Create(entity).Error)
}
// CreateIfNotExist tries to create the record if not exist
func (d *Dalgorm) CreateIfNotExist(entity interface{}, clauses ...dal.Clause)
errors.Error {
+ d.unwrapDynamic(&entity, &clauses)
return d.convertGormError(buildTx(d.db,
clauses).Clauses(clause.OnConflict{DoNothing: true}).Create(entity).Error)
}
// Delete records from database
func (d *Dalgorm) Delete(entity interface{}, clauses ...dal.Clause)
errors.Error {
+ d.unwrapDynamic(&entity, &clauses)
return d.convertGormError(buildTx(d.db, clauses).Delete(entity).Error)
}
// UpdateColumn allows you to update mulitple records
func (d *Dalgorm) UpdateColumn(entityOrTable interface{}, columnName string,
value interface{}, clauses ...dal.Clause) errors.Error {
+ d.unwrapDynamic(&entityOrTable, &clauses)
if expr, ok := value.(dal.DalClause); ok {
value = gorm.Expr(expr.Expr, transformParams(expr.Params)...)
}
- clauses = append(clauses, dal.From(entityOrTable))
return d.convertGormError(buildTx(d.db, clauses).Update(columnName,
value).Error)
}
// UpdateColumns allows you to update multiple columns of mulitple records
func (d *Dalgorm) UpdateColumns(entityOrTable interface{}, set []dal.DalSet,
clauses ...dal.Clause) errors.Error {
+ d.unwrapDynamic(&entityOrTable, &clauses)
updatesSet := make(map[string]interface{})
for _, s := range set {
diff --git a/backend/plugins/github/api/blueprint_v200.go
b/backend/plugins/github/api/blueprint_v200.go
index b6b508930..0ab128e4b 100644
--- a/backend/plugins/github/api/blueprint_v200.go
+++ b/backend/plugins/github/api/blueprint_v200.go
@@ -86,10 +86,17 @@ func makeDataSourcePipelinePlanV200(
stage = coreModels.PipelineStage{}
}
// get repo and scope config from db
- githubRepo, scopeConfig, err :=
scopeHelper.DbHelper().GetScopeAndConfig(connection.ID, bpScope.ScopeId)
+ githubRepo, err := scopeSrv.FindByPk(connection.ID,
bpScope.ScopeId)
if err != nil {
return nil, err
}
+ var scopeConfig *models.GithubScopeConfig
+ if githubRepo.ScopeConfigId != 0 {
+ scopeConfig, err =
scSrv.FindByPk(githubRepo.ScopeConfigId)
+ if err != nil {
+ return nil, err
+ }
+ }
// refdiff
if scopeConfig != nil && scopeConfig.Refdiff != nil {
// add a new task to next stage
diff --git a/backend/plugins/github/api/connection.go
b/backend/plugins/github/api/connection_api.go
similarity index 91%
rename from backend/plugins/github/api/connection.go
rename to backend/plugins/github/api/connection_api.go
index cb508ad58..618d39fe0 100644
--- a/backend/plugins/github/api/connection.go
+++ b/backend/plugins/github/api/connection_api.go
@@ -212,12 +212,7 @@ func TestConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/github/connections [POST]
func PostConnections(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- connection := &models.GithubConnection{}
- err := connectionHelper.Create(connection, input)
- if err != nil {
- return nil, err
- }
- return &plugin.ApiResourceOutput{Body: connection, Status:
http.StatusOK}, nil
+ return connApi.Post(input)
}
// @Summary patch github connection
@@ -229,12 +224,7 @@ func PostConnections(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/github/connections/{connectionId} [PATCH]
func PatchConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- connection := &models.GithubConnection{}
- err := connectionHelper.Patch(connection, input)
- if err != nil {
- return nil, err
- }
- return &plugin.ApiResourceOutput{Body: connection, Status:
http.StatusOK}, nil
+ return connApi.Patch(input)
}
// @Summary delete a github connection
@@ -246,7 +236,7 @@ func PatchConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/github/connections/{connectionId} [DELETE]
func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- return connectionHelper.Delete(&models.GithubConnection{}, input)
+ return connApi.Delete(input)
}
// @Summary get all github connections
@@ -257,13 +247,7 @@ func DeleteConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/github/connections [GET]
func ListConnections(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- var connections []models.GithubConnection
- err := connectionHelper.List(&connections)
- if err != nil {
- return nil, err
- }
-
- return &plugin.ApiResourceOutput{Body: connections}, nil
+ return connApi.GetAll(input)
}
// @Summary get github connection detail
@@ -274,10 +258,5 @@ func ListConnections(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Failure 500 {string} errcode.Error "Internal Error"
// @Router /plugins/github/connections/{connectionId} [GET]
func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
- connection := &models.GithubConnection{}
- err := connectionHelper.First(connection, input.Params)
- if err != nil {
- return nil, err
- }
- return &plugin.ApiResourceOutput{Body: connection}, nil
+ return connApi.GetDetail(input)
}
diff --git a/backend/plugins/github/api/init.go
b/backend/plugins/github/api/init.go
index 2459b86d0..a976677d9 100644
--- a/backend/plugins/github/api/init.go
+++ b/backend/plugins/github/api/init.go
@@ -18,65 +18,40 @@ limitations under the License.
package api
import (
- "strconv"
-
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/core/context"
- "github.com/apache/incubator-devlake/core/dal"
- "github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/helpers/srvhelper"
"github.com/apache/incubator-devlake/plugins/github/models"
"github.com/go-playground/validator/v10"
)
var vld *validator.Validate
+
+// var connSrv *srvhelper.ConnectionSrvHelper[models.GithubConnection,
models.GithubRepo, models.GithubScopeConfig]
+var connApi *api.DsConnectionApiHelper[models.GithubConnection,
models.GithubRepo, models.GithubScopeConfig]
+var scopeSrv *srvhelper.ScopeSrvHelper[models.GithubConnection,
models.GithubRepo, models.GithubScopeConfig]
+var scopeApi *api.DsScopeApiHelper[models.GithubConnection, models.GithubRepo,
models.GithubScopeConfig]
+var scSrv *srvhelper.ScopeConfigSrvHelper[models.GithubConnection,
models.GithubRepo, models.GithubScopeConfig]
+var scApi *api.DsScopeConfigApiHelper[models.GithubConnection,
models.GithubRepo, models.GithubScopeConfig]
var connectionHelper *api.ConnectionApiHelper
-var scopeHelper *api.ScopeApiHelper[models.GithubConnection,
models.GithubRepo, models.GithubScopeConfig]
var basicRes context.BasicRes
-var scHelper *api.ScopeConfigHelper[models.GithubScopeConfig]
var remoteHelper *api.RemoteApiHelper[models.GithubConnection,
models.GithubRepo, repo, plugin.ApiGroup]
func Init(br context.BasicRes, p plugin.PluginMeta) {
-
basicRes = br
- vld = validator.New()
- connectionHelper = api.NewConnectionHelper(
- basicRes,
- vld,
+ _, connApi, scopeSrv, scopeApi, scSrv, scApi = api.NewDataSourceHelpers[
+ models.GithubConnection,
+ models.GithubRepo, models.GithubScopeConfig,
+ ](
+ br,
p.Name(),
+ []string{"full_name"},
)
- params := &api.ReflectionParameters{
- ScopeIdFieldName: "GithubId",
- ScopeIdColumnName: "github_id",
- RawScopeParamName: "Name",
- SearchScopeParamName: "name",
- }
- scopeHelper = api.NewScopeHelper[models.GithubConnection,
models.GithubRepo, models.GithubScopeConfig](
- basicRes,
- vld,
- connectionHelper,
- api.NewScopeDatabaseHelperImpl[models.GithubConnection,
models.GithubRepo, models.GithubScopeConfig](
- basicRes, connectionHelper, params),
- params,
- &api.ScopeHelperOptions{
- GetScopeParamValue: func(db dal.Dal, scopeId string)
(string, errors.Error) {
- id, err :=
errors.Convert01(strconv.ParseInt(scopeId, 10, 64))
- if err != nil {
- return "", err
- }
- repo := models.GithubRepo{
- GithubId: int(id),
- }
- err = db.First(&repo)
- if err != nil {
- return "", err
- }
- return repo.FullName, nil
- },
- },
- )
- scHelper = api.NewScopeConfigHelper[models.GithubScopeConfig](
+ // TODO: refactor remoteHelper
+ vld = validator.New()
+ connectionHelper = api.NewConnectionHelper(
basicRes,
vld,
p.Name(),
diff --git a/backend/plugins/github/api/scope.go
b/backend/plugins/github/api/scope_api.go
similarity index 80%
rename from backend/plugins/github/api/scope.go
rename to backend/plugins/github/api/scope_api.go
index 8df469f7d..902069416 100644
--- a/backend/plugins/github/api/scope.go
+++ b/backend/plugins/github/api/scope_api.go
@@ -24,12 +24,8 @@ import (
"github.com/apache/incubator-devlake/plugins/github/models"
)
-type ScopeRes struct {
- models.GithubRepo
- api.ScopeResDoc[models.GithubScopeConfig]
-}
-
-type ScopeReq api.ScopeReq[models.GithubRepo]
+type PutScopesReqBody api.PutScopesReqBody[models.GithubRepo]
+type ScopeDetail api.ScopeDetail[models.GithubRepo, models.GithubScopeConfig]
// PutScope create or update github repo
// @Summary create or update github repo
@@ -37,13 +33,13 @@ type ScopeReq api.ScopeReq[models.GithubRepo]
// @Tags plugins/github
// @Accept application/json
// @Param connectionId path int true "connection ID"
-// @Param scope body ScopeReq true "json"
-// @Success 200 {object} []ScopeRes
+// @Param scope body PutScopesReqBody true "json"
+// @Success 200 {object} []models.GithubRepo
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/github/connections/{connectionId}/scopes [PUT]
-func PutScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
- return scopeHelper.Put(input)
+func PutScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
+ return scopeApi.PutMultiple(input)
}
// UpdateScope patch to github repo
@@ -54,12 +50,12 @@ func PutScope(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors
// @Param connectionId path int true "connection ID"
// @Param scopeId path int true "scope ID"
// @Param scope body models.GithubRepo true "json"
-// @Success 200 {object} ScopeRes
+// @Success 200 {object} models.GithubRepo
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/github/connections/{connectionId}/scopes/{scopeId} [PATCH]
-func UpdateScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
- return scopeHelper.Update(input)
+func PatchScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
+ return scopeApi.Patch(input)
}
// GetScopeList get Github repos
@@ -71,12 +67,12 @@ func UpdateScope(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, err
// @Param pageSize query int false "page size, default 50"
// @Param page query int false "page size, default 1"
// @Param blueprints query bool false "also return blueprints using these
scopes as part of the payload"
-// @Success 200 {object} []ScopeRes
+// @Success 200 {object} []ScopeDetail
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/github/connections/{connectionId}/scopes [GET]
-func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
- return scopeHelper.GetScopeList(input)
+func GetScopes(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
+ return scopeApi.GetPage(input)
}
// GetScope get one Github repo
@@ -85,12 +81,13 @@ func GetScopeList(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, er
// @Tags plugins/github
// @Param connectionId path int true "connection ID"
// @Param scopeId path int true "scope ID"
-// @Success 200 {object} ScopeRes
+// @Param blueprints query bool false "also return blueprints using these
scopes as part of the payload"
+// @Success 200 {object} ScopeDetail
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/github/connections/{connectionId}/scopes/{scopeId} [GET]
func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
- return scopeHelper.GetScope(input)
+ return scopeApi.GetScopeDetail(input)
}
// DeleteScope delete plugin data associated with the scope and optionally the
scope itself
@@ -100,11 +97,11 @@ func GetScope(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors
// @Param connectionId path int true "connection ID"
// @Param scopeId path int true "scope ID"
// @Param delete_data_only query bool false "Only delete the scope data, not
the scope itself"
-// @Success 200
+// @Success 200 {object} models.GithubRepo
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 409 {object} api.ScopeRefDoc "References exist to this scope"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/github/connections/{connectionId}/scopes/{scopeId} [DELETE]
func DeleteScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
- return scopeHelper.Delete(input)
+ return scopeApi.Delete(input)
}
diff --git a/backend/plugins/github/api/scope_config.go
b/backend/plugins/github/api/scope_config_api.go
similarity index 89%
rename from backend/plugins/github/api/scope_config.go
rename to backend/plugins/github/api/scope_config_api.go
index ee1ee8f8e..351eeb24b 100644
--- a/backend/plugins/github/api/scope_config.go
+++ b/backend/plugins/github/api/scope_config_api.go
@@ -22,7 +22,7 @@ import (
"github.com/apache/incubator-devlake/core/plugin"
)
-// CreateScopeConfig create scope config for Github
+// PostScopeConfig create scope config for Github
// @Summary create scope config for Github
// @Description create scope config for Github
// @Tags plugins/github
@@ -33,11 +33,11 @@ import (
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/github/connections/{connectionId}/scope-configs [POST]
-func CreateScopeConfig(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- return scHelper.Create(input)
+func PostScopeConfig(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ return scApi.Post(input)
}
-// UpdateScopeConfig update scope config for Github
+// PatchScopeConfig update scope config for Github
// @Summary update scope config for Github
// @Description update scope config for Github
// @Tags plugins/github
@@ -49,8 +49,8 @@ func CreateScopeConfig(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutpu
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/github/connections/{connectionId}/scope-configs/{id}
[PATCH]
-func UpdateScopeConfig(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- return scHelper.Update(input)
+func PatchScopeConfig(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
+ return scApi.Patch(input)
}
// GetScopeConfig return one scope config
@@ -64,7 +64,7 @@ func UpdateScopeConfig(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutpu
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/github/connections/{connectionId}/scope-configs/{id} [GET]
func GetScopeConfig(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- return scHelper.Get(input)
+ return scApi.GetDetail(input)
}
// GetScopeConfigList return all scope configs
@@ -79,7 +79,7 @@ func GetScopeConfig(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/github/connections/{connectionId}/scope-configs [GET]
func GetScopeConfigList(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- return scHelper.List(input)
+ return scApi.GetAll(input)
}
// DeleteScopeConfig delete a scope config
@@ -93,5 +93,5 @@ func GetScopeConfigList(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutp
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/github/connections/{connectionId}/scope-configs/{id}
[DELETE]
func DeleteScopeConfig(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
- return scHelper.Delete(input)
+ return scApi.Delete(input)
}
diff --git a/backend/plugins/github/impl/impl.go
b/backend/plugins/github/impl/impl.go
index 2a2d0d4ed..f87dcc63c 100644
--- a/backend/plugins/github/impl/impl.go
+++ b/backend/plugins/github/impl/impl.go
@@ -185,19 +185,19 @@ func (p Github) ApiResources()
map[string]map[string]plugin.ApiResourceHandler {
},
"connections/:connectionId/scopes/:scopeId": {
"GET": api.GetScope,
- "PATCH": api.UpdateScope,
+ "PATCH": api.PatchScope,
"DELETE": api.DeleteScope,
},
"connections/:connectionId/scopes": {
- "GET": api.GetScopeList,
- "PUT": api.PutScope,
+ "GET": api.GetScopes,
+ "PUT": api.PutScopes,
},
"connections/:connectionId/scope-configs": {
- "POST": api.CreateScopeConfig,
+ "POST": api.PostScopeConfig,
"GET": api.GetScopeConfigList,
},
- "connections/:connectionId/scope-configs/:id": {
- "PATCH": api.UpdateScopeConfig,
+ "connections/:connectionId/scope-configs/:scopeConfigId": {
+ "PATCH": api.PatchScopeConfig,
"GET": api.GetScopeConfig,
"DELETE": api.DeleteScopeConfig,
},
diff --git a/backend/python/test/fakeplugin/run.sh
b/backend/python/test/fakeplugin/run.sh
index 6d287d6ba..e02a7ca2f 100755
--- a/backend/python/test/fakeplugin/run.sh
+++ b/backend/python/test/fakeplugin/run.sh
@@ -16,5 +16,8 @@
# limitations under the License.
#
+echo sys path $PATH >&2
+[ -n "$VIRTUAL_ENV" ] && echo "Using virtualenv: $VIRTUAL_ENV" >&2 && .
"$VIRTUAL_ENV/bin/activate"
+
cd "$(dirname "$0")"
poetry run python fakeplugin/main.py "$@"
diff --git a/backend/test/helper/client.go b/backend/test/helper/client.go
index 446ea6d1f..0970db916 100644
--- a/backend/test/helper/client.go
+++ b/backend/test/helper/client.go
@@ -36,7 +36,6 @@ import (
dora "github.com/apache/incubator-devlake/plugins/dora/impl"
org "github.com/apache/incubator-devlake/plugins/org/impl"
refdiff "github.com/apache/incubator-devlake/plugins/refdiff/impl"
- remotePlugin
"github.com/apache/incubator-devlake/server/services/remote/plugin"
"github.com/apache/incubator-devlake/core/config"
corectx "github.com/apache/incubator-devlake/core/context"
@@ -289,13 +288,6 @@ func (d *DevlakeClient) initPlugins(cfg
*LocalClientConfig) {
for _, p := range cfg.Plugins {
require.NoError(d.testCtx, plugin.RegisterPlugin(p.Name(), p))
}
- for _, p := range plugin.AllPlugins() {
- if pi, ok := p.(plugin.PluginInit); ok {
- err := pi.Init(d.basicRes)
- require.NoError(d.testCtx, err)
- }
- }
- remotePlugin.Init(d.basicRes)
}
func (d *DevlakeClient) prepareDB(cfg *LocalClientConfig) {
