This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 14c0b988d fix: convertScopeResponse fix for remote plugins (#5388)
14c0b988d is described below
commit 14c0b988ddda84cf8e509c352a048a8db3c5750f
Author: Keon Amini <[email protected]>
AuthorDate: Tue Jun 6 23:49:40 2023 -0500
fix: convertScopeResponse fix for remote plugins (#5388)
---
backend/go.mod | 166 ++++++++++-----------
backend/server/services/remote/plugin/scope_api.go | 41 +++--
backend/test/e2e/remote/python_plugin_test.go | 3 +-
backend/test/helper/api.go | 4 +-
backend/test/helper/models.go | 6 +-
5 files changed, 107 insertions(+), 113 deletions(-)
diff --git a/backend/go.mod b/backend/go.mod
index 2f998528c..ffc165d96 100644
--- a/backend/go.mod
+++ b/backend/go.mod
@@ -14,6 +14,8 @@ require (
github.com/gocarina/gocsv v0.0.0-20220707092902-b9da1f06c77e
github.com/google/uuid v1.3.0
github.com/iancoleman/strcase v0.2.0
+ github.com/jackc/pgx/v5 v5.3.1
+ github.com/jmespath/go-jmespath v0.4.0
github.com/lib/pq v1.10.2
github.com/libgit2/git2go/v33 v33.0.6
github.com/magiconair/properties v1.8.5
@@ -52,8 +54,6 @@ require (
github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 //
indirect
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
github.com/goccy/go-json v0.10.2 // indirect
- github.com/jackc/pgx/v5 v5.3.1 // indirect
- github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/klauspost/cpuid/v2 v2.2.5 // indirect
github.com/pelletier/go-toml/v2 v2.0.8 // indirect
github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
@@ -61,89 +61,89 @@ require (
)
require (
- github.com/KyleBanks/depth v1.2.1 // indirect
- github.com/Microsoft/go-winio v0.5.0 // indirect
- github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7 //
indirect
- github.com/acomagu/bufpipe v1.0.3 // indirect
- github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e //
indirect
- github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f //
indirect
- github.com/cockroachdb/redact v1.1.3 // indirect
- github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
- github.com/davecgh/go-spew v1.1.1 // indirect
- github.com/emirpasic/gods v1.12.0 // indirect
- github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a //
indirect
- github.com/fsnotify/fsnotify v1.5.1 // indirect
- github.com/getsentry/sentry-go v0.12.0 // indirect
- github.com/gin-contrib/sse v0.1.0 // indirect
- github.com/go-git/gcfg v1.5.0 // indirect
- github.com/go-git/go-billy/v5 v5.3.1 // indirect
- github.com/go-openapi/jsonpointer v0.19.6 // indirect
- github.com/go-openapi/jsonreference v0.20.2 // indirect
- github.com/go-openapi/spec v0.20.9 // indirect
- github.com/go-openapi/swag v0.22.3 // indirect
- github.com/go-playground/locales v0.14.1 // indirect
- github.com/go-playground/universal-translator v0.18.1 // indirect
- github.com/go-sql-driver/mysql v1.7.1 // indirect
- github.com/gogo/googleapis v1.4.1 // indirect
- github.com/gogo/protobuf v1.3.2 // indirect
- github.com/gogo/status v1.1.0 // indirect
+ github.com/KyleBanks/depth v1.2.1
+ github.com/Microsoft/go-winio v0.5.0
+ github.com/ProtonMail/go-crypto v0.0.0-20210428141323-04723f9f07d7
+ github.com/acomagu/bufpipe v1.0.3
+ github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e
+ github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f
+ github.com/cockroachdb/redact v1.1.3
+ github.com/cpuguy83/go-md2man/v2 v2.0.2
+ github.com/davecgh/go-spew v1.1.1
+ github.com/emirpasic/gods v1.12.0
+ github.com/facebookgo/clock v0.0.0-20150410010913-600d898af40a
+ github.com/fsnotify/fsnotify v1.5.1
+ github.com/getsentry/sentry-go v0.12.0
+ github.com/gin-contrib/sse v0.1.0
+ github.com/go-git/gcfg v1.5.0
+ github.com/go-git/go-billy/v5 v5.3.1
+ github.com/go-openapi/jsonpointer v0.19.6
+ github.com/go-openapi/jsonreference v0.20.2
+ github.com/go-openapi/spec v0.20.9
+ github.com/go-openapi/swag v0.22.3
+ github.com/go-playground/locales v0.14.1
+ github.com/go-playground/universal-translator v0.18.1
+ github.com/go-sql-driver/mysql v1.7.1
+ github.com/gogo/googleapis v1.4.1
+ github.com/gogo/protobuf v1.3.2
+ github.com/gogo/status v1.1.0
github.com/golang-jwt/jwt/v5 v5.0.0-rc.1
- github.com/golang/mock v1.6.0 // indirect
- github.com/golang/protobuf v1.5.2 // indirect
- github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect
- github.com/hashicorp/hcl v1.0.0 // indirect
- github.com/imdario/mergo v0.3.12 // indirect
- github.com/inconshreveable/mousetrap v1.0.0 // indirect
- github.com/jackc/pgpassfile v1.0.0 // indirect
- github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a //
indirect
- github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 //
indirect
- github.com/jinzhu/inflection v1.0.0 // indirect
- github.com/jinzhu/now v1.1.5 // indirect
- github.com/josharian/intern v1.0.0 // indirect
- github.com/json-iterator/go v1.1.12 // indirect
- github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351 //
indirect
- github.com/kr/pretty v0.3.0 // indirect
- github.com/kr/text v0.2.0 // indirect
- github.com/leodido/go-urn v1.2.4 // indirect
- github.com/mailru/easyjson v0.7.7 // indirect
- github.com/mattn/go-colorable v0.1.11 // indirect
- github.com/mattn/go-isatty v0.0.19 // indirect
- github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
- github.com/mitchellh/go-homedir v1.1.0 // indirect
- github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd //
indirect
- github.com/modern-go/reflect2 v1.0.2 // indirect
- github.com/pborman/uuid v1.2.1 // indirect
- github.com/pelletier/go-toml v1.9.3 // indirect
- github.com/pkg/errors v0.9.1 // indirect
- github.com/pmezard/go-difflib v1.0.0 // indirect
- github.com/robfig/cron v1.2.0 // indirect
- github.com/rogpeppe/go-internal v1.8.1 // indirect
- github.com/russross/blackfriday/v2 v2.1.0 // indirect
- github.com/sergi/go-diff v1.1.0 // indirect
- github.com/spf13/jwalterweatherman v1.1.0 // indirect
- github.com/spf13/pflag v1.0.6-0.20200504143853-81378bbcd8a1 // indirect
- github.com/stretchr/objx v0.5.0 // indirect
- github.com/subosito/gotenv v1.2.0 // indirect
- github.com/tidwall/match v1.1.1 // indirect
- github.com/tidwall/pretty v1.2.0 // indirect
- github.com/ugorji/go/codec v1.2.11 // indirect
- github.com/xanzy/ssh-agent v0.3.0 // indirect
- go.uber.org/atomic v1.9.0 // indirect
+ github.com/golang/mock v1.6.0
+ github.com/golang/protobuf v1.5.2
+ github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
+ github.com/hashicorp/hcl v1.0.0
+ github.com/imdario/mergo v0.3.12
+ github.com/inconshreveable/mousetrap v1.0.0
+ github.com/jackc/pgpassfile v1.0.0
+ github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a
+ github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99
+ github.com/jinzhu/inflection v1.0.0
+ github.com/jinzhu/now v1.1.5
+ github.com/josharian/intern v1.0.0
+ github.com/json-iterator/go v1.1.12
+ github.com/kevinburke/ssh_config v0.0.0-20201106050909-4977a11b4351
+ github.com/kr/pretty v0.3.0
+ github.com/kr/text v0.2.0
+ github.com/leodido/go-urn v1.2.4
+ github.com/mailru/easyjson v0.7.7
+ github.com/mattn/go-colorable v0.1.11
+ github.com/mattn/go-isatty v0.0.19
+ github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d
+ github.com/mitchellh/go-homedir v1.1.0
+ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd
+ github.com/modern-go/reflect2 v1.0.2
+ github.com/pborman/uuid v1.2.1
+ github.com/pelletier/go-toml v1.9.3
+ github.com/pkg/errors v0.9.1
+ github.com/pmezard/go-difflib v1.0.0
+ github.com/robfig/cron v1.2.0
+ github.com/rogpeppe/go-internal v1.8.1
+ github.com/russross/blackfriday/v2 v2.1.0
+ github.com/sergi/go-diff v1.1.0
+ github.com/spf13/jwalterweatherman v1.1.0
+ github.com/spf13/pflag v1.0.6-0.20200504143853-81378bbcd8a1
+ github.com/stretchr/objx v0.5.0
+ github.com/subosito/gotenv v1.2.0
+ github.com/tidwall/match v1.1.1
+ github.com/tidwall/pretty v1.2.0
+ github.com/ugorji/go/codec v1.2.11
+ github.com/xanzy/ssh-agent v0.3.0
+ go.uber.org/atomic v1.9.0
golang.org/x/mod v0.10.0
- golang.org/x/net v0.10.0 // indirect
- golang.org/x/sys v0.8.0 // indirect
- golang.org/x/term v0.8.0 // indirect
- golang.org/x/text v0.9.0 // indirect
- golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac // indirect
- golang.org/x/tools v0.9.3 // indirect
- google.golang.org/appengine v1.6.7 // indirect
- google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf //
indirect
- google.golang.org/grpc v1.44.0 // indirect
- google.golang.org/protobuf v1.30.0 // indirect
- gopkg.in/ini.v1 v1.62.0 // indirect
- gopkg.in/warnings.v0 v0.1.2 // indirect
- gopkg.in/yaml.v2 v2.4.0 // indirect
- gopkg.in/yaml.v3 v3.0.1 // indirect
+ golang.org/x/net v0.10.0
+ golang.org/x/sys v0.8.0
+ golang.org/x/term v0.8.0
+ golang.org/x/text v0.9.0
+ golang.org/x/time v0.0.0-20210723032227-1f47c861a9ac
+ golang.org/x/tools v0.9.3
+ google.golang.org/appengine v1.6.7
+ google.golang.org/genproto v0.0.0-20220222213610-43724f9ea8cf
+ google.golang.org/grpc v1.44.0
+ google.golang.org/protobuf v1.30.0
+ gopkg.in/ini.v1 v1.62.0
+ gopkg.in/warnings.v0 v0.1.2
+ gopkg.in/yaml.v2 v2.4.0
+ gopkg.in/yaml.v3 v3.0.1
)
//replace github.com/apache/incubator-devlake => ./
diff --git a/backend/server/services/remote/plugin/scope_api.go
b/backend/server/services/remote/plugin/scope_api.go
index 9593890e9..43dbf2b0e 100644
--- a/backend/server/services/remote/plugin/scope_api.go
+++ b/backend/server/services/remote/plugin/scope_api.go
@@ -102,30 +102,25 @@ func (pa *pluginAPI) DeleteScope(input
*plugin.ApiResourceInput) (*plugin.ApiRes
return &plugin.ApiResourceOutput{Body: nil, Status: http.StatusOK}, nil
}
-// convertScopeResponse adapt the "remote" scopes to a serializable
api.ScopeRes
+// convertScopeResponse adapt the "remote" scopes to a serializable
api.ScopeRes. This code is needed because squashed mapstructure don't work
+// with dynamic/runtime structs used by remote plugins
func convertScopeResponse(scopes ...*api.ScopeRes[models.RemoteScope,
models.RemoteTransformation]) ([]map[string]any, errors.Error) {
responses := make([]map[string]any, len(scopes))
- // @keon @camille please help refactoring this, why don't we use
mapstructure?
- // for i, scope := range scopes {
- // resMap := map[string]any{}
- // err := models.MapTo(api.ScopeRes[map[string]any, ]{
- // Scope: nil, //ignore intentionally
- // ScopeConfig: scope.ScopeConfig,
- // Blueprints: scope.Blueprints,
- // }, &resMap)
- // if err != nil {
- // return nil, err
- // }
- // scopeMap := map[string]any{}
- // err = models.MapTo(scope.Scope, &scopeMap)
- // if err != nil {
- // return nil, err
- // }
- // delete(resMap, "Scope")
- // for k, v := range scopeMap {
- // resMap[k] = v
- // }
- // responses[i] = resMap
- // }
+ for i, scope := range scopes {
+ resMap := map[string]any{}
+ err := models.MapTo(scope.ScopeResDoc, &resMap)
+ if err != nil {
+ return nil, err
+ }
+ scopeMap := map[string]any{}
+ err = models.MapTo(scope.Scope, &scopeMap)
+ if err != nil {
+ return nil, err
+ }
+ for k, v := range scopeMap {
+ resMap[k] = v
+ }
+ responses[i] = resMap
+ }
return responses, nil
}
diff --git a/backend/test/e2e/remote/python_plugin_test.go
b/backend/test/e2e/remote/python_plugin_test.go
index 8dfd235b8..04237e439 100644
--- a/backend/test/e2e/remote/python_plugin_test.go
+++ b/backend/test/e2e/remote/python_plugin_test.go
@@ -129,8 +129,7 @@ func TestBlueprintV200_withScopeDeletion(t *testing.T) {
scopesResponse := client.ListScopes(PLUGIN_NAME, params.connection.ID,
true)
require.Equal(t, 1, len(scopesResponse))
require.Equal(t, 1, len(scopesResponse[0].Blueprints))
- bps := client.DeleteScope(PLUGIN_NAME, params.connection.ID,
params.scope.Id, false)
- require.Equal(t, 1, len(bps))
+ client.DeleteScope(PLUGIN_NAME, params.connection.ID, params.scope.Id,
false)
scopesResponse = client.ListScopes(PLUGIN_NAME, params.connection.ID,
true)
require.Equal(t, 0, len(scopesResponse))
bpsResult := client.ListBlueprints()
diff --git a/backend/test/helper/api.go b/backend/test/helper/api.go
index b31926194..f1a859edf 100644
--- a/backend/test/helper/api.go
+++ b/backend/test/helper/api.go
@@ -194,8 +194,8 @@ func (d *DevlakeClient) GetScope(pluginName string,
connectionId uint64, scopeId
}, http.MethodGet,
fmt.Sprintf("%s/plugins/%s/connections/%d/scopes/%s?blueprints=%v", d.Endpoint,
pluginName, connectionId, scopeId, listBlueprints), nil, nil)
}
-func (d *DevlakeClient) DeleteScope(pluginName string, connectionId uint64,
scopeId string, deleteDataOnly bool) []models.Blueprint {
- return sendHttpRequest[[]models.Blueprint](d.testCtx, d.timeout,
debugInfo{
+func (d *DevlakeClient) DeleteScope(pluginName string, connectionId uint64,
scopeId string, deleteDataOnly bool) {
+ sendHttpRequest[any](d.testCtx, d.timeout, debugInfo{
print: true,
inlineJson: false,
}, http.MethodDelete,
fmt.Sprintf("%s/plugins/%s/connections/%d/scopes/%s?delete_data_only=%v",
d.Endpoint, pluginName, connectionId, scopeId, deleteDataOnly), nil, nil)
diff --git a/backend/test/helper/models.go b/backend/test/helper/models.go
index a362f4ce7..a95bfb263 100644
--- a/backend/test/helper/models.go
+++ b/backend/test/helper/models.go
@@ -38,9 +38,9 @@ type (
}
ScopeResponse struct {
- Scope any
- TransformationRuleName string
- Blueprints []*models.Blueprint
+ Scope any
+ ScopeConfig any
+ Blueprints []*models.Blueprint
}
)
