This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 55e6835f8 fix: scope list was in random order (#5639)
55e6835f8 is described below
commit 55e6835f87a752d1a182350fb0b1a6c34ff41c5f
Author: Klesh Wong <[email protected]>
AuthorDate: Thu Jul 6 15:04:13 2023 +0800
fix: scope list was in random order (#5639)
* fix: scripts/compile-plugins.sh: 61: [: Linux: unexpected operato
* fix: scope list was in random order
---
.../pluginhelper/api/scope_generic_helper.go | 40 ++++++++--------------
backend/scripts/compile-plugins.sh | 12 +++----
2 files changed, 20 insertions(+), 32 deletions(-)
diff --git a/backend/helpers/pluginhelper/api/scope_generic_helper.go
b/backend/helpers/pluginhelper/api/scope_generic_helper.go
index 61554d66c..795ed7e61 100644
--- a/backend/helpers/pluginhelper/api/scope_generic_helper.go
+++ b/backend/helpers/pluginhelper/api/scope_generic_helper.go
@@ -62,9 +62,9 @@ type (
Blueprints []*models.Blueprint
`mapstructure:"blueprints,omitempty" json:"blueprints"`
}
// Alias, for swagger purposes
- ScopeRefDoc =
serviceHelper.BlueprintProjectPairs
- ScopeRes[Scope any, ScopeConfig any] struct {
- Scope *Scope
`mapstructure:",squash"` // ideally we need this field to be embedded in the
struct
+ ScopeRefDoc =
serviceHelper.BlueprintProjectPairs
+ ScopeRes[Scope plugin.ToolLayerScope, ScopeConfig any] struct {
+ Scope Scope
`mapstructure:",squash"` // ideally we need this field to be embedded in the
struct
ScopeResDoc[ScopeConfig] `mapstructure:",squash"` // however,
only this type of embeding is supported as of golang 1.20
}
ReflectionParameters struct {
@@ -242,22 +242,19 @@ func (gs *GenericScopeApiHelper[Conn, Scope,
ScopeConfig]) GetScopes(input *plug
}
// return empty array rather than nil in case of no scopes
if len(apiScopes) > 0 && params.loadBlueprints {
- scopesById := gs.mapByScopeId(apiScopes)
+ // fetch blueprints for all scopes in one call since all bps
must be loaded to determine which ones are associated with the scopes
+ // TODO: split bp.settings into separate tables and load only
the ones needed
var scopeIds []string
- for id := range scopesById {
- scopeIds = append(scopeIds, id)
+ for _, apiScope := range apiScopes {
+ // scopeId := fmt.Sprintf("%v",
reflectField(apiScope.Scope, gs.reflectionParams.ScopeIdFieldName).Interface())
+ scopeIds = append(scopeIds, apiScope.Scope.ScopeId())
}
- blueprintMap, err :=
gs.bpManager.GetBlueprintsByScopes(params.connectionId, params.plugin,
scopeIds...)
- if err != nil {
- return nil, errors.Default.Wrap(err, fmt.Sprintf("error
getting blueprints for scopes from connection %d", params.connectionId))
- }
- apiScopes = nil
- for scopeId, scope := range scopesById {
- if bps, ok := blueprintMap[scopeId]; ok {
- scope.Blueprints = bps
- delete(blueprintMap, scopeId)
+ blueprintMap :=
errors.Must1(gs.bpManager.GetBlueprintsByScopes(params.connectionId,
params.plugin, scopeIds...))
+ for _, apiScope := range apiScopes {
+ if bps, ok := blueprintMap[apiScope.Scope.ScopeId()];
ok {
+ apiScope.Blueprints = bps
+ delete(blueprintMap, apiScope.Scope.ScopeId())
}
- apiScopes = append(apiScopes, scope)
}
if len(blueprintMap) > 0 {
var danglingIds []string
@@ -361,7 +358,7 @@ func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig])
addScopeConfig(scopes
apiScopes := make([]*ScopeRes[Scope, ScopeConfig], len(scopes))
for i, scope := range scopes {
apiScopes[i] = &ScopeRes[Scope, ScopeConfig]{
- Scope: scope,
+ Scope: *scope,
}
scIdField := reflectField(scope, "ScopeConfigId")
if scIdField.IsValid() && scIdField.Uint() > 0 {
@@ -387,15 +384,6 @@ func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig])
getScopeReferences(co
return serviceHelper.NewBlueprintProjectPairs(blueprints), nil
}
-func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig]) mapByScopeId(scopes
[]*ScopeRes[Scope, ScopeConfig]) map[string]*ScopeRes[Scope, ScopeConfig] {
- scopeMap := map[string]*ScopeRes[Scope, ScopeConfig]{}
- for _, scope := range scopes {
- scopeId := fmt.Sprintf("%v", reflectField(scope.Scope,
gs.reflectionParams.ScopeIdFieldName).Interface())
- scopeMap[scopeId] = scope
- }
- return scopeMap
-}
-
func (gs *GenericScopeApiHelper[Conn, Scope, ScopeConfig])
extractFromReqParam(input *plugin.ApiResourceInput, withScopeId bool)
(*requestParams, errors.Error) {
connectionId, err := strconv.ParseUint(input.Params["connectionId"],
10, 64)
if err != nil {
diff --git a/backend/scripts/compile-plugins.sh
b/backend/scripts/compile-plugins.sh
index f61645aac..9b4d7fd4f 100644
--- a/backend/scripts/compile-plugins.sh
+++ b/backend/scripts/compile-plugins.sh
@@ -57,13 +57,13 @@ for PLUG in $PLUGINS; do
PIDS="$PIDS $!"
# avoid too many processes causing signal killed
COUNT=$(echo "$PIDS" | wc -w)
- THREADS=1
- if [ "$(uname)" == "Darwin" ]; then
- THREADS=$(sysctl -n hw.physicalcpu)
- elif [ "$(expr substr $(uname -s) 1 5)" == "Linux" ]; then
- THREADS=$(nproc)
+ PARALLELISM=4
+ if command -v nproc >/dev/null 2>&1; then
+ PARALLELISM=$(nproc)
+ elif command -v sysctl >/dev/null 2>&1; then
+ PARALLELISM=$(sysctl -n hw.ncpu)
fi
- if [ "$COUNT" -ge "$THREADS" ]; then
+ if [ "$COUNT" -ge "$PARALLELISM" ]; then
for PID in $PIDS; do
wait $PID
done
