This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch release-v0.19
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/release-v0.19 by this push:
new 306814dc7 cherry-pick(0.19): implement ApiClient reuse in
RemoteApiHelper (#6069)
306814dc7 is described below
commit 306814dc726a8af0ee64a68bfc5b6411621b48be
Author: Leric Zhang <[email protected]>
AuthorDate: Wed Sep 13 12:47:46 2023 +0800
cherry-pick(0.19): implement ApiClient reuse in RemoteApiHelper (#6069)
* feat: try implement common GetHash in BaseConnection (#5993) (#6025)
* fix(gitlab): _raw_gitlab_api_pipeline_details table data not collected
(#6027) (#6032)
---
backend/core/plugin/plugin_connection_abstract.go | 8 ++-
backend/helpers/pluginhelper/api/connection.go | 5 ++
.../helpers/pluginhelper/api/remote_api_helper.go | 70 +++++++++++++++++++---
backend/plugins/gitlab/api/proxy.go | 27 +--------
backend/plugins/gitlab/tasks/pipeline_extractor.go | 2 +-
backend/plugins/sonarqube/models/connection.go | 16 ++---
6 files changed, 83 insertions(+), 45 deletions(-)
diff --git a/backend/core/plugin/plugin_connection_abstract.go
b/backend/core/plugin/plugin_connection_abstract.go
index 6a9b648c8..098dfb1fc 100644
--- a/backend/core/plugin/plugin_connection_abstract.go
+++ b/backend/core/plugin/plugin_connection_abstract.go
@@ -18,10 +18,11 @@ limitations under the License.
package plugin
import (
+ "net/http"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api/apihelperabstract"
"github.com/go-playground/validator/v10"
- "net/http"
)
// ApiConnection represents a API Connection
@@ -31,6 +32,11 @@ type ApiConnection interface {
GetRateLimitPerHour() int
}
+type CacheableConnection interface {
+ ApiConnection
+ GetHash() string
+}
+
// ApiAuthenticator is to be implemented by a Concreate Connection if
Authorization is required
type ApiAuthenticator interface {
// SetupAuthentication is a hook function for connection to set up
authentication for the HTTP request
diff --git a/backend/helpers/pluginhelper/api/connection.go
b/backend/helpers/pluginhelper/api/connection.go
index b39d3a1d6..a4877d14e 100644
--- a/backend/helpers/pluginhelper/api/connection.go
+++ b/backend/helpers/pluginhelper/api/connection.go
@@ -18,6 +18,7 @@ limitations under the License.
package api
import (
+ "fmt"
"strings"
"github.com/apache/incubator-devlake/core/models/common"
@@ -29,6 +30,10 @@ type BaseConnection struct {
common.Model
}
+func (c BaseConnection) GetHash() string {
+ return fmt.Sprintf("%d%v", c.ID, c.UpdatedAt)
+}
+
// RestConnection implements the ApiConnection interface
type RestConnection struct {
Endpoint string `mapstructure:"endpoint" validate:"required"
json:"endpoint"`
diff --git a/backend/helpers/pluginhelper/api/remote_api_helper.go
b/backend/helpers/pluginhelper/api/remote_api_helper.go
index 76a0f540c..80d3eeedb 100644
--- a/backend/helpers/pluginhelper/api/remote_api_helper.go
+++ b/backend/helpers/pluginhelper/api/remote_api_helper.go
@@ -18,13 +18,17 @@ limitations under the License.
package api
import (
+ gocontext "context"
"encoding/base64"
"encoding/json"
"fmt"
- "github.com/apache/incubator-devlake/core/log"
+ "io"
"net/http"
+ "net/url"
"strconv"
+ "github.com/apache/incubator-devlake/core/log"
+
"github.com/apache/incubator-devlake/core/context"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
@@ -65,10 +69,11 @@ type SearchRemoteScopesOutput struct {
// RemoteApiHelper is used to write the CURD of connection
type RemoteApiHelper[Conn plugin.ApiConnection, Scope plugin.ToolLayerScope,
ApiScope plugin.ApiScope, Group plugin.ApiGroup] struct {
- basicRes context.BasicRes
- validator *validator.Validate
- connHelper *ConnectionApiHelper
- logger log.Logger
+ basicRes context.BasicRes
+ validator *validator.Validate
+ connHelper *ConnectionApiHelper
+ httpClientCache map[string]*ApiClient
+ logger log.Logger
}
// NewRemoteHelper creates a ScopeHelper for connection management
@@ -84,10 +89,11 @@ func NewRemoteHelper[Conn plugin.ApiConnection, Scope
plugin.ToolLayerScope, Api
return nil
}
return &RemoteApiHelper[Conn, Scope, ApiScope, Group]{
- basicRes: basicRes,
- validator: vld,
- connHelper: connHelper,
- logger: basicRes.GetLogger(),
+ basicRes: basicRes,
+ validator: vld,
+ connHelper: connHelper,
+ httpClientCache: make(map[string]*ApiClient),
+ logger: basicRes.GetLogger(),
}
}
@@ -122,6 +128,52 @@ const (
TypeMixed string = "mixed"
)
+func (r *RemoteApiHelper[Conn, Scope, ApiScope, Group])
GetApiClient(connection plugin.CacheableConnection) (*ApiClient, errors.Error) {
+ key := connection.GetHash()
+ // empty key means no connection reuse
+ if key == "" {
+ r.logger.Info("No api client reuse")
+ return NewApiClientFromConnection(gocontext.TODO(), r.basicRes,
connection)
+ }
+
+ if client, ok := r.httpClientCache[key]; ok {
+ r.logger.Info("Reused api client")
+ return client, nil
+ }
+ r.logger.Info("Creating new api client")
+ newClient, err := NewApiClientFromConnection(gocontext.TODO(),
r.basicRes, connection)
+ if err != nil {
+ return nil, err
+ }
+ r.httpClientCache[key] = newClient
+ return newClient, nil
+}
+
+func (r *RemoteApiHelper[Conn, Scope, ApiScope, Group]) ProxyApiGet(conn
plugin.CacheableConnection, path string, query url.Values)
(*plugin.ApiResourceOutput, errors.Error) {
+ apiClient, err := r.GetApiClient(conn)
+ if err != nil {
+ return nil, err
+ }
+
+ resp, err := apiClient.Get(path, query, nil)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+
+ body, err := errors.Convert01(io.ReadAll(resp.Body))
+ if err != nil {
+ return nil, err
+ }
+ // verify response body is json
+ var tmp interface{}
+ err = errors.Convert(json.Unmarshal(body, &tmp))
+ if err != nil {
+ return nil, err
+ }
+ return &plugin.ApiResourceOutput{Status: resp.StatusCode, Body:
json.RawMessage(body)}, nil
+}
+
// PrepareFirstPageToken prepares the first page token
func (r *RemoteApiHelper[Conn, Scope, ApiScope, Group])
PrepareFirstPageToken(customInfo string) (*plugin.ApiResourceOutput,
errors.Error) {
outputBody := &FirstPageTokenOutput{}
diff --git a/backend/plugins/gitlab/api/proxy.go
b/backend/plugins/gitlab/api/proxy.go
index f14209c81..f9ee83910 100644
--- a/backend/plugins/gitlab/api/proxy.go
+++ b/backend/plugins/gitlab/api/proxy.go
@@ -18,14 +18,10 @@ limitations under the License.
package api
import (
- "context"
- "encoding/json"
- "io"
"time"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
)
@@ -39,26 +35,5 @@ func Proxy(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Er
if err != nil {
return nil, err
}
- apiClient, err := helper.NewApiClientFromConnection(context.TODO(),
basicRes, connection)
- if err != nil {
- return nil, err
- }
-
- resp, err := apiClient.Get(input.Params["path"], input.Query, nil)
- if err != nil {
- return nil, err
- }
- defer resp.Body.Close()
-
- body, err := errors.Convert01(io.ReadAll(resp.Body))
- if err != nil {
- return nil, err
- }
- // verify response body is json
- var tmp interface{}
- err = errors.Convert(json.Unmarshal(body, &tmp))
- if err != nil {
- return nil, err
- }
- return &plugin.ApiResourceOutput{Status: resp.StatusCode, Body:
json.RawMessage(body)}, nil
+ return remoteHelper.ProxyApiGet(connection, input.Params["path"],
input.Query)
}
diff --git a/backend/plugins/gitlab/tasks/pipeline_extractor.go
b/backend/plugins/gitlab/tasks/pipeline_extractor.go
index c18eea0e3..c07ee745d 100644
--- a/backend/plugins/gitlab/tasks/pipeline_extractor.go
+++ b/backend/plugins/gitlab/tasks/pipeline_extractor.go
@@ -106,7 +106,7 @@ func ExtractApiPipelines(taskCtx plugin.SubTaskContext)
errors.Error {
IsDetailRequired: false,
}
- if gitlabApiPipeline.CreatedAt == nil &&
gitlabApiPipeline.UpdatedAt == nil {
+ if gitlabApiPipeline.StartedAt == nil &&
gitlabApiPipeline.FinishedAt == nil {
gitlabPipeline.IsDetailRequired = true
}
diff --git a/backend/plugins/sonarqube/models/connection.go
b/backend/plugins/sonarqube/models/connection.go
index 3309d5d28..3b8cd3125 100644
--- a/backend/plugins/sonarqube/models/connection.go
+++ b/backend/plugins/sonarqube/models/connection.go
@@ -20,10 +20,11 @@ package models
import (
"encoding/base64"
"fmt"
+ "net/http"
+
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
- "net/http"
)
type SonarqubeAccessToken helper.AccessToken
@@ -43,19 +44,18 @@ func (sat SonarqubeAccessToken) GetEncodedToken() string {
return base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%v:",
sat.Token)))
}
-// This object conforms to what the frontend currently sends.
-type SonarqubeConnection struct {
- helper.BaseConnection `mapstructure:",squash"`
- helper.RestConnection `mapstructure:",squash"`
- SonarqubeAccessToken `mapstructure:",squash"`
-}
-
// SonarqubeConn holds the essential information to connect to the sonarqube
API
type SonarqubeConn struct {
helper.RestConnection `mapstructure:",squash"`
SonarqubeAccessToken `mapstructure:",squash"`
}
+// This object conforms to what the frontend currently sends.
+type SonarqubeConnection struct {
+ helper.BaseConnection `mapstructure:",squash"`
+ SonarqubeConn `mapstructure:",squash"`
+}
+
// This object conforms to what the frontend currently expects.
type SonarqubeResponse struct {
Name string `json:"name"`
