This is an automated email from the ASF dual-hosted git repository.
warren pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 6a6e86685 refactor(framework): add scopeHelper (#4632)
6a6e86685 is described below
commit 6a6e86685c5732dafa9e05e91209e32009092209
Author: Warren Chen <[email protected]>
AuthorDate: Tue Mar 14 14:56:01 2023 +0800
refactor(framework): add scopeHelper (#4632)
* refactor(framework): add scopeHelper
* feat(framework): finish scope helper
* test(framework): add unit test for put
---
backend/helpers/pluginhelper/api/scope_helper.go | 346 +++++++++++++++++++++
.../helpers/pluginhelper/api/scope_helper_test.go | 269 ++++++++++++++++
backend/plugins/github/api/init.go | 6 +
backend/plugins/github/api/scope.go | 123 +-------
backend/plugins/github/impl/impl.go | 2 +-
backend/plugins/gitlab/api/init.go | 12 +-
backend/plugins/gitlab/api/scope.go | 39 +--
7 files changed, 651 insertions(+), 146 deletions(-)
diff --git a/backend/helpers/pluginhelper/api/scope_helper.go
b/backend/helpers/pluginhelper/api/scope_helper.go
new file mode 100644
index 000000000..0901f69d8
--- /dev/null
+++ b/backend/helpers/pluginhelper/api/scope_helper.go
@@ -0,0 +1,346 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "fmt"
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/dal"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/log"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/go-playground/validator/v10"
+ "github.com/mitchellh/mapstructure"
+ "gorm.io/gorm"
+ "strconv"
+ "strings"
+ "time"
+
+ "reflect"
+)
+
+// ScopeApiHelper is used to write the CURD of connection
+type ScopeApiHelper struct {
+ log log.Logger
+ db dal.Dal
+ validator *validator.Validate
+ connHelper *ConnectionApiHelper
+}
+
+// NewScopeHelper creates a ScopeHelper for connection management
+func NewScopeHelper(
+ basicRes context.BasicRes,
+ vld *validator.Validate,
+ connHelper *ConnectionApiHelper,
+) *ScopeApiHelper {
+ if vld == nil {
+ vld = validator.New()
+ }
+ if connHelper == nil {
+ return nil
+ }
+ return &ScopeApiHelper{
+ log: basicRes.GetLogger(),
+ db: basicRes.GetDal(),
+ validator: vld,
+ connHelper: connHelper,
+ }
+}
+
+// Put saves the given scopes to the database. It expects a slice of struct
pointers
+// as the scopes argument. It also expects a fieldName argument, which is used
to extract
+// the connection ID from the input.Params map.
+func (c *ScopeApiHelper) Put(input *plugin.ApiResourceInput, apiScope
interface{}, connection interface{}) errors.Error {
+ err := errors.Convert(mapstructure.Decode(input.Body, apiScope))
+ if err != nil {
+ return errors.BadInput.Wrap(err, "decoding Github repo error")
+ }
+ // Ensure that the scopes argument is a slice
+ v := reflect.ValueOf(apiScope)
+ scopesValue := v.Elem().FieldByName("Data")
+ if scopesValue.Kind() != reflect.Slice {
+ panic("expected a slice")
+ }
+ // Extract the connection ID from the input.Params map
+ connectionId, _ := ExtractParam(input.Params)
+ if connectionId == 0 {
+ return errors.BadInput.New("invalid connectionId or scopeId")
+ }
+ err = c.VerifyConnection(connection, connectionId)
+ if err != nil {
+ return err
+ }
+ // Create a map to keep track of primary key values
+ keeper := make(map[string]struct{})
+
+ // Set the CreatedDate and UpdatedDate fields to the current time for
each scope
+ now := time.Now()
+ for i := 0; i < scopesValue.Len(); i++ {
+ // Get the reflect.Value of the i-th struct pointer in the slice
+ structValue := scopesValue.Index(i)
+
+ // Ensure that the structValue is a pointer to a struct
+ if structValue.Kind() != reflect.Ptr ||
structValue.Elem().Kind() != reflect.Struct {
+ panic("expected a pointer to a struct")
+ }
+
+ // Ensure that the primary key value is unique
+ primaryValueStr :=
ReturnPrimaryKeyValue(structValue.Elem().Interface())
+ if _, ok := keeper[primaryValueStr]; ok {
+ return errors.BadInput.New("duplicated item")
+ } else {
+ keeper[primaryValueStr] = struct{}{}
+ }
+
+ // Set the connection ID, CreatedDate, and UpdatedDate fields
+ SetScopeFields(structValue.Interface(), connectionId, &now,
&now)
+
+ // Verify that the primary key value is valid
+ err = VerifyPrimaryKeyValue(structValue.Elem().Interface())
+ if err != nil {
+ return err
+ }
+ }
+
+ // Save the scopes to the database
+ return c.save(scopesValue.Interface(), c.db.Create)
+}
+
+func (c *ScopeApiHelper) Update(input *plugin.ApiResourceInput, fieldName
string, connection interface{}, scope interface{}) errors.Error {
+ connectionId, scopeId := ExtractParam(input.Params)
+
+ if connectionId == 0 || len(scopeId) == 0 || scopeId == "0" {
+ return errors.BadInput.New("invalid connectionId")
+ }
+ err := c.VerifyConnection(connection, connectionId)
+ if err != nil {
+ return err
+ }
+
+ err = c.db.First(scope, dal.Where(fmt.Sprintf("connection_id = ? AND %s
= ?", fieldName), connectionId, scopeId))
+ if err != nil {
+ return errors.Default.New("getting Scope error")
+ }
+ err = DecodeMapStruct(input.Body, scope)
+ if err != nil {
+ return errors.Default.Wrap(err, "patch scope error")
+ }
+ err = VerifyPrimaryKeyValue(scope)
+ if err != nil {
+ return err
+ }
+ err = c.db.Update(scope)
+ if err != nil {
+ return errors.Default.Wrap(err, "error on saving Scope")
+ }
+ return nil
+}
+
+func (c *ScopeApiHelper) GetScopeList(input *plugin.ApiResourceInput,
connection interface{}, scopes interface{}, rules interface{})
(map[uint64]string, errors.Error) {
+ connectionId, _ := ExtractParam(input.Params)
+ if connectionId == 0 {
+ return nil, errors.BadInput.New("invalid path params")
+ }
+ err := c.VerifyConnection(connection, connectionId)
+ if err != nil {
+ return nil, err
+ }
+ limit, offset := GetLimitOffset(input.Query, "pageSize", "page")
+ err = c.db.All(scopes, dal.Where("connection_id = ?", connectionId),
dal.Limit(limit), dal.Offset(offset))
+ if err != nil {
+ return nil, err
+ }
+
+ scopesValue :=
reflect.ValueOf(reflect.ValueOf(scopes).Elem().Interface())
+ if scopesValue.Kind() != reflect.Slice {
+ panic("expected a slice")
+ }
+ var ruleIds []uint64
+ for i := 0; i < scopesValue.Len(); i++ {
+ // Get the reflect.Value of the i-th struct pointer in the slice
+ structValue := scopesValue.Index(i)
+
+ // Ensure that the structValue is a pointer to a struct
+ if structValue.Kind() != reflect.Ptr ||
structValue.Elem().Kind() != reflect.Struct {
+ panic("expected a pointer to a struct")
+ }
+ ruleId :=
structValue.Elem().FieldByName("TransformationRuleId").Uint()
+ if ruleId > 0 {
+ ruleIds = append(ruleIds, ruleId)
+ }
+ }
+
+ if len(ruleIds) > 0 {
+ err = c.db.All(rules, dal.Where("id IN (?)", ruleIds))
+ if err != nil {
+ return nil, err
+ }
+ }
+ rulesValue := reflect.ValueOf(reflect.ValueOf(rules).Elem().Interface())
+ if scopesValue.Kind() != reflect.Slice {
+ panic("expected a slice")
+ }
+ names := make(map[uint64]string)
+ for i := 0; i < rulesValue.Len(); i++ {
+ // Get the reflect.Value of the i-th struct pointer in the slice
+ structValue := rulesValue.Index(i)
+ names[structValue.FieldByName("ID").Uint()] =
structValue.FieldByName("Name").String()
+ }
+ return names, nil
+}
+
+func (c *ScopeApiHelper) GetScope(input *plugin.ApiResourceInput, fieldName
string, connection interface{}, scope interface{}, rule interface{})
errors.Error {
+ connectionId, scopeId := ExtractParam(input.Params)
+ if connectionId == 0 || len(scopeId) == 0 || scopeId == "0" {
+ return errors.BadInput.New("invalid path params")
+ }
+ err := c.VerifyConnection(connection, connectionId)
+ if err != nil {
+ return err
+ }
+ db := c.db
+ query := dal.Where(fmt.Sprintf("connection_id = ? AND %s = ?",
fieldName), connectionId, scopeId)
+ err = db.First(scope, query)
+ if db.IsErrorNotFound(err) {
+ return errors.NotFound.New("Scope not found")
+ }
+ if err != nil {
+ return err
+ }
+ repoRuleId :=
reflect.ValueOf(scope).Elem().FieldByName("TransformationRuleId").Uint()
+ if repoRuleId > 0 {
+ err = db.First(rule, dal.Where("id = ?", repoRuleId))
+ if err != nil {
+ return errors.NotFound.New("transformationRule not
found")
+ }
+ }
+ return nil
+}
+
+func (c *ScopeApiHelper) VerifyConnection(connection interface{}, connId
uint64) errors.Error {
+ err := c.connHelper.FirstById(&connection, connId)
+ if err != nil {
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ return errors.BadInput.New("Invalid Connection Id")
+ }
+ return err
+ }
+ return nil
+}
+
+func (c *ScopeApiHelper) save(scope interface{}, method func(entity
interface{}, clauses ...dal.Clause) errors.Error) errors.Error {
+ err := c.db.CreateOrUpdate(scope)
+ if err != nil {
+ if c.db.IsDuplicationError(err) {
+ return errors.BadInput.New("the scope already exists")
+ }
+ return err
+ }
+ return nil
+}
+
+func ExtractParam(params map[string]string) (uint64, string) {
+ connectionId, err := strconv.ParseUint(params["connectionId"], 10, 64)
+ if err != nil {
+ return 0, ""
+ }
+ scopeId := params["scopeId"]
+ return connectionId, scopeId
+}
+
+// VerifyPrimaryKeyValue function verifies that the primary key value of a
given struct instance is not zero or empty.
+func VerifyPrimaryKeyValue(i interface{}) errors.Error {
+ var value reflect.Value
+ pType := reflect.TypeOf(i)
+ if pType.Kind() == reflect.Ptr {
+ value = reflect.ValueOf(reflect.ValueOf(i).Elem().Interface())
+ } else {
+ value = reflect.ValueOf(i)
+ }
+ // Loop through the fields of the input struct using reflection
+ for j := 0; j < value.NumField(); j++ {
+ field := value.Field(j)
+ tag := value.Type().Field(j).Tag.Get("gorm")
+
+ // Check if the field is tagged as a primary key using the GORM
tag "primaryKey"
+ if strings.Contains(tag, "primaryKey") {
+ // If the field value is zero or nil, return an error
indicating that the primary key value is invalid
+ if field.Interface() ==
reflect.Zero(field.Type()).Interface() || field.Interface() == nil {
+ return errors.Default.New("primary key value is
zero or empty")
+ }
+ }
+ }
+ // If all primary key values are valid, return nil (no error)
+ return nil
+}
+
+func SetScopeFields(p interface{}, connectionId uint64, createdDate
*time.Time, updatedDate *time.Time) {
+ pType := reflect.TypeOf(p)
+ if pType.Kind() != reflect.Ptr {
+ panic("expected a pointer to a struct")
+ }
+ pValue := reflect.ValueOf(p).Elem()
+
+ // set connectionId
+ connIdField := pValue.FieldByName("ConnectionId")
+ connIdField.SetUint(connectionId)
+
+ // set CreatedDate
+ createdDateField := pValue.FieldByName("CreatedDate")
+ createdDateField.Set(reflect.ValueOf(createdDate))
+
+ // set UpdatedDate
+ updatedDateField := pValue.FieldByName("UpdatedDate")
+ if updatedDate == nil {
+ // if updatedDate is nil, set UpdatedDate to be nil
+ updatedDateField.Set(reflect.Zero(updatedDateField.Type()))
+ } else {
+ // if updatedDate is not nil, set UpdatedDate to be the value
+ updatedDateFieldValue := reflect.ValueOf(updatedDate)
+ updatedDateField.Set(updatedDateFieldValue)
+ }
+}
+
+// ReturnPrimaryKeyValue returns a string containing the primary key value(s)
of a struct, concatenated with "-" between them.
+// This function receives an interface{} type argument p, which can be a
pointer to any struct.
+// The function uses reflection to iterate through the fields of the struct,
and checks if each field is tagged as "primaryKey".
+func ReturnPrimaryKeyValue(p interface{}) string {
+ result := ""
+ // get the type and value of the input interface using reflection
+ t := reflect.TypeOf(p)
+ v := reflect.ValueOf(p)
+ // iterate over each field in the struct type
+ for i := 0; i < t.NumField(); i++ {
+ // get the i-th field
+ field := t.Field(i)
+
+ // check if the field is marked as "primaryKey" in the struct
tag
+ if strings.Contains(string(field.Tag), "primaryKey") {
+ // if this is the first primaryKey field encountered,
set the result to be its value
+ if result == "" {
+ result = fmt.Sprintf("%v",
v.Field(i).Interface())
+ } else {
+ // if this is not the first primaryKey field,
append its value to the result with a "-" separator
+ result = fmt.Sprintf("%s-%v", result,
v.Field(i).Interface())
+ }
+ }
+ }
+
+ // return the final primary key value as a string
+ return result
+}
diff --git a/backend/helpers/pluginhelper/api/scope_helper_test.go
b/backend/helpers/pluginhelper/api/scope_helper_test.go
new file mode 100644
index 000000000..ad29684d9
--- /dev/null
+++ b/backend/helpers/pluginhelper/api/scope_helper_test.go
@@ -0,0 +1,269 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "github.com/apache/incubator-devlake/core/models/common"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/unithelper"
+ mockcontext "github.com/apache/incubator-devlake/mocks/core/context"
+ mockdal "github.com/apache/incubator-devlake/mocks/core/dal"
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/mock"
+ "gorm.io/gorm"
+ "reflect"
+ "testing"
+ "time"
+)
+
+type TestModel struct {
+ ID uint `gorm:"primaryKey"`
+ Name string `gorm:"primaryKey;type:BIGINT NOT NULL"`
+}
+
+type GithubRepo struct {
+ ConnectionId uint64 `json:"connectionId" gorm:"primaryKey"
mapstructure:"connectionId,omitempty"`
+ GithubId int `json:"githubId" gorm:"primaryKey"
mapstructure:"githubId"`
+ Name string `json:"name" gorm:"type:varchar(255)"
mapstructure:"name,omitempty"`
+ HTMLUrl string `json:"HTMLUrl"
gorm:"type:varchar(255)" mapstructure:"HTMLUrl,omitempty"`
+ Description string `json:"description"
mapstructure:"description,omitempty"`
+ TransformationRuleId uint64 `json:"transformationRuleId,omitempty"
mapstructure:"transformationRuleId,omitempty"`
+ OwnerId int `json:"ownerId"
mapstructure:"ownerId,omitempty"`
+ Language string `json:"language"
gorm:"type:varchar(255)" mapstructure:"language,omitempty"`
+ ParentGithubId int `json:"parentId"
mapstructure:"parentGithubId,omitempty"`
+ ParentHTMLUrl string `json:"parentHtmlUrl"
mapstructure:"parentHtmlUrl,omitempty"`
+ CloneUrl string `json:"cloneUrl"
gorm:"type:varchar(255)" mapstructure:"cloneUrl,omitempty"`
+ CreatedDate *time.Time `json:"createdDate" mapstructure:"-"`
+ UpdatedDate *time.Time `json:"updatedDate" mapstructure:"-"`
+ common.NoPKModel `json:"-" mapstructure:"-"`
+}
+
+func (GithubRepo) TableName() string {
+ return "_tool_github_repos"
+}
+
+type GithubConnection struct {
+ common.Model
+ Name string `gorm:"type:varchar(100);uniqueIndex"
json:"name" validate:"required"`
+ Endpoint string `mapstructure:"endpoint" env:"GITHUB_ENDPOINT"
validate:"required"`
+ Proxy string `mapstructure:"proxy" env:"GITHUB_PROXY"`
+ RateLimitPerHour int `comment:"api request rate limit per hour"`
+ Token string `mapstructure:"token" env:"GITHUB_AUTH"
validate:"required" encrypt:"yes"`
+}
+
+func (GithubConnection) TableName() string {
+ return "_tool_github_connections"
+}
+
+type req struct {
+ Data []*GithubRepo `json:"data"`
+}
+
+func TestCheckPrimaryKeyValue(t *testing.T) {
+ testCases := []struct {
+ name string
+ model TestModel
+ wantErr bool
+ }{
+ {
+ name: "valid case",
+ model: TestModel{
+ ID: 1,
+ Name: "test name",
+ },
+ wantErr: false,
+ },
+ {
+ name: "zero value",
+ model: TestModel{
+ ID: 0,
+ Name: "test name",
+ },
+ wantErr: true,
+ },
+ {
+ name: "nil value",
+ model: TestModel{
+ ID: 1,
+ Name: "",
+ },
+ wantErr: true,
+ },
+ }
+
+ for _, tc := range testCases {
+ err := VerifyPrimaryKeyValue(tc.model)
+ if (err != nil) != tc.wantErr {
+ t.Errorf("unexpected error value - got: %v, want: %v",
err, tc.wantErr)
+ }
+
+ }
+}
+
+func TestSetGitlabProjectFields(t *testing.T) {
+ // create a struct
+ var p struct {
+ ConnectionId uint64 `json:"connectionId"
mapstructure:"connectionId" gorm:"primaryKey"`
+ GitlabId int `json:"gitlabId" mapstructure:"gitlabId"
gorm:"primaryKey"`
+
+ CreatedDate *time.Time `json:"createdDate"
mapstructure:"-"`
+ UpdatedDate *time.Time `json:"updatedDate"
mapstructure:"-"`
+ common.NoPKModel `json:"-" mapstructure:"-"`
+ }
+
+ // call SetScopeFields to assign value
+ connectionId := uint64(123)
+ createdDate := time.Now()
+ updatedDate := &createdDate
+ SetScopeFields(&p, connectionId, &createdDate, updatedDate)
+
+ // verify fields
+ if p.ConnectionId != connectionId {
+ t.Errorf("ConnectionId not set correctly, expected: %v, got:
%v", connectionId, p.ConnectionId)
+ }
+
+ if !p.CreatedDate.Equal(createdDate) {
+ t.Errorf("CreatedDate not set correctly, expected: %v, got:
%v", createdDate, p.CreatedDate)
+ }
+
+ if p.UpdatedDate == nil {
+ t.Errorf("UpdatedDate not set correctly, expected: %v, got:
%v", updatedDate, p.UpdatedDate)
+ } else if !p.UpdatedDate.Equal(*updatedDate) {
+ t.Errorf("UpdatedDate not set correctly, expected: %v, got:
%v", updatedDate, p.UpdatedDate)
+ }
+
+ SetScopeFields(&p, connectionId, &createdDate, nil)
+
+ // verify fields
+ if p.ConnectionId != connectionId {
+ t.Errorf("ConnectionId not set correctly, expected: %v, got:
%v", connectionId, p.ConnectionId)
+ }
+
+ if !p.CreatedDate.Equal(createdDate) {
+ t.Errorf("CreatedDate not set correctly, expected: %v, got:
%v", createdDate, p.CreatedDate)
+ }
+
+ if p.UpdatedDate != nil {
+ t.Errorf("UpdatedDate not set correctly, expected: %v, got:
%v", nil, p.UpdatedDate)
+ }
+}
+
+func TestReturnPrimaryKeyValue(t *testing.T) {
+ // Define a test struct with the primaryKey tag on one of its fields.
+ type TestStruct struct {
+ ConnectionId int `json:"connectionId" gorm:"primaryKey"`
+ Id int `json:"id" gorm:"primaryKey"`
+ Name string `json:"name"`
+ CreatedAt time.Time
+ UpdatedAt time.Time
+ DeletedAt gorm.DeletedAt `gorm:"index"`
+ }
+
+ // Create an instance of the test struct.
+ test := TestStruct{
+ ConnectionId: 1,
+ Id: 123,
+ Name: "Test",
+ CreatedAt: time.Now(),
+ }
+
+ // Call the function and check if it returns the correct primary key
value.
+ result := ReturnPrimaryKeyValue(test)
+ expected := "1-123"
+ if result != expected {
+ t.Errorf("ReturnPrimaryKeyValue returned %s, expected %s",
result, expected)
+ }
+
+ // Test with a different struct that has no field with primaryKey tag.
+ type TestStruct2 struct {
+ Id int `json:"id"`
+ Name string `json:"name"`
+ CreatedAt time.Time
+ UpdatedAt time.Time
+ }
+
+ test2 := TestStruct2{
+ Id: 456,
+ Name: "Test 2",
+ CreatedAt: time.Now(),
+ }
+
+ result2 := ReturnPrimaryKeyValue(test2)
+ expected2 := ""
+ if result2 != expected2 {
+ t.Errorf("ReturnPrimaryKeyValue returned %s, expected %s",
result2, expected2)
+ }
+}
+
+func TestScopeApiHelper_Put(t *testing.T) {
+ mockDal := new(mockdal.Dal)
+ mockLogger := unithelper.DummyLogger()
+ mockRes := new(mockcontext.BasicRes)
+
+ mockRes.On("GetDal").Return(mockDal)
+ mockRes.On("GetConfig", mock.Anything).Return("")
+ mockRes.On("GetLogger").Return(mockLogger)
+
+ // we expect total 2 deletion calls after all code got carried out
+ mockDal.On("Delete", mock.Anything, mock.Anything).Return(nil).Twice()
+ mockDal.On("GetPrimaryKeyFields", mock.Anything).Return(
+ []reflect.StructField{
+ {Name: "ID", Type: reflect.TypeOf("")},
+ },
+ )
+ mockDal.On("CreateOrUpdate", mock.Anything, mock.Anything).Return(nil)
+ mockDal.On("First", mock.Anything, mock.Anything).Return(nil)
+ mockDal.On("All", mock.Anything, mock.Anything).Return(nil)
+
+ connHelper := NewConnectionHelper(mockRes, nil)
+
+ // create a mock input, scopes, and connection
+ input := &plugin.ApiResourceInput{Params:
map[string]string{"connectionId": "123"}}
+ scopes := []*GithubRepo{
+ {GithubId: 1, Name: "scope1"},
+ {GithubId: 2, Name: "scope2"},
+ }
+ connection := &GithubConnection{}
+ connection.ID = 3
+ connection.Name = "test"
+
+ // create a mock ScopeApiHelper with a mock database connection
+ apiHelper := &ScopeApiHelper{db: mockDal, connHelper: connHelper}
+ apiReq := req{Data: scopes}
+ // test a successful call to Put
+ err := apiHelper.Put(input, &apiReq, connection)
+ assert.NoError(t, err)
+
+ // test a call to Put with a duplicate primary key value
+ duplicateScopes := []*GithubRepo{
+ {GithubId: 1, Name: "scope1"},
+ {GithubId: 1, Name: "scope2"},
+ }
+ apiReq.Data = duplicateScopes
+ err = apiHelper.Put(input, &apiReq, connection)
+ assert.Error(t, err)
+
+ // test a call to Put with an invalid primary key value
+ invalidScopes := []*GithubRepo{
+ {GithubId: 0, Name: "scope1"},
+ }
+ apiReq.Data = invalidScopes
+
+ err = apiHelper.Put(input, &apiReq, connection)
+ assert.Error(t, err)
+}
diff --git a/backend/plugins/github/api/init.go
b/backend/plugins/github/api/init.go
index d92c2b334..8f0dc3bad 100644
--- a/backend/plugins/github/api/init.go
+++ b/backend/plugins/github/api/init.go
@@ -25,6 +25,7 @@ import (
var vld *validator.Validate
var connectionHelper *api.ConnectionApiHelper
+var scopeHelper *api.ScopeApiHelper
var basicRes context.BasicRes
func Init(br context.BasicRes) {
@@ -34,4 +35,9 @@ func Init(br context.BasicRes) {
basicRes,
vld,
)
+ scopeHelper = api.NewScopeHelper(
+ basicRes,
+ vld,
+ connectionHelper,
+ )
}
diff --git a/backend/plugins/github/api/scope.go
b/backend/plugins/github/api/scope.go
index 75d819e5c..245d6eb39 100644
--- a/backend/plugins/github/api/scope.go
+++ b/backend/plugins/github/api/scope.go
@@ -18,16 +18,10 @@ limitations under the License.
package api
import (
- "github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
- "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/github/models"
"net/http"
- "strconv"
- "time"
-
- "github.com/mitchellh/mapstructure"
)
type apiRepo struct {
@@ -51,33 +45,8 @@ type req struct {
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/github/connections/{connectionId}/scopes [PUT]
func PutScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
- connectionId, _ := extractParam(input.Params)
- if connectionId == 0 {
- return nil, errors.BadInput.New("invalid connectionId")
- }
var repos req
- err := errors.Convert(mapstructure.Decode(input.Body, &repos))
- if err != nil {
- return nil, errors.BadInput.Wrap(err, "decoding Github repo
error")
- }
- keeper := make(map[int]struct{})
- now := time.Now()
- for _, repo := range repos.Data {
- if _, ok := keeper[repo.GithubId]; ok {
- return nil, errors.BadInput.New("duplicated item")
- } else {
- keeper[repo.GithubId] = struct{}{}
- }
- repo.ConnectionId = connectionId
- // Fixme: why do we set this to now?
- repo.CreatedDate = &now
- repo.UpdatedDate = &now
- err = verifyRepo(repo)
- if err != nil {
- return nil, err
- }
- }
- err = basicRes.GetDal().CreateOrUpdate(repos.Data)
+ err := scopeHelper.Put(input, &repos, &models.GithubConnection{})
if err != nil {
return nil, errors.Default.Wrap(err, "error on saving
GithubRepo")
}
@@ -95,28 +64,13 @@ func PutScope(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors
// @Success 200 {object} models.GithubRepo
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
-// @Router /plugins/github/connections/{connectionId}/scopes/{repoId} [PATCH]
+// @Router /plugins/github/connections/{connectionId}/scopes/{scopeId} [PATCH]
func UpdateScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
- connectionId, repoId := extractParam(input.Params)
- if connectionId*repoId == 0 {
- return nil, errors.BadInput.New("invalid connectionId or
repoId")
- }
var repo models.GithubRepo
- err := basicRes.GetDal().First(&repo, dal.Where("connection_id = ? AND
github_id = ?", connectionId, repoId))
- if err != nil {
- return nil, errors.Default.Wrap(err, "getting GithubRepo error")
- }
- err = api.DecodeMapStruct(input.Body, &repo)
+ var conn models.GithubConnection
+ err := scopeHelper.Update(input, "github_id", &conn, &repo)
if err != nil {
- return nil, errors.Default.Wrap(err, "patch github repo error")
- }
- err = verifyRepo(&repo)
- if err != nil {
- return nil, err
- }
- err = basicRes.GetDal().Update(repo)
- if err != nil {
- return nil, errors.Default.Wrap(err, "error on saving
GithubRepo")
+ return &plugin.ApiResourceOutput{Body: nil, Status:
http.StatusInternalServerError}, err
}
return &plugin.ApiResourceOutput{Body: repo, Status: http.StatusOK}, nil
}
@@ -133,36 +87,16 @@ func UpdateScope(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, err
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/github/connections/{connectionId}/scopes/ [GET]
func GetScopeList(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
- var repos []models.GithubRepo
- connectionId, _ := extractParam(input.Params)
- if connectionId == 0 {
- return nil, errors.BadInput.New("invalid path params")
- }
- limit, offset := api.GetLimitOffset(input.Query, "pageSize", "page")
- err := basicRes.GetDal().All(&repos, dal.Where("connection_id = ?",
connectionId), dal.Limit(limit), dal.Offset(offset))
+ var repos []*models.GithubRepo
+ var rules []*models.GithubTransformationRule
+ var conn models.GithubConnection
+ names, err := scopeHelper.GetScopeList(input, conn, &repos, &rules)
if err != nil {
return nil, err
}
- var ruleIds []uint64
- for _, repo := range repos {
- if repo.TransformationRuleId > 0 {
- ruleIds = append(ruleIds, repo.TransformationRuleId)
- }
- }
- var rules []models.GithubTransformationRule
- if len(ruleIds) > 0 {
- err = basicRes.GetDal().All(&rules, dal.Where("id IN (?)",
ruleIds))
- if err != nil {
- return nil, err
- }
- }
- names := make(map[uint64]string)
- for _, rule := range rules {
- names[rule.ID] = rule.Name
- }
var apiRepos []apiRepo
for _, repo := range repos {
- apiRepos = append(apiRepos, apiRepo{repo,
names[repo.TransformationRuleId]})
+ apiRepos = append(apiRepos, apiRepo{*repo,
names[repo.TransformationRuleId]})
}
return &plugin.ApiResourceOutput{Body: apiRepos, Status:
http.StatusOK}, nil
}
@@ -176,43 +110,14 @@ func GetScopeList(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, er
// @Success 200 {object} apiRepo
// @Failure 400 {object} shared.ApiBody "Bad Request"
// @Failure 500 {object} shared.ApiBody "Internal Error"
-// @Router /plugins/github/connections/{connectionId}/scopes/{repoId} [GET]
+// @Router /plugins/github/connections/{connectionId}/scopes/{id} [GET]
func GetScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
var repo models.GithubRepo
- connectionId, repoId := extractParam(input.Params)
- if connectionId*repoId == 0 {
- return nil, errors.BadInput.New("invalid path params")
- }
- db := basicRes.GetDal()
- err := db.First(&repo, dal.Where("connection_id = ? AND github_id = ?",
connectionId, repoId))
- if db.IsErrorNotFound(err) {
- return nil, errors.NotFound.New("record not found")
- }
+ var rule models.GithubTransformationRule
+ var conn models.GithubConnection
+ err := scopeHelper.GetScope(input, "github_id", conn, &repo, &rule)
if err != nil {
return nil, err
}
- var rule models.GithubTransformationRule
- if repo.TransformationRuleId > 0 {
- err = basicRes.GetDal().First(&rule, dal.Where("id = ?",
repo.TransformationRuleId))
- if err != nil {
- return nil, err
- }
- }
return &plugin.ApiResourceOutput{Body: apiRepo{repo, rule.Name},
Status: http.StatusOK}, nil
}
-
-func extractParam(params map[string]string) (uint64, uint64) {
- connectionId, _ := strconv.ParseUint(params["connectionId"], 10, 64)
- repoId, _ := strconv.ParseUint(params["repoId"], 10, 64)
- return connectionId, repoId
-}
-
-func verifyRepo(repo *models.GithubRepo) errors.Error {
- if repo.ConnectionId == 0 {
- return errors.BadInput.New("invalid connectionId")
- }
- if repo.GithubId <= 0 {
- return errors.BadInput.New("invalid github ID")
- }
- return nil
-}
diff --git a/backend/plugins/github/impl/impl.go
b/backend/plugins/github/impl/impl.go
index 756e73097..71dd2dcea 100644
--- a/backend/plugins/github/impl/impl.go
+++ b/backend/plugins/github/impl/impl.go
@@ -204,7 +204,7 @@ func (p Github) ApiResources()
map[string]map[string]plugin.ApiResourceHandler {
"PATCH": api.PatchConnection,
"DELETE": api.DeleteConnection,
},
- "connections/:connectionId/scopes/:repoId": {
+ "connections/:connectionId/scopes/:scopeId": {
"GET": api.GetScope,
"PATCH": api.UpdateScope,
},
diff --git a/backend/plugins/gitlab/api/init.go
b/backend/plugins/gitlab/api/init.go
index cd019a36f..8f0dc3bad 100644
--- a/backend/plugins/gitlab/api/init.go
+++ b/backend/plugins/gitlab/api/init.go
@@ -19,19 +19,25 @@ package api
import (
"github.com/apache/incubator-devlake/core/context"
- helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+ "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/go-playground/validator/v10"
)
var vld *validator.Validate
-var connectionHelper *helper.ConnectionApiHelper
+var connectionHelper *api.ConnectionApiHelper
+var scopeHelper *api.ScopeApiHelper
var basicRes context.BasicRes
func Init(br context.BasicRes) {
basicRes = br
vld = validator.New()
- connectionHelper = helper.NewConnectionHelper(
+ connectionHelper = api.NewConnectionHelper(
basicRes,
vld,
)
+ scopeHelper = api.NewScopeHelper(
+ basicRes,
+ vld,
+ connectionHelper,
+ )
}
diff --git a/backend/plugins/gitlab/api/scope.go
b/backend/plugins/gitlab/api/scope.go
index 6af3c4b1e..4e660b9a2 100644
--- a/backend/plugins/gitlab/api/scope.go
+++ b/backend/plugins/gitlab/api/scope.go
@@ -18,16 +18,13 @@ limitations under the License.
package api
import (
- "net/http"
- "strconv"
- "time"
-
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/plugin"
"github.com/apache/incubator-devlake/helpers/pluginhelper/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
- "github.com/mitchellh/mapstructure"
+ "net/http"
+ "strconv"
)
type apiProject struct {
@@ -51,36 +48,12 @@ type req struct {
// @Failure 500 {object} shared.ApiBody "Internal Error"
// @Router /plugins/gitlab/connections/{connectionId}/scopes [PUT]
func PutScope(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
- connectionId, _ := extractParam(input.Params)
- if connectionId == 0 {
- return nil, errors.BadInput.New("invalid connectionId")
- }
- var projects req
- err := errors.Convert(mapstructure.Decode(input.Body, &projects))
+ var scopes req
+ err := scopeHelper.Put(input, &scopes, &models.GitlabConnection{})
if err != nil {
- return nil, errors.BadInput.Wrap(err, "decoding Gitlab project
error")
- }
- keeper := make(map[int]struct{})
- now := time.Now()
- for _, project := range projects.Data {
- if _, ok := keeper[project.GitlabId]; ok {
- return nil, errors.BadInput.New("duplicated item")
- } else {
- keeper[project.GitlabId] = struct{}{}
- }
- project.ConnectionId = connectionId
- project.CreatedDate = now
- project.UpdatedDate = &now
- err = verifyProject(project)
- if err != nil {
- return nil, err
- }
- }
- err = basicRes.GetDal().CreateOrUpdate(projects.Data)
- if err != nil {
- return nil, errors.Default.Wrap(err, "error on saving
GitlabProject")
+ return nil, errors.Default.Wrap(err, "error on saving
GithubRepo")
}
- return &plugin.ApiResourceOutput{Body: projects.Data, Status:
http.StatusOK}, nil
+ return &plugin.ApiResourceOutput{Body: scopes.Data, Status:
http.StatusOK}, nil
}
// UpdateScope patch to gitlab project
