This is an automated email from the ASF dual-hosted git repository.
zhangliang2022 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 7a9706bd0 feat: add table `_tool_gitlab_transformation_rules` and
support `scope` (#3811)
7a9706bd0 is described below
commit 7a9706bd0bde97a32253e11751db31eab9329c5b
Author: mindlesscloud <[email protected]>
AuthorDate: Wed Nov 30 17:03:43 2022 +0800
feat: add table `_tool_gitlab_transformation_rules` and support `scope`
(#3811)
* feat: dora stage 2
* fix: fix e2e
* feat: pagenation support
* fix: add pagenation param for swagger docs
* refactor: remove default transformation rules
* fix: fix the linter complain
---
plugins/gitlab/api/scope.go | 164 +++++++++++++++++++++
plugins/gitlab/api/transformation_rule.go | 128 ++++++++++++++++
plugins/gitlab/e2e/account_test.go | 5 +-
plugins/gitlab/e2e/issues_test.go | 5 +-
plugins/gitlab/e2e/job_test.go | 5 +-
plugins/gitlab/e2e/mr_commits_test.go | 5 +-
plugins/gitlab/e2e/mr_notes_test.go | 5 +-
plugins/gitlab/e2e/mr_test.go | 5 +-
plugins/gitlab/e2e/pipelines_test.go | 5 +-
plugins/gitlab/e2e/project_test.go | 5 +-
plugins/gitlab/impl/impl.go | 45 +++++-
plugins/gitlab/models/connection.go | 17 +--
...go => 20221125_add_trasformation_rule_table.go} | 32 ++--
.../archived/transformation_rules.go} | 56 +++----
plugins/gitlab/models/migrationscripts/register.go | 1 +
plugins/gitlab/models/project.go | 4 +-
.../{connection.go => transformation_rule.go} | 32 +---
plugins/gitlab/tasks/task_data.go | 46 ++----
.../register.go => helper/pagenation.go} | 36 +++--
19 files changed, 441 insertions(+), 160 deletions(-)
diff --git a/plugins/gitlab/api/scope.go b/plugins/gitlab/api/scope.go
new file mode 100644
index 000000000..47f749c8b
--- /dev/null
+++ b/plugins/gitlab/api/scope.go
@@ -0,0 +1,164 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "net/http"
+ "strconv"
+
+ "github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+ "github.com/apache/incubator-devlake/plugins/gitlab/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/mitchellh/mapstructure"
+)
+
+// PutScope create or update gitlab project
+// @Summary create or update gitlab project
+// @Description Create or update gitlab project
+// @Tags plugins/gitlab
+// @Accept application/json
+// @Param connectionId path int false "connection ID"
+// @Param projectId path int false "project ID"
+// @Param scope body models.GitlabProject true "json"
+// @Success 200 {object} models.GitlabProject
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/gitlab/connections/{connectionId}/scopes/{projectId} [PUT]
+func PutScope(input *core.ApiResourceInput) (*core.ApiResourceOutput,
errors.Error) {
+ connectionId, projectId := extractParam(input.Params)
+ if connectionId*projectId == 0 {
+ return nil, errors.BadInput.New("invalid connectionId or
projectId")
+ }
+ var project models.GitlabProject
+ err := errors.Convert(mapstructure.Decode(input.Body, &project))
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "decoding Gitlab project
error")
+ }
+ err = verifyProject(&project)
+ if err != nil {
+ return nil, err
+ }
+ err = BasicRes.GetDal().CreateOrUpdate(project)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "error on saving
GitlabProject")
+ }
+ return &core.ApiResourceOutput{Body: project, Status: http.StatusOK},
nil
+}
+
+// UpdateScope patch to gitlab project
+// @Summary patch to gitlab project
+// @Description patch to gitlab project
+// @Tags plugins/gitlab
+// @Accept application/json
+// @Param connectionId path int false "connection ID"
+// @Param projectId path int false "project ID"
+// @Param scope body models.GitlabProject true "json"
+// @Success 200 {object} models.GitlabProject
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/gitlab/connections/{connectionId}/scopes/{projectId}
[PATCH]
+func UpdateScope(input *core.ApiResourceInput) (*core.ApiResourceOutput,
errors.Error) {
+ connectionId, projectId := extractParam(input.Params)
+ if connectionId*projectId == 0 {
+ return nil, errors.BadInput.New("invalid connectionId or
projectId")
+ }
+ var project models.GitlabProject
+ err := BasicRes.GetDal().First(&project, dal.Where("connection_id = ?
AND gitlab_id = ?", connectionId, projectId))
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "getting GitlabProject
error")
+ }
+ err = helper.DecodeMapStruct(input.Body, &project)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "patch gitlab project
error")
+ }
+ err = verifyProject(&project)
+ if err != nil {
+ return nil, err
+ }
+ err = BasicRes.GetDal().Update(project)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "error on saving
GitlabProject")
+ }
+ return &core.ApiResourceOutput{Body: project, Status: http.StatusOK},
nil
+}
+
+// GetScopeList get Gitlab projects
+// @Summary get Gitlab projects
+// @Description get Gitlab projects
+// @Tags plugins/gitlab
+// @Param connectionId path int false "connection ID"
+// @Success 200 {object} []models.GitlabProject
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/gitlab/connections/{connectionId}/scopes/ [GET]
+func GetScopeList(input *core.ApiResourceInput) (*core.ApiResourceOutput,
errors.Error) {
+ var projects []models.GitlabProject
+ connectionId, _ := extractParam(input.Params)
+ if connectionId == 0 {
+ return nil, errors.BadInput.New("invalid path params")
+ }
+ limit, offset := helper.GetLimitOffset(input.Query, "pageSize", "page")
+ err := BasicRes.GetDal().All(&projects, dal.Where("connection_id = ?",
connectionId), dal.Limit(limit), dal.Offset(offset))
+ if err != nil {
+ return nil, err
+ }
+ return &core.ApiResourceOutput{Body: projects, Status: http.StatusOK},
nil
+}
+
+// GetScope get one Gitlab project
+// @Summary get one Gitlab project
+// @Description get one Gitlab project
+// @Tags plugins/gitlab
+// @Param connectionId path int false "connection ID"
+// @Param projectId path int false "project ID"
+// @Param pageSize query int false "page size, default 50"
+// @Param page query int false "page size, default 1"
+// @Success 200 {object} models.GitlabProject
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/gitlab/connections/{connectionId}/scopes/{projectId} [GET]
+func GetScope(input *core.ApiResourceInput) (*core.ApiResourceOutput,
errors.Error) {
+ var project models.GitlabProject
+ connectionId, projectId := extractParam(input.Params)
+ if connectionId*projectId == 0 {
+ return nil, errors.BadInput.New("invalid path params")
+ }
+ err := BasicRes.GetDal().First(&project, dal.Where("connection_id = ?
AND gitlab_id = ?", connectionId, projectId))
+ if err != nil {
+ return nil, err
+ }
+ return &core.ApiResourceOutput{Body: project, Status: http.StatusOK},
nil
+}
+
+func extractParam(params map[string]string) (uint64, uint64) {
+ connectionId, _ := strconv.ParseUint(params["connectionId"], 10, 64)
+ projectId, _ := strconv.ParseUint(params["projectId"], 10, 64)
+ return connectionId, projectId
+}
+
+func verifyProject(project *models.GitlabProject) errors.Error {
+ if project.ConnectionId == 0 {
+ return errors.BadInput.New("invalid connectionId")
+ }
+ if project.GitlabId <= 0 {
+ return errors.BadInput.New("invalid projectId")
+ }
+ return nil
+}
diff --git a/plugins/gitlab/api/transformation_rule.go
b/plugins/gitlab/api/transformation_rule.go
new file mode 100644
index 000000000..fbc72b6da
--- /dev/null
+++ b/plugins/gitlab/api/transformation_rule.go
@@ -0,0 +1,128 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+ "net/http"
+ "strconv"
+
+ "github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
+ "github.com/apache/incubator-devlake/plugins/gitlab/models"
+ "github.com/apache/incubator-devlake/plugins/helper"
+ "github.com/mitchellh/mapstructure"
+)
+
+// CreateTransformationRule create transformation rule for Gitlab
+// @Summary create transformation rule for Gitlab
+// @Description create transformation rule for Gitlab
+// @Tags plugins/gitlab
+// @Accept application/json
+// @Param transformationRule body models.TransformationRules true
"transformation rule"
+// @Success 200 {object} models.TransformationRules
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/gitlab/transformation_rules [POST]
+func CreateTransformationRule(input *core.ApiResourceInput)
(*core.ApiResourceOutput, errors.Error) {
+ var rule models.TransformationRules
+ err := mapstructure.Decode(input.Body, &rule)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "error in decoding
transformation rule")
+ }
+ err = BasicRes.GetDal().Create(&rule)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "error on saving
TransformationRule")
+ }
+ return &core.ApiResourceOutput{Body: rule, Status: http.StatusOK}, nil
+}
+
+// UpdateTransformationRule update transformation rule for Gitlab
+// @Summary update transformation rule for Gitlab
+// @Description update transformation rule for Gitlab
+// @Tags plugins/gitlab
+// @Accept application/json
+// @Param id path int true "id"
+// @Param transformationRule body models.TransformationRules true
"transformation rule"
+// @Success 200 {object} models.TransformationRules
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/gitlab/transformation_rules/{id} [PATCH]
+func UpdateTransformationRule(input *core.ApiResourceInput)
(*core.ApiResourceOutput, errors.Error) {
+ transformationRuleId, err := strconv.ParseUint(input.Params["id"], 10,
64)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "the transformation rule
ID should be an integer")
+ }
+ var old models.TransformationRules
+ err = BasicRes.GetDal().First(&old, dal.Where("id = ?",
transformationRuleId))
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "error on saving
TransformationRule")
+ }
+ err = helper.DecodeMapStruct(input.Body, &old)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "error decoding map into
transformationRule")
+ }
+ old.ID = transformationRuleId
+ err = BasicRes.GetDal().Update(&old, dal.Where("id = ?",
transformationRuleId))
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "error on saving
TransformationRule")
+ }
+ return &core.ApiResourceOutput{Body: old, Status: http.StatusOK}, nil
+}
+
+// GetTransformationRule return one transformation rule
+// @Summary return one transformation rule
+// @Description return one transformation rule
+// @Tags plugins/gitlab
+// @Param id path int true "id"
+// @Success 200 {object} models.TransformationRules
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/gitlab/transformation_rules/{id} [GET]
+func GetTransformationRule(input *core.ApiResourceInput)
(*core.ApiResourceOutput, errors.Error) {
+ transformationRuleId, err := strconv.ParseUint(input.Params["id"], 10,
64)
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "the transformation rule
ID should be an integer")
+ }
+ var rule models.TransformationRules
+ err = BasicRes.GetDal().First(&rule, dal.Where("id = ?",
transformationRuleId))
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "error on get
TransformationRule")
+ }
+ return &core.ApiResourceOutput{Body: rule, Status: http.StatusOK}, nil
+}
+
+// GetTransformationRuleList return all transformation rules
+// @Summary return all transformation rules
+// @Description return all transformation rules
+// @Tags plugins/gitlab
+// @Param pageSize query int false "page size, default 50"
+// @Param page query int false "page size, default 1"
+// @Success 200 {object} []models.TransformationRules
+// @Failure 400 {object} shared.ApiBody "Bad Request"
+// @Failure 500 {object} shared.ApiBody "Internal Error"
+// @Router /plugins/gitlab/transformation_rules [GET]
+func GetTransformationRuleList(input *core.ApiResourceInput)
(*core.ApiResourceOutput, errors.Error) {
+ var rules []models.TransformationRules
+ limit, offset := helper.GetLimitOffset(input.Query, "pageSize", "page")
+ err := BasicRes.GetDal().All(&rules, dal.Limit(limit),
dal.Offset(offset))
+ if err != nil {
+ return nil, errors.Default.Wrap(err, "error on get
TransformationRule list")
+ }
+ return &core.ApiResourceOutput{Body: rules, Status: http.StatusOK}, nil
+}
diff --git a/plugins/gitlab/e2e/account_test.go
b/plugins/gitlab/e2e/account_test.go
index 0f8235381..8994455f0 100644
--- a/plugins/gitlab/e2e/account_test.go
+++ b/plugins/gitlab/e2e/account_test.go
@@ -34,8 +34,9 @@ func TestGitlabAccountDataFlow(t *testing.T) {
taskData := &tasks.GitlabTaskData{
Options: &tasks.GitlabOptions{
- ConnectionId: 1,
- ProjectId: 12345678,
+ ConnectionId: 1,
+ ProjectId: 12345678,
+ TransformationRules: new(models.TransformationRules),
},
}
diff --git a/plugins/gitlab/e2e/issues_test.go
b/plugins/gitlab/e2e/issues_test.go
index 3472ab6d3..4574ebe56 100644
--- a/plugins/gitlab/e2e/issues_test.go
+++ b/plugins/gitlab/e2e/issues_test.go
@@ -35,8 +35,9 @@ func TestGitlabIssueDataFlow(t *testing.T) {
taskData := &tasks.GitlabTaskData{
Options: &tasks.GitlabOptions{
- ConnectionId: 1,
- ProjectId: 12345678,
+ ConnectionId: 1,
+ ProjectId: 12345678,
+ TransformationRules: new(models.TransformationRules),
},
}
// import raw data table
diff --git a/plugins/gitlab/e2e/job_test.go b/plugins/gitlab/e2e/job_test.go
index c45a0b74a..111b4b98b 100644
--- a/plugins/gitlab/e2e/job_test.go
+++ b/plugins/gitlab/e2e/job_test.go
@@ -36,8 +36,9 @@ func TestGitlabJobDataFlow(t *testing.T) {
taskData := &tasks.GitlabTaskData{
Options: &tasks.GitlabOptions{
- ConnectionId: 1,
- ProjectId: 44,
+ ConnectionId: 1,
+ ProjectId: 44,
+ TransformationRules: new(models.TransformationRules),
},
}
// import raw data table
diff --git a/plugins/gitlab/e2e/mr_commits_test.go
b/plugins/gitlab/e2e/mr_commits_test.go
index 3e79b9c33..89235b110 100644
--- a/plugins/gitlab/e2e/mr_commits_test.go
+++ b/plugins/gitlab/e2e/mr_commits_test.go
@@ -35,8 +35,9 @@ func TestGitlabMrCommitDataFlow(t *testing.T) {
taskData := &tasks.GitlabTaskData{
Options: &tasks.GitlabOptions{
- ConnectionId: 1,
- ProjectId: 12345678,
+ ConnectionId: 1,
+ ProjectId: 12345678,
+ TransformationRules: new(models.TransformationRules),
},
}
diff --git a/plugins/gitlab/e2e/mr_notes_test.go
b/plugins/gitlab/e2e/mr_notes_test.go
index 5d72fd78c..27d8c0dde 100644
--- a/plugins/gitlab/e2e/mr_notes_test.go
+++ b/plugins/gitlab/e2e/mr_notes_test.go
@@ -35,8 +35,9 @@ func TestGitlabMrNoteDataFlow(t *testing.T) {
taskData := &tasks.GitlabTaskData{
Options: &tasks.GitlabOptions{
- ConnectionId: 1,
- ProjectId: 12345678,
+ ConnectionId: 1,
+ ProjectId: 12345678,
+ TransformationRules: new(models.TransformationRules),
},
}
// import raw data table
diff --git a/plugins/gitlab/e2e/mr_test.go b/plugins/gitlab/e2e/mr_test.go
index 5d39bf957..1abba267d 100644
--- a/plugins/gitlab/e2e/mr_test.go
+++ b/plugins/gitlab/e2e/mr_test.go
@@ -35,8 +35,9 @@ func TestGitlabMrDataFlow(t *testing.T) {
taskData := &tasks.GitlabTaskData{
Options: &tasks.GitlabOptions{
- ConnectionId: 1,
- ProjectId: 12345678,
+ ConnectionId: 1,
+ ProjectId: 12345678,
+ TransformationRules: new(models.TransformationRules),
},
}
// import raw data table
diff --git a/plugins/gitlab/e2e/pipelines_test.go
b/plugins/gitlab/e2e/pipelines_test.go
index c8a6c24d6..1cfaadce8 100644
--- a/plugins/gitlab/e2e/pipelines_test.go
+++ b/plugins/gitlab/e2e/pipelines_test.go
@@ -36,8 +36,9 @@ func TestGitlabPipelineDataFlow(t *testing.T) {
taskData := &tasks.GitlabTaskData{
Options: &tasks.GitlabOptions{
- ConnectionId: 1,
- ProjectId: 12345678,
+ ConnectionId: 1,
+ ProjectId: 12345678,
+ TransformationRules: new(models.TransformationRules),
},
}
// import raw data table
diff --git a/plugins/gitlab/e2e/project_test.go
b/plugins/gitlab/e2e/project_test.go
index 2a705583f..858514443 100644
--- a/plugins/gitlab/e2e/project_test.go
+++ b/plugins/gitlab/e2e/project_test.go
@@ -39,8 +39,9 @@ func TestGitlabProjectDataFlow(t *testing.T) {
taskData := &tasks.GitlabTaskData{
Options: &tasks.GitlabOptions{
- ConnectionId: 1,
- ProjectId: 12345678,
+ ConnectionId: 1,
+ ProjectId: 12345678,
+ TransformationRules: new(models.TransformationRules),
},
}
diff --git a/plugins/gitlab/impl/impl.go b/plugins/gitlab/impl/impl.go
index 388205635..2bb7f8862 100644
--- a/plugins/gitlab/impl/impl.go
+++ b/plugins/gitlab/impl/impl.go
@@ -22,6 +22,7 @@ import (
"github.com/apache/incubator-devlake/errors"
"github.com/apache/incubator-devlake/plugins/core"
+ "github.com/apache/incubator-devlake/plugins/core/dal"
"github.com/apache/incubator-devlake/plugins/gitlab/api"
"github.com/apache/incubator-devlake/plugins/gitlab/models"
"github.com/apache/incubator-devlake/plugins/gitlab/models/migrationscripts"
@@ -32,7 +33,7 @@ import (
"gorm.io/gorm"
)
-type GitlabImpl interface {
+var _ interface {
core.PluginMeta
core.PluginInit
core.PluginTask
@@ -41,9 +42,8 @@ type GitlabImpl interface {
core.PluginBlueprintV100
core.DataSourcePluginBlueprintV200
core.CloseablePluginTask
-}
-
-var _ GitlabImpl = (*Gitlab)(nil)
+ core.PluginSource
+} = (*Gitlab)(nil)
type Gitlab string
@@ -52,6 +52,18 @@ func (plugin Gitlab) Init(config *viper.Viper, logger
core.Logger, db *gorm.DB)
return nil
}
+func (plugin Gitlab) Connection() interface{} {
+ return &models.GitlabConnection{}
+}
+
+func (plugin Gitlab) Scope() interface{} {
+ return &models.GitlabProject{}
+}
+
+func (plugin Gitlab) TransformationRule() interface{} {
+ return &models.TransformationRules{}
+}
+
func (plugin Gitlab) MakeDataSourcePipelinePlanV200(connectionId uint64,
scopes []*core.BlueprintScopeV200) (pp core.PipelinePlan, sc []core.Scope, err
errors.Error) {
return api.MakeDataSourcePipelinePlanV200(connectionId, scopes)
}
@@ -141,7 +153,14 @@ func (plugin Gitlab) PrepareTaskData(taskCtx
core.TaskContext, options map[strin
if err != nil {
return nil, err
}
-
+ if op.TransformationRules == nil && op.TransformationRuleId != 0 {
+ var transformationRule models.TransformationRules
+ err = taskCtx.GetDal().First(&transformationRule, dal.Where("id
= ?", op.TransformationRuleId))
+ if err != nil {
+ return nil, errors.BadInput.Wrap(err, "fail to get
transformationRule")
+ }
+ op.TransformationRules = &transformationRule
+ }
return &tasks.GitlabTaskData{
Options: op,
ApiClient: apiClient,
@@ -174,6 +193,22 @@ func (plugin Gitlab) ApiResources()
map[string]map[string]core.ApiResourceHandle
"DELETE": api.DeleteConnection,
"GET": api.GetConnection,
},
+ "connections/:connectionId/scopes/:projectId": {
+ "GET": api.GetScope,
+ "PUT": api.PutScope,
+ "PATCH": api.UpdateScope,
+ },
+ "connections/:connectionId/scopes": {
+ "GET": api.GetScopeList,
+ },
+ "transformation_rules": {
+ "POST": api.CreateTransformationRule,
+ "GET": api.GetTransformationRuleList,
+ },
+ "transformation_rules/:id": {
+ "PATCH": api.UpdateTransformationRule,
+ "GET": api.GetTransformationRule,
+ },
"connections/:connectionId/proxy/rest/*path": {
"GET": api.Proxy,
},
diff --git a/plugins/gitlab/models/connection.go
b/plugins/gitlab/models/connection.go
index 6fd3db1ec..bd270742c 100644
--- a/plugins/gitlab/models/connection.go
+++ b/plugins/gitlab/models/connection.go
@@ -17,7 +17,9 @@ limitations under the License.
package models
-import "github.com/apache/incubator-devlake/plugins/helper"
+import (
+ "github.com/apache/incubator-devlake/plugins/helper"
+)
// This object conforms to what the frontend currently sends.
type GitlabConnection struct {
@@ -44,19 +46,6 @@ type ApiUserResponse struct {
Name string `json:"name"`
}
-type TransformationRules struct {
- PrType string `mapstructure:"prType" json:"prType"`
- PrComponent string `mapstructure:"prComponent"
json:"prComponent"`
- PrBodyClosePattern string `mapstructure:"prBodyClosePattern"
json:"prBodyClosePattern"`
- IssueSeverity string `mapstructure:"issueSeverity"
json:"issueSeverity"`
- IssuePriority string `mapstructure:"issuePriority"
json:"issuePriority"`
- IssueComponent string `mapstructure:"issueComponent"
json:"issueComponent"`
- IssueTypeBug string `mapstructure:"issueTypeBug"
json:"issueTypeBug"`
- IssueTypeIncident string `mapstructure:"issueTypeIncident"
json:"issueTypeIncident"`
- IssueTypeRequirement string `mapstructure:"issueTypeRequirement"
json:"issueTypeRequirement"`
- DeploymentPattern string `mapstructure:"deploymentPattern"
json:"deploymentPattern"`
-}
-
func (GitlabConnection) TableName() string {
return "_tool_gitlab_connections"
}
diff --git a/plugins/gitlab/models/migrationscripts/register.go
b/plugins/gitlab/models/migrationscripts/20221125_add_trasformation_rule_table.go
similarity index 50%
copy from plugins/gitlab/models/migrationscripts/register.go
copy to
plugins/gitlab/models/migrationscripts/20221125_add_trasformation_rule_table.go
index a91a1fcdd..10e57d250 100644
--- a/plugins/gitlab/models/migrationscripts/register.go
+++
b/plugins/gitlab/models/migrationscripts/20221125_add_trasformation_rule_table.go
@@ -18,16 +18,30 @@ limitations under the License.
package migrationscripts
import (
+ "github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
"github.com/apache/incubator-devlake/plugins/core"
+
"github.com/apache/incubator-devlake/plugins/gitlab/models/migrationscripts/archived"
)
-// All return all the migration scripts
-func All() []core.MigrationScript {
- return []core.MigrationScript{
- new(addInitTables),
- new(addGitlabCI),
- new(addPipelineID),
- new(addPipelineProjects),
- new(fixDurationToFloat8),
- }
+type gitlabProject20221125 struct {
+ TransformationRuleId uint64
+}
+
+func (gitlabProject20221125) TableName() string {
+ return "_tool_gitlab_projects"
+}
+
+type addTransformationRule20221125 struct{}
+
+func (*addTransformationRule20221125) Up(basicRes core.BasicRes) errors.Error {
+ return migrationhelper.AutoMigrateTables(basicRes,
&gitlabProject20221125{}, &archived.TransformationRules{})
+}
+
+func (*addTransformationRule20221125) Version() uint64 {
+ return 20221125102500
+}
+
+func (*addTransformationRule20221125) Name() string {
+ return "add table _tool_gitlab_transformation_rules, add
transformation_rule_id to _tool_gitlab_projects"
}
diff --git a/plugins/gitlab/models/connection.go
b/plugins/gitlab/models/migrationscripts/archived/transformation_rules.go
similarity index 59%
copy from plugins/gitlab/models/connection.go
copy to plugins/gitlab/models/migrationscripts/archived/transformation_rules.go
index 6fd3db1ec..aa0616697 100644
--- a/plugins/gitlab/models/connection.go
+++ b/plugins/gitlab/models/migrationscripts/archived/transformation_rules.go
@@ -15,48 +15,26 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package models
+package archived
-import "github.com/apache/incubator-devlake/plugins/helper"
-
-// This object conforms to what the frontend currently sends.
-type GitlabConnection struct {
- helper.RestConnection `mapstructure:",squash"`
- helper.AccessToken `mapstructure:",squash"`
-}
-
-type TestConnectionRequest struct {
- Endpoint string `json:"endpoint"`
- Proxy string `json:"proxy"`
- helper.AccessToken `mapstructure:",squash"`
-}
-
-// This object conforms to what the frontend currently expects.
-type GitlabResponse struct {
- Name string `json:"name"`
- ID int `json:"id"`
- GitlabConnection
-}
-
-// Using User because it requires authentication.
-type ApiUserResponse struct {
- Id int
- Name string `json:"name"`
-}
+import (
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+)
type TransformationRules struct {
- PrType string `mapstructure:"prType" json:"prType"`
- PrComponent string `mapstructure:"prComponent"
json:"prComponent"`
- PrBodyClosePattern string `mapstructure:"prBodyClosePattern"
json:"prBodyClosePattern"`
- IssueSeverity string `mapstructure:"issueSeverity"
json:"issueSeverity"`
- IssuePriority string `mapstructure:"issuePriority"
json:"issuePriority"`
- IssueComponent string `mapstructure:"issueComponent"
json:"issueComponent"`
- IssueTypeBug string `mapstructure:"issueTypeBug"
json:"issueTypeBug"`
- IssueTypeIncident string `mapstructure:"issueTypeIncident"
json:"issueTypeIncident"`
- IssueTypeRequirement string `mapstructure:"issueTypeRequirement"
json:"issueTypeRequirement"`
- DeploymentPattern string `mapstructure:"deploymentPattern"
json:"deploymentPattern"`
+ archived.Model
+ PrType string `mapstructure:"prType" json:"prType"
gorm:"type:varchar(255)"`
+ PrComponent string `mapstructure:"prComponent"
json:"prComponent" gorm:"type:varchar(255)"`
+ PrBodyClosePattern string `mapstructure:"prBodyClosePattern"
json:"prBodyClosePattern" gorm:"type:varchar(255)"`
+ IssueSeverity string `mapstructure:"issueSeverity"
json:"issueSeverity" gorm:"type:varchar(255)"`
+ IssuePriority string `mapstructure:"issuePriority"
json:"issuePriority" gorm:"type:varchar(255)"`
+ IssueComponent string `mapstructure:"issueComponent"
json:"issueComponent" gorm:"type:varchar(255)"`
+ IssueTypeBug string `mapstructure:"issueTypeBug"
json:"issueTypeBug" gorm:"type:varchar(255)"`
+ IssueTypeIncident string `mapstructure:"issueTypeIncident"
json:"issueTypeIncident" gorm:"type:varchar(255)"`
+ IssueTypeRequirement string `mapstructure:"issueTypeRequirement"
json:"issueTypeRequirement" gorm:"type:varchar(255)"`
+ DeploymentPattern string `mapstructure:"deploymentPattern"
json:"deploymentPattern" gorm:"type:varchar(255)"`
}
-func (GitlabConnection) TableName() string {
- return "_tool_gitlab_connections"
+func (t TransformationRules) TableName() string {
+ return "_tool_gitlab_transformation_rules"
}
diff --git a/plugins/gitlab/models/migrationscripts/register.go
b/plugins/gitlab/models/migrationscripts/register.go
index a91a1fcdd..6f373bd94 100644
--- a/plugins/gitlab/models/migrationscripts/register.go
+++ b/plugins/gitlab/models/migrationscripts/register.go
@@ -29,5 +29,6 @@ func All() []core.MigrationScript {
new(addPipelineID),
new(addPipelineProjects),
new(fixDurationToFloat8),
+ new(addTransformationRule20221125),
}
}
diff --git a/plugins/gitlab/models/project.go b/plugins/gitlab/models/project.go
index 1bcb63eca..8b71bf440 100644
--- a/plugins/gitlab/models/project.go
+++ b/plugins/gitlab/models/project.go
@@ -24,8 +24,8 @@ import (
)
type GitlabProject struct {
- ConnectionId uint64 `gorm:"primaryKey"`
-
+ ConnectionId uint64 `gorm:"primaryKey"`
+ TransformationRuleId uint64
GitlabId int `gorm:"primaryKey"`
Name string `gorm:"type:varchar(255)"`
Description string
diff --git a/plugins/gitlab/models/connection.go
b/plugins/gitlab/models/transformation_rule.go
similarity index 67%
copy from plugins/gitlab/models/connection.go
copy to plugins/gitlab/models/transformation_rule.go
index 6fd3db1ec..b65706079 100644
--- a/plugins/gitlab/models/connection.go
+++ b/plugins/gitlab/models/transformation_rule.go
@@ -17,34 +17,10 @@ limitations under the License.
package models
-import "github.com/apache/incubator-devlake/plugins/helper"
-
-// This object conforms to what the frontend currently sends.
-type GitlabConnection struct {
- helper.RestConnection `mapstructure:",squash"`
- helper.AccessToken `mapstructure:",squash"`
-}
-
-type TestConnectionRequest struct {
- Endpoint string `json:"endpoint"`
- Proxy string `json:"proxy"`
- helper.AccessToken `mapstructure:",squash"`
-}
-
-// This object conforms to what the frontend currently expects.
-type GitlabResponse struct {
- Name string `json:"name"`
- ID int `json:"id"`
- GitlabConnection
-}
-
-// Using User because it requires authentication.
-type ApiUserResponse struct {
- Id int
- Name string `json:"name"`
-}
+import "github.com/apache/incubator-devlake/models/common"
type TransformationRules struct {
+ common.Model
PrType string `mapstructure:"prType" json:"prType"`
PrComponent string `mapstructure:"prComponent"
json:"prComponent"`
PrBodyClosePattern string `mapstructure:"prBodyClosePattern"
json:"prBodyClosePattern"`
@@ -57,6 +33,6 @@ type TransformationRules struct {
DeploymentPattern string `mapstructure:"deploymentPattern"
json:"deploymentPattern"`
}
-func (GitlabConnection) TableName() string {
- return "_tool_gitlab_connections"
+func (t TransformationRules) TableName() string {
+ return "_tool_gitlab_transformation_rules"
}
diff --git a/plugins/gitlab/tasks/task_data.go
b/plugins/gitlab/tasks/task_data.go
index a76a2f288..fa887204c 100644
--- a/plugins/gitlab/tasks/task_data.go
+++ b/plugins/gitlab/tasks/task_data.go
@@ -21,17 +21,17 @@ import (
"time"
"github.com/apache/incubator-devlake/errors"
-
"github.com/apache/incubator-devlake/plugins/gitlab/models"
"github.com/apache/incubator-devlake/plugins/helper"
)
type GitlabOptions struct {
- ConnectionId uint64 `json:"connectionId"`
- ProjectId int `json:"projectId"`
- Tasks []string `json:"tasks,omitempty"`
- Since string
- models.TransformationRules `mapstructure:"transformationRules"
json:"transformationRules"`
+ ConnectionId uint64 `json:"connectionId"`
+ ProjectId int `json:"projectId"`
+ TransformationRuleId uint64 `json:"transformationRuleId"`
+ Tasks []string `json:"tasks,omitempty"`
+ Since string
+ *models.TransformationRules `mapstructure:"transformationRules"
json:"transformationRules"`
}
type GitlabTaskData struct {
@@ -50,39 +50,11 @@ func DecodeAndValidateTaskOptions(options
map[string]interface{}) (*GitlabOption
if op.ProjectId == 0 {
return nil, errors.BadInput.New("ProjectId is required for
Gitlab execution")
}
- if op.PrType == "" {
- op.PrType = "type/(.*)$"
- }
- if op.PrComponent == "" {
- op.PrComponent = "component/(.*)$"
- }
- if op.PrBodyClosePattern == "" {
- op.PrBodyClosePattern =
"(?mi)(fix|close|resolve|fixes|closes|resolves|fixed|closed|resolved)[\\s]*.*(((and
)?(#|https:\\/\\/github.com\\/%s\\/%s\\/issues\\/)\\d+[ ]*)+)"
- }
- if op.IssueSeverity == "" {
- op.IssueSeverity = "severity/(.*)$"
- }
- if op.IssuePriority == "" {
- op.IssuePriority = "^(highest|high|medium|low)$"
- }
- if op.IssueComponent == "" {
- op.IssueComponent = "component/(.*)$"
- }
- if op.IssueTypeBug == "" {
- op.IssueTypeBug = "^(bug|failure|error)$"
- }
- if op.IssueTypeIncident == "" {
- op.IssueTypeIncident = ""
- }
- if op.IssueTypeRequirement == "" {
- op.IssueTypeRequirement =
"^(feat|feature|proposal|requirement)$"
- }
- if op.DeploymentPattern == "" {
- op.DeploymentPattern = "(?i)deploy"
- }
- // find the needed GitHub now
if op.ConnectionId == 0 {
return nil, errors.BadInput.New("connectionId is invalid")
}
+ if op.TransformationRules == nil && op.TransformationRuleId == 0 {
+ op.TransformationRules = new(models.TransformationRules)
+ }
return &op, nil
}
diff --git a/plugins/gitlab/models/migrationscripts/register.go
b/plugins/helper/pagenation.go
similarity index 53%
copy from plugins/gitlab/models/migrationscripts/register.go
copy to plugins/helper/pagenation.go
index a91a1fcdd..84560b854 100644
--- a/plugins/gitlab/models/migrationscripts/register.go
+++ b/plugins/helper/pagenation.go
@@ -15,19 +15,35 @@ See the License for the specific language governing
permissions and
limitations under the License.
*/
-package migrationscripts
+package helper
import (
- "github.com/apache/incubator-devlake/plugins/core"
+ "net/url"
+ "strconv"
)
-// All return all the migration scripts
-func All() []core.MigrationScript {
- return []core.MigrationScript{
- new(addInitTables),
- new(addGitlabCI),
- new(addPipelineID),
- new(addPipelineProjects),
- new(fixDurationToFloat8),
+const pageSize = 50
+
+func getPageParam(q url.Values, pageSizeKey, pageKey string) (size int, page
int) {
+ if ps := q[pageSizeKey]; len(ps) > 0 {
+ size, _ = strconv.Atoi(ps[0])
+ }
+ if p := q[pageKey]; len(p) > 0 {
+ page, _ = strconv.Atoi(p[0])
+ }
+ if size < 1 {
+ size = pageSize
}
+ if page < 1 {
+ page = 1
+ }
+ return size, page
+}
+
+// GetLimitOffset extract page and page size, then calculus the limit and
offset from them
+func GetLimitOffset(q url.Values, pageSizeKey, pageKey string) (limit int,
offset int) {
+ size, page := getPageParam(q, pageSizeKey, pageKey)
+ limit = size
+ offset = (page - 1) * limit
+ return limit, offset
}
