This is an automated email from the ASF dual-hosted git repository.
likyh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 63f3aaf65 Feat(whole): use gorm tag replace encrypt/decrypt (#4227)
63f3aaf65 is described below
commit 63f3aaf655b68872d4bf072f9fb1d927e517c3f4
Author: Warren Chen <[email protected]>
AuthorDate: Wed Jan 18 15:05:38 2023 +0800
Feat(whole): use gorm tag replace encrypt/decrypt (#4227)
* feat(framwork): use gorm tag to rplace
* fix(connections): replace dec or enc by gorm tag
* style(whole): clear unused annotation
* fix(framework): update for review
---
backend/core/models/blueprint.go | 25 +---
backend/core/models/pipeline.go | 52 +++------
backend/core/runner/run_pipeline.go | 2 +-
backend/core/runner/run_task.go | 4 +-
backend/helpers/pluginhelper/api/connection.go | 45 +------
backend/impls/dalgorm/encdec_serializer.go | 24 +++-
backend/plugins/dora/impl/impl.go | 1 -
backend/plugins/pagerduty/models/connection.go | 1 -
backend/plugins/sonarqube/api/blueprint.go | 3 -
backend/plugins/sonarqube/api/connection.go | 12 +-
backend/plugins/sonarqube/impl/impl.go | 1 -
backend/plugins/sonarqube/tasks/task_data.go | 1 -
backend/plugins/tapd/models/connection.go | 13 +--
.../20230115_encode_connection_token.go | 65 +++++++++++
.../tapd/models/migrationscripts/register.go | 1 +
backend/plugins/zentao/api/blueprint.go | 2 -
backend/plugins/zentao/api/connection.go | 43 ++++---
backend/plugins/zentao/impl/impl.go | 1 -
.../plugins/zentao/models/archived/connection.go | 4 +-
backend/plugins/zentao/models/connection.go | 4 +-
backend/plugins/zentao/tasks/account_collector.go | 6 +-
backend/plugins/zentao/tasks/account_convertor.go | 1 -
backend/plugins/zentao/tasks/api_client.go | 2 -
backend/plugins/zentao/tasks/bug_collector.go | 6 +-
backend/plugins/zentao/tasks/bug_convertor.go | 1 -
.../plugins/zentao/tasks/department_collector.go | 7 +-
.../plugins/zentao/tasks/execution_collector.go | 3 +-
.../plugins/zentao/tasks/execution_convertor.go | 1 -
backend/plugins/zentao/tasks/product_collector.go | 4 +-
backend/plugins/zentao/tasks/product_convertor.go | 1 -
backend/plugins/zentao/tasks/project_collector.go | 3 +-
backend/plugins/zentao/tasks/story_collector.go | 6 +-
backend/plugins/zentao/tasks/task_collector.go | 6 +-
backend/plugins/zentao/tasks/task_data.go | 1 -
backend/server/services/blueprint.go | 59 ++--------
backend/server/services/blueprint_helper.go | 130 ++++-----------------
backend/server/services/pipeline.go | 46 +++-----
backend/server/services/pipeline_helper.go | 116 +++---------------
backend/server/services/pipeline_runner.go | 4 +-
backend/server/services/project.go | 4 +-
backend/test/services/pipeline_runner_e2e_test.go | 2 +-
41 files changed, 237 insertions(+), 476 deletions(-)
diff --git a/backend/core/models/blueprint.go b/backend/core/models/blueprint.go
index c6c922bac..69438acf7 100644
--- a/backend/core/models/blueprint.go
+++ b/backend/core/models/blueprint.go
@@ -35,14 +35,14 @@ type Blueprint struct {
Name string `json:"name" validate:"required"`
ProjectName string `json:"projectName"
gorm:"type:varchar(255)"`
Mode string `json:"mode" gorm:"varchar(20)"
validate:"required,oneof=NORMAL ADVANCED"`
- Plan json.RawMessage `json:"plan"`
+ Plan json.RawMessage `json:"plan" gorm:"serializer:encdec"`
Enable bool `json:"enable"`
//please check this https://crontab.guru/ for detail
CronConfig string `json:"cronConfig" format:"* * * * *"
example:"0 0 * * 1"`
IsManual bool `json:"isManual"`
SkipOnFail bool `json:"skipOnFail"`
- Labels []string `json:"labels"`
- Settings json.RawMessage `json:"settings"
swaggertype:"array,string" example:"please check api:
/blueprints/<PLUGIN_NAME>/blueprint-setting"`
+ Labels []string `json:"labels" gorm:"-"`
+ Settings json.RawMessage `json:"settings"
swaggertype:"array,string" example:"please check api:
/blueprints/<PLUGIN_NAME>/blueprint-setting" gorm:"serializer:encdec"`
common.Model `swaggerignore:"true"`
}
@@ -64,24 +64,7 @@ func (bp *Blueprint) UnmarshalPlan() (plugin.PipelinePlan,
errors.Error) {
return plan, nil
}
-// @Description CronConfig
-type DbBlueprint struct {
- Name string `json:"name" validate:"required"`
- ProjectName string `json:"projectName" gorm:"type:varchar(255)"`
- Mode string `json:"mode" gorm:"varchar(20)"
validate:"required,oneof=NORMAL ADVANCED"`
- Plan string `json:"plan" encrypt:"yes"`
- Enable bool `json:"enable"`
- //please check this https://crontab.guru/ for detail
- CronConfig string `json:"cronConfig" format:"* * * * *" example:"0 0
* * 1"`
- IsManual bool `json:"isManual"`
- SkipOnFail bool `json:"skipOnFail"`
- Settings string `json:"settings" encrypt:"yes"
swaggertype:"array,string" example:"please check api:
/blueprints/<PLUGIN_NAME>/blueprint-setting"`
- common.Model `swaggerignore:"true"`
-
- Labels []DbBlueprintLabel `json:"-" gorm:"-"`
-}
-
-func (DbBlueprint) TableName() string {
+func (Blueprint) TableName() string {
return "_devlake_blueprints"
}
diff --git a/backend/core/models/pipeline.go b/backend/core/models/pipeline.go
index 3d1feef28..13e9b974d 100644
--- a/backend/core/models/pipeline.go
+++ b/backend/core/models/pipeline.go
@@ -18,30 +18,29 @@ limitations under the License.
package models
import (
+ "encoding/json"
"time"
"github.com/apache/incubator-devlake/core/models/common"
"github.com/apache/incubator-devlake/core/plugin"
-
- "gorm.io/datatypes"
)
type Pipeline struct {
common.Model
- Name string `json:"name" gorm:"index"`
- BlueprintId uint64 `json:"blueprintId"`
- Plan datatypes.JSON `json:"plan"`
- TotalTasks int `json:"totalTasks"`
- FinishedTasks int `json:"finishedTasks"`
- BeganAt *time.Time `json:"beganAt"`
- FinishedAt *time.Time `json:"finishedAt" gorm:"index"`
- Status string `json:"status"`
- Message string `json:"message"`
- ErrorName string `json:"errorName"`
- SpentSeconds int `json:"spentSeconds"`
- Stage int `json:"stage"`
- Labels []string `json:"labels"`
- SkipOnFail bool `json:"skipOnFail"`
+ Name string `json:"name" gorm:"index"`
+ BlueprintId uint64 `json:"blueprintId"`
+ Plan json.RawMessage `json:"plan" gorm:"serializer:encdec"`
+ TotalTasks int `json:"totalTasks"`
+ FinishedTasks int `json:"finishedTasks"`
+ BeganAt *time.Time `json:"beganAt"`
+ FinishedAt *time.Time `json:"finishedAt" gorm:"index"`
+ Status string `json:"status"`
+ Message string `json:"message"`
+ ErrorName string `json:"errorName"`
+ SpentSeconds int `json:"spentSeconds"`
+ Stage int `json:"stage"`
+ Labels []string `json:"labels" gorm:"-"`
+ SkipOnFail bool `json:"skipOnFail"`
}
// We use a 2D array because the request body must be an array of a set of
tasks
@@ -54,26 +53,7 @@ type NewPipeline struct {
BlueprintId uint64
}
-type DbPipeline struct {
- common.Model
- Name string `json:"name" gorm:"index"`
- BlueprintId uint64 `json:"blueprintId"`
- Plan string `json:"plan" encrypt:"yes"`
- TotalTasks int `json:"totalTasks"`
- FinishedTasks int `json:"finishedTasks"`
- BeganAt *time.Time `json:"beganAt"`
- FinishedAt *time.Time `json:"finishedAt" gorm:"index"`
- Status string `json:"status"`
- Message string `json:"message"`
- ErrorName string `json:"errorName"`
- SpentSeconds int `json:"spentSeconds"`
- Stage int `json:"stage"`
- SkipOnFail bool `json:"skipOnFail"`
-
- Labels []DbPipelineLabel `json:"-" gorm:"-"`
-}
-
-func (DbPipeline) TableName() string {
+func (Pipeline) TableName() string {
return "_devlake_pipelines"
}
diff --git a/backend/core/runner/run_pipeline.go
b/backend/core/runner/run_pipeline.go
index 0313847fa..20816de7e 100644
--- a/backend/core/runner/run_pipeline.go
+++ b/backend/core/runner/run_pipeline.go
@@ -62,7 +62,7 @@ func runPipelineTasks(
db := basicRes.GetDal()
log := basicRes.GetLogger()
// load pipeline from db
- dbPipeline := &models.DbPipeline{}
+ dbPipeline := &models.Pipeline{}
err := db.First(dbPipeline, dal.Where("id = ?", pipelineId))
if err != nil {
return err
diff --git a/backend/core/runner/run_task.go b/backend/core/runner/run_task.go
index def6c039d..0e4d4678d 100644
--- a/backend/core/runner/run_task.go
+++ b/backend/core/runner/run_task.go
@@ -48,7 +48,7 @@ func RunTask(
if task.Status == models.TASK_COMPLETED {
return errors.Default.New("invalid task status")
}
- dbPipeline := &models.DbPipeline{}
+ dbPipeline := &models.Pipeline{}
if err := db.First(dbPipeline, dal.Where("id = ? ", task.PipelineId));
err != nil {
return err
}
@@ -107,7 +107,7 @@ func RunTask(
}
// update finishedTasks
dbe := db.UpdateColumn(
- &models.DbPipeline{},
+ &models.Pipeline{},
"finished_tasks", dal.Expr("finished_tasks + 1"),
dal.Where("id=?", task.PipelineId),
)
diff --git a/backend/helpers/pluginhelper/api/connection.go
b/backend/helpers/pluginhelper/api/connection.go
index d2e77c9fb..a8843ae1d 100644
--- a/backend/helpers/pluginhelper/api/connection.go
+++ b/backend/helpers/pluginhelper/api/connection.go
@@ -41,8 +41,8 @@ type BaseConnection struct {
// BasicAuth FIXME ...
type BasicAuth struct {
- Username string `mapstructure:"username" validate:"required"
json:"username" encrypt:"yes"`
- Password string `mapstructure:"password" validate:"required"
json:"password"`
+ Username string `mapstructure:"username" validate:"required"
json:"username"`
+ Password string `mapstructure:"password" validate:"required"
json:"password" gorm:"serializer:encdec"`
}
// GetEncodedToken FIXME ...
@@ -52,7 +52,7 @@ func (ba BasicAuth) GetEncodedToken() string {
// AccessToken FIXME ...
type AccessToken struct {
- Token string `mapstructure:"token" validate:"required" json:"token"
encrypt:"yes"`
+ Token string `mapstructure:"token" validate:"required" json:"token"
gorm:"serializer:encdec"`
}
// AppKey FIXME ...
@@ -132,25 +132,12 @@ func (c *ConnectionApiHelper) First(connection
interface{}, params map[string]st
// FirstById finds connection from db by id and decrypt it
func (c *ConnectionApiHelper) FirstById(connection interface{}, id uint64)
errors.Error {
- err := c.db.First(connection, dal.Where("id = ?", id))
- if err != nil {
- return err
- }
- c.decrypt(connection)
- return nil
+ return c.db.First(connection, dal.Where("id = ?", id))
}
// List returns all connections with password/token decrypted
func (c *ConnectionApiHelper) List(connections interface{}) errors.Error {
- err := c.db.All(connections)
- if err != nil {
- return err
- }
- conns := reflect.ValueOf(connections).Elem()
- for i := 0; i < conns.Len(); i++ {
- c.decrypt(conns.Index(i).Addr().Interface())
- }
- return nil
+ return c.db.All(connections)
}
// Delete connection
@@ -163,8 +150,6 @@ func (c *ConnectionApiHelper) merge(connection interface{},
body map[string]inte
}
func (c *ConnectionApiHelper) save(connection interface{}) errors.Error {
- c.encrypt(connection)
-
err := c.db.CreateOrUpdate(connection)
if err != nil {
if strings.Contains(strings.ToLower(err.Error()), "duplicate") {
@@ -172,29 +157,9 @@ func (c *ConnectionApiHelper) save(connection interface{})
errors.Error {
}
return err
}
-
- c.decrypt(connection)
return nil
}
-func (c *ConnectionApiHelper) decrypt(connection interface{}) {
- err := UpdateEncryptFields(connection, func(encrypted string) (string,
errors.Error) {
- return plugin.Decrypt(c.encKey, encrypted)
- })
- if err != nil {
- c.log.Error(err, "failed to decrypt")
- }
-}
-
-func (c *ConnectionApiHelper) encrypt(connection interface{}) {
- err := UpdateEncryptFields(connection, func(plaintext string) (string,
errors.Error) {
- return plugin.Encrypt(c.encKey, plaintext)
- })
- if err != nil {
- c.log.Error(err, "failed to encrypt")
- }
-}
-
// UpdateEncryptFields update fields of val with tag `encrypt:"yes|true"`
func UpdateEncryptFields(val interface{}, update func(in string) (string,
errors.Error)) errors.Error {
v := reflect.ValueOf(val)
diff --git a/backend/impls/dalgorm/encdec_serializer.go
b/backend/impls/dalgorm/encdec_serializer.go
index 53a198889..af3011107 100644
--- a/backend/impls/dalgorm/encdec_serializer.go
+++ b/backend/impls/dalgorm/encdec_serializer.go
@@ -19,6 +19,7 @@ package dalgorm
import (
"context"
+ "encoding/json"
"fmt"
"github.com/apache/incubator-devlake/core/plugin"
"reflect"
@@ -36,6 +37,7 @@ type EncDecSerializer struct {
// Scan implements serializer interface
func (es *EncDecSerializer) Scan(ctx context.Context, field *schema.Field, dst
reflect.Value, dbValue interface{}) (err error) {
+ fieldValue := reflect.New(field.FieldType)
if dbValue != nil {
var base64str string
switch v := dbValue.(type) {
@@ -51,14 +53,32 @@ func (es *EncDecSerializer) Scan(ctx context.Context, field
*schema.Field, dst r
if err != nil {
return err
}
- field.ReflectValueOf(ctx, dst).SetString(decrypted)
+ switch fieldValue.Elem().Kind() {
+ case reflect.Slice:
+ bytes := []byte(decrypted)
+ _ = json.Unmarshal(bytes, fieldValue.Interface())
+ field.ReflectValueOf(ctx, dst).Set(fieldValue.Elem())
+ case reflect.String:
+ field.ReflectValueOf(ctx, dst).SetString(decrypted)
+ default:
+ return fmt.Errorf("failed to decrypt value: %#v",
dbValue)
+ }
}
return nil
}
// Value implements serializer interface
func (es *EncDecSerializer) Value(ctx context.Context, field *schema.Field,
dst reflect.Value, fieldValue interface{}) (interface{}, error) {
- return plugin.Encrypt(es.encKey, fieldValue.(string))
+ var target string
+ switch v := fieldValue.(type) {
+ case json.RawMessage:
+ target = string(v)
+ case string:
+ target = v
+ default:
+ return nil, fmt.Errorf("failed to encrypt value: %#v",
fieldValue)
+ }
+ return plugin.Encrypt(es.encKey, target)
}
// Init the encdec serializer
diff --git a/backend/plugins/dora/impl/impl.go
b/backend/plugins/dora/impl/impl.go
index 42ca7c2fc..892343acd 100644
--- a/backend/plugins/dora/impl/impl.go
+++ b/backend/plugins/dora/impl/impl.go
@@ -80,7 +80,6 @@ func (p Dora) Settings() interface{} {
}
func (p Dora) SubTaskMetas() []plugin.SubTaskMeta {
- // TODO add your sub task here
return []plugin.SubTaskMeta{
tasks.EnrichTaskEnvMeta,
tasks.CalculateChangeLeadTimeMeta,
diff --git a/backend/plugins/pagerduty/models/connection.go
b/backend/plugins/pagerduty/models/connection.go
index b9d9d63f5..9b3b8ec94 100644
--- a/backend/plugins/pagerduty/models/connection.go
+++ b/backend/plugins/pagerduty/models/connection.go
@@ -21,7 +21,6 @@ import (
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
-// TODO Please modify the following code to fit your needs
// This object conforms to what the frontend currently sends.
type PagerDutyConnection struct {
helper.BaseConnection `mapstructure:",squash"`
diff --git a/backend/plugins/sonarqube/api/blueprint.go
b/backend/plugins/sonarqube/api/blueprint.go
index b98582a66..5d26b6b27 100644
--- a/backend/plugins/sonarqube/api/blueprint.go
+++ b/backend/plugins/sonarqube/api/blueprint.go
@@ -35,9 +35,6 @@ func MakePipelinePlan(subtaskMetas []plugin.SubTaskMeta,
connectionId uint64, sc
return nil, errors.Default.Wrap(err, "error
unmarshalling task options")
}
taskOptions["connectionId"] = connectionId
-
- //TODO Add transformation rules to task options
-
/*
var transformationRules tasks.TransformationRules
if len(scopeElem.Transformation) > 0 {
diff --git a/backend/plugins/sonarqube/api/connection.go
b/backend/plugins/sonarqube/api/connection.go
index c66a85173..0dea3fcbf 100644
--- a/backend/plugins/sonarqube/api/connection.go
+++ b/backend/plugins/sonarqube/api/connection.go
@@ -122,14 +122,14 @@ func ListConnections(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
return &plugin.ApiResourceOutput{Body: connections, Status:
http.StatusOK}, nil
}
-//TODO Please modify the folowing code to adapt to your plugin
/*
GET /plugins/Sonarqube/connections/:connectionId
-{
- "name": "Sonarqube data connection name",
- "endpoint": "Sonarqube api endpoint, i.e. http://host:port/api/";,
- "token": "Sonarqube user token"
-}
+
+ {
+ "name": "Sonarqube data connection name",
+ "endpoint": "Sonarqube api endpoint, i.e.
http://host:port/api/";,
+ "token": "Sonarqube user token"
+ }
*/
func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
connection := &models.SonarqubeConnection{}
diff --git a/backend/plugins/sonarqube/impl/impl.go
b/backend/plugins/sonarqube/impl/impl.go
index 99b0e661b..04679ab82 100644
--- a/backend/plugins/sonarqube/impl/impl.go
+++ b/backend/plugins/sonarqube/impl/impl.go
@@ -50,7 +50,6 @@ func (p Sonarqube) Init(br context.BasicRes) errors.Error {
}
func (p Sonarqube) SubTaskMetas() []plugin.SubTaskMeta {
- // TODO add your sub task here
return []plugin.SubTaskMeta{}
}
diff --git a/backend/plugins/sonarqube/tasks/task_data.go
b/backend/plugins/sonarqube/tasks/task_data.go
index 08b11e839..67dbdbd01 100644
--- a/backend/plugins/sonarqube/tasks/task_data.go
+++ b/backend/plugins/sonarqube/tasks/task_data.go
@@ -26,7 +26,6 @@ type SonarqubeApiParams struct {
}
type SonarqubeOptions struct {
- // TODO add some custom options here if necessary
// options means some custom params required by plugin running.
// Such As How many rows do your want
// You can use it in sub tasks and you need pass it in main.go and
pipelines.
diff --git a/backend/plugins/tapd/models/connection.go
b/backend/plugins/tapd/models/connection.go
index d5d2608e2..77be688e6 100644
--- a/backend/plugins/tapd/models/connection.go
+++ b/backend/plugins/tapd/models/connection.go
@@ -18,8 +18,6 @@ limitations under the License.
package models
import (
- "encoding/base64"
- "fmt"
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
@@ -35,18 +33,9 @@ type WorkspaceResponse struct {
Value string
}
-type BasicAuth struct {
- Username string `mapstructure:"username" validate:"required"
json:"username"`
- Password string `mapstructure:"password" validate:"required"
json:"password"`
-}
-
-func (ba BasicAuth) GetEncodedToken() string {
- return base64.StdEncoding.EncodeToString([]byte(fmt.Sprintf("%v:%v",
ba.Username, ba.Password)))
-}
-
type TapdConnection struct {
helper.RestConnection `mapstructure:",squash"`
- BasicAuth `mapstructure:",squash"`
+ helper.BasicAuth `mapstructure:",squash"`
}
type TapdConnectionDetail struct {
diff --git
a/backend/plugins/tapd/models/migrationscripts/20230115_encode_connection_token.go
b/backend/plugins/tapd/models/migrationscripts/20230115_encode_connection_token.go
new file mode 100644
index 000000000..7b7c245a6
--- /dev/null
+++
b/backend/plugins/tapd/models/migrationscripts/20230115_encode_connection_token.go
@@ -0,0 +1,65 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "github.com/apache/incubator-devlake/core/config"
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+
"github.com/apache/incubator-devlake/plugins/tapd/models/migrationscripts/archived"
+)
+
+type encodeConnToken struct{}
+
+func (script *encodeConnToken) Up(basicRes context.BasicRes) errors.Error {
+ encKey := config.GetConfig().GetString(plugin.EncodeKeyEnvStr)
+ if encKey == "" {
+ return errors.BadInput.New("jira v0.11 invalid encKey")
+ }
+
+ return migrationhelper.TransformColumns(
+ basicRes,
+ script,
+ "_tool_tapd_connections",
+ []string{},
+ func(old *archived.TapdConnection) (*archived.TapdConnection,
errors.Error) {
+ var err errors.Error
+ conn := &archived.TapdConnection{}
+ conn.ID = old.ID
+ conn.Name = old.Name
+ conn.Endpoint = old.Endpoint
+ conn.Proxy = old.Proxy
+ conn.RateLimitPerHour = old.RateLimitPerHour
+ conn.Username = old.Username
+ conn.Password, err = plugin.Encrypt(encKey,
old.Password)
+ if err != nil {
+ return nil, err
+ }
+ return conn, nil
+ })
+}
+
+func (*encodeConnToken) Version() uint64 {
+ return 20230115201138
+}
+
+func (*encodeConnToken) Name() string {
+ return "Tapd encode connection token"
+}
diff --git a/backend/plugins/tapd/models/migrationscripts/register.go
b/backend/plugins/tapd/models/migrationscripts/register.go
index ec054748c..04bf2dbea 100644
--- a/backend/plugins/tapd/models/migrationscripts/register.go
+++ b/backend/plugins/tapd/models/migrationscripts/register.go
@@ -25,5 +25,6 @@ import (
func All() []plugin.MigrationScript {
return []plugin.MigrationScript{
new(addInitTables),
+ new(encodeConnToken),
}
}
diff --git a/backend/plugins/zentao/api/blueprint.go
b/backend/plugins/zentao/api/blueprint.go
index fb6df52bd..fc0625597 100644
--- a/backend/plugins/zentao/api/blueprint.go
+++ b/backend/plugins/zentao/api/blueprint.go
@@ -36,8 +36,6 @@ func MakePipelinePlan(subtaskMetas []plugin.SubTaskMeta,
connectionId uint64, sc
}
taskOptions["connectionId"] = connectionId
- //TODO Add transformation rules to task options
-
/*
var transformationRules tasks.JiraTransformationRule
if len(scopeElem.Transformation) > 0 {
diff --git a/backend/plugins/zentao/api/connection.go
b/backend/plugins/zentao/api/connection.go
index a6df6a273..f92d88262 100644
--- a/backend/plugins/zentao/api/connection.go
+++ b/backend/plugins/zentao/api/connection.go
@@ -27,7 +27,6 @@ import (
"net/http"
)
-// TODO Please modify the following code to fit your needs
func TestConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
// process input
var params models.TestConnectionRequest
@@ -67,15 +66,15 @@ func TestConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
return nil, nil
}
-//TODO Please modify the folowing code to adapt to your plugin
/*
POST /plugins/Zentao/connections
-{
- "name": "Zentao data connection name",
- "endpoint": "Zentao api endpoint, i.e. https://example.com";,
- "username": "username, usually should be email address",
- "password": "Zentao api access token"
-}
+
+ {
+ "name": "Zentao data connection name",
+ "endpoint": "Zentao api endpoint, i.e. https://example.com";,
+ "username": "username, usually should be email address",
+ "password": "Zentao api access token"
+ }
*/
func PostConnections(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
// update from request and save to database
@@ -87,15 +86,15 @@ func PostConnections(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
return &plugin.ApiResourceOutput{Body: connection, Status:
http.StatusOK}, nil
}
-//TODO Please modify the folowing code to adapt to your plugin
/*
PATCH /plugins/Zentao/connections/:connectionId
-{
- "name": "Zentao data connection name",
- "endpoint": "Zentao api endpoint, i.e. https://example.com";,
- "username": "username, usually should be email address",
- "password": "Zentao api access token"
-}
+
+ {
+ "name": "Zentao data connection name",
+ "endpoint": "Zentao api endpoint, i.e. https://example.com";,
+ "username": "username, usually should be email address",
+ "password": "Zentao api access token"
+ }
*/
func PatchConnection(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput, errors.Error) {
connection := &models.ZentaoConnection{}
@@ -131,15 +130,15 @@ func ListConnections(input *plugin.ApiResourceInput)
(*plugin.ApiResourceOutput,
return &plugin.ApiResourceOutput{Body: connections, Status:
http.StatusOK}, nil
}
-//TODO Please modify the folowing code to adapt to your plugin
/*
GET /plugins/Zentao/connections/:connectionId
-{
- "name": "Zentao data connection name",
- "endpoint": "Zentao api endpoint, i.e.
https://merico.atlassian.net/rest";,
- "username": "username, usually should be email address",
- "password": "Zentao api access token"
-}
+
+ {
+ "name": "Zentao data connection name",
+ "endpoint": "Zentao api endpoint, i.e.
https://merico.atlassian.net/rest";,
+ "username": "username, usually should be email address",
+ "password": "Zentao api access token"
+ }
*/
func GetConnection(input *plugin.ApiResourceInput) (*plugin.ApiResourceOutput,
errors.Error) {
connection := &models.ZentaoConnection{}
diff --git a/backend/plugins/zentao/impl/impl.go
b/backend/plugins/zentao/impl/impl.go
index 89247f522..66359d068 100644
--- a/backend/plugins/zentao/impl/impl.go
+++ b/backend/plugins/zentao/impl/impl.go
@@ -49,7 +49,6 @@ func (p Zentao) Init(basicRes context.BasicRes) errors.Error {
}
func (p Zentao) SubTaskMetas() []plugin.SubTaskMeta {
- // TODO add your sub task here
return []plugin.SubTaskMeta{
tasks.CollectProductMeta,
tasks.ExtractProductMeta,
diff --git a/backend/plugins/zentao/models/archived/connection.go
b/backend/plugins/zentao/models/archived/connection.go
index f04d9a252..3b6caa485 100644
--- a/backend/plugins/zentao/models/archived/connection.go
+++ b/backend/plugins/zentao/models/archived/connection.go
@@ -21,12 +21,10 @@ import (
"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
)
-// TODO Please modify the following code to fit your needs
// This object conforms to what the frontend currently sends.
type ZentaoConnection struct {
RestConnection `mapstructure:",squash"`
- //TODO you may need to use api.BasicAuth instead of api.AccessToken
- BasicAuth `mapstructure:",squash"`
+ BasicAuth `mapstructure:",squash"`
}
type TestConnectionRequest struct {
diff --git a/backend/plugins/zentao/models/connection.go
b/backend/plugins/zentao/models/connection.go
index 75c7c3bfa..de1dedb73 100644
--- a/backend/plugins/zentao/models/connection.go
+++ b/backend/plugins/zentao/models/connection.go
@@ -21,12 +21,10 @@ import (
helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
)
-// TODO Please modify the following code to fit your needs
// This object conforms to what the frontend currently sends.
type ZentaoConnection struct {
helper.RestConnection `mapstructure:",squash"`
- //TODO you may need to use api.BasicAuth instead of api.AccessToken
- helper.BasicAuth `mapstructure:",squash"`
+ helper.BasicAuth `mapstructure:",squash"`
}
type TestConnectionRequest struct {
diff --git a/backend/plugins/zentao/tasks/account_collector.go
b/backend/plugins/zentao/tasks/account_collector.go
index ac279531a..176073cc5 100644
--- a/backend/plugins/zentao/tasks/account_collector.go
+++ b/backend/plugins/zentao/tasks/account_collector.go
@@ -44,10 +44,8 @@ func CollectAccount(taskCtx plugin.SubTaskContext)
errors.Error {
},
Table: RAW_ACCOUNT_TABLE,
},
- ApiClient: data.ApiClient,
-
- PageSize: 100,
- // TODO write which api would you want request
+ ApiClient: data.ApiClient,
+ PageSize: 100,
UrlTemplate: "/users",
Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
diff --git a/backend/plugins/zentao/tasks/account_convertor.go
b/backend/plugins/zentao/tasks/account_convertor.go
index 80d4099bc..eb7b44e15 100644
--- a/backend/plugins/zentao/tasks/account_convertor.go
+++ b/backend/plugins/zentao/tasks/account_convertor.go
@@ -67,7 +67,6 @@ func ConvertAccount(taskCtx plugin.SubTaskContext)
errors.Error {
},
Convert: func(inputRow interface{}) ([]interface{},
errors.Error) {
toolEntity := inputRow.(*models.ZentaoAccount)
-
domainEntity := &crossdomain.Account{
DomainEntity: domainlayer.DomainEntity{
Id:
accountIdGen.Generate(toolEntity.ConnectionId, toolEntity.ID),
diff --git a/backend/plugins/zentao/tasks/api_client.go
b/backend/plugins/zentao/tasks/api_client.go
index 09b7b9a0d..871b4a9ec 100644
--- a/backend/plugins/zentao/tasks/api_client.go
+++ b/backend/plugins/zentao/tasks/api_client.go
@@ -33,7 +33,6 @@ func NewZentaoApiClient(taskCtx plugin.TaskContext,
connection *models.ZentaoCon
if err != nil {
return nil, err
}
-
// request for access token
tokenReqBody := &models.ApiAccessTokenRequest{
Account: connection.Username,
@@ -60,7 +59,6 @@ func NewZentaoApiClient(taskCtx plugin.TaskContext,
connection *models.ZentaoCon
apiClient.SetHeaders(map[string]string{
"Token": fmt.Sprintf("%v", tokenResBody.Token),
})
-
// create rate limit calculator
rateLimiter := &api.ApiRateLimitCalculator{
UserRateLimitPerHour: connection.RateLimitPerHour,
diff --git a/backend/plugins/zentao/tasks/bug_collector.go
b/backend/plugins/zentao/tasks/bug_collector.go
index 5f7681d8c..0b8ca0255 100644
--- a/backend/plugins/zentao/tasks/bug_collector.go
+++ b/backend/plugins/zentao/tasks/bug_collector.go
@@ -44,10 +44,8 @@ func CollectBug(taskCtx plugin.SubTaskContext) errors.Error {
},
Table: RAW_BUG_TABLE,
},
- ApiClient: data.ApiClient,
-
- PageSize: 100,
- // TODO write which api would you want request
+ ApiClient: data.ApiClient,
+ PageSize: 100,
UrlTemplate: "/products/{{ .Params.ProductId }}/bugs",
Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
diff --git a/backend/plugins/zentao/tasks/bug_convertor.go
b/backend/plugins/zentao/tasks/bug_convertor.go
index efab59665..fefd7145f 100644
--- a/backend/plugins/zentao/tasks/bug_convertor.go
+++ b/backend/plugins/zentao/tasks/bug_convertor.go
@@ -108,7 +108,6 @@ func ConvertBug(taskCtx plugin.SubTaskContext) errors.Error
{
return results, nil
},
})
-
if err != nil {
return err
}
diff --git a/backend/plugins/zentao/tasks/department_collector.go
b/backend/plugins/zentao/tasks/department_collector.go
index 43c77ee43..294673ba2 100644
--- a/backend/plugins/zentao/tasks/department_collector.go
+++ b/backend/plugins/zentao/tasks/department_collector.go
@@ -44,10 +44,8 @@ func CollectDepartment(taskCtx plugin.SubTaskContext)
errors.Error {
},
Table: RAW_DEPARTMENT_TABLE,
},
- ApiClient: data.ApiClient,
-
- PageSize: 100,
- // TODO write which api would you want request
+ ApiClient: data.ApiClient,
+ PageSize: 100,
UrlTemplate: "/users",
Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
@@ -70,7 +68,6 @@ func CollectDepartment(taskCtx plugin.SubTaskContext)
errors.Error {
if err != nil {
return err
}
-
return collector.Execute()
}
diff --git a/backend/plugins/zentao/tasks/execution_collector.go
b/backend/plugins/zentao/tasks/execution_collector.go
index 1ca9f9d52..d6775c2c5 100644
--- a/backend/plugins/zentao/tasks/execution_collector.go
+++ b/backend/plugins/zentao/tasks/execution_collector.go
@@ -45,8 +45,7 @@ func CollectExecution(taskCtx plugin.SubTaskContext)
errors.Error {
},
Table: RAW_EXECUTION_TABLE,
},
- ApiClient: data.ApiClient,
- // TODO write which api would you want request
+ ApiClient: data.ApiClient,
UrlTemplate: "executions/{{ .Params.ExecutionId }}",
Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
diff --git a/backend/plugins/zentao/tasks/execution_convertor.go
b/backend/plugins/zentao/tasks/execution_convertor.go
index 06d1073e4..fbc78d563 100644
--- a/backend/plugins/zentao/tasks/execution_convertor.go
+++ b/backend/plugins/zentao/tasks/execution_convertor.go
@@ -78,7 +78,6 @@ func ConvertExecutions(taskCtx plugin.SubTaskContext)
errors.Error {
CreatedDate:
toolExecution.OpenedDate.ToNullableTime(),
Type: toolExecution.Type,
}
-
results := make([]interface{}, 0)
results = append(results, domainBoard)
return results, nil
diff --git a/backend/plugins/zentao/tasks/product_collector.go
b/backend/plugins/zentao/tasks/product_collector.go
index 30418a00c..56fbf839f 100644
--- a/backend/plugins/zentao/tasks/product_collector.go
+++ b/backend/plugins/zentao/tasks/product_collector.go
@@ -45,8 +45,7 @@ func CollectProduct(taskCtx plugin.SubTaskContext)
errors.Error {
},
Table: RAW_PRODUCT_TABLE,
},
- ApiClient: data.ApiClient,
- // TODO write which api would you want request
+ ApiClient: data.ApiClient,
UrlTemplate: "products/{{ .Params.ProductId }}",
Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
@@ -66,7 +65,6 @@ func CollectProduct(taskCtx plugin.SubTaskContext)
errors.Error {
if err != nil {
return err
}
-
return collector.Execute()
}
diff --git a/backend/plugins/zentao/tasks/product_convertor.go
b/backend/plugins/zentao/tasks/product_convertor.go
index 75b97d63e..eeb66baa5 100644
--- a/backend/plugins/zentao/tasks/product_convertor.go
+++ b/backend/plugins/zentao/tasks/product_convertor.go
@@ -77,7 +77,6 @@ func ConvertProducts(taskCtx plugin.SubTaskContext)
errors.Error {
CreatedDate:
toolProduct.CreatedDate.ToNullableTime(),
Type: toolProduct.Type,
}
-
results := make([]interface{}, 0)
results = append(results, domainBoard)
return results, nil
diff --git a/backend/plugins/zentao/tasks/project_collector.go
b/backend/plugins/zentao/tasks/project_collector.go
index 39bbee1b7..579075a14 100644
--- a/backend/plugins/zentao/tasks/project_collector.go
+++ b/backend/plugins/zentao/tasks/project_collector.go
@@ -44,8 +44,7 @@ func CollectProject(taskCtx plugin.SubTaskContext)
errors.Error {
},
Table: RAW_PROJECT_TABLE,
},
- ApiClient: data.ApiClient,
- // TODO write which api would you want request
+ ApiClient: data.ApiClient,
UrlTemplate: "projects",
Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
diff --git a/backend/plugins/zentao/tasks/story_collector.go
b/backend/plugins/zentao/tasks/story_collector.go
index 870a27a10..6c05b5250 100644
--- a/backend/plugins/zentao/tasks/story_collector.go
+++ b/backend/plugins/zentao/tasks/story_collector.go
@@ -44,10 +44,8 @@ func CollectStory(taskCtx plugin.SubTaskContext)
errors.Error {
},
Table: RAW_STORY_TABLE,
},
- ApiClient: data.ApiClient,
-
- PageSize: 100,
- // TODO write which api would you want request
+ ApiClient: data.ApiClient,
+ PageSize: 100,
UrlTemplate: "/products/{{ .Params.ProductId }}/stories",
Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
diff --git a/backend/plugins/zentao/tasks/task_collector.go
b/backend/plugins/zentao/tasks/task_collector.go
index a6a62515f..cb0974989 100644
--- a/backend/plugins/zentao/tasks/task_collector.go
+++ b/backend/plugins/zentao/tasks/task_collector.go
@@ -44,9 +44,8 @@ func CollectTask(taskCtx plugin.SubTaskContext) errors.Error {
},
Table: RAW_TASK_TABLE,
},
- ApiClient: data.ApiClient,
- PageSize: 100,
- // TODO write which api would you want request
+ ApiClient: data.ApiClient,
+ PageSize: 100,
UrlTemplate: "/executions/{{ .Params.ExecutionId }}/tasks",
Query: func(reqData *api.RequestData) (url.Values,
errors.Error) {
query := url.Values{}
@@ -69,7 +68,6 @@ func CollectTask(taskCtx plugin.SubTaskContext) errors.Error {
if err != nil {
return err
}
-
return collector.Execute()
}
diff --git a/backend/plugins/zentao/tasks/task_data.go
b/backend/plugins/zentao/tasks/task_data.go
index 18a2e8a4c..81b9936da 100644
--- a/backend/plugins/zentao/tasks/task_data.go
+++ b/backend/plugins/zentao/tasks/task_data.go
@@ -31,7 +31,6 @@ type ZentaoApiParams struct {
}
type ZentaoOptions struct {
- // TODO add some custom options here if necessary
// options means some custom params required by plugin running.
// Such As How many rows do your want
// You can use it in sub tasks and you need pass it in main.go and
pipelines.
diff --git a/backend/server/services/blueprint.go
b/backend/server/services/blueprint.go
index 813ed6e1e..e8def9bb8 100644
--- a/backend/server/services/blueprint.go
+++ b/backend/server/services/blueprint.go
@@ -48,16 +48,10 @@ func CreateBlueprint(blueprint *models.Blueprint)
errors.Error {
if err != nil {
return err
}
- dbBlueprint := parseDbBlueprint(blueprint)
- dbBlueprint, err = encryptDbBlueprint(dbBlueprint)
+ err = SaveDbBlueprint(blueprint)
if err != nil {
return err
}
- err = SaveDbBlueprint(dbBlueprint)
- if err != nil {
- return err
- }
- blueprint.Model = dbBlueprint.Model
err = ReloadBlueprints(cronManager)
if err != nil {
return errors.Internal.Wrap(err, "error reloading blueprints")
@@ -67,36 +61,22 @@ func CreateBlueprint(blueprint *models.Blueprint)
errors.Error {
// GetBlueprints returns a paginated list of Blueprints based on `query`
func GetBlueprints(query *BlueprintQuery) ([]*models.Blueprint, int64,
errors.Error) {
- dbBlueprints, count, err := GetDbBlueprints(query)
+ blueprints, count, err := GetDbBlueprints(query)
if err != nil {
return nil, 0, errors.Convert(err)
}
- blueprints := make([]*models.Blueprint, 0)
- for _, dbBlueprint := range dbBlueprints {
- dbBlueprint, err = decryptDbBlueprint(dbBlueprint)
- if err != nil {
- return nil, 0, err
- }
- blueprint := parseBlueprint(dbBlueprint)
- blueprints = append(blueprints, blueprint)
- }
return blueprints, count, nil
}
// GetBlueprint returns the detail of a given Blueprint ID
func GetBlueprint(blueprintId uint64) (*models.Blueprint, errors.Error) {
- dbBlueprint, err := GetDbBlueprint(blueprintId)
+ blueprint, err := GetDbBlueprint(blueprintId)
if err != nil {
if db.IsErrorNotFound(err) {
return nil, errors.NotFound.New("blueprint not found")
}
return nil, errors.Internal.Wrap(err, "error getting the
blueprint from database")
}
- dbBlueprint, err = decryptDbBlueprint(dbBlueprint)
- if err != nil {
- return nil, err
- }
- blueprint := parseBlueprint(dbBlueprint)
return blueprint, nil
}
@@ -105,7 +85,7 @@ func GetBlueprintByProjectName(projectName string)
(*models.Blueprint, errors.Er
if projectName == "" {
return nil, errors.Internal.New("can not use the empty
projectName to search the unique blueprint")
}
- dbBlueprint, err := GetDbBlueprintByProjectName(projectName)
+ blueprint, err := GetDbBlueprintByProjectName(projectName)
if err != nil {
// Allow specific projectName to fail to find the corresponding
blueprint
if db.IsErrorNotFound(err) {
@@ -113,15 +93,13 @@ func GetBlueprintByProjectName(projectName string)
(*models.Blueprint, errors.Er
}
return nil, errors.Internal.Wrap(err, fmt.Sprintf("error
getting the blueprint from database with project %s", projectName))
}
- dbBlueprint, err = decryptDbBlueprint(dbBlueprint)
- if err != nil {
- return nil, err
- }
- blueprint := parseBlueprint(dbBlueprint)
return blueprint, nil
}
func validateBlueprintAndMakePlan(blueprint *models.Blueprint) errors.Error {
+ if len(blueprint.Settings) == 0 {
+ blueprint.Settings = nil
+ }
// validation
err := vld.Struct(blueprint)
if err != nil {
@@ -175,7 +153,6 @@ func validateBlueprintAndMakePlan(blueprint
*models.Blueprint) errors.Error {
return errors.Default.Wrap(err, "failed to markshal
plan")
}
}
-
return nil
}
@@ -185,14 +162,7 @@ func saveBlueprint(blueprint *models.Blueprint)
(*models.Blueprint, errors.Error
if err != nil {
return nil, errors.BadInput.WrapRaw(err)
}
-
- // save
- dbBlueprint := parseDbBlueprint(blueprint)
- dbBlueprint, err = encryptDbBlueprint(dbBlueprint)
- if err != nil {
- return nil, err
- }
- err = SaveDbBlueprint(dbBlueprint)
+ err = SaveDbBlueprint(blueprint)
if err != nil {
return nil, err
}
@@ -236,7 +206,7 @@ func PatchBlueprint(id uint64, body map[string]interface{})
(*models.Blueprint,
func ReloadBlueprints(c *cron.Cron) errors.Error {
enable := true
isManual := false
- dbBlueprints, _, err := GetDbBlueprints(&BlueprintQuery{Enable:
&enable, IsManual: &isManual})
+ blueprints, _, err := GetDbBlueprints(&BlueprintQuery{Enable: &enable,
IsManual: &isManual})
if err != nil {
return err
}
@@ -244,12 +214,7 @@ func ReloadBlueprints(c *cron.Cron) errors.Error {
c.Remove(e.ID)
}
c.Stop()
- for _, dbBlueprint := range dbBlueprints {
- dbBlueprint, err = decryptDbBlueprint(dbBlueprint)
- if err != nil {
- return err
- }
- blueprint := parseBlueprint(dbBlueprint)
+ for _, blueprint := range blueprints {
if err != nil {
blueprintLog.Error(err, failToCreateCronJob)
return err
@@ -266,10 +231,10 @@ func ReloadBlueprints(c *cron.Cron) errors.Error {
return errors.Default.Wrap(err, "created cron job
failed")
}
}
- if len(dbBlueprints) > 0 {
+ if len(blueprints) > 0 {
c.Start()
}
- logger.Info("total %d blueprints were scheduled", len(dbBlueprints))
+ logger.Info("total %d blueprints were scheduled", len(blueprints))
return nil
}
diff --git a/backend/server/services/blueprint_helper.go
b/backend/server/services/blueprint_helper.go
index d0403398f..adf5f6f39 100644
--- a/backend/server/services/blueprint_helper.go
+++ b/backend/server/services/blueprint_helper.go
@@ -19,33 +19,35 @@ package services
import (
"fmt"
- "github.com/apache/incubator-devlake/core/config"
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models"
- "github.com/apache/incubator-devlake/core/plugin"
)
// SaveDbBlueprint accepts a Blueprint instance and upsert it to database
-func SaveDbBlueprint(dbBlueprint *models.DbBlueprint) errors.Error {
+func SaveDbBlueprint(blueprint *models.Blueprint) errors.Error {
var err error
- if dbBlueprint.ID != 0 {
- err = db.Update(&dbBlueprint)
+ if blueprint.ID != 0 {
+ err = db.Update(&blueprint)
} else {
- err = db.Create(&dbBlueprint)
+ err = db.Create(&blueprint)
}
if err != nil {
return errors.Default.Wrap(err, "error creating DB blueprint")
}
- err = db.Delete(&models.DbBlueprintLabel{}, dal.Where(`blueprint_id =
?`, dbBlueprint.ID))
+ err = db.Delete(&models.DbBlueprintLabel{}, dal.Where(`blueprint_id =
?`, blueprint.ID))
if err != nil {
return errors.Default.Wrap(err, "error delete DB blueprint's
old labelModels")
}
- if len(dbBlueprint.Labels) > 0 {
- for i := range dbBlueprint.Labels {
- dbBlueprint.Labels[i].BlueprintId = dbBlueprint.ID
+ if len(blueprint.Labels) > 0 {
+ blueprintLabels := make([]*models.DbBlueprintLabel, 0)
+ for i := range blueprint.Labels {
+ blueprintLabels = append(blueprintLabels,
&models.DbBlueprintLabel{
+ BlueprintId: blueprint.ID,
+ Name: blueprint.Labels[i],
+ })
}
- err = db.Create(&dbBlueprint.Labels)
+ err = db.Create(&blueprintLabels)
if err != nil {
return errors.Default.Wrap(err, "error creating DB
blueprint's labelModels")
}
@@ -54,9 +56,9 @@ func SaveDbBlueprint(dbBlueprint *models.DbBlueprint)
errors.Error {
}
// GetDbBlueprints returns a paginated list of Blueprints based on `query`
-func GetDbBlueprints(query *BlueprintQuery) ([]*models.DbBlueprint, int64,
errors.Error) {
+func GetDbBlueprints(query *BlueprintQuery) ([]*models.Blueprint, int64,
errors.Error) {
// process query parameters
- clauses := []dal.Clause{dal.From(&models.DbBlueprint{})}
+ clauses := []dal.Clause{dal.From(&models.Blueprint{})}
if query.Enable != nil {
clauses = append(clauses, dal.Where("enable = ?",
*query.Enable))
}
@@ -82,7 +84,7 @@ func GetDbBlueprints(query *BlueprintQuery)
([]*models.DbBlueprint, int64, error
dal.Offset(query.GetSkip()),
dal.Limit(query.GetPageSize()),
)
- dbBlueprints := make([]*models.DbBlueprint, 0)
+ dbBlueprints := make([]*models.Blueprint, 0)
err = db.All(&dbBlueprints, clauses...)
if err != nil {
return nil, 0, errors.Default.Wrap(err, "error getting DB count
of blueprints")
@@ -100,25 +102,25 @@ func GetDbBlueprints(query *BlueprintQuery)
([]*models.DbBlueprint, int64, error
}
// GetDbBlueprint returns the detail of a given Blueprint ID
-func GetDbBlueprint(dbBlueprintId uint64) (*models.DbBlueprint, errors.Error) {
- dbBlueprint := &models.DbBlueprint{}
- err := db.First(dbBlueprint, dal.Where("id = ?", dbBlueprintId))
+func GetDbBlueprint(blueprintId uint64) (*models.Blueprint, errors.Error) {
+ blueprint := &models.Blueprint{}
+ err := db.First(blueprint, dal.Where("id = ?", blueprintId))
if err != nil {
if db.IsErrorNotFound(err) {
return nil, errors.NotFound.Wrap(err, "could not find
blueprint in DB")
}
return nil, errors.Default.Wrap(err, "error getting blueprint
from DB")
}
- err = fillBlueprintDetail(dbBlueprint)
+ err = fillBlueprintDetail(blueprint)
if err != nil {
return nil, err
}
- return dbBlueprint, nil
+ return blueprint, nil
}
// GetDbBlueprintByProjectName returns the detail of a given projectName
-func GetDbBlueprintByProjectName(projectName string) (*models.DbBlueprint,
errors.Error) {
- dbBlueprint := &models.DbBlueprint{}
+func GetDbBlueprintByProjectName(projectName string) (*models.Blueprint,
errors.Error) {
+ dbBlueprint := &models.Blueprint{}
err := db.First(dbBlueprint, dal.Where("project_name = ?", projectName))
if err != nil {
if db.IsErrorNotFound(err) {
@@ -133,90 +135,8 @@ func GetDbBlueprintByProjectName(projectName string)
(*models.DbBlueprint, error
return dbBlueprint, nil
}
-// parseBlueprint
-func parseBlueprint(dbBlueprint *models.DbBlueprint) *models.Blueprint {
- labelList := []string{}
- for _, labelModel := range dbBlueprint.Labels {
- labelList = append(labelList, labelModel.Name)
- }
- blueprint := models.Blueprint{
- Name: dbBlueprint.Name,
- ProjectName: dbBlueprint.ProjectName,
- Mode: dbBlueprint.Mode,
- Plan: []byte(dbBlueprint.Plan),
- Enable: dbBlueprint.Enable,
- CronConfig: dbBlueprint.CronConfig,
- IsManual: dbBlueprint.IsManual,
- SkipOnFail: dbBlueprint.SkipOnFail,
- Settings: []byte(dbBlueprint.Settings),
- Model: dbBlueprint.Model,
- Labels: labelList,
- }
- if len(blueprint.Settings) == 0 {
- blueprint.Settings = nil
- }
- return &blueprint
-}
-
-// parseDbBlueprint
-func parseDbBlueprint(blueprint *models.Blueprint) *models.DbBlueprint {
- dbBlueprint := models.DbBlueprint{
- Name: blueprint.Name,
- ProjectName: blueprint.ProjectName,
- Mode: blueprint.Mode,
- Plan: string(blueprint.Plan),
- Enable: blueprint.Enable,
- CronConfig: blueprint.CronConfig,
- IsManual: blueprint.IsManual,
- SkipOnFail: blueprint.SkipOnFail,
- Settings: string(blueprint.Settings),
- Model: blueprint.Model,
- }
- dbBlueprint.Labels = []models.DbBlueprintLabel{}
- for _, label := range blueprint.Labels {
- dbBlueprint.Labels = append(dbBlueprint.Labels,
models.DbBlueprintLabel{
- // NOTICE: BlueprintId may be nil
- BlueprintId: blueprint.ID,
- Name: label,
- })
- }
- return &dbBlueprint
-}
-
-// encryptDbBlueprint
-func encryptDbBlueprint(dbBlueprint *models.DbBlueprint) (*models.DbBlueprint,
errors.Error) {
- encKey := config.GetConfig().GetString(plugin.EncodeKeyEnvStr)
- planEncrypt, err := plugin.Encrypt(encKey, dbBlueprint.Plan)
- if err != nil {
- return nil, err
- }
- dbBlueprint.Plan = planEncrypt
- settingsEncrypt, err := plugin.Encrypt(encKey, dbBlueprint.Settings)
- dbBlueprint.Settings = settingsEncrypt
- if err != nil {
- return nil, err
- }
- return dbBlueprint, nil
-}
-
-// decryptDbBlueprint
-func decryptDbBlueprint(dbBlueprint *models.DbBlueprint) (*models.DbBlueprint,
errors.Error) {
- encKey := config.GetConfig().GetString(plugin.EncodeKeyEnvStr)
- plan, err := plugin.Decrypt(encKey, dbBlueprint.Plan)
- if err != nil {
- return nil, err
- }
- dbBlueprint.Plan = plan
- settings, err := plugin.Decrypt(encKey, dbBlueprint.Settings)
- dbBlueprint.Settings = settings
- if err != nil {
- return nil, err
- }
- return dbBlueprint, nil
-}
-
-func fillBlueprintDetail(blueprint *models.DbBlueprint) errors.Error {
- err := db.All(&blueprint.Labels, dal.Where("blueprint_id = ?",
blueprint.ID))
+func fillBlueprintDetail(blueprint *models.Blueprint) errors.Error {
+ err := db.Pluck("name", &blueprint.Labels,
dal.From(&models.DbBlueprintLabel{}), dal.Where("blueprint_id = ?",
blueprint.ID))
if err != nil {
return errors.Internal.Wrap(err, "error getting the blueprint
labels from database")
}
diff --git a/backend/server/services/pipeline.go
b/backend/server/services/pipeline.go
index 4f6e02c4e..cd52e9daa 100644
--- a/backend/server/services/pipeline.go
+++ b/backend/server/services/pipeline.go
@@ -74,7 +74,7 @@ func pipelineServiceInit() {
} else {
// standalone mode: reset pipeline status
err := db.UpdateColumn(
- &models.DbPipeline{},
+ &models.Pipeline{},
"status", models.TASK_FAILED,
dal.Where("status = ?", models.TASK_RUNNING),
)
@@ -110,34 +110,29 @@ func pipelineServiceInit() {
// CreatePipeline and return the model
func CreatePipeline(newPipeline *models.NewPipeline) (*models.Pipeline,
errors.Error) {
- dbPipeline, err := CreateDbPipeline(newPipeline)
+ pipeline, err := CreateDbPipeline(newPipeline)
if err != nil {
return nil, errors.Convert(err)
}
- dbPipeline, err = decryptDbPipeline(dbPipeline)
+ err = fillPipelineDetail(pipeline)
if err != nil {
- return nil, err
+ return nil, errors.Convert(err)
}
- pipeline := parsePipeline(dbPipeline)
return pipeline, nil
}
// GetPipelines by query
func GetPipelines(query *PipelineQuery) ([]*models.Pipeline, int64,
errors.Error) {
- dbPipelines, i, err := GetDbPipelines(query)
+ pipelines, i, err := GetDbPipelines(query)
if err != nil {
return nil, 0, errors.Convert(err)
}
- pipelines := make([]*models.Pipeline, 0)
- for _, dbPipeline := range dbPipelines {
- dbPipeline, err = decryptDbPipeline(dbPipeline)
+ for _, p := range pipelines {
+ err = fillPipelineDetail(p)
if err != nil {
return nil, 0, err
}
- pipeline := parsePipeline(dbPipeline)
- pipelines = append(pipelines, pipeline)
}
-
return pipelines, i, nil
}
@@ -147,12 +142,11 @@ func GetPipeline(pipelineId uint64) (*models.Pipeline,
errors.Error) {
if err != nil {
return nil, err
}
- dbPipeline, err = decryptDbPipeline(dbPipeline)
+ err = fillPipelineDetail(dbPipeline)
if err != nil {
return nil, err
}
- pipeline := parsePipeline(dbPipeline)
- return pipeline, nil
+ return dbPipeline, nil
}
// GetPipelineLogsArchivePath creates an archive for the logs of this pipeline
and returns its file path
@@ -182,7 +176,7 @@ func RunPipelineInQueue(pipelineMaxParallel int64) {
panic(err)
}
globalPipelineLog.Info("get lock and wait next pipeline")
- dbPipeline := &models.DbPipeline{}
+ dbPipeline := &models.Pipeline{}
for {
cronLocker.Lock()
// prepare query to find an appropriate pipeline to
execute
@@ -212,7 +206,7 @@ func RunPipelineInQueue(pipelineMaxParallel int64) {
}
// mark the pipeline running
- err = db.UpdateColumns(&models.DbPipeline{}, []dal.DalSet{
+ err = db.UpdateColumns(&models.Pipeline{}, []dal.DalSet{
{ColumnName: "status", Value: models.TASK_RUNNING},
{ColumnName: "message", Value: ""},
{ColumnName: "began_at", Value: time.Now()},
@@ -221,17 +215,15 @@ func RunPipelineInQueue(pipelineMaxParallel int64) {
panic(err)
}
- // load pipeline
- dbPipeline, err = GetDbPipeline(dbPipeline.ID)
+ // add pipelineParallelLabels to runningParallelLabels
+ var pipelineParallelLabels []string
+ err = fillPipelineDetail(dbPipeline)
if err != nil {
panic(err)
}
-
- // add pipelineParallelLabels to runningParallelLabels
- var pipelineParallelLabels []string
for _, dbLabel := range dbPipeline.Labels {
- if strings.HasPrefix(dbLabel.Name, `parallel/`) {
- pipelineParallelLabels =
append(pipelineParallelLabels, dbLabel.Name)
+ if strings.HasPrefix(dbLabel, `parallel/`) {
+ pipelineParallelLabels =
append(pipelineParallelLabels, dbLabel)
}
}
runningParallelLabelLock.Lock()
@@ -262,7 +254,7 @@ func watchTemporalPipelines() {
// run forever
for range ticker.C {
// load all running pipeline from database
- runningDbPipelines := make([]models.DbPipeline, 0)
+ runningDbPipelines := make([]models.Pipeline, 0)
err := db.All(&runningDbPipelines, dal.Where("status =
?", models.TASK_RUNNING))
if err != nil {
panic(err)
@@ -380,7 +372,7 @@ func CancelPipeline(pipelineId uint64) errors.Error {
// prevent RunPipelineInQueue from consuming pending pipelines
cronLocker.Lock()
defer cronLocker.Unlock()
- pipeline := &models.DbPipeline{}
+ pipeline := &models.Pipeline{}
err := db.First(pipeline, dal.Where("id = ?", pipelineId))
if err != nil {
return errors.BadInput.New("pipeline not found")
@@ -516,7 +508,7 @@ func RerunPipeline(pipelineId uint64, task *models.Task)
([]*models.Task, errors
}
// mark pipline rerun
- err = db.UpdateColumn(&models.DbPipeline{},
+ err = db.UpdateColumn(&models.Pipeline{},
"status", models.TASK_RERUN,
dal.Where("id = ?", pipelineId),
)
diff --git a/backend/server/services/pipeline_helper.go
b/backend/server/services/pipeline_helper.go
index b49ddf819..5884bf532 100644
--- a/backend/server/services/pipeline_helper.go
+++ b/backend/server/services/pipeline_helper.go
@@ -23,19 +23,18 @@ import (
"github.com/apache/incubator-devlake/core/dal"
"github.com/apache/incubator-devlake/core/errors"
"github.com/apache/incubator-devlake/core/models"
- "github.com/apache/incubator-devlake/core/plugin"
)
// ErrBlueprintRunning indicates there is a running pipeline with the
specified blueprint_id
var ErrBlueprintRunning = errors.Default.New("the blueprint is running")
// CreateDbPipeline returns a NewPipeline
-func CreateDbPipeline(newPipeline *models.NewPipeline) (*models.DbPipeline,
errors.Error) {
+func CreateDbPipeline(newPipeline *models.NewPipeline) (*models.Pipeline,
errors.Error) {
cronLocker.Lock()
defer cronLocker.Unlock()
if newPipeline.BlueprintId > 0 {
count, err := db.Count(
- dal.From(&models.DbPipeline{}),
+ dal.From(&models.Pipeline{}),
dal.Where("blueprint_id = ? AND status IN ?",
newPipeline.BlueprintId, models.PendingTaskStatus),
)
if err != nil {
@@ -50,22 +49,18 @@ func CreateDbPipeline(newPipeline *models.NewPipeline)
(*models.DbPipeline, erro
return nil, err
}
// create pipeline object from posted data
- dbPipeline := &models.DbPipeline{
+ dbPipeline := &models.Pipeline{
Name: newPipeline.Name,
FinishedTasks: 0,
Status: models.TASK_CREATED,
Message: "",
SpentSeconds: 0,
- Plan: string(planByte),
+ Plan: planByte,
SkipOnFail: newPipeline.SkipOnFail,
}
if newPipeline.BlueprintId != 0 {
dbPipeline.BlueprintId = newPipeline.BlueprintId
}
- dbPipeline, err = encryptDbPipeline(dbPipeline)
- if err != nil {
- return nil, err
- }
// save pipeline to database
if err := db.Create(&dbPipeline); err != nil {
@@ -73,15 +68,15 @@ func CreateDbPipeline(newPipeline *models.NewPipeline)
(*models.DbPipeline, erro
return nil, errors.Internal.Wrap(err, "create pipeline failed")
}
- dbPipeline.Labels = []models.DbPipelineLabel{}
+ labels := make([]models.DbPipelineLabel, 0)
for _, label := range newPipeline.Labels {
- dbPipeline.Labels = append(dbPipeline.Labels,
models.DbPipelineLabel{
+ labels = append(labels, models.DbPipelineLabel{
PipelineId: dbPipeline.ID,
Name: label,
})
}
- if len(dbPipeline.Labels) > 0 {
- if err := db.Create(&dbPipeline.Labels); err != nil {
+ if len(newPipeline.Labels) > 0 {
+ if err := db.Create(&labels); err != nil {
globalPipelineLog.Error(err, "create pipeline's
labelModels failed: %v", err)
return nil, errors.Internal.Wrap(err, "create
pipeline's labelModels failed")
}
@@ -125,9 +120,9 @@ func CreateDbPipeline(newPipeline *models.NewPipeline)
(*models.DbPipeline, erro
}
// GetDbPipelines by query
-func GetDbPipelines(query *PipelineQuery) ([]*models.DbPipeline, int64,
errors.Error) {
+func GetDbPipelines(query *PipelineQuery) ([]*models.Pipeline, int64,
errors.Error) {
// process query parameters
- clauses := []dal.Clause{dal.From(&models.DbPipeline{})}
+ clauses := []dal.Clause{dal.From(&models.Pipeline{})}
if query.BlueprintId != 0 {
clauses = append(clauses, dal.Where("blueprint_id = ?",
query.BlueprintId))
}
@@ -139,8 +134,8 @@ func GetDbPipelines(query *PipelineQuery)
([]*models.DbPipeline, int64, errors.E
}
if query.Label != "" {
clauses = append(clauses,
- dal.Join("LEFT JOIN _devlake_pipeline_labels bl ON
bl.pipeline_id = _devlake_pipelines.id"),
- dal.Where("bl.name = ?", query.Label),
+ dal.Join("LEFT JOIN _devlake_pipeline_labels pl ON
pl.pipeline_id = _devlake_pipelines.id"),
+ dal.Where("pl.name = ?", query.Label),
)
}
@@ -156,7 +151,7 @@ func GetDbPipelines(query *PipelineQuery)
([]*models.DbPipeline, int64, errors.E
dal.Offset(query.GetSkip()),
dal.Limit(query.GetPageSize()),
)
- dbPipelines := make([]*models.DbPipeline, 0)
+ dbPipelines := make([]*models.Pipeline, 0)
err = db.All(&dbPipelines, clauses...)
if err != nil {
return nil, 0, errors.Default.Wrap(err, "error getting DB count
of pipelines")
@@ -174,8 +169,8 @@ func GetDbPipelines(query *PipelineQuery)
([]*models.DbPipeline, int64, errors.E
}
// GetDbPipeline by id
-func GetDbPipeline(pipelineId uint64) (*models.DbPipeline, errors.Error) {
- dbPipeline := &models.DbPipeline{}
+func GetDbPipeline(pipelineId uint64) (*models.Pipeline, errors.Error) {
+ dbPipeline := &models.Pipeline{}
err := db.First(dbPipeline, dal.Where("id = ?", pipelineId))
if err != nil {
if db.IsErrorNotFound(err) {
@@ -190,85 +185,8 @@ func GetDbPipeline(pipelineId uint64) (*models.DbPipeline,
errors.Error) {
return dbPipeline, nil
}
-// parsePipeline converts DbPipeline to Pipeline
-func parsePipeline(dbPipeline *models.DbPipeline) *models.Pipeline {
- labelList := []string{}
- for _, labelModel := range dbPipeline.Labels {
- labelList = append(labelList, labelModel.Name)
- }
- pipeline := models.Pipeline{
- Model: dbPipeline.Model,
- Name: dbPipeline.Name,
- BlueprintId: dbPipeline.BlueprintId,
- Plan: []byte(dbPipeline.Plan),
- TotalTasks: dbPipeline.TotalTasks,
- FinishedTasks: dbPipeline.FinishedTasks,
- BeganAt: dbPipeline.BeganAt,
- FinishedAt: dbPipeline.FinishedAt,
- Status: dbPipeline.Status,
- Message: dbPipeline.Message,
- ErrorName: dbPipeline.ErrorName,
- SpentSeconds: dbPipeline.SpentSeconds,
- Stage: dbPipeline.Stage,
- SkipOnFail: dbPipeline.SkipOnFail,
- Labels: labelList,
- }
- return &pipeline
-}
-
-// parseDbPipeline converts Pipeline to DbPipeline
-// nolint:unused
-func parseDbPipeline(pipeline *models.Pipeline) *models.DbPipeline {
- dbPipeline := models.DbPipeline{
- Model: pipeline.Model,
- Name: pipeline.Name,
- BlueprintId: pipeline.BlueprintId,
- Plan: string(pipeline.Plan),
- TotalTasks: pipeline.TotalTasks,
- FinishedTasks: pipeline.FinishedTasks,
- BeganAt: pipeline.BeganAt,
- FinishedAt: pipeline.FinishedAt,
- Status: pipeline.Status,
- Message: pipeline.Message,
- SpentSeconds: pipeline.SpentSeconds,
- Stage: pipeline.Stage,
- SkipOnFail: pipeline.SkipOnFail,
- }
- dbPipeline.Labels = []models.DbPipelineLabel{}
- for _, label := range pipeline.Labels {
- dbPipeline.Labels = append(dbPipeline.Labels,
models.DbPipelineLabel{
- // NOTICE: PipelineId may be nil
- PipelineId: pipeline.ID,
- Name: label,
- })
- }
- return &dbPipeline
-}
-
-// encryptDbPipeline encrypts dbPipeline.Plan
-func encryptDbPipeline(dbPipeline *models.DbPipeline) (*models.DbPipeline,
errors.Error) {
- encKey := cfg.GetString(plugin.EncodeKeyEnvStr)
- planEncrypt, err := plugin.Encrypt(encKey, dbPipeline.Plan)
- if err != nil {
- return nil, err
- }
- dbPipeline.Plan = planEncrypt
- return dbPipeline, nil
-}
-
-// encryptDbPipeline decrypts dbPipeline.Plan
-func decryptDbPipeline(dbPipeline *models.DbPipeline) (*models.DbPipeline,
errors.Error) {
- encKey := cfg.GetString(plugin.EncodeKeyEnvStr)
- plan, err := plugin.Decrypt(encKey, dbPipeline.Plan)
- if err != nil {
- return nil, err
- }
- dbPipeline.Plan = plan
- return dbPipeline, nil
-}
-
-func fillPipelineDetail(pipeline *models.DbPipeline) errors.Error {
- err := db.All(&pipeline.Labels, dal.Where("pipeline_id = ?",
pipeline.ID))
+func fillPipelineDetail(pipeline *models.Pipeline) errors.Error {
+ err := basicRes.GetDal().Pluck("name", &pipeline.Labels,
dal.From(&models.DbPipelineLabel{}), dal.Where("pipeline_id = ?", pipeline.ID))
if err != nil {
return errors.Internal.Wrap(err, "error getting the pipeline
labels from database")
}
diff --git a/backend/server/services/pipeline_runner.go
b/backend/server/services/pipeline_runner.go
index 5ab990c2b..cb3d27f4a 100644
--- a/backend/server/services/pipeline_runner.go
+++ b/backend/server/services/pipeline_runner.go
@@ -148,7 +148,7 @@ func runPipeline(pipelineId uint64) errors.Error {
// 1. TASK_COMPLETED: all tasks were executed sucessfully
// 2. TASK_FAILED: SkipOnFail=false with failed task(s)
// 3. TASK_PARTIAL: SkipOnFail=true with failed task(s)
-func ComputePipelineStatus(pipeline *models.DbPipeline, isCancelled bool)
(string, errors.Error) {
+func ComputePipelineStatus(pipeline *models.Pipeline, isCancelled bool)
(string, errors.Error) {
tasks, err := GetLatestTasksOfPipeline(pipeline)
if err != nil {
return "", err
@@ -182,7 +182,7 @@ func ComputePipelineStatus(pipeline *models.DbPipeline,
isCancelled bool) (strin
}
// GetLatestTasksOfPipeline returns latest tasks (reran tasks are excluding)
of specified pipeline
-func GetLatestTasksOfPipeline(pipeline *models.DbPipeline) ([]*models.Task,
errors.Error) {
+func GetLatestTasksOfPipeline(pipeline *models.Pipeline) ([]*models.Task,
errors.Error) {
task := &models.Task{}
cursor, err := db.Cursor(
dal.From(task),
diff --git a/backend/server/services/project.go
b/backend/server/services/project.go
index 0c5c00c36..f57c82de0 100644
--- a/backend/server/services/project.go
+++ b/backend/server/services/project.go
@@ -205,7 +205,7 @@ func PatchProject(name string, body map[string]interface{})
(*models.ApiOutputPr
// Blueprint
err = tx.UpdateColumn(
- &models.DbBlueprint{},
+ &models.Blueprint{},
"project_name", project.Name,
dal.Where("project_name = ?", name),
)
@@ -222,7 +222,7 @@ func PatchProject(name string, body map[string]interface{})
(*models.ApiOutputPr
// Blueprint
err = tx.UpdateColumn(
- &models.DbBlueprint{},
+ &models.Blueprint{},
"enable", projectInput.Enable,
dal.Where("project_name = ?", name),
)
diff --git a/backend/test/services/pipeline_runner_e2e_test.go
b/backend/test/services/pipeline_runner_e2e_test.go
index 337fae224..e3b3768ea 100644
--- a/backend/test/services/pipeline_runner_e2e_test.go
+++ b/backend/test/services/pipeline_runner_e2e_test.go
@@ -28,7 +28,7 @@ import (
func TestComputePipelineStatus(t *testing.T) {
db := services.GetBasicRes().GetDal()
// insert fake tasks to datbase
- pipeline := &models.DbPipeline{
+ pipeline := &models.Pipeline{
TotalTasks: 3,
}
err := db.Create(pipeline)
