This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 010c3fdf1 feat: encrypt task.options in database (#4005)
010c3fdf1 is described below
commit 010c3fdf117283752f67a92f23fc8497a940a76c
Author: Klesh Wong <[email protected]>
AuthorDate: Thu Dec 22 09:28:12 2022 +0800
feat: encrypt task.options in database (#4005)
---
impl/dalgorm/encdec_serializer.go | 67 ++++++++++++++++++++++
models/migrationscripts/20221221_encrypt_task.go | 73 ++++++++++++++++++++++++
models/migrationscripts/register.go | 1 +
models/task.go | 4 +-
plugins/core/plugin_model.go | 1 +
plugins/core/plugin_utils.go | 3 +-
runner/basic_res.go | 14 +++++
runner/directrun.go | 2 +-
services/task.go | 2 +-
9 files changed, 162 insertions(+), 5 deletions(-)
diff --git a/impl/dalgorm/encdec_serializer.go
b/impl/dalgorm/encdec_serializer.go
new file mode 100644
index 000000000..bb0242d80
--- /dev/null
+++ b/impl/dalgorm/encdec_serializer.go
@@ -0,0 +1,67 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dalgorm
+
+import (
+ "context"
+ "fmt"
+ "reflect"
+
+ "github.com/apache/incubator-devlake/plugins/core"
+ "gorm.io/gorm/schema"
+)
+
+var _ schema.SerializerInterface = (*EncDecSerializer)(nil)
+
+// EncDecSerializer is responsible for field encryption/decryption in
Application Level
+// Ref: https://gorm.io/docs/serializer.html
+type EncDecSerializer struct {
+ encKey string
+}
+
+// Scan implements serializer interface
+func (es *EncDecSerializer) Scan(ctx context.Context, field *schema.Field, dst
reflect.Value, dbValue interface{}) (err error) {
+ if dbValue != nil {
+ var base64str string
+ switch v := dbValue.(type) {
+ case []byte:
+ base64str = string(v)
+ case string:
+ base64str = v
+ default:
+ return fmt.Errorf("failed to decrypt value: %#v",
dbValue)
+ }
+
+ decrypted, err := core.Decrypt(es.encKey, base64str)
+ if err != nil {
+ return err
+ }
+ field.ReflectValueOf(ctx, dst).SetString(decrypted)
+ }
+ return nil
+}
+
+// Value implements serializer interface
+func (es *EncDecSerializer) Value(ctx context.Context, field *schema.Field,
dst reflect.Value, fieldValue interface{}) (interface{}, error) {
+ return core.Encrypt(es.encKey, fieldValue.(string))
+}
+
+// Init the encdec serializer
+func Init(encKey string) {
+ schema.RegisterSerializer("encdec", &EncDecSerializer{encKey: encKey})
+}
diff --git a/models/migrationscripts/20221221_encrypt_task.go
b/models/migrationscripts/20221221_encrypt_task.go
new file mode 100644
index 000000000..c2a079021
--- /dev/null
+++ b/models/migrationscripts/20221221_encrypt_task.go
@@ -0,0 +1,73 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+ "encoding/json"
+
+ "github.com/apache/incubator-devlake/errors"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+ "github.com/apache/incubator-devlake/models/migrationscripts/archived"
+ "github.com/apache/incubator-devlake/plugins/core"
+)
+
+var _ core.MigrationScript = (*encryptTask221221)(nil)
+
+type encryptTask221221 struct{}
+
+type srcTaskEncryption221221 struct {
+ archived.Model
+ Options json.RawMessage
+}
+
+type dstTaskEncryption221221 struct {
+ archived.Model
+ Options string
+}
+
+func (script *encryptTask221221) Up(basicRes core.BasicRes) errors.Error {
+ encKey := basicRes.GetConfig(core.EncodeKeyEnvStr)
+ if encKey == "" {
+ return errors.BadInput.New("invalid encKey")
+ }
+ err := migrationhelper.TransformColumns(
+ basicRes,
+ script,
+ "_devlake_tasks",
+ []string{"options"},
+ func(src *srcTaskEncryption221221) (*dstTaskEncryption221221,
errors.Error) {
+ options, err := core.Encrypt(encKey,
string(src.Options))
+ if err != nil {
+ return nil, err
+ }
+ return &dstTaskEncryption221221{
+ Model: src.Model,
+ Options: options,
+ }, nil
+ },
+ )
+ return err
+}
+
+func (*encryptTask221221) Version() uint64 {
+ return 20221221162121
+}
+
+func (*encryptTask221221) Name() string {
+ return "encrypt task.options"
+}
diff --git a/models/migrationscripts/register.go
b/models/migrationscripts/register.go
index 0db6254ff..9d0e01fd0 100644
--- a/models/migrationscripts/register.go
+++ b/models/migrationscripts/register.go
@@ -66,5 +66,6 @@ func All() []core.MigrationScript {
new(addCollectorMeta20221125),
new(addOriginalProject),
new(addErrorName),
+ new(encryptTask221221),
}
}
diff --git a/models/task.go b/models/task.go
index 9ad823447..f1933a35c 100644
--- a/models/task.go
+++ b/models/task.go
@@ -51,7 +51,7 @@ type Task struct {
common.Model
Plugin string `json:"plugin" gorm:"index"`
Subtasks datatypes.JSON `json:"subtasks"`
- Options datatypes.JSON `json:"options"`
+ Options string `json:"options"
gorm:"serializer:encdec"`
Status string `json:"status"`
Message string `json:"message"`
ErrorName string `json:"errorName"`
@@ -102,6 +102,6 @@ func (task *Task) GetSubTasks() ([]string, errors.Error) {
func (task *Task) GetOptions() (map[string]interface{}, errors.Error) {
var options map[string]interface{}
- err := errors.Convert(json.Unmarshal(task.Options, &options))
+ err := errors.Convert(json.Unmarshal([]byte(task.Options), &options))
return options, err
}
diff --git a/plugins/core/plugin_model.go b/plugins/core/plugin_model.go
index 5fdb7de06..cb0847a5c 100644
--- a/plugins/core/plugin_model.go
+++ b/plugins/core/plugin_model.go
@@ -17,6 +17,7 @@ limitations under the License.
package core
+// TODO: replace with dal.Tabler
type Tabler interface {
TableName() string
}
diff --git a/plugins/core/plugin_utils.go b/plugins/core/plugin_utils.go
index 137e03a37..c888546ec 100644
--- a/plugins/core/plugin_utils.go
+++ b/plugins/core/plugin_utils.go
@@ -24,9 +24,10 @@ import (
"crypto/sha256"
"encoding/base64"
"fmt"
- "github.com/apache/incubator-devlake/errors"
"math/rand"
"time"
+
+ "github.com/apache/incubator-devlake/errors"
)
const EncodeKeyEnvStr = "ENCODE_KEY"
diff --git a/runner/basic_res.go b/runner/basic_res.go
index b58aaacbf..e8ac2416d 100644
--- a/runner/basic_res.go
+++ b/runner/basic_res.go
@@ -18,6 +18,9 @@ limitations under the License.
package runner
import (
+ "fmt"
+ "sync"
+
"github.com/apache/incubator-devlake/config"
"github.com/apache/incubator-devlake/impl"
"github.com/apache/incubator-devlake/impl/dalgorm"
@@ -26,15 +29,26 @@ import (
"gorm.io/gorm"
)
+var app_lock sync.Mutex
+var app_inited bool
+
// CreateAppBasicRes returns a application level BasicRes instance based on
.env/environment variables
// it is useful because multiple places need BasicRes including `main.go`
`directrun` and `worker`
+// keep in mind this function can be called only once
func CreateAppBasicRes() core.BasicRes {
+ app_lock.Lock()
+ if app_inited {
+ panic(fmt.Errorf("CreateAppBasicRes can be called once"))
+ }
+ app_inited = true
+ app_lock.Unlock()
cfg := config.GetConfig()
log := logger.Global
db, err := NewGormDb(cfg, logger.Global)
if err != nil {
panic(err)
}
+ dalgorm.Init(cfg.GetString(core.EncodeKeyEnvStr))
return CreateBasicRes(cfg, log, db)
}
diff --git a/runner/directrun.go b/runner/directrun.go
index 7794aa88c..484870802 100644
--- a/runner/directrun.go
+++ b/runner/directrun.go
@@ -88,7 +88,7 @@ func DirectRun(cmd *cobra.Command, args []string, pluginTask
core.PluginTask, op
}
task := &models.Task{
Plugin: cmd.Use,
- Options: optionsJson,
+ Options: string(optionsJson),
Subtasks: subtasksJson,
}
err = RunPluginSubTasks(
diff --git a/services/task.go b/services/task.go
index 0630e0ddc..7b93897e7 100644
--- a/services/task.go
+++ b/services/task.go
@@ -56,7 +56,7 @@ func CreateTask(newTask *models.NewTask) (*models.Task,
errors.Error) {
task := &models.Task{
Plugin: newTask.Plugin,
Subtasks: s,
- Options: b,
+ Options: string(b),
Status: models.TASK_CREATED,
Message: "",
PipelineId: newTask.PipelineId,
