This is an automated email from the ASF dual-hosted git repository.
klesh pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-devlake.git
The following commit(s) were added to refs/heads/main by this push:
new 4c7d77062 Fix azure devops migration from 0.17 to 0.18 (#5957)
4c7d77062 is described below
commit 4c7d770621626298b232593dbaac0380a69be1e3
Author: Camille Teruel <[email protected]>
AuthorDate: Wed Aug 30 11:28:19 2023 +0200
Fix azure devops migration from 0.17 to 0.18 (#5957)
* fix: Fix AzDO connection migration 0.17 -> 0.18
In 0.17, all connection fields are encrypted.
In 0.18 only `token` is, so we need a migration of AzDo connection to
decrypt all fields but `token`.
* fix: Allow setting entities=None to ScopeConfig
Allow setting `entities` field to `None` in `ScopeConfig` and fallback to
default value.
* feat: Add RenameColumn migration op
* fix: Add missing AzDO migrations from 0.17 to 0.18
Now that migrations have been made manual, we need to add all the
migrations for
all that changed from 0.17 to 0.18 that were automatic before...
Those are all required migrations collected with `git diff v0.17.0
v0.18.0-beta7 -- backend/python/plugins/azuredevops/azuredevops/models.py`:
* Rename GitRepository.transformation_rule_id -> scope_config_id
* Add GitRepository.provider
* Rename Job.startTime -> start_time
* Rename Job.finishTime -> start_time
---------
Co-authored-by: Camille Teruel <[email protected]>
---
.../plugins/azuredevops/azuredevops/migrations.py | 13 ++-
backend/python/pydevlake/pydevlake/model.py | 6 ++
backend/server/services/remote/models/migration.go | 26 +++++-
.../azuredevops_20230714_decrypt_azdo_fields.go | 96 ++++++++++++++++++++++
.../remote/models/migrationscripts/register.go | 1 +
5 files changed, 137 insertions(+), 5 deletions(-)
diff --git a/backend/python/plugins/azuredevops/azuredevops/migrations.py
b/backend/python/plugins/azuredevops/azuredevops/migrations.py
index c017dcb58..98fd3b245 100644
--- a/backend/python/plugins/azuredevops/azuredevops/migrations.py
+++ b/backend/python/plugins/azuredevops/azuredevops/migrations.py
@@ -164,9 +164,14 @@ def add_entities_column_to_scope_config(b:
MigrationScriptBuilder):
def add_raw_data_params_table_to_scope(b: MigrationScriptBuilder):
b.execute(f'''UPDATE _tool_azuredevops_gitrepositories SET _raw_data_table
= '_raw_azuredevops_scopes' WHERE 1=1''')
+
@migration(20230802000001, name="rename startTime/finishTime to
start_time/finish_time")
def rename_starttime_and_finishtime_for_job(b: MigrationScriptBuilder):
- b.execute(f'ALTER TABLE _tool_azuredevops_jobs RENAME COLUMN startTime TO
start_time', Dialect.MYSQL, ignore_error=True)
- b.execute(f'ALTER TABLE _tool_azuredevops_jobs RENAME COLUMN finishTime TO
finish_time', Dialect.MYSQL, ignore_error=True)
- b.execute(f'ALTER TABLE _tool_azuredevops_jobs RENAME COLUMN `startTime`
TO start_time', Dialect.POSTGRESQL, ignore_error=True)
- b.execute(f'ALTER TABLE _tool_azuredevops_jobs RENAME COLUMN `finishTime`
TO finish_time', Dialect.POSTGRESQL, ignore_error=True)
\ No newline at end of file
+ b.rename_column('_tool_azuredevops_jobs', 'startTime', 'start_time')
+ b.rename_column('_tool_azuredevops_jobs', 'finishTime', 'finish_time')
+
+
+@migration(20230825150421, name="add missing migrations from 0.17 to 0.18")
+def add_missing_migrations_0_17_to_0_18(b: MigrationScriptBuilder):
+ b.rename_column('_tool_azuredevops_gitrepositories',
'transformation_rule_id', 'scope_config_id')
+ b.add_column('_tool_azuredevops_gitrepositories', 'provider',
'varchar(255)')
diff --git a/backend/python/pydevlake/pydevlake/model.py
b/backend/python/pydevlake/pydevlake/model.py
index 105b2aeca..aa1b7771b 100644
--- a/backend/python/pydevlake/pydevlake/model.py
+++ b/backend/python/pydevlake/pydevlake/model.py
@@ -89,6 +89,12 @@ class ScopeConfig(ToolTable, Model):
domain_types: list[DomainType] = Field(default=list(DomainType),
alias="entities")
connection_id: Optional[int]
+ @validator('domain_types', pre=True, always=True)
+ def set_default_domain_types(cls, v):
+ if v is None:
+ return list(DomainType)
+ return v
+
class RawModel(SQLModel):
id: int = Field(primary_key=True)
diff --git a/backend/server/services/remote/models/migration.go
b/backend/server/services/remote/models/migration.go
index ce7b4f1e6..2dccb7b23 100644
--- a/backend/server/services/remote/models/migration.go
+++ b/backend/server/services/remote/models/migration.go
@@ -101,6 +101,28 @@ func (o DropTableOperation) Execute(basicRes
context.BasicRes) errors.Error {
var _ Operation = (*DropTableOperation)(nil)
+type RenameColumnOperation struct {
+ Table string `json:"table"`
+ OldName string `json:"old_name"`
+ NewName string `json:"new_name"`
+}
+
+func (o RenameColumnOperation) Execute(basicRes context.BasicRes) errors.Error
{
+ db := basicRes.GetDal()
+ if !db.HasColumn(o.Table, o.OldName) {
+ return nil
+ }
+ if db.HasColumn(o.Table, o.NewName) {
+ err := db.DropColumns(o.Table, o.NewName)
+ if err != nil {
+ return err
+ }
+ }
+ return db.RenameColumn(o.Table, o.OldName, o.NewName)
+}
+
+var _ Operation = (*RenameTableOperation)(nil)
+
type RenameTableOperation struct {
OldName string `json:"old_name"`
NewName string `json:"new_name"`
@@ -178,12 +200,14 @@ func (s *RemoteMigrationScript) UnmarshalJSON(data
[]byte) error {
operation = &DropColumnOperation{}
case "drop_table":
operation = &DropTableOperation{}
+ case "rename_column":
+ operation = &RenameColumnOperation{}
case "rename_table":
operation = &RenameTableOperation{}
case "create_table":
operation = &CreateTableOperation{}
default:
- return errors.BadInput.New("unsupported operation type")
+ return errors.BadInput.New("unsupported operation type:
" + operationType)
}
err = json.Unmarshal(operationRaw, operation)
if err != nil {
diff --git
a/backend/server/services/remote/models/migrationscripts/azuredevops/azuredevops_20230714_decrypt_azdo_fields.go
b/backend/server/services/remote/models/migrationscripts/azuredevops/azuredevops_20230714_decrypt_azdo_fields.go
new file mode 100644
index 000000000..d02cb10b0
--- /dev/null
+++
b/backend/server/services/remote/models/migrationscripts/azuredevops/azuredevops_20230714_decrypt_azdo_fields.go
@@ -0,0 +1,96 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package azuredevops
+
+import (
+ "github.com/apache/incubator-devlake/core/context"
+ "github.com/apache/incubator-devlake/core/errors"
+
"github.com/apache/incubator-devlake/core/models/migrationscripts/archived"
+ "github.com/apache/incubator-devlake/core/plugin"
+ "github.com/apache/incubator-devlake/helpers/migrationhelper"
+)
+
+var _ plugin.MigrationScript = (*DecryptConnectionFields)(nil)
+
+type azureDevopsConnection20230825 struct {
+ archived.Model
+ Name string
+ Token string
+ Proxy *string
+ Organization *string
+}
+
+type DecryptConnectionFields struct{}
+
+func (script *DecryptConnectionFields) Up(basicRes context.BasicRes)
errors.Error {
+ encryptionSecret := basicRes.GetConfig(plugin.EncodeKeyEnvStr)
+ if encryptionSecret == "" {
+ return errors.BadInput.New("invalid encryptionSecret")
+ }
+
+ err := migrationhelper.TransformColumns(
+ basicRes,
+ script,
+ "_tool_azuredevops_azuredevopsconnections",
+ []string{"name", "proxy", "organization"},
+ func(src *azureDevopsConnection20230825)
(*azureDevopsConnection20230825, errors.Error) {
+ encName := src.Name
+ name, err := plugin.Decrypt(encryptionSecret, encName)
+ if err != nil {
+ return src, nil
+ }
+ src.Name = name
+
+ if src.Proxy != nil {
+ encProxy := *src.Proxy
+ decProxy, err :=
plugin.Decrypt(encryptionSecret, encProxy)
+ if err != nil {
+ return src, nil
+ }
+ if decProxy == "" {
+ src.Proxy = nil
+ } else {
+ src.Proxy = &decProxy
+ }
+ }
+
+ if src.Organization != nil {
+ encOrg := *src.Organization
+ decOrg, err := plugin.Decrypt(encryptionSecret,
encOrg)
+ if err != nil {
+ return src, nil
+ }
+ if decOrg == "" {
+ src.Organization = nil
+ } else {
+ src.Organization = &decOrg
+ }
+ }
+ return src, nil
+ },
+ )
+ return err
+}
+
+func (*DecryptConnectionFields) Version() uint64 {
+ return 20230825090504
+}
+
+func (script *DecryptConnectionFields) Name() string {
+ return "Decrypt Azure DevOps connection fields"
+}
diff --git a/backend/server/services/remote/models/migrationscripts/register.go
b/backend/server/services/remote/models/migrationscripts/register.go
index af1e9f949..15565c30a 100644
--- a/backend/server/services/remote/models/migrationscripts/register.go
+++ b/backend/server/services/remote/models/migrationscripts/register.go
@@ -25,6 +25,7 @@ import (
var allMigrations = map[string][]plugin.MigrationScript{
"azuredevops": {
&azuredevops.AddRawDataForScope{},
+ &azuredevops.DecryptConnectionFields{},
},
}
