JinyLeeChina edited a comment on issue #6597:
URL:
https://github.com/apache/dolphinscheduler/issues/6597#issuecomment-952508320
## Tables to modify
t_ds_command
t_ds_error_command
t_ds_process_definition
t_ds_process_instance
t_ds_project
t_ds_schedules
t_ds_task_instance
## Tables to add
t_ds_alert_plugin_instance
t_ds_environment
t_ds_environment_worker_group_relation
t_ds_plugin_define
t_ds_process_definition_log
t_ds_process_task_relation
t_ds_process_task_relation_log
t_ds_task_definition
t_ds_task_definition_log
## Table structure change no data migration
### t_ds_command
```sql
alter table t_ds_command change process_definition_id
process_definition_code bigint(20) NOT NULL COMMENT 'process definition code';
alter table t_ds_command add environment_code bigint(20) DEFAULT '-1'
COMMENT 'environment code' AFTER worker_group;
alter table t_ds_command add dry_run tinyint(4) DEFAULT '0' COMMENT 'dry run
flag：0 normal, 1 dry run' AFTER environment_code;
alter table t_ds_command add process_definition_version int(11) DEFAULT '0'
COMMENT 'process definition version' AFTER process_definition_code;
alter table t_ds_command add process_instance_id int(11) DEFAULT '0' COMMENT
'process instance id' AFTER process_definition_version;
alter table t_ds_command add KEY `priority_id_index`
(`process_instance_priority`,`id`) USING BTREE;
```
### t_ds_error_command
```sql
alter table t_ds_error_command change process_definition_id
process_definition_code bigint(20) NOT NULL COMMENT 'process definition code';
alter table t_ds_error_command add environment_code bigint(20) DEFAULT '-1'
COMMENT 'environment code' AFTER worker_group;
alter table t_ds_error_command add dry_run tinyint(4) DEFAULT '0' COMMENT
'dry run flag：0 normal, 1 dry run' AFTER message;
alter table t_ds_error_command add process_definition_version int(11)
DEFAULT '0' COMMENT 'process definition version' AFTER process_definition_code;
alter table t_ds_error_command add process_instance_id int(11) DEFAULT '0'
COMMENT 'process instance id' AFTER process_definition_version;
```
## Data migration to be determined
### t_ds_process_instance
```sql
alter table t_ds_process_instance change process_definition_id
process_definition_code bigint(20) NOT NULL COMMENT 'process definition code';
alter table t_ds_process_instance add process_definition_version int(11)
DEFAULT '0' COMMENT 'process definition version' AFTER process_definition_code;
alter table t_ds_process_instance add environment_code bigint(20) DEFAULT
'-1' COMMENT 'environment code' AFTER worker_group;
alter table t_ds_process_instance add var_pool longtext COMMENT 'var_pool'
AFTER tenant_id;
alter table t_ds_process_instance add dry_run tinyint(4) DEFAULT '0' COMMENT
'dry run flag：0 normal, 1 dry run' AFTER var_pool;
alter table t_ds_process_instance drop KEY `process_instance_index`;
alter table t_ds_process_instance add KEY `process_instance_index`
(`process_definition_code`,`id`) USING BTREE;
alter table t_ds_process_instance drop process_instance_json;
alter table t_ds_process_instance drop locations;
alter table t_ds_process_instance drop connects;
alter table t_ds_process_instance drop dependence_schedule_times;
```
### t_ds_task_instance
```sql
alter table t_ds_task_instance change process_definition_id task_code
bigint(20) NOT NULL COMMENT 'task definition code';
alter table t_ds_task_instance add task_definition_version int(11) DEFAULT
'0' COMMENT 'task definition version' AFTER task_code;
alter table t_ds_task_instance add task_params text COMMENT 'job custom
parameters' AFTER app_link;
alter table t_ds_task_instance add environment_code bigint(20) DEFAULT '-1'
COMMENT 'environment code' AFTER worker_group;
alter table t_ds_task_instance add environment_config text COMMENT 'this
config contains many environment variables config' AFTER environment_code;
alter table t_ds_task_instance add first_submit_time datetime DEFAULT NULL
COMMENT 'task first submit time' AFTER executor_id;
alter table t_ds_task_instance add delay_time int(4) DEFAULT '0' COMMENT
'task delay execution time' AFTER first_submit_time;
alter table t_ds_task_instance add var_pool longtext COMMENT 'var_pool'
AFTER delay_time;
alter table t_ds_task_instance add dry_run tinyint(4) DEFAULT '0' COMMENT
'dry run flag：0 normal, 1 dry run' AFTER var_pool;
alter table t_ds_task_instance drop KEY `task_instance_index`;
alter table t_ds_task_instance drop task_json;
```
## Table structure changes and data migration
### t_ds_project
```sql
alter table t_ds_project add `code` bigint(20) NOT NULL COMMENT 'encoding'
AFTER `name`;
```
`The generated code is inserted directly into the table`
### t_ds_schedules
```sql
alter table t_ds_schedules change process_definition_id
process_definition_code bigint(20) NOT NULL COMMENT 'process definition code';
alter table t_ds_schedules add timezone_id varchar(40) DEFAULT NULL COMMENT
'timezoneId' AFTER end_time;
alter table t_ds_schedules add environment_code bigint(20) DEFAULT '-1'
COMMENT 'environment code' AFTER worker_group;
```
`From t_ ds_ process_definition table obtains the mapping relationship
between ID and CODE, and then inserts it into the table`
### t_ds_process_definition
```sql
alter table t_ds_process_definition add `code` bigint(20) NOT NULL COMMENT
'encoding' AFTER `id`;
alter table t_ds_process_definition add `project_code` bigint(20) NOT NULL
COMMENT 'encoding' AFTER `project_id`;
alter table t_ds_process_definition add `warning_group_id` int(11) DEFAULT
NULL COMMENT 'alert group id' AFTER `locations`;
alter table t_ds_process_definition add UNIQUE KEY `process_unique`
(`name`,`project_code`) USING BTREE;
alter table t_ds_process_definition modify `description` text COMMENT
'description' after `version`;
alter table t_ds_process_definition modify `release_state` tinyint(4)
DEFAULT NULL COMMENT 'process definition release state：0:offline,1:online'
after `project_code`;
alter table t_ds_process_definition modify `create_time` datetime DEFAULT
NULL COMMENT 'create time' after `tenant_id`;
```
`Execute the following statements after converting the data`
```sql
alter table t_ds_process_definition drop primary key;
ALTER TABLE t_ds_process_definition ADD PRIMARY KEY (`id`,`code`);
ALTER TABLE t_ds_process_definition drop KEY `process_definition_unique`;
ALTER TABLE t_ds_process_definition drop KEY `process_definition_index`;
alter table t_ds_process_definition drop project_id;
alter table t_ds_process_definition drop process_definition_json;
alter table t_ds_process_definition drop connects;
alter table t_ds_process_definition drop receivers;
alter table t_ds_process_definition drop receivers_cc;
alter table t_ds_process_definition drop modify_by;
alter table t_ds_process_definition drop resource_ids;
```
## JSON Splitting
```json
{"tenantId":1,"globalParams":[],"timeout":120,"tasks":[{"conditionResult":{"successNode":[""],"failedNode":[""]},"description":"","runFlag":"FORBIDDEN","type":"SHELL","params":{"rawScript":"run_etl
jobId=2653
env=prod","localParams":[],"resourceList":[]},"timeout":{"enable":true,"interval":30,"strategy":"WARN"},"maxRetryTimes":"0","taskInstancePriority":"MEDIUM","name":"test_1","dependence":{},"retryInterval":"1","preTasks":[],"workerGroup":"prod","id":"tasks-76841"}]}
```
JSON field | t_ds_process_definition field
-- | --
tenantId | tenant_id
globalParams | global_params
timeout | timeout
JSON field | t_ds_task_definition field
-- | --
conditionResult | Put the whole into task_params
description | description
runFlag | flag (The int value of enumeration)
type | task_type
params | task_params
timeout.interval | timeout
timeout.enable | timeout_flag (The int value of enumeration)
timeout.strategy | timeout_notify_strategy (The int value of enumeration)
maxRetryTimes | fail_retry_times
taskInstancePriority | task_priority (The int value of enumeration)
name | name
dependence | Put the whole into task_params
retryInterval | fail_retry_interval
workerGroup | worker_group
| resource_ids from t_ ds_ process_definition table get
`t_ds_process_task_relation maps from preTasks`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
