JinyLeeChina opened a new issue #4417:
URL: https://github.com/apache/incubator-dolphinscheduler/issues/4417
Original demand #4325
## 1、Table structure design
```sql
--工作流定义表
CREATE TABLE `t_ds_process_definition` (
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'self-increasing id',
`code` bigint(20) NOT NULL COMMENT 'encoding',
`name` varchar(200) DEFAULT NULL COMMENT 'process definition name',
`version` int(11) DEFAULT NULL COMMENT 'process definition version',
`description` text COMMENT 'description',
`project_code` bigint(20) NOT NULL COMMENT 'project code',
`release_state` tinyint(4) DEFAULT NULL COMMENT 'process definition
release state：0:offline,1:online',
`user_id` int(11) DEFAULT NULL COMMENT 'process definition creator id',
`global_params` text COMMENT 'global parameters',
`flag` tinyint(4) DEFAULT NULL COMMENT '0 not available, 1 available',
`receivers` text COMMENT 'receivers',
`receivers_cc` text COMMENT 'cc',
`timeout` int(11) DEFAULT '0' COMMENT 'time out',
`tenant_id` int(11) NOT NULL DEFAULT '-1' COMMENT 'tenant id',
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT
'creation time',
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE
CURRENT_TIMESTAMP COMMENT 'modification time',
PRIMARY KEY (`id`,`code`),
UNIQUE KEY `process_unique` (`name`,`project_code`) USING BTREE
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
--作业定义表
CREATE TABLE `t_ds_task_definition` (
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'self-increasing id',
`code` bigint(20) NOT NULL COMMENT 'encoding',
`name` varchar(200) DEFAULT NULL COMMENT 'task definition name',
`version` int(11) DEFAULT NULL COMMENT 'task definition version',
`description` text COMMENT 'description',
`project_code` bigint(20) NOT NULL COMMENT 'project code',
`user_id` int(11) DEFAULT NULL COMMENT 'task definition creator id',
`task_type` varchar(30) DEFAULT NULL COMMENT 'job type',
`task_params` text COMMENT 'job custom parameters',
`run_flag` tinyint(2) DEFAULT NULL COMMENT '0 not available, 1 available',
`task_priority` tinyint(4) DEFAULT NULL COMMENT 'job priority',
`worker_group` varchar(200) DEFAULT NULL COMMENT 'worker grouping',
`fail_retry_times` int(11) DEFAULT NULL COMMENT 'number of failed retries',
`fail_retry_interval` int(11) DEFAULT NULL COMMENT 'failed retry interval',
`timeout_flag` tinyint(1) DEFAULT '0' COMMENT 'timeout flag:0 close, 1
open',
`timeout_notify_strategy` tinyint(4) DEFAULT NULL COMMENT 'timeout
notification policy: 0 warning, 1 fail',
`timeout_duration` int(11) DEFAULT '0' COMMENT 'timeout length',
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT
'creation time',
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE
CURRENT_TIMESTAMP COMMENT 'modification time',
PRIMARY KEY (`id`,`code`),
UNIQUE KEY `task_unique` (`name`,`project_code`) USING BTREE
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
--工作流作业关系表：t_ds_process_task_relation
CREATE TABLE `t_ds_process_task_relation` (
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'self-increasing id',
`name` varchar(200) DEFAULT NULL COMMENT 'relation name',
`version` int(11) DEFAULT NULL COMMENT 'relation version',
`project_code` bigint(20) NOT NULL COMMENT 'project code',
`process_definition_code` bigint(20) NOT NULL COMMENT 'process code',
`pre_project_code` bigint(20) NOT NULL COMMENT 'pre process code',
`pre_task_code` bigint(20) NOT NULL COMMENT 'pre task code',
`condition_type` tinyint(2) DEFAULT NULL COMMENT 'condition type : 0 none,
1 judge 2 delay',
`condition_params` text COMMENT 'condition params(json)',
`post_project_code` bigint(20) NOT NULL COMMENT 'post process code',
`post_task_code` bigint(20) NOT NULL COMMENT 'post task code',
`locations` text COMMENT 'node location information',
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT
'creation time',
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE
CURRENT_TIMESTAMP COMMENT 'modification time',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
--工作流定义日志表
CREATE TABLE `t_ds_process_definition_log` (
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'self-increasing id',
`code` bigint(20) NOT NULL COMMENT 'encoding',
`name` varchar(200) DEFAULT NULL COMMENT 'process definition name',
`version` int(11) DEFAULT NULL COMMENT 'process definition version',
`description` text COMMENT 'description',
`project_code` bigint(20) NOT NULL COMMENT 'project code',
`release_state` tinyint(4) DEFAULT NULL COMMENT 'process definition
release state：0:offline,1:online',
`user_id` int(11) DEFAULT NULL COMMENT 'process definition creator id',
`global_params` text COMMENT 'global parameters',
`flag` tinyint(4) DEFAULT NULL COMMENT '0 not available, 1 available',
`receivers` text COMMENT 'receivers',
`receivers_cc` text COMMENT 'cc',
`timeout` int(11) DEFAULT '0' COMMENT 'time out',
`tenant_id` int(11) NOT NULL DEFAULT '-1' COMMENT 'tenant id',
`operator` int(11) DEFAULT NULL COMMENT 'operator user id',
`operate_time` timestamp DEFAULT NULL COMMENT 'operate time',
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT
'creation time',
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE
CURRENT_TIMESTAMP COMMENT 'modification time',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
--作业定义日志表
CREATE TABLE `t_ds_task_definition_log` (
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'self-increasing id',
`code` bigint(20) NOT NULL COMMENT 'encoding',
`name` varchar(200) DEFAULT NULL COMMENT 'task definition name',
`version` int(11) DEFAULT NULL COMMENT 'task definition version',
`description` text COMMENT 'description',
`project_code` bigint(20) NOT NULL COMMENT 'project code',
`user_id` int(11) DEFAULT NULL COMMENT 'task definition creator id',
`task_type` varchar(30) DEFAULT NULL COMMENT 'job type',
`task_params` text COMMENT 'job custom parameters',
`run_flag` tinyint(2) DEFAULT NULL COMMENT '0 not available, 1 available',
`task_priority` tinyint(4) DEFAULT NULL COMMENT 'job priority',
`worker_group` varchar(200) DEFAULT NULL COMMENT 'worker grouping',
`fail_retry_times` int(11) DEFAULT NULL COMMENT 'number of failed retries',
`fail_retry_interval` int(11) DEFAULT NULL COMMENT 'failed retry interval',
`timeout_flag` tinyint(1) DEFAULT '0' COMMENT 'timeout flag:0 close, 1
open',
`timeout_notify_strategy` tinyint(4) DEFAULT NULL COMMENT 'timeout
notification policy: 0 warning, 1 fail',
`timeout_duration` int(11) DEFAULT '0' COMMENT 'timeout length',
`operator` int(11) DEFAULT NULL COMMENT 'operator user id',
`operate_time` timestamp DEFAULT NULL COMMENT 'operate time',
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT
'creation time',
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE
CURRENT_TIMESTAMP COMMENT 'modification time',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
--工作流作业关系日志表
CREATE TABLE `t_ds_process_task_relation_log` (
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'self-increasing id',
`name` varchar(200) DEFAULT NULL COMMENT 'relation name',
`version` int(11) DEFAULT NULL COMMENT 'relation version',
`project_code` bigint(20) NOT NULL COMMENT 'project code',
`process_definition_code` bigint(20) NOT NULL COMMENT 'process code',
`pre_project_code` bigint(20) NOT NULL COMMENT 'pre process code',
`pre_task_code` bigint(20) NOT NULL COMMENT 'pre task code',
`condition_type` tinyint(2) DEFAULT NULL COMMENT 'condition type : 0 none,
1 judge 2 delay',
`condition_params` text COMMENT 'condition params(json)',
`post_project_code` bigint(20) NOT NULL COMMENT 'post process code',
`post_task_code` bigint(20) NOT NULL COMMENT 'post task code',
`locations` text COMMENT 'node location information',
`operator` int(11) DEFAULT NULL COMMENT 'operator user id',
`operate_time` timestamp DEFAULT NULL COMMENT 'operate time',
`create_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT
'creation time',
`update_time` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE
CURRENT_TIMESTAMP COMMENT 'modification time',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
--工作流实例表修改： 新增工作流定义版本号字段
CREATE TABLE `t_ds_process_instance` (
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'key',
`name` varchar(255) DEFAULT NULL COMMENT 'process instance name',
`version` int(11) DEFAULT NULL COMMENT 'process definition version',
`process_definition_code` bigint(20) not NULL COMMENT 'process definition
code',
`state` tinyint(4) DEFAULT NULL COMMENT 'process instance Status: 0 commit
succeeded, 1 running, 2 prepare to pause, 3 pause, 4 prepare to stop, 5 stop, 6
fail, 7 succeed, 8 need fault tolerance, 9 kill, 10 wait for thread, 11 wait
for dependency to complete',
`recovery` tinyint(4) DEFAULT NULL COMMENT 'process instance failover
flag：0:normal,1:failover instance',
`start_time` datetime DEFAULT NULL COMMENT 'process instance start time',
`end_time` datetime DEFAULT NULL COMMENT 'process instance end time',
`run_times` int(11) DEFAULT NULL COMMENT 'process instance run times',
`host` varchar(45) DEFAULT NULL COMMENT 'process instance host',
`command_type` tinyint(4) DEFAULT NULL COMMENT 'command type',
`command_param` text COMMENT 'json command parameters',
`task_depend_type` tinyint(4) DEFAULT NULL COMMENT 'task depend type. 0:
only current node,1:before the node,2:later nodes',
`max_try_times` tinyint(4) DEFAULT '0' COMMENT 'max try times',
`failure_strategy` tinyint(4) DEFAULT '0' COMMENT 'failure strategy. 0:end
the process when node failed,1:continue running the other nodes when node
failed',
`warning_type` tinyint(4) DEFAULT '0' COMMENT 'warning type. 0:no
warning,1:warning if process success,2:warning if process failed,3:warning if
success',
`warning_group_id` int(11) DEFAULT NULL COMMENT 'warning group id',
`schedule_time` datetime DEFAULT NULL COMMENT 'schedule time',
`command_start_time` datetime DEFAULT NULL COMMENT 'command start time',
`global_params` text COMMENT 'global parameters',
`flag` tinyint(4) DEFAULT '1' COMMENT 'flag',
`update_time` timestamp NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE
CURRENT_TIMESTAMP,
`is_sub_process` int(11) DEFAULT '0' COMMENT 'flag, whether the process is
sub process',
`executor_id` int(11) NOT NULL COMMENT 'executor id',
`history_cmd` text COMMENT 'history commands of process instance
operation',
`process_instance_priority` int(11) DEFAULT NULL COMMENT 'process instance
priority. 0 Highest,1 High,2 Medium,3 Low,4 Lowest',
`worker_group` varchar(64) DEFAULT NULL COMMENT 'worker group id',
`timeout` int(11) DEFAULT '0' COMMENT 'time out',
`tenant_id` int(11) NOT NULL DEFAULT '-1' COMMENT 'tenant id',
PRIMARY KEY (`id`),
KEY `process_instance_index` (`process_definition_code`,`id`) USING BTREE,
KEY `start_time_index` (`start_time`) USING BTREE
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
--任务实例表
CREATE TABLE `t_ds_task_instance` (
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'key',
`name` varchar(255) DEFAULT NULL COMMENT 'task name',
`version` int(11) DEFAULT NULL COMMENT 'task definition version',
`task_type` varchar(64) DEFAULT NULL COMMENT 'task type',
`process_definition_code` bigint(20) NOT NULL COMMENT 'process definition
code',
`process_instance_id` int(11) DEFAULT NULL COMMENT 'process instance id',
`task_json` longtext COMMENT 'task content json',
`state` tinyint(4) DEFAULT NULL COMMENT 'Status: 0 commit succeeded, 1
running, 2 prepare to pause, 3 pause, 4 prepare to stop, 5 stop, 6 fail, 7
succeed, 8 need fault tolerance, 9 kill, 10 wait for thread, 11 wait for
dependency to complete',
`submit_time` datetime DEFAULT NULL COMMENT 'task submit time',
`start_time` datetime DEFAULT NULL COMMENT 'task start time',
`end_time` datetime DEFAULT NULL COMMENT 'task end time',
`host` varchar(45) DEFAULT NULL COMMENT 'host of task running on',
`execute_path` varchar(200) DEFAULT NULL COMMENT 'task execute path in the
host',
`log_path` varchar(200) DEFAULT NULL COMMENT 'task log path',
`alert_flag` tinyint(4) DEFAULT NULL COMMENT 'whether alert',
`retry_times` int(4) DEFAULT '0' COMMENT 'task retry times',
`pid` int(4) DEFAULT NULL COMMENT 'pid of task',
`app_link` text COMMENT 'yarn app id',
`flag` tinyint(4) DEFAULT '1' COMMENT '0 not available, 1 available',
`retry_interval` int(4) DEFAULT NULL COMMENT 'retry interval when task
failed ',
`max_retry_times` int(2) DEFAULT NULL COMMENT 'max retry times',
`task_instance_priority` int(11) DEFAULT NULL COMMENT 'task instance
priority:0 Highest,1 High,2 Medium,3 Low,4 Lowest',
`worker_group` varchar(64) DEFAULT NULL COMMENT 'worker group id',
`executor_id` int(11) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `process_instance_id` (`process_instance_id`) USING BTREE,
KEY `task_instance_index` (`process_definition_id`,`process_instance_id`)
USING BTREE,
CONSTRAINT `foreign_key_instance_id` FOREIGN KEY (`process_instance_id`)
REFERENCES `t_ds_process_instance` (`id`) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
```
## 2、API operation logic
#### 1 save operation
In the same transaction, save to t_ds_process_definition and
t_ds_process_definition_log, task and relation tables are similar
#### 2 update operation
In the same transaction, modify t_ds_process_definition, and version + 1,
then insert the data from t_ds_process_definition into
t_ds_process_definition_log, task and relation tables are similar
#### 3 delete operation
Delete the data from t_ds_process_definition directly, task and relation
tables are similar
#### 4 switch operation
Overwrite current version from t_ds_process_definition_log into
t_ds_process_definition, task and relation tables are similar
## 3 code design
Code generation uses snowflake algorithm
## 4 version design
The version is displayed by default self-increasing, The first version is
the number 1, and the page display uses a V before the number, such as V1 and V2
## 5 the subtask
#### 1、process definition api
#### 2、task definition api
#### 3、relation definition api
#### 4、process instance and task instance api
#### 5、service module
#### 6、server module
#### 7、ui module
[process definition
json拆分方案.pdf](https://github.com/apache/incubator-dolphinscheduler/files/5794699/process.definition.json.pdf)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
