This is an automated email from the ASF dual-hosted git repository. gaojun2048 pushed a commit to branch alert_plugin_design in repository https://gitbox.apache.org/repos/asf/incubator-dolphinscheduler.git
commit 10eacaede97129d0cb0d557e0d4e2f6272f05dd8 Merge: a7aa58e e15d83c Author: gaojun2048 <[email protected]> AuthorDate: Tue Sep 8 15:22:07 2020 +0800 Merge branch 'break60-alert_plugin_design' into alert_plugin_design .github/ISSUE_TEMPLATE/bug_report.md | 2 + .github/ISSUE_TEMPLATE/feature_request.md | 2 + .github/ISSUE_TEMPLATE/improvement_suggestion.md | 2 + .github/ISSUE_TEMPLATE/question.md | 2 + .github/ISSUE_TEMPLATE/test.md | 2 + .github/workflows/ci_e2e.yml | 4 +- .github/workflows/ci_ut.yml | 27 + .gitignore | 1 + .../DOLPHIN/1.2.1/package/scripts/params.py | 4 +- .../1.2.1/package/templates/dolphin-daemon.j2 | 2 +- .../DOLPHIN/1.3.0/package/scripts/params.py | 4 +- .../1.3.0/package/templates/dolphin-daemon.sh.j2 | 2 +- docker/build/Dockerfile | 4 +- docker/build/README.md | 70 +- docker/build/README_zh_CN.md | 70 +- .../conf/dolphinscheduler/alert.properties.tpl | 14 +- .../conf/dolphinscheduler/common.properties.tpl | 12 +- .../dolphinscheduler/datasource.properties.tpl | 15 +- .../dolphinscheduler/logback/logback-alert.xml | 2 +- .../conf/dolphinscheduler/worker.properties.tpl | 5 +- .../conf/dolphinscheduler/zookeeper.properties.tpl | 2 +- docker/build/conf/nginx/dolphinscheduler.conf | 3 + docker/build/startup-init-conf.sh | 24 +- docker/build/startup.sh | 40 +- docker/docker-swarm/docker-compose.yml | 41 +- docker/docker-swarm/docker-stack.yml | 53 +- docker/kubernetes/dolphinscheduler/Chart.yaml | 2 +- docker/kubernetes/dolphinscheduler/README.md | 42 +- .../kubernetes/dolphinscheduler/requirements.yaml | 25 - .../dolphinscheduler/templates/NOTES.txt | 13 - .../dolphinscheduler/templates/_helpers.tpl | 8 + .../configmap-dolphinscheduler-alert.yaml | 1 + ...yaml => configmap-dolphinscheduler-common.yaml} | 26 +- .../configmap-dolphinscheduler-worker.yaml | 1 + .../deployment-dolphinscheduler-alert.yaml | 98 +- .../templates/deployment-dolphinscheduler-api.yaml | 99 +- .../deployment-dolphinscheduler-frontend.yaml | 17 + .../statefulset-dolphinscheduler-master.yaml | 99 +- .../statefulset-dolphinscheduler-worker.yaml | 174 ++- docker/kubernetes/dolphinscheduler/values.yaml | 88 +- .../alert/utils/EnterpriseWeChatUtils.java | 119 +- .../dolphinscheduler/alert/utils/MailUtils.java | 11 +- .../src/main/resources/alert.properties | 8 +- .../alert/utils/EnterpriseWeChatUtilsTest.java | 247 ++-- dolphinscheduler-api/pom.xml | 6 +- .../dolphinscheduler/api/ApiApplicationServer.java | 1 - .../api/controller/AccessTokenController.java | 30 +- .../api/controller/DataSourceController.java | 10 +- .../api/controller/LoggerController.java | 25 +- .../controller/ProcessDefinitionController.java | 334 +++-- .../api/controller/ProcessInstanceController.java | 42 +- .../api/controller/ProjectController.java | 20 + .../api/controller/SchedulerController.java | 4 +- .../api/controller/TenantController.java | 62 +- .../api/controller/UsersController.java | 56 +- .../api/controller/WorkFlowLineageController.java | 81 ++ .../dolphinscheduler/api/dto/TaskCountDto.java | 26 +- .../apache/dolphinscheduler/api/enums/Status.java | 330 ++--- .../api/exceptions/ApiExceptionHandler.java | 5 +- .../api/interceptor/LoginHandlerInterceptor.java | 14 +- .../api/service/AccessTokenService.java | 126 +- .../api/service/AlertGroupService.java | 14 +- .../api/service/BaseDAGService.java | 54 - .../dolphinscheduler/api/service/BaseService.java | 17 +- .../api/service/DataAnalysisService.java | 317 +---- .../api/service/DataSourceService.java | 163 +-- .../api/service/ExecutorService.java | 12 +- .../api/service/LoggerService.java | 109 +- .../api/service/MonitorService.java | 8 +- .../api/service/ProcessDefinitionService.java | 1435 ++------------------ .../service/ProcessDefinitionVersionService.java | 70 + .../api/service/ProcessInstanceService.java | 195 ++- .../api/service/ProjectService.java | 328 +---- .../dolphinscheduler/api/service/QueueService.java | 8 +- .../api/service/ResourcesService.java | 22 +- .../api/service/SchedulerService.java | 19 +- .../api/service/SessionService.java | 110 +- .../api/service/TaskInstanceService.java | 63 +- .../api/service/TenantService.java | 379 +----- .../api/service/UdfFuncService.java | 8 +- .../dolphinscheduler/api/service/UsersService.java | 136 +- .../api/service/WorkFlowLineageService.java | 97 ++ .../api/service/WorkerGroupService.java | 9 +- .../AccessTokenServiceImpl.java} | 39 +- .../DataAnalysisServiceImpl.java} | 245 ++-- .../api/service/impl/LoggerServiceImpl.java | 146 ++ .../ProcessDefinitionServiceImpl.java} | 637 ++++++--- .../impl/ProcessDefinitionVersionServiceImpl.java | 181 +++ .../ProjectServiceImpl.java} | 120 +- .../api/service/impl/SessionServiceImpl.java | 158 +++ .../api/service/impl/TenantServiceImpl.java | 331 +++++ .../dolphinscheduler/api/utils/ZooKeeperState.java | 1 + .../api/utils/ZookeeperMonitor.java | 5 +- .../src/main/resources/i18n/messages.properties | 12 +- .../main/resources/i18n/messages_en_US.properties | 10 +- .../main/resources/i18n/messages_zh_CN.properties | 11 +- .../ProcessDefinitionControllerTest.java | 264 ++-- .../controller/ProcessInstanceControllerTest.java | 82 +- .../api/controller/UsersControllerTest.java | 40 +- .../controller/WorkFlowLineageControllerTest.java | 69 + .../api/service/AccessTokenServiceTest.java | 109 +- .../api/service/AlertGroupServiceTest.java | 2 +- .../api/service/BaseDAGServiceTest.java | 50 - .../api/service/DataAnalysisServiceTest.java | 78 +- .../api/service/DataSourceServiceTest.java | 224 ++- .../api/service/ExecutorService2Test.java | 63 +- .../api/service/ExecutorServiceTest.java | 2 +- .../api/service/LoggerServiceTest.java | 44 +- .../api/service/ProcessDefinitionServiceTest.java | 919 ++++++++----- .../ProcessDefinitionVersionServiceTest.java | 274 ++++ .../api/service/ProcessInstanceServiceTest.java | 145 +- .../api/service/ProjectServiceTest.java | 217 +-- .../api/service/SchedulerServiceTest.java | 34 +- .../api/service/SessionServiceTest.java | 11 +- .../api/service/TaskInstanceServiceTest.java | 60 +- .../api/service/TenantServiceTest.java | 133 +- .../api/service/UsersServiceTest.java | 100 +- .../api/service/WorkFlowLineageServiceTest.java | 88 ++ .../api/service/WorkerGroupServiceTest.java | 4 +- dolphinscheduler-common/pom.xml | 5 + .../apache/dolphinscheduler/common/Constants.java | 35 +- .../dolphinscheduler/common/enums/AlertEvent.java | 7 +- .../common/enums/AlertWarnLevel.java | 7 +- .../dolphinscheduler/common/enums/DbType.java | 4 +- .../common/enums/ExecutionStatus.java | 111 +- .../common/enums/TaskStateType.java | 12 +- .../dolphinscheduler/common/enums/TaskType.java | 4 +- .../dolphinscheduler/common/model/TaskNode.java | 50 +- .../common/shell/AbstractShell.java | 4 +- .../common/thread/ThreadUtils.java | 4 +- .../dolphinscheduler/common/utils/CommonUtils.java | 46 +- .../dolphinscheduler/common/utils/DateUtils.java | 444 +++++- .../dolphinscheduler/common/utils/HadoopUtils.java | 9 +- .../dolphinscheduler/common/utils/HttpUtils.java | 177 ++- .../dolphinscheduler/common/utils/JSONUtils.java | 3 +- .../common/utils/KerberosHttpClient.java | 156 +++ .../common/utils/ParameterUtils.java | 2 +- .../common/utils/PropertyUtils.java | 10 + .../dolphinscheduler/common/utils/StringUtils.java | 4 + .../common/utils/TaskParametersUtils.java | 2 + .../dolphinscheduler/common/utils/TriFunction.java | 11 +- .../src/main/resources/common.properties | 6 +- .../ExecutionStatusTest.java} | 19 +- .../common/utils/CommonUtilsTest.java | 39 + .../common/utils/FileUtilsTest.java | 14 + .../common/utils/HttpUtilsTest.java | 61 +- .../common/utils/KerberosHttpClientTest.java | 46 + .../common/utils/PropertyUtilsTest.java | 2 +- .../org/apache/dolphinscheduler/dao/AlertDao.java | 85 +- .../dao/datasource/BaseDataSource.java | 8 +- .../dao/datasource/DataSourceFactory.java | 5 + .../dao/datasource/MySQLDataSource.java | 2 + .../dao/datasource/PrestoDataSource.java | 28 +- .../dao/datasource/SpringConnectionFactory.java | 14 + .../dao/entity/ProcessAlertContent.java | 236 ++++ .../dao/entity/ProcessDefinition.java | 32 +- .../dao/entity/ProcessDefinitionVersion.java | 329 +++++ .../dolphinscheduler/dao/entity/Project.java | 101 +- .../dao/entity/ServerAlertContent.java | 85 ++ .../dolphinscheduler/dao/entity/TaskInstance.java | 156 ++- .../dolphinscheduler/dao/entity/UdfFunc.java | 35 +- .../dao/entity/WorkFlowLineage.java | 94 ++ .../dao/entity/WorkFlowRelation.java | 26 +- .../dao/mapper/ProcessDefinitionMapper.java | 24 +- .../dao/mapper/ProcessDefinitionVersionMapper.java | 69 + .../dao/mapper/ProcessInstanceMapper.java | 13 + .../dao/mapper/WorkFlowLineageMapper.java | 33 +- .../dolphinscheduler/dao/utils/DagHelper.java | 36 +- .../dao/mapper/ProcessDefinitionMapper.xml | 6 + .../dao/mapper/ProcessDefinitionVersionMapper.xml | 47 + .../dao/mapper/ProcessInstanceMapper.xml | 10 + .../dao/mapper/WorkFlowLineageMapper.xml | 103 ++ .../dao/datasource/BaseDataSourceTest.java | 49 + .../dao/datasource/MySQLDataSourceTest.java | 35 + .../dao/entity/TaskInstanceTest.java | 32 + .../dolphinscheduler/dao/entity/UdfFuncTest.java | 64 + .../dao/mapper/AccessTokenMapperTest.java | 76 +- .../dao/mapper/AlertMapperTest.java | 1 - .../dao/mapper/CommandMapperTest.java | 1 - .../dao/mapper/DataSourceMapperTest.java | 1 - .../dao/mapper/ProcessDefinitionMapperTest.java | 54 +- .../mapper/ProcessDefinitionVersionMapperTest.java | 172 +++ .../dao/mapper/ProcessInstanceMapperTest.java | 87 +- .../dao/mapper/ResourceMapperTest.java | 85 +- .../dao/mapper/TaskInstanceMapperTest.java | 10 +- .../dao/mapper/WorkFlowLineageMapperTest.java | 62 + dolphinscheduler-dist/pom.xml | 1 + dolphinscheduler-dist/release-docs/LICENSE | 3 + dolphinscheduler-dist/release-docs/NOTICE | 7 + .../release-docs/licenses/LICENSE-presto-jdbc.txt | 201 +++ .../ui-licenses/LICENSE-@form-create-element-ui | 21 + .../licenses/ui-licenses/LICENSE-element-ui | 21 + .../src/main/assembly/dolphinscheduler-binary.xml | 1 + .../src/main/assembly/dolphinscheduler-nginx.xml | 1 + .../remote/NettyRemotingClient.java | 188 +-- .../remote/NettyRemotingServer.java | 135 +- .../remote/codec/NettyDecoder.java | 1 + .../dolphinscheduler/remote/utils/Constants.java | 10 + .../apache/dolphinscheduler/remote/utils/Host.java | 63 +- .../utils/{Constants.java => NettyUtils.java} | 41 +- .../dolphinscheduler/remote/NettyUtilTest.java | 23 +- .../builder/TaskExecutionContextBuilder.java | 2 + .../server/entity/SQLTaskExecutionContext.java | 4 +- .../server/entity/TaskExecutionContext.java | 171 ++- .../server/master/MasterServer.java | 4 +- .../cache/impl/TaskInstanceCacheManagerImpl.java | 2 +- .../master/dispatch/host/CommonHostManager.java | 7 +- .../master/dispatch/host/RandomHostManager.java | 2 +- .../dispatch/host/RoundRobinHostManager.java | 2 +- .../dispatch/host/assign/RandomSelector.java | 45 +- .../dispatch/host/assign/RoundRobinSelector.java | 120 +- .../server/master/future/TaskFuture.java | 21 +- .../processor/queue/TaskResponseService.java | 2 + .../server/master/registry/MasterRegistry.java | 59 +- .../master/runner/ConditionsTaskExecThread.java | 3 +- .../master/runner/DependentTaskExecThread.java | 3 +- .../master/runner/MasterBaseTaskExecThread.java | 21 +- .../server/master/runner/MasterExecThread.java | 119 +- .../master/runner/MasterSchedulerService.java | 37 +- .../server/master/runner/MasterTaskExecThread.java | 17 +- .../master/runner/SubProcessTaskExecThread.java | 2 +- .../server/registry/ZookeeperNodeManager.java | 4 +- .../server/utils/AlertManager.java | 160 +-- .../server/utils/ProcessUtils.java | 78 +- .../impl/TaskExecutionContextCacheManagerImpl.java | 2 +- .../server/worker/config/WorkerConfig.java | 12 + .../worker/processor/TaskExecuteProcessor.java | 39 +- .../server/worker/registry/WorkerRegistry.java | 28 +- .../server/worker/runner/TaskExecuteThread.java | 98 +- .../worker/task/AbstractCommandExecutor.java | 32 +- .../server/worker/task/TaskManager.java | 2 + .../server/worker/task/sql/SqlTask.java | 26 +- .../src/main/resources/config/install_config.conf | 12 +- .../src/main/resources/worker.properties | 3 + .../server/entity/SQLTaskExecutionContextTest.java | 189 +++ .../server/log/LoggerServerTest.java | 62 +- .../server/master/ConditionsTaskTest.java | 173 ++- .../server/master/DependentTaskTest.java | 2 +- .../server/master/MasterExecThreadTest.java | 21 +- .../consumer/TaskPriorityQueueConsumerTest.java | 24 +- .../master/dispatch/ExecutorDispatcherTest.java | 5 +- .../executor/NettyExecutorManagerTest.java | 3 +- .../dispatch/host/RoundRobinHostManagerTest.java | 4 +- .../dispatch/host/assign/RandomSelectorTest.java | 16 +- .../host/assign/RoundRobinSelectorTest.java | 51 +- .../processor/queue/TaskResponseServiceTest.java | 8 +- .../server/master/registry/MasterRegistryTest.java | 16 +- .../master/runner/MasterTaskExecThreadTest.java | 70 +- .../server/registry/ZookeeperNodeManagerTest.java | 12 +- .../server/utils/ProcessUtilsTest.java | 7 +- .../worker/processor/TaskCallbackServiceTest.java | 11 +- .../server/worker/registry/WorkerRegistryTest.java | 1 + .../worker/runner/TaskExecuteThreadTest.java | 171 +++ .../service/log/LogClientService.java | 37 +- .../dolphinscheduler/service/log/LogPromise.java | 1 + .../service/process/ProcessService.java | 115 +- .../service/quartz/QuartzExecutors.java | 23 +- .../service/zk/AbstractZKClient.java | 2 +- .../service/zk/CuratorZookeeperClient.java | 119 ++ .../service/zk/ZookeeperCachedOperator.java | 3 +- .../service/zk/ZookeeperOperator.java | 91 +- .../src/main/resources/logback-zookeeper.xml | 8 +- .../service/zk/CuratorZookeeperClientTest.java | 67 + dolphinscheduler-ui/build/config.js | 3 +- dolphinscheduler-ui/package.json | 6 +- dolphinscheduler-ui/src/js/conf/home/index.js | 8 + .../src/js/conf/home/pages/dag/_source/config.js | 13 +- .../src/js/conf/home/pages/dag/_source/dag.scss | 9 +- .../src/js/conf/home/pages/dag/_source/dag.vue | 216 ++- .../pages/dag/_source/formModel/formLineModel.vue | 122 ++ .../home/pages/dag/_source/formModel/formModel.vue | 54 +- .../_source/formModel/tasks/_source/nodeStatus.vue | 6 +- .../pages/dag/_source/formModel/tasks/flink.vue | 4 +- .../home/pages/dag/_source/formModel/tasks/mr.vue | 4 +- .../pages/dag/_source/formModel/tasks/python.vue | 2 +- .../pages/dag/_source/formModel/tasks/shell.vue | 2 +- .../pages/dag/_source/formModel/tasks/spark.vue | 4 +- .../formModel/tasks/{shell.vue => waterdrop.vue} | 306 +++-- .../home/pages/dag/_source/jumpAffirm/index.js | 2 +- .../home/pages/dag/_source/plugIn/jsPlumbHandle.js | 60 +- .../js/conf/home/pages/dag/_source/plugIn/util.js | 18 +- .../home/pages/dag/_source/startingParam/index.vue | 5 - .../src/js/conf/home/pages/dag/_source/udp/udp.vue | 12 +- .../conf/home/pages/dag/img/toolbar_WATERDROP.png | Bin 0 -> 1675 bytes .../pages/list/_source/createDataSource.vue | 7 + .../home/pages/monitor/pages/servers/master.vue | 2 +- .../home/pages/monitor/pages/servers/worker.vue | 2 +- .../src/js/conf/home/pages/projects/index.vue | 4 +- .../pages/_source/instanceConditions/common.js | 5 +- .../pages/definition/pages/list/_source/list.vue | 125 +- .../pages/definition/pages/list/_source/start.vue | 6 - .../pages/definition/pages/list/_source/timing.vue | 6 - .../definition/pages/list/_source/versions.vue | 255 ++++ .../pages/index/_source/taskStatusCount.vue | 147 ++ .../conf/home/pages/projects/pages/index/index.vue | 8 +- .../pages/instance/pages/list/_source/list.vue | 4 +- .../projects/pages/kinship/_source/graphGrid.vue | 63 + .../pages/kinship/_source/graphGridOption.js | 145 ++ .../projects/pages/kinship/_source/img/dag_bg.png | Bin 0 -> 5101 bytes .../home/pages/projects/pages/kinship/index.vue | 169 +++ .../pages/udf/pages/function/_source/createUdf.vue | 4 +- .../security/pages/warningGroups/_source/list.vue | 51 +- .../src/js/conf/home/router/index.js | 8 + .../src/js/conf/home/store/dag/actions.js | 67 +- .../src/js/conf/home/store/dag/mutations.js | 8 + .../src/js/conf/home/store/dag/state.js | 5 + .../src/js/conf/home/store/datasource/actions.js | 4 +- .../src/js/conf/home/store/index.js | 2 + .../src/js/conf/home/store/kinship/actions.js | 89 ++ .../src/js/conf/home/store/kinship/getters.js | 7 +- .../src/js/conf/home/store/kinship/index.js | 16 +- .../src/js/conf/home/store/kinship/mutations.js | 9 +- .../src/js/conf/home/store/kinship/state.js | 11 +- .../src/js/conf/home/store/projects/actions.js | 2 +- dolphinscheduler-ui/src/js/conf/login/index.js | 8 + .../components/secondaryMenu/_source/menu.js | 11 +- .../src/js/module/components/transfer/resource.vue | 4 +- .../src/js/module/i18n/locale/en_US.js | 32 +- .../src/js/module/i18n/locale/zh_CN.js | 30 +- dolphinscheduler-ui/src/sass/common/index.scss | 4 + .../dolphinscheduler/common/BrowserCommon.java | 44 +- .../ProcessInstanceData.java} | 9 +- .../{CreateProjectData.java => ProjectData.java} | 5 +- .../dolphinscheduler/data/project/TimingData.java | 5 +- ...teWorkflowData.java => WorkflowDefineData.java} | 14 +- .../data/security/AlertManageData.java | 14 +- .../data/security/QueueManageData.java | 6 +- .../data/security/TokenManageData.java | 4 + .../data/security/UserManageData.java | 3 + .../ProcessInstanceLocator.java} | 23 +- ...eateProjectLocator.java => ProjectLocator.java} | 12 +- .../locator/project/TimingLocator.java | 16 +- ...flowLocator.java => WorkflowDefineLocator.java} | 29 +- .../locator/security/AlertManageLocator.java | 3 +- .../locator/security/QueueManageLocator.java | 2 + .../locator/security/TenantManageLocator.java | 2 + .../locator/security/TokenManageLocator.java | 4 + .../locator/security/UserManageLocator.java | 6 +- .../page/project/CreateWorkflowPage.java | 226 --- .../page/project/ProcessInstancePage.java | 39 +- .../{CreateProjectPage.java => ProjectPage.java} | 30 +- .../page/project/RunWorkflowPage.java | 8 +- .../dolphinscheduler/page/project/TimingPage.java | 43 +- .../page/project/WorkflowDefinePage.java | 242 ++++ .../page/security/AlertManagePage.java | 17 +- .../page/security/QueueManagePage.java | 23 +- .../page/security/TenantManagePage.java | 6 +- .../page/security/TokenManagePage.java | 35 +- .../page/security/UserManagePage.java | 16 +- .../testcase/testDeleteData/TestDeleteProject.java | 6 +- .../testDeleteData/TestDeleteWorkflow.java | 26 +- ...CreateProject.java => TestProcessInstance.java} | 25 +- .../{TestCreateProject.java => TestProject.java} | 12 +- .../testcase/testProject/TestRunWorkflow.java | 10 +- ...CreateWorkflow.java => TestWorkflowDefine.java} | 36 +- .../testcase/testSecurity/TestTokenManage.java | 2 +- .../testcase/testSecurity/TestUserManage.java | 2 +- e2e/testng.xml | 41 +- install.sh | 7 +- pom.xml | 37 +- .../dolphinscheduler-ui/.gitignore | 6 + script/dolphinscheduler-daemon.sh | 8 +- script/remove-zk-node.sh | 8 +- script/scp-hosts.sh | 8 +- script/status-all.sh | 41 + .../mysql/dolphinscheduler_dml.sql | 2 +- .../postgresql/dolphinscheduler_dml.sql | 2 +- sql/dolphinscheduler-postgre.sql | 33 +- sql/dolphinscheduler_mysql.sql | 31 +- .../1.2.0_schema/mysql/dolphinscheduler_dml.sql | 3 +- .../1.3.0_schema/mysql/dolphinscheduler_ddl.sql | 1 - .../1.3.0_schema/mysql/dolphinscheduler_dml.sql | 3 +- .../postgresql/dolphinscheduler_dml.sql | 3 +- .../1.3.3_schema/mysql/dolphinscheduler_ddl.sql | 89 ++ .../1.3.3_schema/mysql/dolphinscheduler_dml.sql | 10 +- .../postgresql/dolphinscheduler_ddl.sql | 82 ++ .../postgresql/dolphinscheduler_dml.sql | 10 +- style/checkstyle.xml | 66 +- style/intellij-java-code-style.xml | 101 ++ tools/dependencies/known-dependencies.txt | 1 + 380 files changed, 14561 insertions(+), 7121 deletions(-)
